c8ctl-plugin-nano 1.70.2 → 1.70.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/agent-instance.mjs +180 -22
- package/c8ctl-plugin.js +1534 -78
- package/package.json +8 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -95,6 +95,110 @@ import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecret
|
|
|
95
95
|
const requireFromHere = createRequire(import.meta.url);
|
|
96
96
|
const pluginDir = dirname(fileURLToPath(import.meta.url));
|
|
97
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Build the ordered list of `require` resolvers used to read the HOST c8ctl / Camunda
|
|
100
|
+
* SDK package manifests (issue #289 review). The host SDK lives in the host **c8ctl
|
|
101
|
+
* CLI's** dependency graph, NOT this plugin's. In the managed layout the plugin loads
|
|
102
|
+
* from `<data>/plugins/node_modules`, a SIBLING of `@camunda8/cli`; in the global
|
|
103
|
+
* layout the SDK is nested BENEATH the CLI package. Node resolution never searches a
|
|
104
|
+
* sibling package's `node_modules`, so a probe rooted only at this module commonly
|
|
105
|
+
* reads `null` for both versions — exactly when it needs to name the old host SDK.
|
|
106
|
+
* Resolve from the host graph instead:
|
|
107
|
+
* 1. a `require` rooted at the real c8ctl entry (`C8CTL_NANO_ENTRY` / `process.argv[1]`),
|
|
108
|
+
* canonicalized through symlinks — the CLI and its nested SDK live in THIS graph;
|
|
109
|
+
* 2. a `require` rooted at the resolved `@camunda8/cli` package itself, so an SDK
|
|
110
|
+
* nested beneath the CLI (not hoisted to the entry root) still resolves;
|
|
111
|
+
* 3. this module's own graph as a last-resort fallback (works in a hoisted/global
|
|
112
|
+
* layout where the SDK is visible from here).
|
|
113
|
+
* Every step is guarded: an unresolvable root is skipped, never thrown.
|
|
114
|
+
*
|
|
115
|
+
* The entry is `realpathSync`'d first because a globally installed CLI is launched
|
|
116
|
+
* through a launcher symlink (e.g. `/usr/local/bin/c8ctl` → `.../lib/node_modules/@camunda8/cli/bin/...`),
|
|
117
|
+
* which Node preserves verbatim in `process.argv[1]`. Rooting `createRequire` at the
|
|
118
|
+
* raw symlink would search `/usr/local/bin/node_modules` and miss the nested host SDK
|
|
119
|
+
* entirely — exactly the global layout this probe exists to diagnose.
|
|
120
|
+
*/
|
|
121
|
+
export function buildHostSdkResolvers(entryArg) {
|
|
122
|
+
const resolvers = [];
|
|
123
|
+
const seen = new Set();
|
|
124
|
+
const push = (rq) => {
|
|
125
|
+
if (typeof rq === 'function' && !seen.has(rq)) {
|
|
126
|
+
seen.add(rq);
|
|
127
|
+
resolvers.push(rq);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
const rawEntry = entryArg !== undefined ? entryArg : (process.env.C8CTL_NANO_ENTRY || process.argv[1]);
|
|
131
|
+
let entry = rawEntry;
|
|
132
|
+
if (entry) {
|
|
133
|
+
try {
|
|
134
|
+
entry = realpathSync(entry);
|
|
135
|
+
} catch {
|
|
136
|
+
/* not a real path yet (or already canonical) — fall back to the raw entry */
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
let entryRequire = null;
|
|
140
|
+
if (entry) {
|
|
141
|
+
try {
|
|
142
|
+
entryRequire = createRequire(entry);
|
|
143
|
+
push(entryRequire);
|
|
144
|
+
} catch {
|
|
145
|
+
/* unresolvable entry — skip */
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const rq of [entryRequire, requireFromHere]) {
|
|
149
|
+
if (!rq) continue;
|
|
150
|
+
try {
|
|
151
|
+
const cliPkg = rq.resolve('@camunda8/cli/package.json');
|
|
152
|
+
push(createRequire(cliPkg));
|
|
153
|
+
} catch {
|
|
154
|
+
/* CLI not resolvable from this root — skip */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
push(requireFromHere);
|
|
158
|
+
return resolvers;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Best-effort host c8ctl / Camunda SDK versions (issue #289). The host SDK lives in the
|
|
163
|
+
* host c8ctl CLI's dependency graph, which this plugin's (sibling) `node_modules` cannot
|
|
164
|
+
* reach, so the probe walks an ordered chain of resolvers rooted at the host CLI entry /
|
|
165
|
+
* package first and this module only as a fallback (see `buildHostSdkResolvers`) — the
|
|
166
|
+
* first resolver to yield a version wins. Every read is guarded: an unresolvable package
|
|
167
|
+
* (hoisting layout, bundling) simply yields null rather than throwing. Tests may inject a
|
|
168
|
+
* single `requireFn` (used as the sole resolver) or an explicit `resolvers` array.
|
|
169
|
+
*/
|
|
170
|
+
export function probeHostSdkInfo(requireFn, resolvers) {
|
|
171
|
+
const chain = Array.isArray(resolvers) && resolvers.length > 0
|
|
172
|
+
? resolvers
|
|
173
|
+
: requireFn
|
|
174
|
+
? [requireFn]
|
|
175
|
+
: buildHostSdkResolvers();
|
|
176
|
+
const read = (spec) => {
|
|
177
|
+
for (const rq of chain) {
|
|
178
|
+
try {
|
|
179
|
+
const v = rq(`${spec}/package.json`)?.version;
|
|
180
|
+
if (typeof v === 'string' && v.trim() !== '') return String(v);
|
|
181
|
+
} catch {
|
|
182
|
+
/* try the next resolver in the chain */
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return null;
|
|
186
|
+
};
|
|
187
|
+
return {
|
|
188
|
+
cliVersion: read('@camunda8/cli'),
|
|
189
|
+
sdkVersion: read('@camunda8/orchestration-cluster-api'),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Render host SDK versions for a log line (issue #289). Never throws. */
|
|
194
|
+
export function formatHostSdkInfo(info) {
|
|
195
|
+
const cli = info?.cliVersion ? `@camunda8/cli ${info.cliVersion}` : '@camunda8/cli (version unknown)';
|
|
196
|
+
const sdk = info?.sdkVersion
|
|
197
|
+
? `@camunda8/orchestration-cluster-api ${info.sdkVersion}`
|
|
198
|
+
: '@camunda8/orchestration-cluster-api (version unknown)';
|
|
199
|
+
return `${cli}, ${sdk}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
98
202
|
/**
|
|
99
203
|
* Read the bundled-binary marker (nanobpmn-binary.json) written by the upstream
|
|
100
204
|
* release pipeline. Records which nanobpmn build the shipped binaries came from.
|
|
@@ -352,6 +456,22 @@ function activityMarkerReadyFor(act, pid) {
|
|
|
352
456
|
return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
|
|
353
457
|
}
|
|
354
458
|
|
|
459
|
+
/**
|
|
460
|
+
* Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
|
|
461
|
+
* All three must hold (#293): the worker exists, its PID is alive, AND its
|
|
462
|
+
* activity marker proves its SIGUSR1 reload listener is installed (a finite
|
|
463
|
+
* `readyAt` + matching pid, via activityMarkerReadyFor). workAgent installs that
|
|
464
|
+
* listener only once its activation loop is up, which is the same handshake that
|
|
465
|
+
* stamps `readyAt`. Signalling a freshly spawned worker BEFORE it installs the
|
|
466
|
+
* handler delivers SIGUSR1 to Node's DEFAULT action — starting the inspector
|
|
467
|
+
* instead of reloading — so a not-yet-ready worker is skipped (it re-reads the
|
|
468
|
+
* fresh token at its own startup prime anyway). Pure so it can be unit-tested.
|
|
469
|
+
*/
|
|
470
|
+
function reloadCredentialsReady(worker, activity, pidAlive) {
|
|
471
|
+
if (!worker || !pidAlive) return false;
|
|
472
|
+
return activityMarkerReadyFor(activity, worker.pid);
|
|
473
|
+
}
|
|
474
|
+
|
|
355
475
|
/**
|
|
356
476
|
* Deterministic control-socket path shared by the daemon and every client.
|
|
357
477
|
* Derived from a hash of the (possibly overridden) state home so distinct
|
|
@@ -4451,10 +4571,28 @@ function makeSecretResolver(kind) {
|
|
|
4451
4571
|
return null; // unknown → caller reports a clear error
|
|
4452
4572
|
}
|
|
4453
4573
|
|
|
4574
|
+
// Force a fresh `gh auth token` read, bypassing the TTL cache, and
|
|
4575
|
+
// return the token (or null). Injectable into resolveJobSecrets so a default
|
|
4576
|
+
// github gh-cache MISS re-reads at job-resolve time — catching a `gh auth login`
|
|
4577
|
+
// the worker-lifetime refresh timer has not re-primed yet — instead of trusting
|
|
4578
|
+
// provisionRepo's (non-universal) auth-failure recovery (Copilot review on #294).
|
|
4579
|
+
// ASYNC (spawn, never spawnSync): resolveJobSecrets runs on the activated-job
|
|
4580
|
+
// path after recordJobStart, so a SYNCHRONOUS `gh auth token` + scopes probe
|
|
4581
|
+
// here would block the shared event loop (and the lock-extension heartbeat) for
|
|
4582
|
+
// up to ~15s — the very hot-path stall the async refresh exists to prevent
|
|
4583
|
+
// (Copilot review on #294). refreshGhAuthTokenAsync also defers to any refresh
|
|
4584
|
+
// already in flight instead of starting a second concurrent read.
|
|
4585
|
+
async function defaultForceRefreshGhAuthToken() {
|
|
4586
|
+
const r = await refreshGhAuthTokenAsync({ force: true });
|
|
4587
|
+
return (r && r.token) || null;
|
|
4588
|
+
}
|
|
4589
|
+
|
|
4454
4590
|
// Resolve the names a job needs (setup.secretRefs, plus the repo/PR credential
|
|
4455
4591
|
// when allowPr). Returns resolved values + a list of names that were missing so
|
|
4456
|
-
// the caller can fail the job with a clear provisioning error.
|
|
4457
|
-
|
|
4592
|
+
// the caller can fail the job with a clear provisioning error. ASYNC: the
|
|
4593
|
+
// gh-cache-miss force-refresh awaits a spawned (non-blocking) gh read, so the
|
|
4594
|
+
// caller must `await` this (the workAgent job runner already is async).
|
|
4595
|
+
async function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli, forceRefreshGhAuthToken = defaultForceRefreshGhAuthToken } = {}) {
|
|
4458
4596
|
const names = new Set();
|
|
4459
4597
|
for (const n of envelope.setup?.secretRefs || []) if (n) names.add(n);
|
|
4460
4598
|
const resolved = {};
|
|
@@ -4466,7 +4604,10 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
|
|
|
4466
4604
|
}
|
|
4467
4605
|
// The github clone/push credential is resolved with a gh-CLI fallback so a
|
|
4468
4606
|
// default GITHUB_TOKEN isn't reported "missing" merely because it isn't in the
|
|
4469
|
-
// env when `gh auth login` provides it. A
|
|
4607
|
+
// env when `gh auth login` provides it. A default github credential whose gh
|
|
4608
|
+
// cache is a MISS is force-re-read here (bypassing the TTL cache); it is
|
|
4609
|
+
// reported missing only when that forced read ALSO finds no token. A custom
|
|
4610
|
+
// authRef stays strict.
|
|
4470
4611
|
if (envelope.task?.allowPr) {
|
|
4471
4612
|
const provider = envelope.repository?.provider || 'github';
|
|
4472
4613
|
const authRef = envelope.repository?.authRef;
|
|
@@ -4476,16 +4617,35 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
|
|
|
4476
4617
|
// so provisioning sheds rather than silently borrowing the default/gh token.
|
|
4477
4618
|
if (!missing.includes('repository.authRef')) missing.push('repository.authRef');
|
|
4478
4619
|
} else {
|
|
4479
|
-
const ghAuthRef =
|
|
4480
|
-
? ref.name
|
|
4481
|
-
: (provider === 'github' ? 'GITHUB_TOKEN' : undefined);
|
|
4620
|
+
const ghAuthRef = ghCredentialEnvName({ provider, authRef });
|
|
4482
4621
|
if (ghAuthRef) {
|
|
4483
4622
|
names.add(ghAuthRef);
|
|
4484
|
-
const token =
|
|
4623
|
+
const { token, source } = githubCloneTokenWithSource({ provider, authRef, secretResolver: resolver, ghAuthToken });
|
|
4485
4624
|
const missingIdx = missing.indexOf(ghAuthRef);
|
|
4486
4625
|
if (token) {
|
|
4487
4626
|
resolved[ghAuthRef] = token;
|
|
4488
4627
|
if (missingIdx !== -1) missing.splice(missingIdx, 1);
|
|
4628
|
+
} else if (source === 'gh-cache') {
|
|
4629
|
+
// A DEFAULT github credential whose gh-CLI cache is a MISS (null).
|
|
4630
|
+
// provisionRepo's null → fresh auth-failure recovery is NOT universal:
|
|
4631
|
+
// container jobs never call provisionRepo, and an anonymous clone of a
|
|
4632
|
+
// PUBLIC repo SUCCEEDS (no auth failure → no forced re-read), so the
|
|
4633
|
+
// harness would otherwise reach an `allowPr` job with GITHUB_TOKEN
|
|
4634
|
+
// ABSENT instead of failing early. FORCE a fresh `gh auth token` read
|
|
4635
|
+
// HERE (catching a `gh auth login` the worker-lifetime refresh timer
|
|
4636
|
+
// has not re-primed yet): populate + drop from `missing` only if a
|
|
4637
|
+
// token now exists; otherwise KEEP it missing so the job fails with a
|
|
4638
|
+
// clear provisioning error rather than silently handing the harness no
|
|
4639
|
+
// credential (Copilot review on #294). Priming the cache here also
|
|
4640
|
+
// feeds provisioning's own re-resolve the same fresh token. A
|
|
4641
|
+
// custom/invalid authRef reports source 'none' and still fails strict.
|
|
4642
|
+
const fresh = await forceRefreshGhAuthToken();
|
|
4643
|
+
if (fresh) {
|
|
4644
|
+
resolved[ghAuthRef] = fresh;
|
|
4645
|
+
if (missingIdx !== -1) missing.splice(missingIdx, 1);
|
|
4646
|
+
} else if (missingIdx === -1) {
|
|
4647
|
+
missing.push(ghAuthRef);
|
|
4648
|
+
}
|
|
4489
4649
|
} else if (missingIdx === -1) {
|
|
4490
4650
|
missing.push(ghAuthRef);
|
|
4491
4651
|
}
|
|
@@ -4779,42 +4939,690 @@ function credArgs() {
|
|
|
4779
4939
|
// a trimmed token, or null when gh is missing / not logged in. The token is fed to
|
|
4780
4940
|
// git via GIT_ASKPASS only (never argv/URL/helper), preserving the ephemeral-token
|
|
4781
4941
|
// guarantee.
|
|
4782
|
-
// Memoized for the process lifetime: this is a synchronous spawnSync (up to a
|
|
4783
|
-
// 10s timeout) that can be reached per job, so consult the CLI at most once per
|
|
4784
|
-
// worker run rather than blocking a handler on every job. A sentinel
|
|
4785
|
-
// distinguishes "not yet computed" from
|
|
4786
|
-
// a cached null (gh missing / not logged in).
|
|
4787
4942
|
//
|
|
4788
|
-
//
|
|
4789
|
-
//
|
|
4790
|
-
//
|
|
4791
|
-
//
|
|
4792
|
-
//
|
|
4793
|
-
//
|
|
4943
|
+
// Caching (issue #293): the lookup is a synchronous spawnSync (up to a 10s
|
|
4944
|
+
// timeout), so it cannot run on the per-job hot path — but a PROCESS-LIFETIME
|
|
4945
|
+
// memo goes stale the moment the operator refreshes the gh login (re-login, a new
|
|
4946
|
+
// scope, a rotation), and nothing told the worker (or the operator) it was now
|
|
4947
|
+
// handing every job the OLD token. The cache is therefore a RECORD
|
|
4948
|
+
// (`{ token, fingerprint, scopes, since, lastCheck }`) with a TTL:
|
|
4949
|
+
// - `ghAuthTokenFromCli()` (the per-job hot path) serves the PRIMED cache
|
|
4950
|
+
// record VERBATIM — even once `lastCheck` is older than
|
|
4951
|
+
// GH_AUTH_TOKEN_TTL_MS — and NEVER triggers a synchronous TTL re-read, so a
|
|
4952
|
+
// job never pays `gh auth token`'s spawn cost (Copilot review on #293). The
|
|
4953
|
+
// sole synchronous read is for a still-UNSET cache (a worker that never
|
|
4954
|
+
// primed), and even that defers to any async refresh already in flight. The
|
|
4955
|
+
// off-path refreshers below keep a primed record current.
|
|
4956
|
+
// - `nano work` PRIMES at startup (`primeGhAuthToken()`) AND re-primes on a
|
|
4957
|
+
// timer (`startGhAuthTokenRefresh`) so the TTL re-read happens OFF the job
|
|
4958
|
+
// path. The timer's refresh is ASYNC (spawn, never spawnSync) and DEFERRED
|
|
4959
|
+
// while a job is active, so it can never block the event loop / lock
|
|
4960
|
+
// heartbeat (Copilot review on #293) — a refreshed login is therefore
|
|
4961
|
+
// adopted within one TTL only when the worker is idle at a tick. Under
|
|
4962
|
+
// sustained back-to-back load the defer is BOUNDED by
|
|
4963
|
+
// GH_AUTH_TOKEN_MAX_DEFER_MS, after which the tick refreshes DESPITE an
|
|
4964
|
+
// active job (still async) to cap staleness at a few intervals. When the
|
|
4965
|
+
// identity changes it republishes the activity marker via an onChange
|
|
4966
|
+
// callback.
|
|
4967
|
+
// - A git/gh auth FAILURE (clone/push/PR-reconcile — see isGitAuthFailure)
|
|
4968
|
+
// forces one re-read (`refreshGhAuthToken({ force, budgetMs })`, bounded by
|
|
4969
|
+
// the caller's remaining lease budget so the synchronous re-read can't
|
|
4970
|
+
// overrun the lock heartbeat) and a single retry, so a rotated token
|
|
4971
|
+
// self-heals instead of failing the job. The re-read is
|
|
4972
|
+
// NON-destructive (it never invalidates the cache first), so a transient
|
|
4973
|
+
// re-read failure keeps the still-valid cached credential.
|
|
4974
|
+
// - `primeGhAuthToken({ logger })` logs the token's FINGERPRINT (a short sha256
|
|
4975
|
+
// prefix — never the value) and its OAuth scopes at startup and whenever the
|
|
4976
|
+
// token changes, so "this worker has a stale token" is a one-line check; the
|
|
4977
|
+
// same record rides the supervisor activity marker (`ghAuthTokenInfo()`) into
|
|
4978
|
+
// `supervisor status`, and `supervisor reload-credentials` re-reads it
|
|
4979
|
+
// fleet-wide (async; the old token is served until the new one lands)
|
|
4980
|
+
// without restarting workers.
|
|
4981
|
+
// A sentinel distinguishes "not yet computed" from a cached null (gh missing /
|
|
4982
|
+
// not logged in).
|
|
4794
4983
|
const GH_AUTH_TOKEN_UNSET = Symbol('gh-auth-token-unset');
|
|
4984
|
+
// How long a cached gh token is trusted before a re-read. 5 minutes bounds the
|
|
4985
|
+
// staleness window after a `gh auth login`/`refresh` while keeping `gh auth
|
|
4986
|
+
// token` (a cheap keychain read) off the per-job path. Override with
|
|
4987
|
+
// NANO_GH_TOKEN_TTL_MS (0 = re-read on every lookup, honouring the spawn cost).
|
|
4988
|
+
const GH_AUTH_TOKEN_TTL_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_TTL_MS, 5 * 60_000));
|
|
4989
|
+
// Upper bound on how long the maintenance timer may KEEP DEFERRING its refresh
|
|
4990
|
+
// while jobs run back-to-back. The tick normally defers while a job is active
|
|
4991
|
+
// (`isIdle`) so it never even spawns `gh` mid-job — but that defer is only an
|
|
4992
|
+
// optimization, because the refresh is async (spawn, never spawnSync) and so
|
|
4993
|
+
// cannot block the event loop / lock heartbeat even when it DOES run mid-job.
|
|
4994
|
+
// Under SUSTAINED saturation the worker is never idle at a tick, so an unbounded
|
|
4995
|
+
// defer would leave an expired, non-null cache reaching every subsequent job for
|
|
4996
|
+
// the worker's whole busy lifetime — defeating issue #293's "keep the token
|
|
4997
|
+
// fresh for the worker's lifetime" goal (Copilot review on #294). Bound it: once
|
|
4998
|
+
// a PRIMED cache is stale beyond this window, the tick refreshes DESPITE an
|
|
4999
|
+
// active job (still async/non-blocking), capping staleness at a few intervals
|
|
5000
|
+
// instead of the busy lifetime. Derived from the TTL (4 intervals) and overridable
|
|
5001
|
+
// via NANO_GH_TOKEN_MAX_DEFER_MS. Priming an UNSET cache is NEVER forced mid-job
|
|
5002
|
+
// (the defer there still holds), so a worker that starts logged-out stays anonymous
|
|
5003
|
+
// until it goes idle, exactly as before.
|
|
5004
|
+
const GH_AUTH_TOKEN_MAX_DEFER_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_MAX_DEFER_MS, 4 * GH_AUTH_TOKEN_TTL_MS));
|
|
5005
|
+
// The gh subprocesses below must never block on an interactive prompt inside a
|
|
5006
|
+
// worker (a missing/expired login would otherwise hang the event loop for the
|
|
5007
|
+
// full timeout). GH_PROMPT_DISABLED fails fast instead; GH_NO_UPDATE_NOTIFIER
|
|
5008
|
+
// keeps gh's upgrade check out of the captured output.
|
|
5009
|
+
const GH_NONINTERACTIVE_ENV = () => ({ ...process.env, GH_PROMPT_DISABLED: '1', GH_NO_UPDATE_NOTIFIER: '1' });
|
|
4795
5010
|
let ghAuthTokenCache = GH_AUTH_TOKEN_UNSET;
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
5011
|
+
// A Promise while an ASYNC gh-token refresh (the periodic timer tick or the
|
|
5012
|
+
// `reload-credentials` SIGUSR1 handler) is actually spawning `gh` subprocesses;
|
|
5013
|
+
// null otherwise. This is the serialization point that keeps a refresh off the
|
|
5014
|
+
// job path (Copilot review on #293): while it is set, the SYNCHRONOUS per-job
|
|
5015
|
+
// lookup (`ghAuthTokenFromCli`) must NOT start its own blocking `spawnSync`
|
|
5016
|
+
// re-read — it hands out the currently-cached token and lets the in-flight
|
|
5017
|
+
// async refresh update the cache, so an async refresh overlapping a freshly
|
|
5018
|
+
// activated job can never degrade into a 15s event-loop/lease-heartbeat stall.
|
|
5019
|
+
// It also prevents the timer and the SIGUSR1 handler from racing two concurrent
|
|
5020
|
+
// reads: a second async refresh awaits the first instead of spawning again.
|
|
5021
|
+
let ghAuthRefreshInFlight = null;
|
|
5022
|
+
// Monotonic generation counter bumped on EVERY write to `ghAuthTokenCache`
|
|
5023
|
+
// (`writeGhAuthCache` below is the single mutation point). It fences a stale
|
|
5024
|
+
// async refresh against a newer forced recovery (Copilot review on #293): the
|
|
5025
|
+
// synchronous clone/push auth-failure recovery (a forced `refreshGhAuthToken({
|
|
5026
|
+
// force })`) CANNOT await an already-running async timer
|
|
5027
|
+
// refresh (sync code can't await the `ghAuthRefreshInFlight` promise). So an
|
|
5028
|
+
// async refresh can read the OLD token, yield across its scopes probe, the sync
|
|
5029
|
+
// recovery can adopt the ROTATED token in the meantime, and then the async
|
|
5030
|
+
// refresh's late `adoptGhAuthTokenAsync` would overwrite the cache with its
|
|
5031
|
+
// now-stale read — resurrecting the revoked credential. `refreshGhAuthTokenAsync`
|
|
5032
|
+
// snapshots this generation BEFORE it starts reading and `adoptGhAuthTokenAsync`
|
|
5033
|
+
// refuses to commit when the generation has advanced (a newer definitive write
|
|
5034
|
+
// landed), so an older read can never clobber a newer forced recovery.
|
|
5035
|
+
let ghAuthCacheGeneration = 0;
|
|
5036
|
+
// The single mutation point for `ghAuthTokenCache`. Every write (adopt, preserve,
|
|
5037
|
+
// invalidate/reset) goes through here so the generation fence above sees them all.
|
|
5038
|
+
function writeGhAuthCache(record) {
|
|
5039
|
+
ghAuthTokenCache = record;
|
|
5040
|
+
ghAuthCacheGeneration++;
|
|
5041
|
+
return record;
|
|
5042
|
+
}
|
|
5043
|
+
// Test-only seam: the `gh auth token` reader `refreshGhAuthToken` consults when
|
|
5044
|
+
// no per-call `run` is injected. Production leaves this unset (the real
|
|
5045
|
+
// spawnSync of the gh CLI); a test installs a stub so the clone/push
|
|
5046
|
+
// invalidation-and-retry paths — which refresh through the MODULE cache, not a
|
|
5047
|
+
// per-call `run` — can rotate the token deterministically. Always a function
|
|
5048
|
+
// returning a trimmed token-or-null; reset with `resetGhAuthTokenReader()`.
|
|
5049
|
+
let ghAuthTokenReader = null;
|
|
5050
|
+
function setGhAuthTokenReader(fn) { ghAuthTokenReader = typeof fn === 'function' ? fn : null; }
|
|
5051
|
+
function resetGhAuthTokenReader() { ghAuthTokenReader = null; }
|
|
5052
|
+
|
|
5053
|
+
// Sentinel distinguishing a TRANSIENT `gh auth token` read failure from a
|
|
5054
|
+
// definitive logout (Copilot review on #293). The readers used to collapse a
|
|
5055
|
+
// timeout / spawn error / unrecognised nonzero exit into `null`, which the
|
|
5056
|
+
// refresh path then adopted as a real "logged out" change — discarding a
|
|
5057
|
+
// previously usable cached token, stamping the cache fresh with `null`, and
|
|
5058
|
+
// dropping the worker (and its clone-auth recovery) to anonymous until the next
|
|
5059
|
+
// interval. A read that returns this sentinel instead means "could not
|
|
5060
|
+
// determine"; `adoptGhAuthToken*` preserves the prior credential rather than
|
|
5061
|
+
// wiping it, and only a DEFINITIVE result (a real token, or a confirmed logout)
|
|
5062
|
+
// ever changes the cache.
|
|
5063
|
+
const GH_AUTH_READ_FAILED = Symbol('gh-auth-read-failed');
|
|
5064
|
+
// gh's not-logged-in signatures ("no oauth token found for github.com",
|
|
5065
|
+
// "You are not logged into any GitHub hosts"). Matched on stderr ONLY for a
|
|
5066
|
+
// nonzero exit, so an unrelated nonzero failure (keychain error, transient
|
|
5067
|
+
// network) stays a transient read failure rather than a spurious logout.
|
|
5068
|
+
const GH_LOGGED_OUT_RE = /no oauth token|no token found|not logged in/i;
|
|
5069
|
+
// Map one completed (or failed) `gh auth token` invocation to:
|
|
5070
|
+
// - a trimmed token string → logged in
|
|
5071
|
+
// - null → the read DEFINITIVELY established no login (clean exit with empty
|
|
5072
|
+
// output, or a nonzero exit whose stderr carries gh's logged-out message)
|
|
5073
|
+
// - GH_AUTH_READ_FAILED → the read could not be completed (spawn error,
|
|
5074
|
+
// timeout — spawnSync sets `.error` — or a nonzero exit WITHOUT a
|
|
5075
|
+
// logged-out signal), so the caller must preserve any prior cached token.
|
|
5076
|
+
function classifyGhAuthRead({ error = null, status = null, stdout = '', stderr = '' } = {}) {
|
|
5077
|
+
if (error) return GH_AUTH_READ_FAILED;
|
|
5078
|
+
if (status === 0) return (String(stdout ?? '').trim()) || null;
|
|
5079
|
+
if (GH_LOGGED_OUT_RE.test(String(stderr ?? ''))) return null;
|
|
5080
|
+
return GH_AUTH_READ_FAILED;
|
|
5081
|
+
}
|
|
5082
|
+
|
|
5083
|
+
function readGhAuthTokenDefault(timeoutMs = 10_000) {
|
|
5084
|
+
let r;
|
|
4799
5085
|
try {
|
|
4800
|
-
|
|
4801
|
-
const tok = r.status === 0 ? (r.stdout || '').trim() : '';
|
|
4802
|
-
token = tok || null;
|
|
5086
|
+
r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: Math.max(1, timeoutMs), env: GH_NONINTERACTIVE_ENV() });
|
|
4803
5087
|
} catch {
|
|
5088
|
+
return GH_AUTH_READ_FAILED;
|
|
5089
|
+
}
|
|
5090
|
+
return classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
|
|
5091
|
+
}
|
|
5092
|
+
|
|
5093
|
+
// A short, non-reversible identity for a token so logs/status can show WHEN the
|
|
5094
|
+
// credential changed (and that two workers hold the SAME one) without ever
|
|
5095
|
+
// printing a usable fragment of the secret. 12 hex chars of sha256.
|
|
5096
|
+
function ghTokenFingerprint(token) {
|
|
5097
|
+
if (!token) return null;
|
|
5098
|
+
try { return createHash('sha256').update(String(token)).digest('hex').slice(0, 12); }
|
|
5099
|
+
catch { return null; }
|
|
5100
|
+
}
|
|
5101
|
+
|
|
5102
|
+
// Read the token's OAuth scopes from the live `X-OAuth-Scopes` response header
|
|
5103
|
+
// (the issue's own diagnostic). Best-effort and bounded: a single lightweight
|
|
5104
|
+
// api.github.com ping, failure → null (scopes simply go unreported). The token
|
|
5105
|
+
// rides the Authorization HEADER — never argv — so it can't leak via `ps`.
|
|
5106
|
+
function ghTokenScopes(token, { timeoutMs = 5_000, run = spawnSync } = {}) {
|
|
5107
|
+
if (!token) return null;
|
|
5108
|
+
try {
|
|
5109
|
+
const r = run('gh', ['api', '-i', '--include', 'user'], { encoding: 'utf8', timeout: timeoutMs, env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
|
|
5110
|
+
const out = `${r.stdout || ''}\n${r.stderr || ''}`;
|
|
5111
|
+
const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
|
|
5112
|
+
return m ? m[1].trim() : null;
|
|
5113
|
+
} catch {
|
|
5114
|
+
return null;
|
|
5115
|
+
}
|
|
5116
|
+
}
|
|
5117
|
+
|
|
5118
|
+
// Fold one (re)read result into the cache record. Shared by the sync and async
|
|
5119
|
+
// refresh paths so both maintain identical cache semantics. Returns
|
|
5120
|
+
// { token, changed, snapshotChanged } — `changed` is true when the token VALUE
|
|
5121
|
+
// differs from what the cache previously held (including null → token and token
|
|
5122
|
+
// → null), so a caller doing credential-rotation recovery fires exactly when the
|
|
5123
|
+
// credential the worker hands out actually rotates. `snapshotChanged` is the
|
|
5124
|
+
// broader "did the OBSERVABLE snapshot move" flag: `changed` OR a same-token
|
|
5125
|
+
// scopes self-repair (a prior null scopes reading re-probed to a real value) —
|
|
5126
|
+
// both of which change what `supervisor status`/the activity marker should show,
|
|
5127
|
+
// so the log + republish paths gate on IT, not on `changed` (Copilot review on
|
|
5128
|
+
// #293). `scopes` is read only when the token changed, on the first populate, or
|
|
5129
|
+
// to re-probe a cached null reading.
|
|
5130
|
+
function adoptGhAuthToken(token, { scopesProbe = ghTokenScopes } = {}) {
|
|
5131
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5132
|
+
const now = Date.now();
|
|
5133
|
+
if (token === GH_AUTH_READ_FAILED) {
|
|
5134
|
+
// Transient read failure: never discard a usable cached credential. Preserve
|
|
5135
|
+
// the prior record but bump `lastCheck` so the per-job hot path keeps serving
|
|
5136
|
+
// it via the TTL short-circuit (no blocking re-read on every lookup); the
|
|
5137
|
+
// off-path refresh timer retries on its own cadence. With no usable prior
|
|
5138
|
+
// token, fall back to anonymous (null) as before.
|
|
5139
|
+
if (prev && prev.token != null) {
|
|
5140
|
+
writeGhAuthCache({ ...prev, lastCheck: now });
|
|
5141
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5142
|
+
}
|
|
5143
|
+
token = null;
|
|
5144
|
+
}
|
|
5145
|
+
const changed = !prev || prev.token !== token;
|
|
5146
|
+
// On a CHANGE re-read the scopes; otherwise carry the previous reading forward
|
|
5147
|
+
// (a TTL re-read that finds the same token must not pay the scopes ping, and
|
|
5148
|
+
// must not silently DROP the scopes it already reported). BUT a prior probe
|
|
5149
|
+
// that failed transiently stored `scopes: null` — carrying that null forward
|
|
5150
|
+
// would leave the GH AUTH visibility unrepaired until the token rotates or the
|
|
5151
|
+
// worker restarts (Copilot review on #293). Re-probe when the cached scope
|
|
5152
|
+
// value is MISSING (null) while a token is present; reuse only a SUCCESSFUL
|
|
5153
|
+
// (non-null) prior reading.
|
|
5154
|
+
const scopes = token ? ((changed || !(prev && prev.scopes)) ? scopesProbe(token) : prev.scopes) : null;
|
|
5155
|
+
// `snapshotChanged` is true when the OBSERVABLE identity (the token-free
|
|
5156
|
+
// fingerprint + scopes that ghAuthTokenInfo/describeGhAuthToken surface) moved
|
|
5157
|
+
// — not only when the token VALUE rotated. A scopes self-repair (a prior
|
|
5158
|
+
// transient probe cached `scopes:null`, now re-probed to a real value) leaves
|
|
5159
|
+
// the token unchanged yet DOES change what `supervisor status` should show, so
|
|
5160
|
+
// the republish/log path must fire on it too; gating republishing on `changed`
|
|
5161
|
+
// alone leaves an idle worker reporting `?` scopes indefinitely (Copilot review
|
|
5162
|
+
// on #293).
|
|
5163
|
+
const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
|
|
5164
|
+
writeGhAuthCache({
|
|
5165
|
+
token,
|
|
5166
|
+
fingerprint: ghTokenFingerprint(token),
|
|
5167
|
+
scopes: scopes ?? null,
|
|
5168
|
+
// `since` tracks the TOKEN's age (when THIS value was first seen) so status
|
|
5169
|
+
// can answer "how stale is the credential"; `lastCheck` tracks the CACHE's
|
|
5170
|
+
// age (when we last asked gh) so the TTL is honoured.
|
|
5171
|
+
since: prev && prev.token === token && token != null ? prev.since : now,
|
|
5172
|
+
lastCheck: now,
|
|
5173
|
+
});
|
|
5174
|
+
return { token, changed, snapshotChanged };
|
|
5175
|
+
}
|
|
5176
|
+
|
|
5177
|
+
// The async twin of adoptGhAuthToken: identical cache semantics, but the scopes
|
|
5178
|
+
// probe is awaited so it may return a Promise (the async ghTokenScopesAsync).
|
|
5179
|
+
// `generation` (when provided by refreshGhAuthTokenAsync) is the cache generation
|
|
5180
|
+
// snapshotted BEFORE this refresh started reading. If it no longer matches at
|
|
5181
|
+
// commit time, a NEWER definitive write (a forced sync clone/push recovery, an
|
|
5182
|
+
// invalidate, or another refresh) landed while we were reading/probing — so this
|
|
5183
|
+
// read is stale and MUST NOT be committed, or it would resurrect the superseded
|
|
5184
|
+
// token (Copilot review on #293). We leave the newer cache in place and report
|
|
5185
|
+
// it as superseded.
|
|
5186
|
+
async function adoptGhAuthTokenAsync(token, { scopesProbe, generation = null } = {}) {
|
|
5187
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5188
|
+
const now = Date.now();
|
|
5189
|
+
const superseded = () => {
|
|
5190
|
+
const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5191
|
+
return { token: c ? c.token : null, changed: false, snapshotChanged: false, superseded: true };
|
|
5192
|
+
};
|
|
5193
|
+
if (token === GH_AUTH_READ_FAILED) {
|
|
5194
|
+
// Transient read failure (see adoptGhAuthToken): preserve the prior token and
|
|
5195
|
+
// bump `lastCheck` so the hot path keeps serving it; anonymous only with no prior.
|
|
5196
|
+
if (prev && prev.token != null) {
|
|
5197
|
+
if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
|
|
5198
|
+
writeGhAuthCache({ ...prev, lastCheck: now });
|
|
5199
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5200
|
+
}
|
|
4804
5201
|
token = null;
|
|
4805
5202
|
}
|
|
4806
|
-
|
|
4807
|
-
|
|
5203
|
+
const changed = !prev || prev.token !== token;
|
|
5204
|
+
// Same re-probe-on-missing rule as the sync twin: a same-token refresh reuses a
|
|
5205
|
+
// successful prior scope reading but RE-PROBES when the cached value is null
|
|
5206
|
+
// (a prior transient probe failure), so the GH AUTH visibility self-repairs.
|
|
5207
|
+
const scopes = token ? ((changed || !(prev && prev.scopes)) ? await scopesProbe(token) : prev.scopes) : null;
|
|
5208
|
+
// See adoptGhAuthToken: publish when the observable snapshot (fingerprint +
|
|
5209
|
+
// scopes) moved, not only when the token value rotated, so a same-token scopes
|
|
5210
|
+
// self-repair still republishes the activity marker (Copilot review on #293).
|
|
5211
|
+
const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
|
|
5212
|
+
// Generation fence: re-check AFTER the (awaited) scopes probe, the last point a
|
|
5213
|
+
// forced recovery could have slipped a newer credential in underneath us.
|
|
5214
|
+
if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
|
|
5215
|
+
writeGhAuthCache({
|
|
5216
|
+
token,
|
|
5217
|
+
fingerprint: ghTokenFingerprint(token),
|
|
5218
|
+
scopes: scopes ?? null,
|
|
5219
|
+
since: prev && prev.token === token && token != null ? prev.since : now,
|
|
5220
|
+
lastCheck: now,
|
|
5221
|
+
});
|
|
5222
|
+
return { token, changed, snapshotChanged };
|
|
5223
|
+
}
|
|
5224
|
+
|
|
5225
|
+
// One (re)read of `gh auth token`, folding the result into the cache record.
|
|
5226
|
+
// Returns { token, changed } (see adoptGhAuthToken). SYNCHRONOUS (spawnSync) —
|
|
5227
|
+
// safe off the job path (the startup prime and the clone/push auth-failure
|
|
5228
|
+
// recovery, where the caller is already blocked waiting on git) but it blocks
|
|
5229
|
+
// the event loop while it runs, so the periodic refresh AND the SIGUSR1
|
|
5230
|
+
// reload-credentials handler use refreshGhAuthTokenAsync instead.
|
|
5231
|
+
function refreshGhAuthToken({ force = false, run = null, scopesProbe, budgetMs = null } = {}) {
|
|
5232
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5233
|
+
const now = Date.now();
|
|
5234
|
+
if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
|
|
5235
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5236
|
+
}
|
|
5237
|
+
// When a caller on the ACTIVE-JOB path (the clone/push auth-failure recovery)
|
|
5238
|
+
// threads its remaining lease budget, bound BOTH synchronous gh subprocesses
|
|
5239
|
+
// (the `gh auth token` read AND the scopes probe) to it so this forced
|
|
5240
|
+
// spawnSync re-read can never block the event loop / lock heartbeat past the
|
|
5241
|
+
// caller's lease-safe window — the finalizeGit/clone git ops are already
|
|
5242
|
+
// netDeadline-bounded, and this read must participate in the same budget
|
|
5243
|
+
// (Copilot review on #294). Absent a budget (startup prime, UNSET-cache hot
|
|
5244
|
+
// path, unit tests) the historical 10s read / 5s scopes caps stand.
|
|
5245
|
+
const haveBudget = Number.isFinite(budgetMs) && budgetMs > 0;
|
|
5246
|
+
const deadline = haveBudget ? now + budgetMs : null;
|
|
5247
|
+
const readTimeout = haveBudget ? Math.max(1, Math.min(10_000, budgetMs)) : 10_000;
|
|
5248
|
+
let token = null;
|
|
5249
|
+
if (run) {
|
|
5250
|
+
// Explicit per-call spawn injection (unit tests of this function in isolation).
|
|
5251
|
+
try {
|
|
5252
|
+
const r = run('gh', ['auth', 'token'], { encoding: 'utf8', timeout: readTimeout, env: GH_NONINTERACTIVE_ENV() });
|
|
5253
|
+
token = classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
|
|
5254
|
+
} catch {
|
|
5255
|
+
token = GH_AUTH_READ_FAILED;
|
|
5256
|
+
}
|
|
5257
|
+
} else {
|
|
5258
|
+
// The module reader (production: the real gh CLI; tests: setGhAuthTokenReader).
|
|
5259
|
+
token = (ghAuthTokenReader || readGhAuthTokenDefault)(readTimeout);
|
|
5260
|
+
}
|
|
5261
|
+
// The scopes probe gets whatever of the budget the read left (still capped at
|
|
5262
|
+
// 5s); a read that consumed the whole budget leaves it ~1ms so the probe fails
|
|
5263
|
+
// fast to null (scopes are best-effort) rather than overrunning the lease.
|
|
5264
|
+
const scopesTimeout = haveBudget ? Math.max(1, Math.min(5_000, deadline - Date.now())) : 5_000;
|
|
5265
|
+
// The scopes probe defaults to the live `gh api -i user` ping — but when a
|
|
5266
|
+
// test installed a token READER (hermetic, no network) and did not inject a
|
|
5267
|
+
// scopesProbe, skip the ping so a stubbed refresh never hits the network.
|
|
5268
|
+
const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((t) => ghTokenScopes(t, { timeoutMs: scopesTimeout })));
|
|
5269
|
+
return adoptGhAuthToken(token, { scopesProbe: probe });
|
|
5270
|
+
}
|
|
5271
|
+
|
|
5272
|
+
// Async `gh auth token` read (spawn, never spawnSync) for the periodic refresh
|
|
5273
|
+
// path — the whole point of the timer is that it must NOT block the worker's
|
|
5274
|
+
// event loop (and with it the lock heartbeat) while a job runs. Same 10s bound
|
|
5275
|
+
// and non-interactive env as the sync reader; resolves a trimmed token-or-null.
|
|
5276
|
+
// `_spawn` is a test-only seam (production uses node:child_process.spawn).
|
|
5277
|
+
function readGhAuthTokenAsync({ _spawn = spawn } = {}) {
|
|
5278
|
+
return new Promise((resolve) => {
|
|
5279
|
+
let child;
|
|
5280
|
+
try {
|
|
5281
|
+
child = _spawn('gh', ['auth', 'token'], { env: GH_NONINTERACTIVE_ENV() });
|
|
5282
|
+
} catch {
|
|
5283
|
+
resolve(GH_AUTH_READ_FAILED);
|
|
5284
|
+
return;
|
|
5285
|
+
}
|
|
5286
|
+
let out = '';
|
|
5287
|
+
let err = '';
|
|
5288
|
+
let settled = false;
|
|
5289
|
+
const finish = (token) => {
|
|
5290
|
+
if (settled) return;
|
|
5291
|
+
settled = true;
|
|
5292
|
+
clearTimeout(timer);
|
|
5293
|
+
resolve(token);
|
|
5294
|
+
};
|
|
5295
|
+
// Same 10s bound as the sync reader. On expiry SIGTERM the child (matching
|
|
5296
|
+
// spawnSync's timeout semantics) and resolve the transient sentinel — a
|
|
5297
|
+
// wedged gh must never pin the refresh chain, NOR masquerade as a logout.
|
|
5298
|
+
const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(GH_AUTH_READ_FAILED); }, 10_000);
|
|
5299
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
5300
|
+
child.stdout?.on('data', (d) => { out += d; });
|
|
5301
|
+
// stderr is now CAPTURED (not just drained) so classifyGhAuthRead can tell a
|
|
5302
|
+
// definitive logout (gh's "no oauth token found" message) from a transient
|
|
5303
|
+
// nonzero failure that must preserve the cached credential.
|
|
5304
|
+
child.stderr?.on('data', (d) => { err += d; });
|
|
5305
|
+
child.on('error', () => finish(GH_AUTH_READ_FAILED));
|
|
5306
|
+
child.on('close', (code) => finish(classifyGhAuthRead({ status: code, stdout: out, stderr: err })));
|
|
5307
|
+
});
|
|
5308
|
+
}
|
|
5309
|
+
|
|
5310
|
+
// Async scopes probe mirroring ghTokenScopes (same 5s default bound, same
|
|
5311
|
+
// header parse, same GH_TOKEN-in-env delivery so the token never touches argv).
|
|
5312
|
+
// `_spawn` is a test-only seam.
|
|
5313
|
+
function ghTokenScopesAsync(token, { timeoutMs = 5_000, _spawn = spawn } = {}) {
|
|
5314
|
+
if (!token) return Promise.resolve(null);
|
|
5315
|
+
return new Promise((resolve) => {
|
|
5316
|
+
let child;
|
|
5317
|
+
try {
|
|
5318
|
+
child = _spawn('gh', ['api', '-i', '--include', 'user'], { env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
|
|
5319
|
+
} catch {
|
|
5320
|
+
resolve(null);
|
|
5321
|
+
return;
|
|
5322
|
+
}
|
|
5323
|
+
let out = '';
|
|
5324
|
+
let settled = false;
|
|
5325
|
+
const finish = (scopes) => {
|
|
5326
|
+
if (settled) return;
|
|
5327
|
+
settled = true;
|
|
5328
|
+
clearTimeout(timer);
|
|
5329
|
+
resolve(scopes);
|
|
5330
|
+
};
|
|
5331
|
+
const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(null); }, timeoutMs);
|
|
5332
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
5333
|
+
child.stdout?.on('data', (d) => { out += d; });
|
|
5334
|
+
child.stderr?.on('data', (d) => { out += d; });
|
|
5335
|
+
child.on('error', () => finish(null));
|
|
5336
|
+
child.on('close', () => {
|
|
5337
|
+
const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
|
|
5338
|
+
finish(m ? m[1].trim() : null);
|
|
5339
|
+
});
|
|
5340
|
+
});
|
|
5341
|
+
}
|
|
5342
|
+
|
|
5343
|
+
// The async twin of refreshGhAuthToken for the periodic timer: same TTL
|
|
5344
|
+
// short-circuit and the same cache record, but the gh subprocesses are spawned
|
|
5345
|
+
// (never spawnSync) so a refresh that DOES run while a job is active yields the
|
|
5346
|
+
// event loop instead of blocking the lock heartbeat (Copilot review on #293).
|
|
5347
|
+
// When a test installed a token READER (hermetic, no network) and did not
|
|
5348
|
+
// inject a scopesProbe, the scopes ping is skipped — mirroring the sync path.
|
|
5349
|
+
async function refreshGhAuthTokenAsync({ force = false, scopesProbe, _spawn } = {}) {
|
|
5350
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5351
|
+
const now = Date.now();
|
|
5352
|
+
if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
|
|
5353
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5354
|
+
}
|
|
5355
|
+
// Serialize against any refresh already spawning (another timer tick, or the
|
|
5356
|
+
// reload-credentials SIGUSR1 handler): await IT and report its (already
|
|
5357
|
+
// applied) cache rather than starting a second concurrent `gh` read. This is
|
|
5358
|
+
// what lets the synchronous per-job path safely defer to an in-flight async
|
|
5359
|
+
// refresh instead of blocking (Copilot review on #293).
|
|
5360
|
+
// LOOP (not a one-shot `if`): re-check the marker after EVERY await so forced
|
|
5361
|
+
// refreshes stay serialized. A plain `if` let multiple forced callers all await
|
|
5362
|
+
// the SAME in-flight promise and then fall through together, each overwriting
|
|
5363
|
+
// ghAuthRefreshInFlight with its own read; the first to settle then clears the
|
|
5364
|
+
// marker (its `finally` nulls it) while another forced read is still active, so
|
|
5365
|
+
// the hot path (ghAuthTokenFromCli) sees no marker and resumes a blocking
|
|
5366
|
+
// synchronous re-read — the very stall the marker exists to prevent. Re-checking
|
|
5367
|
+
// in a loop means a forced caller that finds a NEW refresh in flight (started by
|
|
5368
|
+
// another forced caller that fell through first) awaits THAT one too, so at most
|
|
5369
|
+
// one forced read runs at a time and the marker is never cleared out from under
|
|
5370
|
+
// an active read (Copilot review on #293).
|
|
5371
|
+
while (ghAuthRefreshInFlight) {
|
|
5372
|
+
try { await ghAuthRefreshInFlight; } catch { /* swallowed — the owner logs/handles it */ }
|
|
5373
|
+
// A NON-forced caller (a periodic timer tick) is happy with whatever the
|
|
5374
|
+
// in-flight refresh just applied. A FORCED caller (the reload-credentials
|
|
5375
|
+
// SIGUSR1 handler) must NOT be: the in-flight read may have captured the OLD
|
|
5376
|
+
// token microseconds before the operator's `gh auth refresh`/`login` landed,
|
|
5377
|
+
// so returning its result would report "reloaded" while leaving the stale
|
|
5378
|
+
// credential cached until the next TTL. Loop back: once no refresh is in
|
|
5379
|
+
// flight, fall through to start a fresh forced read AFTER the signal.
|
|
5380
|
+
if (!force) {
|
|
5381
|
+
const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5382
|
+
return { token: c ? c.token : null, changed: false, snapshotChanged: false };
|
|
5383
|
+
}
|
|
5384
|
+
}
|
|
5385
|
+
// Honor the module reader seam when a test installed one (mirroring the sync
|
|
5386
|
+
// path) so a stubbed refresh never spawns; production (reader null) spawns.
|
|
5387
|
+
// Snapshot the cache generation BEFORE the read begins: if a forced sync
|
|
5388
|
+
// recovery commits a newer credential while we read/probe, adoptGhAuthTokenAsync
|
|
5389
|
+
// sees the generation advance and refuses to clobber it (Copilot review on #293).
|
|
5390
|
+
const startGen = ghAuthCacheGeneration;
|
|
5391
|
+
const run = (async () => {
|
|
5392
|
+
const token = ghAuthTokenReader ? ((ghAuthTokenReader() || null)) : await readGhAuthTokenAsync({ _spawn });
|
|
5393
|
+
const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((tok) => ghTokenScopesAsync(tok, { _spawn })));
|
|
5394
|
+
return adoptGhAuthTokenAsync(token, { scopesProbe: probe, generation: startGen });
|
|
5395
|
+
})();
|
|
5396
|
+
// Publish the in-flight marker BEFORE the first await inside `run` has a chance
|
|
5397
|
+
// to yield, so a job activated on the very next tick already sees it. The
|
|
5398
|
+
// marker never rejects (so a concurrent awaiter can't throw on it); the real
|
|
5399
|
+
// result/rejection is surfaced to this caller below.
|
|
5400
|
+
ghAuthRefreshInFlight = run.then(() => {}, () => {});
|
|
5401
|
+
try {
|
|
5402
|
+
return await run;
|
|
5403
|
+
} finally {
|
|
5404
|
+
ghAuthRefreshInFlight = null;
|
|
5405
|
+
}
|
|
4808
5406
|
}
|
|
4809
5407
|
|
|
4810
|
-
|
|
4811
|
-
//
|
|
4812
|
-
//
|
|
4813
|
-
|
|
4814
|
-
|
|
5408
|
+
function ghAuthTokenFromCli() {
|
|
5409
|
+
// If an async refresh is already spawning (a timer tick or the
|
|
5410
|
+
// reload-credentials SIGUSR1 handler), do NOT start a blocking synchronous
|
|
5411
|
+
// re-read on this per-job path — that is exactly the 15s event-loop/lease
|
|
5412
|
+
// stall the async refresh exists to avoid (Copilot review on #293). Hand out
|
|
5413
|
+
// the currently-cached token; the in-flight refresh updates the cache for the
|
|
5414
|
+
// next lookup, and the clone/push auth-failure recovery still force-re-reads
|
|
5415
|
+
// if the handed-out token turns out stale. A not-yet-primed cache (UNSET)
|
|
5416
|
+
// yields null — the same anonymous fallback the refresh would briefly produce.
|
|
5417
|
+
if (ghAuthRefreshInFlight) {
|
|
5418
|
+
return ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache.token;
|
|
5419
|
+
}
|
|
5420
|
+
if (ghAuthTokenCache === GH_AUTH_TOKEN_UNSET) return refreshGhAuthToken({ force: true }).token;
|
|
5421
|
+
// A PRIMED cache is served verbatim — NEVER a synchronous TTL-triggered
|
|
5422
|
+
// re-read on this per-job hot path (Copilot review on #293). The async timer
|
|
5423
|
+
// DEFERS a tick while a job is active, so a skipped tick can leave the cache
|
|
5424
|
+
// past its TTL when the next job arrives; a `refreshGhAuthToken()` here would
|
|
5425
|
+
// then run `spawnSync('gh auth token')` + the synchronous scopes probe inline,
|
|
5426
|
+
// blocking the event loop (and the lock heartbeat) for up to 15s. Startup
|
|
5427
|
+
// priming, the async maintenance timer, and the clone/push auth-failure
|
|
5428
|
+
// recovery (which force-re-reads if the handed-out token turns out stale) own
|
|
5429
|
+
// every refresh; this path only hands out what they populated.
|
|
5430
|
+
return ghAuthTokenCache.token;
|
|
5431
|
+
}
|
|
5432
|
+
|
|
5433
|
+
// Drop the cached gh token so the next lookup re-reads the CLI. A cache-reset
|
|
5434
|
+
// primitive — exported and used by the tests to reset state between cases. NO
|
|
5435
|
+
// production path calls it anymore: both the git clone/push auth-failure
|
|
5436
|
+
// recovery (recoverGhTokenAfterAuthFailure) and the supervisor's
|
|
5437
|
+
// `reload-credentials` broadcast (SIGUSR1) were made deliberately
|
|
5438
|
+
// NON-destructive (a forced re-read that preserves/atomically-swaps the cached
|
|
5439
|
+
// record), so neither drops the cache first (Copilot review on #294).
|
|
5440
|
+
function invalidateGhAuthToken() {
|
|
5441
|
+
writeGhAuthCache(GH_AUTH_TOKEN_UNSET);
|
|
5442
|
+
}
|
|
5443
|
+
|
|
5444
|
+
// Recover from a git/gh auth failure on a gh-cache-derived credential: force a
|
|
5445
|
+
// fresh `gh auth token` read and return it IFF it differs from the credential
|
|
5446
|
+
// that just FAILED (`failingToken`), otherwise null.
|
|
5447
|
+
//
|
|
5448
|
+
// Deliberately does NOT `invalidateGhAuthToken()` first (Copilot review on
|
|
5449
|
+
// #294). An invalidate-before-reread blows the cached record away, so if the
|
|
5450
|
+
// forced re-read then fails TRANSIENTLY (gh momentarily unavailable), there is
|
|
5451
|
+
// no prior record left for `adoptGhAuthToken` to preserve — it falls back to
|
|
5452
|
+
// `{token:null}` and the worker is left "logged out", ERASING a still-usable —
|
|
5453
|
+
// and possibly NEWER (a `reload-credentials` SIGUSR1 swap that landed mid-job)
|
|
5454
|
+
// — cached credential. `refreshGhAuthToken({ force })` already re-reads past the
|
|
5455
|
+
// TTL, and `adoptGhAuthToken` PRESERVES the prior cached token on a transient
|
|
5456
|
+
// read failure, so a transient hiccup keeps the good token (and we even retry
|
|
5457
|
+
// with a newer cached one) instead of discarding it. This mirrors the SIGUSR1
|
|
5458
|
+
// reload-credentials handler, which is likewise deliberately NON-destructive.
|
|
5459
|
+
//
|
|
5460
|
+
// The retry is gated on `fresh !== failingToken` — NOT on the re-read's
|
|
5461
|
+
// `changed` flag (which compares against the CACHE): when the cache already
|
|
5462
|
+
// holds the newer token (the `reload-credentials` swap case), a forced re-read
|
|
5463
|
+
// returns it with `changed:false`, yet that newer token is exactly the
|
|
5464
|
+
// credential we want to retry with. `fresh !== failingToken` also covers the
|
|
5465
|
+
// genuine-denial case (the re-read returns the SAME failing token → null → no
|
|
5466
|
+
// retry) and the null→fresh recovery (a logged-out worker re-reads a real token
|
|
5467
|
+
// after `gh auth login`).
|
|
5468
|
+
// `budgetMs` (threaded by the clone/push recovery call sites from their
|
|
5469
|
+
// remaining lease budget — provTimeoutMs()/netTimeoutMs()) bounds the forced
|
|
5470
|
+
// synchronous `gh auth token` + scopes probe so this on-the-active-job-path
|
|
5471
|
+
// re-read participates in the caller's lease-safe window instead of blocking the
|
|
5472
|
+
// event loop / lock heartbeat for the full 15s (Copilot review on #294).
|
|
5473
|
+
function recoverGhTokenAfterAuthFailure(failingToken, { budgetMs = null } = {}) {
|
|
5474
|
+
const { token: fresh } = refreshGhAuthToken({ force: true, budgetMs });
|
|
5475
|
+
return fresh && fresh !== failingToken ? fresh : null;
|
|
5476
|
+
}
|
|
5477
|
+
|
|
5478
|
+
// A snapshot of the cached token's observable identity for logs and the
|
|
5479
|
+
// supervisor activity marker. Never carries the token value. `ageMs` is the
|
|
5480
|
+
// TOKEN's age (since this value was first seen); `lastCheckAgoMs` is how long
|
|
5481
|
+
// ago gh was last consulted (TTL freshness). Returns null when the cache has
|
|
5482
|
+
// never been populated (a worker that has not primed yet).
|
|
5483
|
+
function ghAuthTokenInfo(now = Date.now()) {
|
|
5484
|
+
const c = ghAuthTokenCache;
|
|
5485
|
+
if (c === GH_AUTH_TOKEN_UNSET) return null;
|
|
5486
|
+
return {
|
|
5487
|
+
present: c.token != null,
|
|
5488
|
+
fingerprint: c.fingerprint ?? null,
|
|
5489
|
+
scopes: c.scopes ?? null,
|
|
5490
|
+
since: c.since ?? null,
|
|
5491
|
+
ageMs: Number.isFinite(c.since) ? Math.max(0, now - c.since) : null,
|
|
5492
|
+
lastCheckAgoMs: Number.isFinite(c.lastCheck) ? Math.max(0, now - c.lastCheck) : null,
|
|
5493
|
+
};
|
|
5494
|
+
}
|
|
5495
|
+
|
|
5496
|
+
// One-line, token-free description for logs: `gho_…` →
|
|
5497
|
+
// "gh auth token: fingerprint ab12cd34ef56 (scopes: gist, read:org, repo; age 2m)".
|
|
5498
|
+
function describeGhAuthToken(info) {
|
|
5499
|
+
if (!info) return 'gh auth token: (not yet read)';
|
|
5500
|
+
if (!info.present) return 'gh auth token: none (gh missing or not logged in)';
|
|
5501
|
+
const bits = [];
|
|
5502
|
+
bits.push(`fingerprint ${info.fingerprint || 'unknown'}`);
|
|
5503
|
+
bits.push(info.scopes ? `scopes: ${info.scopes}` : 'scopes: unknown');
|
|
5504
|
+
if (info.ageMs != null) bits.push(`age ${formatDuration(info.ageMs)}`);
|
|
5505
|
+
return `gh auth token: ${bits.join('; ')}`;
|
|
5506
|
+
}
|
|
5507
|
+
|
|
5508
|
+
// Warm the gh-token cache, off the job-handling path. Safe to call any number of
|
|
5509
|
+
// times: within the TTL it is a no-op cache hit; past it (or after an
|
|
5510
|
+
// invalidation) it re-reads the CLI. With a `logger` it reports the token's
|
|
5511
|
+
// fingerprint + scopes on the FIRST populate and whenever the observable
|
|
5512
|
+
// snapshot moves — a rotated token OR a scopes self-repair (a prior null reading
|
|
5513
|
+
// re-probed to a real value) — the "which credential is this worker holding?"
|
|
5514
|
+
// line from issue #293. Returns true once the cache is populated.
|
|
5515
|
+
function primeGhAuthToken({ logger = null } = {}) {
|
|
5516
|
+
const { snapshotChanged } = refreshGhAuthToken();
|
|
5517
|
+
if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
|
|
5518
|
+
logger.info(describeGhAuthToken(ghAuthTokenInfo()));
|
|
5519
|
+
}
|
|
4815
5520
|
return ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET;
|
|
4816
5521
|
}
|
|
4817
5522
|
|
|
5523
|
+
// Keep the cache fresh for the worker's lifetime: re-prime on a TTL cadence so
|
|
5524
|
+
// a `gh auth login`/`refresh` the operator runs while the fleet is up is adopted
|
|
5525
|
+
// within one interval — WITHOUT ever blocking the job path (Copilot review on
|
|
5526
|
+
// #293). Two guards make that true:
|
|
5527
|
+
// 1. The refresh itself is ASYNC (refreshGhAuthTokenAsync — spawn, never
|
|
5528
|
+
// spawnSync), so even when a tick fires mid-job the gh subprocesses yield
|
|
5529
|
+
// the event loop instead of freezing the lock heartbeat for up to 15s.
|
|
5530
|
+
// 2. `isIdle` (optional) lets the worker DEFER a tick while a job is active:
|
|
5531
|
+
// the refresh is then simply adopted by the next idle tick (staleness is
|
|
5532
|
+
// bounded by the job length + one interval, and the sync per-job paths —
|
|
5533
|
+
// TTL lookup, auth-failure recovery — remain as the safety net). That defer
|
|
5534
|
+
// is itself BOUNDED (`maxStaleMs`): once a PRIMED cache is stale beyond the
|
|
5535
|
+
// bound, the tick refreshes DESPITE an active job (still async/non-blocking),
|
|
5536
|
+
// so sustained back-to-back jobs can't keep deferring the refresh for the
|
|
5537
|
+
// worker's whole busy lifetime and serve an expired non-null cache to every
|
|
5538
|
+
// job (Copilot review on #294). An UNSET cache is never primed mid-job — the
|
|
5539
|
+
// defer there still holds — so a logged-out worker stays anonymous until idle.
|
|
5540
|
+
// Ticks never overlap (a slow gh can't stack refreshes), a tick failure is
|
|
5541
|
+
// swallowed (the next tick retries), and the change log fires from here so a
|
|
5542
|
+
// rotation is visible even when no job runs. `onChange` (optional) fires on a
|
|
5543
|
+
// genuine observable-snapshot change — a rotated token OR a scopes self-repair
|
|
5544
|
+
// (a prior null scopes reading re-probed to a real value), both of which move
|
|
5545
|
+
// what `supervisor status` should show — so the caller can republish the
|
|
5546
|
+
// supervisor activity marker; an idle worker otherwise keeps reporting the
|
|
5547
|
+
// pre-refresh fingerprint/scopes indefinitely (Copilot review on #293).
|
|
5548
|
+
//
|
|
5549
|
+
// The timer is UNREF'd (like the other maintenance/watchdog intervals): this is
|
|
5550
|
+
// a background credential refresh, NOT a serving handle, so it must never be the
|
|
5551
|
+
// thing that keeps the process alive. `unref()` does not starve the cadence —
|
|
5552
|
+
// contrary to a common misconception, it does not deprioritise or defer the
|
|
5553
|
+
// timer; it only drops the timer's claim on the event loop's "keep the process
|
|
5554
|
+
// alive" accounting. A busy worker's poll/heartbeat fibers keep the loop running,
|
|
5555
|
+
// so an unref'd interval still fires exactly on cadence. The one case where
|
|
5556
|
+
// unref matters is the dangerous one: if the worker's supervisor runtime has
|
|
5557
|
+
// unexpectedly terminated (its serving handles gone), a REF'd refresh timer
|
|
5558
|
+
// would keep the PID alive forever, masking the dead worker so the daemon never
|
|
5559
|
+
// observes the exit and never restarts the non-serving worker — whereas an
|
|
5560
|
+
// unref'd timer lets the now-idle process exit and be restarted. (The `--auto`
|
|
5561
|
+
// poll timer is deliberately the opposite — ref'd — because it IS a serving
|
|
5562
|
+
// handle that must keep a 0-poller worker alive to retry.) The returned handle
|
|
5563
|
+
// carries a `stop()` (clearInterval) so tests can halt it deterministically.
|
|
5564
|
+
function startGhAuthTokenRefresh({ logger = null, intervalMs = GH_AUTH_TOKEN_TTL_MS, isIdle = null, onChange = null, maxStaleMs = GH_AUTH_TOKEN_MAX_DEFER_MS, _spawn } = {}) {
|
|
5565
|
+
// Clamp to a 1s floor so a test can pass a tiny interval; an explicit 0 is
|
|
5566
|
+
// honoured (→ the floor), not treated as "use the default".
|
|
5567
|
+
const raw = Number.isFinite(Number(intervalMs)) ? Number(intervalMs) : GH_AUTH_TOKEN_TTL_MS;
|
|
5568
|
+
const every = Math.max(1000, raw || 1000);
|
|
5569
|
+
// The bounded-defer ceiling: a non-finite/negative value disables the bound
|
|
5570
|
+
// (pure defer-while-busy, the pre-#294 behaviour).
|
|
5571
|
+
const staleBound = Number.isFinite(Number(maxStaleMs)) && Number(maxStaleMs) >= 0 ? Number(maxStaleMs) : Infinity;
|
|
5572
|
+
let inFlight = false;
|
|
5573
|
+
const tick = async () => {
|
|
5574
|
+
if (inFlight) return; // a slow gh must not stack overlapping refreshes
|
|
5575
|
+
if (isIdle && !isIdle()) {
|
|
5576
|
+
// A job is active — normally defer so we don't even spawn gh mid-job. But
|
|
5577
|
+
// bound that defer: a PRIMED cache stale beyond `staleBound` must refresh
|
|
5578
|
+
// anyway (still async/non-blocking) rather than reach every subsequent job
|
|
5579
|
+
// for the worker's whole busy lifetime under sustained saturation (Copilot
|
|
5580
|
+
// review on #294). An UNSET cache is never primed mid-job (no bound applies),
|
|
5581
|
+
// so a logged-out worker stays anonymous until it goes idle, as before.
|
|
5582
|
+
const cached = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5583
|
+
const staleBeyondBound = cached && (Date.now() - cached.lastCheck) >= staleBound;
|
|
5584
|
+
if (!staleBeyondBound) return; // defer to the next idle tick
|
|
5585
|
+
}
|
|
5586
|
+
inFlight = true;
|
|
5587
|
+
try {
|
|
5588
|
+
const { snapshotChanged } = await refreshGhAuthTokenAsync({ _spawn });
|
|
5589
|
+
if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
|
|
5590
|
+
logger.info(describeGhAuthToken(ghAuthTokenInfo()));
|
|
5591
|
+
}
|
|
5592
|
+
if (snapshotChanged && onChange) {
|
|
5593
|
+
try { onChange(); } catch { /* best effort — republishing is advisory */ }
|
|
5594
|
+
}
|
|
5595
|
+
} catch {
|
|
5596
|
+
// A failed tick (gh wedged, spawn error) must never crash the worker or
|
|
5597
|
+
// surface as an unhandled rejection — the next tick retries.
|
|
5598
|
+
} finally {
|
|
5599
|
+
inFlight = false;
|
|
5600
|
+
}
|
|
5601
|
+
};
|
|
5602
|
+
const t = setInterval(() => { tick(); }, every);
|
|
5603
|
+
// Unref so a dead worker runtime (serving handles gone) can exit and be
|
|
5604
|
+
// restarted — this maintenance refresh must not keep a non-serving PID alive
|
|
5605
|
+
// (see the header comment). Guarded for fake-timer test harnesses.
|
|
5606
|
+
if (typeof t.unref === 'function') t.unref();
|
|
5607
|
+
t.stop = () => clearInterval(t);
|
|
5608
|
+
return t;
|
|
5609
|
+
}
|
|
5610
|
+
|
|
5611
|
+
// Classify a git/gh failure as an AUTH (or auth-scope) failure — the cases where
|
|
5612
|
+
// the cached credential, not the request, is the problem: a 401/403, git's
|
|
5613
|
+
// credential prompts on an anonymous/rejected clone, or GitHub's OAuth-scope
|
|
5614
|
+
// refusal ("refusing to allow an OAuth App to create or update workflow …
|
|
5615
|
+
// without `workflow` scope" — the exact escalation from issue #293). Matched on
|
|
5616
|
+
// the ALREADY-REDACTED detail string (gitErrorDetail/describeGitFailure output),
|
|
5617
|
+
// so the token value is never inspected here. Deliberately conservative: a
|
|
5618
|
+
// non-fast-forward reject, a missing repo (404) or a network timeout are NOT
|
|
5619
|
+
// auth failures and must not trigger a token drop + retry.
|
|
5620
|
+
function isGitAuthFailure(detail) {
|
|
5621
|
+
const s = String(detail ?? '');
|
|
5622
|
+
if (!s) return false;
|
|
5623
|
+
return /HTTP\s*40[13]\b|\b40[13]\s*\((Unauthorized|Forbidden)\)|returned error:\s*40[13]\b|could not read (Username|Password)|Authentication failed|authentication required|Permission to .+ denied|refusing to allow an OAuth App|without .?workflow.? scope|Bad credentials|Resource not accessible by (integration|personal access token)/i.test(s);
|
|
5624
|
+
}
|
|
5625
|
+
|
|
4818
5626
|
// Read the operator's own git identity from their GLOBAL config, using the real
|
|
4819
5627
|
// host environment (process.env) rather than a job's sanitized gitEnv — so it
|
|
4820
5628
|
// resolves even on the anonymous clone path, where gitEnv points
|
|
@@ -4942,6 +5750,82 @@ function normalizeAuthRef(authRef) {
|
|
|
4942
5750
|
return { kind: 'custom', name: trimmed };
|
|
4943
5751
|
}
|
|
4944
5752
|
|
|
5753
|
+
// The env-var NAME the github clone/push credential is forwarded to the harness
|
|
5754
|
+
// under. Single source of truth so `resolveJobSecrets` (which builds the
|
|
5755
|
+
// `resolved` secret map) and the clone-recovery rotation glue (which keeps that
|
|
5756
|
+
// map in sync when the token rotates) can NEVER drift on the key: a custom
|
|
5757
|
+
// authRef uses its own name; the default github credential uses GITHUB_TOKEN.
|
|
5758
|
+
// Returns null when no credential is forwarded (a non-github provider with no
|
|
5759
|
+
// custom ref, or a blank/invalid authRef).
|
|
5760
|
+
function ghCredentialEnvName({ provider, authRef }) {
|
|
5761
|
+
const prov = provider || 'github';
|
|
5762
|
+
const ref = normalizeAuthRef(authRef);
|
|
5763
|
+
if (ref.kind === 'invalid') return null;
|
|
5764
|
+
if (ref.kind === 'custom') return ref.name;
|
|
5765
|
+
return prov === 'github' ? 'GITHUB_TOKEN' : null;
|
|
5766
|
+
}
|
|
5767
|
+
|
|
5768
|
+
// When clone recovery ROTATES the cache-derived credential during provisioning
|
|
5769
|
+
// (a stale gh token re-read after an auth failure), the `resolved` secret map —
|
|
5770
|
+
// built BEFORE provisioning and later forwarded to the harness as `secretEnv` —
|
|
5771
|
+
// still holds the STALE token, so an allowPr harness's own `gh`/`git` runs would
|
|
5772
|
+
// use a credential the clone already proved dead (Copilot review on #293). Sync
|
|
5773
|
+
// the map so it hands the harness the SAME credential provisioning just adopted.
|
|
5774
|
+
// Guard rails: only a 'gh-cache'-sourced credential may be swapped (an explicit
|
|
5775
|
+
// `env`/`custom` identity is deliberate and never silently replaced), only when
|
|
5776
|
+
// a token actually exists, and ONLY when that key was already forwarded (never
|
|
5777
|
+
// inject a brand-new secret the job did not request). Pure: mutates and returns
|
|
5778
|
+
// `resolved`.
|
|
5779
|
+
function applyRotatedGhCredential({ resolved, provider, authRef, source, token }) {
|
|
5780
|
+
if (!resolved || source !== 'gh-cache' || !token) return resolved;
|
|
5781
|
+
const key = ghCredentialEnvName({ provider, authRef });
|
|
5782
|
+
if (key && Object.prototype.hasOwnProperty.call(resolved, key)) resolved[key] = token;
|
|
5783
|
+
return resolved;
|
|
5784
|
+
}
|
|
5785
|
+
|
|
5786
|
+
// Resolve the github clone/push credential AND record where it came from. The
|
|
5787
|
+
// default credential (env GITHUB_TOKEN) falls back to the gh CLI's stored token
|
|
5788
|
+
// so `gh auth login` setups work without exporting GITHUB_TOKEN. A custom
|
|
5789
|
+
// `authRef` is honored strictly (env/secret resolver only, no gh fallback) so a
|
|
5790
|
+
// misconfigured named secret surfaces as missing rather than silently borrowing
|
|
5791
|
+
// the operator's gh login. An authRef that is present but blank is a
|
|
5792
|
+
// misconfiguration and yields no token (never the gh fallback). `ghAuthToken`
|
|
5793
|
+
// is injectable for testing.
|
|
5794
|
+
// The returned `source` matters downstream: the #293 auth-failure recovery
|
|
5795
|
+
// (force-re-read `gh auth token`, retry once — NON-destructive, never drops the
|
|
5796
|
+
// cache first) is only legitimate when
|
|
5797
|
+
// the token CAME from that cache ('gh-cache'). Retrying an explicit
|
|
5798
|
+
// GITHUB_TOKEN ('env') or a custom authRef ('custom') failure with the
|
|
5799
|
+
// operator's unrelated gh login would silently swap the configured identity
|
|
5800
|
+
// mid-job (Copilot review on #293), so those sources never take the fallback.
|
|
5801
|
+
// A DEFAULT github lookup that found NO cached gh token ALSO reports
|
|
5802
|
+
// 'gh-cache' (with `token: null`) rather than 'none': it is still the gh-cache
|
|
5803
|
+
// credential path, merely a cache MISS. Tagging it 'gh-cache' keeps the
|
|
5804
|
+
// clone/push auth-failure recovery ELIGIBLE so a worker that cached
|
|
5805
|
+
// "logged out" can still self-heal (a forced re-read recovers null → fresh
|
|
5806
|
+
// token after the operator runs `gh auth login`) instead of failing the job
|
|
5807
|
+
// anonymous and waiting for the periodic TTL refresh (Copilot review on #293).
|
|
5808
|
+
// 'none' is reserved for a non-github provider and an invalid/blank authRef —
|
|
5809
|
+
// neither may borrow the gh login.
|
|
5810
|
+
// Returns { token, source: 'env' | 'custom' | 'gh-cache' | 'none' }.
|
|
5811
|
+
function githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
|
|
5812
|
+
const prov = provider || 'github';
|
|
5813
|
+
const ref = normalizeAuthRef(authRef);
|
|
5814
|
+
if (ref.kind === 'invalid') return { token: null, source: 'none' };
|
|
5815
|
+
const usesDefault = prov === 'github' && ref.kind === 'default';
|
|
5816
|
+
const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
|
|
5817
|
+
const resolved = name ? (secretResolver.resolve(name) || null) : null;
|
|
5818
|
+
if (resolved) return { token: resolved, source: ref.kind === 'custom' ? 'custom' : 'env' };
|
|
5819
|
+
if (!resolved && usesDefault) {
|
|
5820
|
+
const token = ghAuthToken() || null;
|
|
5821
|
+
// A hit AND a miss both report 'gh-cache' (see the header): the miss keeps
|
|
5822
|
+
// the auth-failure recovery eligible so a post-`gh auth login` forced
|
|
5823
|
+
// re-read can still recover this job.
|
|
5824
|
+
return { token, source: 'gh-cache' };
|
|
5825
|
+
}
|
|
5826
|
+
return { token: null, source: 'none' };
|
|
5827
|
+
}
|
|
5828
|
+
|
|
4945
5829
|
// Resolve the github clone/push credential. The default credential (env
|
|
4946
5830
|
// GITHUB_TOKEN) falls back to the gh CLI's stored token so `gh auth login`
|
|
4947
5831
|
// setups work without exporting GITHUB_TOKEN. A custom `authRef` is honored
|
|
@@ -4949,16 +5833,10 @@ function normalizeAuthRef(authRef) {
|
|
|
4949
5833
|
// secret surfaces as missing rather than silently borrowing the operator's gh
|
|
4950
5834
|
// login. An authRef that is present but blank is a misconfiguration and yields no
|
|
4951
5835
|
// token (never the gh fallback). `ghAuthToken` is injectable for testing.
|
|
4952
|
-
// Returns a token or null.
|
|
5836
|
+
// Returns a token or null. (Callers that need the credential's SOURCE — e.g. to
|
|
5837
|
+
// gate the #293 gh-token auth-failure retry — use githubCloneTokenWithSource.)
|
|
4953
5838
|
function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
|
|
4954
|
-
|
|
4955
|
-
const ref = normalizeAuthRef(authRef);
|
|
4956
|
-
if (ref.kind === 'invalid') return null;
|
|
4957
|
-
const usesDefault = prov === 'github' && ref.kind === 'default';
|
|
4958
|
-
const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
|
|
4959
|
-
let token = name ? (secretResolver.resolve(name) || null) : null;
|
|
4960
|
-
if (!token && usesDefault) token = ghAuthToken() || null;
|
|
4961
|
-
return token;
|
|
5839
|
+
return githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken }).token;
|
|
4962
5840
|
}
|
|
4963
5841
|
|
|
4964
5842
|
// Clone repo into <runDir>/workspace and check out / create the working branch.
|
|
@@ -4973,11 +5851,11 @@ function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghA
|
|
|
4973
5851
|
// resume sides agree on which no-create push targets are recoverable (#270).
|
|
4974
5852
|
const CONVENTIONAL_BASE_BRANCHES = new Set(['main', 'master']);
|
|
4975
5853
|
|
|
4976
|
-
function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, _runGit = null }) {
|
|
5854
|
+
function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, credentialSource = null, _runGit = null }) {
|
|
4977
5855
|
// Test-only seam mirroring finalizeGit: route every git call through `runGitFn` so a
|
|
4978
5856
|
// deterministic test can observe the per-call timeouts drawn from the shared
|
|
4979
5857
|
// provisioning budget below. Defaults to the module `runGit` (production unchanged).
|
|
4980
|
-
|
|
5858
|
+
let runGitFn = _runGit || runGit;
|
|
4981
5859
|
// #222: the git ops below are blocking spawnSync calls that cannot observe an
|
|
4982
5860
|
// AbortSignal mid-call, so provisioning is made signal-aware by rechecking
|
|
4983
5861
|
// BETWEEN operations: if a lock-loss race won, bail before the NEXT side effect
|
|
@@ -5003,7 +5881,9 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5003
5881
|
// already-lost lease without starting any repository side effect.
|
|
5004
5882
|
throwIfAborted('entry');
|
|
5005
5883
|
const askpass = writeAskpass(runDir, token);
|
|
5006
|
-
const
|
|
5884
|
+
// `let` (not const): the #293 clone auth-failure retry swaps in a FRESH token
|
|
5885
|
+
// (rewritten askpass + GIT_TOKEN) and re-runs the clone with an updated env.
|
|
5886
|
+
let gitEnv = {
|
|
5007
5887
|
...process.env,
|
|
5008
5888
|
GIT_TERMINAL_PROMPT: '0',
|
|
5009
5889
|
GIT_CONFIG_NOSYSTEM: '1',
|
|
@@ -5091,8 +5971,80 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5091
5971
|
const detailNote = detail && detail !== 'unknown error' ? ` — last git output: ${detail}` : '';
|
|
5092
5972
|
throw new ProvisionError(`git clone timed out after ${effectiveTimeoutMs}ms — the repo may be too large, or the network stalled; raise the timeout (repository.cloneTimeoutMs in the envelope, or the worker's --clone-timeout flag) or scope the clone with filter/singleBranch/depth${detailNote}`);
|
|
5093
5973
|
}
|
|
5094
|
-
|
|
5974
|
+
const detail = gitErrorDetail(clone, token);
|
|
5975
|
+
// #293: an AUTH failure here usually means the cached gh token went stale
|
|
5976
|
+
// (the operator re-logged in / rotated it after this worker primed). Force a
|
|
5977
|
+
// fresh re-read and retry the clone ONCE with it before reporting failure —
|
|
5978
|
+
// a rotated credential self-heals instead of shedding the job. Only when the
|
|
5979
|
+
// re-read yields a token DIFFERENT from the one that failed (a same-token
|
|
5980
|
+
// retry can't fix a genuine permission denial). The recovery is
|
|
5981
|
+
// NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
|
|
5982
|
+
// the cache first, so a transiently-failing re-read can't erase a still-valid
|
|
5983
|
+
// — possibly newer — cached credential (Copilot review on #294). The retry
|
|
5984
|
+
// re-runs the whole clone with the new token threaded through the askpass
|
|
5985
|
+
// env, re-anchored to the remaining provisioning budget.
|
|
5986
|
+
// GATED on the credential's SOURCE (Copilot review on #293): this fallback
|
|
5987
|
+
// replaces the failed token with the operator's `gh auth token`, so it is
|
|
5988
|
+
// only legitimate when the failed token CAME from that same gh cache
|
|
5989
|
+
// ('gh-cache'). An explicit GITHUB_TOKEN ('env') or custom authRef
|
|
5990
|
+
// ('custom') is a deliberately configured identity — silently swapping it
|
|
5991
|
+
// for the operator's gh login would violate githubCloneToken's documented
|
|
5992
|
+
// precedence and could act as a different identity, so those sources fail
|
|
5993
|
+
// without the gh retry.
|
|
5994
|
+
// The `token` guard is deliberately ABSENT (Copilot review on #293): a
|
|
5995
|
+
// default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
|
|
5996
|
+
// a worker that cached "logged out" recovers null → fresh token after the
|
|
5997
|
+
// operator runs `gh auth login` instead of failing the job anonymous. That
|
|
5998
|
+
// recovery must REBUILD the credential plumbing the anonymous path omitted
|
|
5999
|
+
// (the askpass helper, GIT_ASKPASS/GIT_TOKEN, and the username-embedded
|
|
6000
|
+
// remote) — a bare GIT_TOKEN swap leaves the clone unauthenticated.
|
|
6001
|
+
if (credentialSource === 'gh-cache' && isGitAuthFailure(detail)) {
|
|
6002
|
+
const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: provTimeoutMs() });
|
|
6003
|
+
if (fresh) {
|
|
6004
|
+
logger?.info?.(`git clone auth failed${cs} — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the clone once`);
|
|
6005
|
+
throwIfAborted('clone-auth-retry');
|
|
6006
|
+
// Rebuild the credential env for the fresh token. On the null→fresh
|
|
6007
|
+
// recovery the original env was the ANONYMOUS one (no askpass helper,
|
|
6008
|
+
// GIT_CONFIG_GLOBAL neutralized), so re-mint the askpass helper and
|
|
6009
|
+
// restore global config; on a stale-token rotation this is a no-op
|
|
6010
|
+
// re-point of the same GIT_TOKEN.
|
|
6011
|
+
const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
|
|
6012
|
+
const retryAskpass = writeAskpass(runDir, fresh);
|
|
6013
|
+
if (retryAskpass) {
|
|
6014
|
+
retryEnv.GIT_ASKPASS = retryAskpass;
|
|
6015
|
+
// The anonymous path set GIT_CONFIG_GLOBAL=/dev/null so a no-token
|
|
6016
|
+
// clone can't inherit operator credentials; the fresh token is ours,
|
|
6017
|
+
// so global config (e.g. http.proxy) may apply again.
|
|
6018
|
+
delete retryEnv.GIT_CONFIG_GLOBAL;
|
|
6019
|
+
}
|
|
6020
|
+
// Rebuild the clone argv so the remote carries the fresh credential's
|
|
6021
|
+
// username (authUrl embeds x-access-token only when a token is present);
|
|
6022
|
+
// the anonymous remote had no username, so git would never consult the
|
|
6023
|
+
// askpass helper for it. `workspaceDir` stays last; the remote is the
|
|
6024
|
+
// argument immediately before it (cloneArgs.push(remote, workspaceDir)).
|
|
6025
|
+
const retryCloneArgs = cloneArgs.slice();
|
|
6026
|
+
retryCloneArgs[retryCloneArgs.length - 2] = authUrl(repo.url, repo.provider || 'github', true);
|
|
6027
|
+
const retry = runGitFn(retryCloneArgs, { env: retryEnv, timeoutMs: provTimeoutMs() });
|
|
6028
|
+
if (retry.status === 0) {
|
|
6029
|
+
// Adopt the fresh token for the REST of provisioning + the returned
|
|
6030
|
+
// gitEnv (finalizeGit pushes with it) so the whole job runs on the
|
|
6031
|
+
// rotated credential.
|
|
6032
|
+
token = fresh;
|
|
6033
|
+
gitEnv = retryEnv;
|
|
6034
|
+
return provisionRepoAfterClone();
|
|
6035
|
+
}
|
|
6036
|
+
throw new ProvisionError(`git clone failed after a fresh gh token retry: ${gitErrorDetail(retry, fresh)}`);
|
|
6037
|
+
}
|
|
6038
|
+
}
|
|
6039
|
+
throw new ProvisionError(`git clone failed: ${detail}`);
|
|
5095
6040
|
}
|
|
6041
|
+
return provisionRepoAfterClone();
|
|
6042
|
+
|
|
6043
|
+
// Everything after a successful clone — hoisted into a closure so the
|
|
6044
|
+
// auth-retry path above re-enters it without duplicating the body. The clone
|
|
6045
|
+
// result itself is not read below (only its success mattered); the workspace
|
|
6046
|
+
// on disk is the real output.
|
|
6047
|
+
function provisionRepoAfterClone() {
|
|
5096
6048
|
|
|
5097
6049
|
// #222: the clone (the dominant blocking op and the first repo side effect) is
|
|
5098
6050
|
// done. Recheck the abort signal before the remaining side effects/network ops
|
|
@@ -5626,7 +6578,16 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5626
6578
|
// the authoritative result so eligibility (auto checkpointing) doesn't
|
|
5627
6579
|
// under-detect an authenticated clone.
|
|
5628
6580
|
const authenticated = !!token || isAuthenticatedRemote(repo.url);
|
|
5629
|
-
|
|
6581
|
+
// `effectiveToken` is the credential the workspace ACTUALLY runs on after any
|
|
6582
|
+
// #293 auth-failure retry rotated it mid-provisioning (the local `token` is
|
|
6583
|
+
// reassigned on adoption). The caller (workAgent) must use THIS — not its own
|
|
6584
|
+
// pre-provisioning resolution — for the post-provisioning credential users
|
|
6585
|
+
// (finalizeGit's PR reconcile/push, the checkpoint GC/ref-delete askpass), or
|
|
6586
|
+
// a recovered clone would keep failing with the stale credential (Copilot
|
|
6587
|
+
// review on #293). `credentialSource` is echoed back so finalizeGit can gate
|
|
6588
|
+
// its own push-retry on the same source rule.
|
|
6589
|
+
return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, effectiveToken: token || null, credentialSource, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
|
|
6590
|
+
} // end provisionRepoAfterClone
|
|
5630
6591
|
}
|
|
5631
6592
|
|
|
5632
6593
|
// Look up a PR for this branch (2a does NOT open it — the harness does, driven
|
|
@@ -5978,6 +6939,17 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5978
6939
|
restored,
|
|
5979
6940
|
priorNotRestored,
|
|
5980
6941
|
flushTimeoutMs: cfg.flushTimeoutMs,
|
|
6942
|
+
// Adopt a credential the job ROTATED after this checkpointer was created —
|
|
6943
|
+
// e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
|
|
6944
|
+
// (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
|
|
6945
|
+
// its own askpass purely from `token`, so re-binding `token` here is enough
|
|
6946
|
+
// to make the cleanup push authenticate with the live credential instead of
|
|
6947
|
+
// the one the remote already rejected — otherwise the WIP ref lingers until a
|
|
6948
|
+
// later GC (Copilot review on #293). Ignored when the token is unchanged or
|
|
6949
|
+
// falsy; the live checkpointer has already stopped by the time this is called.
|
|
6950
|
+
updateCredential(nextToken) {
|
|
6951
|
+
if (nextToken && nextToken !== token) token = nextToken;
|
|
6952
|
+
},
|
|
5981
6953
|
// True when the workspace still holds checkpoint content NOT captured by a
|
|
5982
6954
|
// commit, so the pushed branch does not represent it (an uncommitted restored
|
|
5983
6955
|
// or working delta that lives only on the WIP ref). The discard-on-ack gate
|
|
@@ -6069,7 +7041,7 @@ function computeRelayCloseReason(result, runCompleted, gitResult) {
|
|
|
6069
7041
|
// branch.push), and reconcile the agent-opened PR (when task.allowPr). A push
|
|
6070
7042
|
// failure is reported (pushError) rather than thrown — the process model decides
|
|
6071
7043
|
// what to do next, and re-running the agent would be non-idempotent.
|
|
6072
|
-
function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, _runGit = null }) {
|
|
7044
|
+
function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, credentialSource = null, _runGit = null, _pushGit = null, _reconcilePr = null, _postAttribution = null }) {
|
|
6073
7045
|
const log = logger || getLogger(); // route observability through the injected logger (tests) or the host logger
|
|
6074
7046
|
// Test-only seam: the late remote-reachability FILTER scans (movedStray /
|
|
6075
7047
|
// unpushedBranchCommits / final stranded) AND the remote-tip VERIFICATION calls
|
|
@@ -6080,8 +7052,25 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6080
7052
|
// probes cannot be made to fail deterministically. Defaults to the module `runGit`, so
|
|
6081
7053
|
// production behaviour is unchanged.
|
|
6082
7054
|
const lateScanGit = _runGit || runGit;
|
|
7055
|
+
// Test-only seam for the branch PUSH itself (the initial push AND the single
|
|
7056
|
+
// post-auth-failure retry). Routed separately from `lateScanGit` so a
|
|
7057
|
+
// hermetic test can drive the central push-recovery branches — changed-token
|
|
7058
|
+
// retry success, the unchanged-token guard, the credential-source gate, and
|
|
7059
|
+
// retry-still-fails — without a live remote, while the local enumeration and
|
|
7060
|
+
// late remote-reachability scans keep their own runner (Copilot review on
|
|
7061
|
+
// #293). Defaults to the module `runGit`, so production behaviour is unchanged.
|
|
7062
|
+
const pushGit = _pushGit || runGit;
|
|
7063
|
+
const reconcilePr = _reconcilePr || reconcileAgentPr;
|
|
7064
|
+
const postAttribution = _postAttribution || postAgentAttribution;
|
|
6083
7065
|
const cs = corr ? ` [${corr}]` : ''; // #229 cross-channel correlation suffix
|
|
6084
7066
|
const out = { branch: workingBranch, baseSha: startSha || null, headSha: null, commits: [], pushed: false, remote: null, pr: null };
|
|
7067
|
+
// The credential this finalize was HANDED. The push auth-failure recovery below
|
|
7068
|
+
// may rotate `token` to a freshly re-read gh credential mid-finalize; capturing
|
|
7069
|
+
// the original lets the return expose `rotatedToken` so the CALLER can propagate
|
|
7070
|
+
// the rotation to subsystems that captured the pre-rotation credential — notably
|
|
7071
|
+
// the WIP-checkpoint cleanup, whose `discardAfterAck()` would otherwise push a
|
|
7072
|
+
// ref deletion with the already-rejected token (Copilot review on #293).
|
|
7073
|
+
const originalToken = token;
|
|
6085
7074
|
|
|
6086
7075
|
// Bound EVERY finalize git op against ONE shared deadline derived from the recovery
|
|
6087
7076
|
// window (budgetMs) — the LOCAL graph scans below AND the sequential NETWORK ops
|
|
@@ -6465,7 +7454,100 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6465
7454
|
// qualified heads ref resolves unambiguously; --set-upstream still tracks the
|
|
6466
7455
|
// local branch.
|
|
6467
7456
|
const pushTimeout = netTimeoutMs();
|
|
6468
|
-
|
|
7457
|
+
let push = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
|
|
7458
|
+
// #293: an AUTH rejection (401/403, or the OAuth `workflow`-scope refusal)
|
|
7459
|
+
// usually means the cached gh token went stale mid-run. Force a fresh
|
|
7460
|
+
// re-read and retry the push ONCE with it before stranding the commits — a
|
|
7461
|
+
// rotated credential self-heals instead of a `pushFailed`. Only when the
|
|
7462
|
+
// re-read yields a token DIFFERENT from the one that failed (a same-token
|
|
7463
|
+
// retry can't fix a genuine permission denial), and the retry runs with the
|
|
7464
|
+
// fresh token in GIT_TOKEN so the askpass hands it to git. The recovery is
|
|
7465
|
+
// NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
|
|
7466
|
+
// the cache first, so a transiently-failing re-read can't erase a still-valid
|
|
7467
|
+
// — possibly newer — cached credential (Copilot review on #294).
|
|
7468
|
+
// GATED on the credential's SOURCE (Copilot review on #293): only a token
|
|
7469
|
+
// that CAME from the gh cache ('gh-cache') may be replaced by a fresh
|
|
7470
|
+
// `gh auth token` read — an explicit GITHUB_TOKEN ('env') or custom authRef
|
|
7471
|
+
// ('custom') is a deliberately configured identity that must fail as
|
|
7472
|
+
// configured, never be silently swapped for the operator's gh login.
|
|
7473
|
+
// The `token` guard is deliberately ABSENT (Copilot review on #293): a
|
|
7474
|
+
// default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
|
|
7475
|
+
// a worker that cached "logged out" recovers null → fresh token after the
|
|
7476
|
+
// operator runs `gh auth login` instead of stranding the commits. That
|
|
7477
|
+
// recovery must REBUILD the credential plumbing the anonymous gitEnv lacked
|
|
7478
|
+
// (the askpass helper + GIT_ASKPASS, and global git config) — a bare
|
|
7479
|
+
// GIT_TOKEN swap leaves the push unauthenticated.
|
|
7480
|
+
// The credential that PRODUCED the ORIGINAL `push` — preserved separately so
|
|
7481
|
+
// the failure report below redacts the value that actually appeared in that
|
|
7482
|
+
// push's output, even once `token` is swapped to the fresh credential for
|
|
7483
|
+
// subsequent authenticated operations after a retry (Copilot review on #293).
|
|
7484
|
+
let pushFailureToken = token;
|
|
7485
|
+
// The timeout that bounded the reported `push` — promoted alongside
|
|
7486
|
+
// pushFailureToken when a failed retry is promoted to the reported result, so
|
|
7487
|
+
// describeGitFailure attributes the RETRY's timeout, not the original push's
|
|
7488
|
+
// (Copilot review on #293).
|
|
7489
|
+
let pushFailureTimeout = pushTimeout;
|
|
7490
|
+
if (push.status !== 0 && credentialSource === 'gh-cache' && isGitAuthFailure(describeGitFailure('git push', push, { token, timeoutMs: pushTimeout }))) {
|
|
7491
|
+
const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: netTimeoutMs() });
|
|
7492
|
+
if (fresh) {
|
|
7493
|
+
log.info?.(`finalizeGit${cs}: git push auth failed — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the push once`);
|
|
7494
|
+
const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
|
|
7495
|
+
// Reuse the clone-time askpass helper when gitEnv already carries one
|
|
7496
|
+
// (the token-backed path): the helper emits GIT_TOKEN at runtime, so
|
|
7497
|
+
// pointing GIT_TOKEN at the fresh credential is enough — no rewrite.
|
|
7498
|
+
// On the null→fresh recovery the original gitEnv was the ANONYMOUS one
|
|
7499
|
+
// (no askpass helper, GIT_CONFIG_GLOBAL neutralized), so mint one and
|
|
7500
|
+
// restore global config. Mint it in the PARENT run directory
|
|
7501
|
+
// (dirname(workspaceDir) === runDir), NEVER inside workspaceDir: writing
|
|
7502
|
+
// askpass.js/.sh into the checked-out repo after the final commits leaves
|
|
7503
|
+
// untracked files that make hasUncommittedChanges() retain the WIP
|
|
7504
|
+
// checkpoint and pollute a preserved workspace (Copilot review on #294).
|
|
7505
|
+
if (!retryEnv.GIT_ASKPASS) {
|
|
7506
|
+
const retryAskpass = writeAskpass(dirname(workspaceDir), fresh);
|
|
7507
|
+
if (retryAskpass) {
|
|
7508
|
+
retryEnv.GIT_ASKPASS = retryAskpass;
|
|
7509
|
+
delete retryEnv.GIT_CONFIG_GLOBAL;
|
|
7510
|
+
}
|
|
7511
|
+
}
|
|
7512
|
+
const retryTimeout = netTimeoutMs();
|
|
7513
|
+
const retry = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: retryEnv, timeoutMs: retryTimeout });
|
|
7514
|
+
// Rebind the git env to the FRESH credential whenever the retry ran —
|
|
7515
|
+
// regardless of its exit code — so the post-push remote-tip verification
|
|
7516
|
+
// below (`ls-remote`/`fetch`, all `env: gitEnv`) queries origin with the
|
|
7517
|
+
// credential the retry used, not the REJECTED old token. A retry that
|
|
7518
|
+
// exits nonzero because the server accepted the ref before a client-side
|
|
7519
|
+
// timeout would otherwise be verified with the stale token, ls-remote
|
|
7520
|
+
// would fail to authenticate, and a landed push would be misreported as
|
|
7521
|
+
// pushFailed/stranded (Copilot review on #293).
|
|
7522
|
+
gitEnv = retryEnv;
|
|
7523
|
+
if (retry.status === 0) {
|
|
7524
|
+
// Adopt the fresh token for the rest of finalization (the PR reconcile
|
|
7525
|
+
// + attribution below read `token`) and record the success.
|
|
7526
|
+
token = fresh;
|
|
7527
|
+
push = retry;
|
|
7528
|
+
} else {
|
|
7529
|
+
// The retry ALSO exited nonzero. It may still have LANDED the ref before
|
|
7530
|
+
// a client-side timeout (the transport-hiccup case verified below); if it
|
|
7531
|
+
// did, `landed` flips on and pushError is never reported. But if it did
|
|
7532
|
+
// NOT land, the RETRY — not the original stale-token rejection — is the
|
|
7533
|
+
// failure that actually stranded the commits: the original failure is
|
|
7534
|
+
// already RECOVERED (we re-read the token and retried), so reporting it
|
|
7535
|
+
// would point at a resolved cause instead of the real one. Promote `retry`
|
|
7536
|
+
// to the reported push result and carry ITS redaction credential (`fresh`,
|
|
7537
|
+
// which appeared in the retry's output) and ITS timeout into the final
|
|
7538
|
+
// error. The ORIGINAL rejected token must NOT be carried into the
|
|
7539
|
+
// subsequent authenticated operations (PR reconcile + attribution read
|
|
7540
|
+
// `token`): a landed retry would otherwise reconcile/attribute with the
|
|
7541
|
+
// rejected old token and fail despite the push. Adopt `fresh` for all
|
|
7542
|
+
// later ops; `gitEnv` is already rebound above (Copilot review on #293).
|
|
7543
|
+
push = retry;
|
|
7544
|
+
pushFailureToken = fresh;
|
|
7545
|
+
pushFailureTimeout = retryTimeout;
|
|
7546
|
+
token = fresh;
|
|
7547
|
+
log.warn?.(`finalizeGit${cs}: push still failed with the fresh gh token — ${oneLineLog(describeGitFailure('git push', retry, { token: fresh, timeoutMs: retryTimeout }))}`);
|
|
7548
|
+
}
|
|
7549
|
+
}
|
|
7550
|
+
}
|
|
6469
7551
|
if (push.status === 0) out.pushed = true;
|
|
6470
7552
|
else {
|
|
6471
7553
|
// A nonzero `git push` does NOT prove the ref was not updated: a timeout or
|
|
@@ -6547,7 +7629,7 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6547
7629
|
// new commit in this throwaway workspace. Surface the at-risk SHAs explicitly
|
|
6548
7630
|
// (issue #231, defense #2) so the failure is a hard, actionable signal with
|
|
6549
7631
|
// the commits to recover — not a soft pushError that reads as "completed".
|
|
6550
|
-
out.pushError = describeGitFailure('git push', push, { token, timeoutMs:
|
|
7632
|
+
out.pushError = describeGitFailure('git push', push, { token: pushFailureToken, timeoutMs: pushFailureTimeout });
|
|
6551
7633
|
out.pushFailed = true;
|
|
6552
7634
|
// Union the work-branch commit list into the stranded set: out.commits is now
|
|
6553
7635
|
// the promoted branchCommits, but if HEAD sat off the branch and promotion
|
|
@@ -6627,13 +7709,18 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6627
7709
|
// envelope + completion note carry only the `pushFailed`/`strandedCommits`
|
|
6628
7710
|
// recovery signal, not a misleading PR reference.
|
|
6629
7711
|
if (workingBranch && envelope.task?.allowPr && !out.pushFailed) {
|
|
6630
|
-
out.pr =
|
|
7712
|
+
out.pr = reconcilePr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
|
|
6631
7713
|
// Record the agent's authorship on the PR (commits carry the operator's
|
|
6632
7714
|
// identity now, so this preserves the machine-generated provenance).
|
|
6633
7715
|
if (out.pr?.found && out.pr.number && (envelope.repository?.provider || 'github') === 'github') {
|
|
6634
|
-
out.attribution =
|
|
7716
|
+
out.attribution = postAttribution({ workspaceDir, token, number: out.pr.number });
|
|
6635
7717
|
}
|
|
6636
7718
|
}
|
|
7719
|
+
// Surface a mid-finalize credential rotation (the push auth-failure recovery
|
|
7720
|
+
// re-read a fresh gh token) so the caller can propagate it to subsystems that
|
|
7721
|
+
// captured the pre-rotation credential — the WIP-checkpoint cleanup in
|
|
7722
|
+
// particular (Copilot review on #293). Absent when the token was never rotated.
|
|
7723
|
+
if (token && token !== originalToken) out.rotatedToken = token;
|
|
6637
7724
|
return out;
|
|
6638
7725
|
}
|
|
6639
7726
|
|
|
@@ -9578,9 +10665,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
|
|
|
9578
10665
|
* `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
|
|
9579
10666
|
* longer running here, so it rides the marker as a flagged, observational entry
|
|
9580
10667
|
* without making an idle worker look busy or inflating the drain's in-flight count.
|
|
9581
|
-
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object }} fields
|
|
10668
|
+
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
|
|
9582
10669
|
*/
|
|
9583
|
-
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }) {
|
|
10670
|
+
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
|
|
9584
10671
|
const jobList = Array.isArray(jobs) ? jobs : [];
|
|
9585
10672
|
return {
|
|
9586
10673
|
pid,
|
|
@@ -9593,6 +10680,11 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }
|
|
|
9593
10680
|
// SDK client and begun leasing), the producer stamps this; null until then.
|
|
9594
10681
|
// The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
|
|
9595
10682
|
readyAt: readyAt ?? null,
|
|
10683
|
+
// #293: the cached gh auth token's observable identity — { present,
|
|
10684
|
+
// fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
|
|
10685
|
+
// `supervisor status` can show the credential's age/scopes per worker. Null
|
|
10686
|
+
// on a worker that has not primed the cache yet (or predates the field).
|
|
10687
|
+
ghAuth: ghAuth ?? null,
|
|
9596
10688
|
};
|
|
9597
10689
|
}
|
|
9598
10690
|
|
|
@@ -10193,10 +11285,27 @@ async function workAgent(req, flags, ctx) {
|
|
|
10193
11285
|
// for why per-job setup.env is deliberately excluded), so PATH resolves a stable
|
|
10194
11286
|
// representative harness binary.
|
|
10195
11287
|
const agentInstanceProbeOff = String(process.env.NANO_AGENT_INSTANCE || '').trim().toLowerCase() === 'off';
|
|
11288
|
+
// Harness transport protocol, resolved with env-override-then-profile precedence and
|
|
11289
|
+
// reduced to the EFFECTIVE protocol the worker actually runs (a container sandbox is
|
|
11290
|
+
// pipe-only regardless of `protocol`). Computed here — ahead of the durable-transcript
|
|
11291
|
+
// diagnostic below and the #275 mismatch refusal later — so the startup "ACP note" is
|
|
11292
|
+
// keyed on the transport the worker WILL use, not a brittle reparse of the command
|
|
11293
|
+
// line (issue #289 review).
|
|
11294
|
+
const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
|
|
11295
|
+
const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
|
|
11296
|
+
const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
|
|
10196
11297
|
const sdkSupportsAgentInstance =
|
|
10197
11298
|
!!camunda &&
|
|
10198
11299
|
typeof camunda.createAgentInstance === 'function' &&
|
|
10199
11300
|
typeof camunda.updateAgentInstance === 'function';
|
|
11301
|
+
// #289: best-effort host c8ctl / SDK versions, so the startup warning and the
|
|
11302
|
+
// per-job "producer unavailable" diagnostic can name the package that must be
|
|
11303
|
+
// upgraded (the macbook workers ran @camunda8/cli 4.1.0-alpha.2 bundling
|
|
11304
|
+
// @camunda8/orchestration-cluster-api 9.1.5, which has no AgentInstance verbs).
|
|
11305
|
+
// Resolved from the HOST c8ctl CLI's dependency graph (rooted at the real entry /
|
|
11306
|
+
// CLI package), not this plugin's sibling node_modules, so the nested host SDK
|
|
11307
|
+
// actually resolves (issue #289 review). Null when unresolvable.
|
|
11308
|
+
const hostSdkInfo = probeHostSdkInfo();
|
|
10200
11309
|
// The producer's `usable` gate also requires the ACP classifier (agent-instance.mjs);
|
|
10201
11310
|
// without it the producer is inert regardless of SDK support, so exclude the probe too.
|
|
10202
11311
|
const acpClassifierAvailable = typeof agenticSessionAcp?.classifyUpdate === 'function';
|
|
@@ -10267,13 +11376,113 @@ async function workAgent(req, flags, ctx) {
|
|
|
10267
11376
|
const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
|
|
10268
11377
|
logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
|
|
10269
11378
|
logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
|
|
11379
|
+
// #289: fail LOUDLY at worker start — not silently per job — when this worker
|
|
11380
|
+
// cannot record a durable AgentInstance transcript on an agentic profile. Every
|
|
11381
|
+
// `nano work` profile runs an agent harness, so any worker that services external
|
|
11382
|
+
// agent jobs needs the host c8ctl SDK's createAgentInstance/updateAgentInstance
|
|
11383
|
+
// verbs. When the host SDK lacks them (e.g. @camunda8/cli 4.1.0-alpha.2 bundling
|
|
11384
|
+
// @camunda8/orchestration-cluster-api 9.1.5), EVERY job silently records no
|
|
11385
|
+
// transcript — the exact macbook-fleet defect. Name the host versions and the fix so
|
|
11386
|
+
// it is actionable from the startup log alone, rather than a per-job "producer
|
|
11387
|
+
// unavailable" line that merges two causes. NANO_AGENT_INSTANCE=off suppresses the
|
|
11388
|
+
// warning (transcripts were intentionally disabled).
|
|
11389
|
+
if (!agentInstanceProbeOff) {
|
|
11390
|
+
if (!sdkSupportsAgentInstance) {
|
|
11391
|
+
logger.warn(
|
|
11392
|
+
`[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the host c8ctl SDK lacks ` +
|
|
11393
|
+
`createAgentInstance/updateAgentInstance, so NO AgentInstance transcript will be ` +
|
|
11394
|
+
`recorded for any job this worker runs. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}. ` +
|
|
11395
|
+
`Upgrade the host c8ctl (the plugin version is irrelevant — the SDK ships with c8ctl): ` +
|
|
11396
|
+
`@camunda8/cli must bundle @camunda8/orchestration-cluster-api >= 10.0.0-alpha.45 ` +
|
|
11397
|
+
`(the first with the AgentInstance verbs). Set NANO_AGENT_INSTANCE=off to silence this ` +
|
|
11398
|
+
`if transcripts are intentionally disabled.`,
|
|
11399
|
+
);
|
|
11400
|
+
} else if (!acpClassifierAvailable) {
|
|
11401
|
+
logger.warn(
|
|
11402
|
+
`[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the ACP classifier ` +
|
|
11403
|
+
`(agenticSessionAcp.classifyUpdate) is unavailable, so no AgentInstance transcript ` +
|
|
11404
|
+
`will be recorded. This is unexpected (it is a static import) — the plugin install may ` +
|
|
11405
|
+
`be corrupt. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}.`,
|
|
11406
|
+
);
|
|
11407
|
+
} else {
|
|
11408
|
+
// Transcripts CAN be recorded, but only for an ACP harness: a non-ACP harness
|
|
11409
|
+
// emits no session/update stream for the producer to classify, so it records
|
|
11410
|
+
// only the opening CONFIGURATION turn (no body). Surface this once at startup so
|
|
11411
|
+
// an operator running e.g. `copilot --allow-all` (no --acp) knows why a run has a
|
|
11412
|
+
// header-only transcript. Key the note on the EFFECTIVE transport the worker will
|
|
11413
|
+
// actually use (`effectiveProtocol === 'acp'`), NOT a reparse of the command line:
|
|
11414
|
+
// a `protocol: acp` profile is driven over ACP even when the command carries no
|
|
11415
|
+
// literal `--acp` (`ensureAcpFlag` appends it at spawn), and a hand-reparse also
|
|
11416
|
+
// misses supported selectors (`acp` subcommand, `*-acp` adapter, `--experimental-acp`)
|
|
11417
|
+
// while a container sandbox runs pipe-only regardless of `protocol` (issue #289 review).
|
|
11418
|
+
const protoNote = effectiveProtocol === 'acp'
|
|
11419
|
+
? ''
|
|
11420
|
+
: ` NOTE: the harness does not run in ACP mode (effective protocol "${effectiveProtocol}") — a non-ACP harness emits no ACP session/update stream, so only the opening CONFIGURATION turn is recorded (no body). Run the harness in ACP mode (--protocol acp on the host executor) for a full transcript.`;
|
|
11421
|
+
logger.info(`[${name}] Durable agent transcripts enabled (host SDK: ${formatHostSdkInfo(hostSdkInfo)}).${protoNote}`);
|
|
11422
|
+
}
|
|
11423
|
+
}
|
|
10270
11424
|
// Warm the gh-token cache now, off the job-handling path: githubCloneToken()
|
|
10271
11425
|
// may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
|
|
10272
11426
|
// credential fallback, and doing that inside a job handler would block the
|
|
10273
11427
|
// event loop — stalling the lock heartbeat and delaying the job itself.
|
|
10274
11428
|
// Priming here pays that cost once at startup so every later lookup is a warm
|
|
10275
|
-
// cache hit.
|
|
10276
|
-
|
|
11429
|
+
// cache hit. Pass the logger so the token's fingerprint + scopes (never the
|
|
11430
|
+
// value) land in the worker log at startup — issue #293's "which credential is
|
|
11431
|
+
// this worker actually holding?" line.
|
|
11432
|
+
primeGhAuthToken({ logger });
|
|
11433
|
+
// …and keep it fresh for the worker's lifetime (issue #293): a process-lifetime
|
|
11434
|
+
// memo went silently stale the moment the operator refreshed the gh login, and
|
|
11435
|
+
// every job kept getting the OLD token until a restart. The TTL timer re-reads
|
|
11436
|
+
// `gh auth token` — ASYNCHRONOUSLY (spawn, never spawnSync) and DEFERRED while
|
|
11437
|
+
// a job is active (`isIdle`) — so the refresh can never block the event loop /
|
|
11438
|
+
// lock heartbeat the way a synchronous prime on the interval could (Copilot
|
|
11439
|
+
// review on #293). When the cached identity actually changes, `onChange`
|
|
11440
|
+
// republishes the activity marker so `supervisor status` shows the adopted
|
|
11441
|
+
// credential even while the worker sits idle. The supervisor's
|
|
11442
|
+
// `reload-credentials` broadcast (SIGUSR1) forces the same re-read on demand
|
|
11443
|
+
// (non-destructive — an atomic swap, never a drop).
|
|
11444
|
+
// `activeJobs`/`writeActivity` are declared below this point but are only
|
|
11445
|
+
// dereferenced when a tick fires (≥1s later) or a change lands — long after
|
|
11446
|
+
// the whole module body has run — so the closures capture the initialized
|
|
11447
|
+
// bindings (TDZ-safe). The handle is tracked so the stop-signal teardown can
|
|
11448
|
+
// clear it (a ref'd interval would otherwise keep the process alive past
|
|
11449
|
+
// SIGTERM/SIGINT — the --auto liveness test relies on the poll timer being
|
|
11450
|
+
// the sole keepalive).
|
|
11451
|
+
const ghRefreshTimer = startGhAuthTokenRefresh({
|
|
11452
|
+
logger,
|
|
11453
|
+
isIdle: () => activeJobs.size === 0,
|
|
11454
|
+
onChange: () => writeActivity(),
|
|
11455
|
+
});
|
|
11456
|
+
// `supervisor reload-credentials` asks every worker to re-read its gh token
|
|
11457
|
+
// without draining in-flight jobs. The daemon signals SIGUSR1; the handler
|
|
11458
|
+
// re-reads the CLI ASYNCHRONOUSLY (spawn, never spawnSync) and serialized
|
|
11459
|
+
// against the TTL timer, so even a worker mid-job keeps its event loop / lock
|
|
11460
|
+
// heartbeat responsive while the new credential is adopted. Deliberately NOT
|
|
11461
|
+
// the SIGUSR2 drain path — reloading a credential must not park the fleet.
|
|
11462
|
+
// Windows cannot deliver SIGUSR1 (Node terminates the process), so the daemon
|
|
11463
|
+
// refuses the op there. `writeActivityRef` is late-bound to `writeActivity`
|
|
11464
|
+
// once it is defined below so the handler can republish the token's
|
|
11465
|
+
// age/scopes once the async re-read settles.
|
|
11466
|
+
let writeActivityRef = null;
|
|
11467
|
+
if (process.platform !== 'win32') {
|
|
11468
|
+
process.on('SIGUSR1', () => {
|
|
11469
|
+
// Use the ASYNC refresh (spawn, never spawnSync) so re-reading/replacing
|
|
11470
|
+
// the cached credential can never pause the event loop / lock heartbeat of a
|
|
11471
|
+
// worker servicing a job — the whole promise of `reload-credentials` is
|
|
11472
|
+
// that it does NOT disrupt in-flight jobs (Copilot review on #293). It is
|
|
11473
|
+
// serialized against the periodic timer via `ghAuthRefreshInFlight`.
|
|
11474
|
+
// Deliberately NOT `invalidateGhAuthToken()` first: the async `force` read
|
|
11475
|
+
// re-reads `gh auth token` regardless of TTL and swaps the cache ATOMICALLY
|
|
11476
|
+
// when the new value lands, so the worker keeps serving the still-valid old
|
|
11477
|
+
// token for the ~10s read window instead of handing out null meanwhile.
|
|
11478
|
+
refreshGhAuthTokenAsync({ force: true })
|
|
11479
|
+
.then(({ changed }) => {
|
|
11480
|
+
logger.info(`Credentials reload requested — re-read the gh auth token: ${describeGhAuthToken(ghAuthTokenInfo())}${changed ? ' (token changed)' : ''}`);
|
|
11481
|
+
if (writeActivityRef) writeActivityRef();
|
|
11482
|
+
})
|
|
11483
|
+
.catch(() => { /* best effort — a wedged gh must not crash the worker */ });
|
|
11484
|
+
});
|
|
11485
|
+
}
|
|
10277
11486
|
logger.info('Polling for work — press Ctrl-C to stop.');
|
|
10278
11487
|
|
|
10279
11488
|
// When launched under the supervisor, report per-job activity — which job(s)
|
|
@@ -10458,7 +11667,10 @@ async function workAgent(req, flags, ctx) {
|
|
|
10458
11667
|
// Bound the last-activation guard map (TTL + settle-aware size cap). Shared
|
|
10459
11668
|
// with recordJobStart so a standalone worker still prunes it.
|
|
10460
11669
|
pruneLastActivation(nowMs);
|
|
10461
|
-
|
|
11670
|
+
// #293: publish the cached gh token's identity (fingerprint/scopes/age —
|
|
11671
|
+
// never the value) so `supervisor status` can answer "is this worker holding
|
|
11672
|
+
// a stale token?" in one line.
|
|
11673
|
+
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
|
|
10462
11674
|
const tmp = `${activityFile}.${process.pid}.tmp`;
|
|
10463
11675
|
try {
|
|
10464
11676
|
mkdirSync(dirname(activityFile), { recursive: true });
|
|
@@ -10469,6 +11681,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
10469
11681
|
/* best effort — activity is advisory, never fail a job over it */
|
|
10470
11682
|
}
|
|
10471
11683
|
};
|
|
11684
|
+
// Late-bind the SIGUSR1 (reload-credentials) handler's republish hook now that
|
|
11685
|
+
// writeActivity exists (#293).
|
|
11686
|
+
writeActivityRef = writeActivity;
|
|
10472
11687
|
// The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
|
|
10473
11688
|
// host connection carries presence/steer/ownership/transcript for EVERY
|
|
10474
11689
|
// supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
|
|
@@ -10745,14 +11960,14 @@ async function workAgent(req, flags, ctx) {
|
|
|
10745
11960
|
logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
|
|
10746
11961
|
}
|
|
10747
11962
|
|
|
10748
|
-
// #110: the role's
|
|
10749
|
-
//
|
|
10750
|
-
//
|
|
10751
|
-
//
|
|
10752
|
-
//
|
|
10753
|
-
//
|
|
10754
|
-
|
|
10755
|
-
|
|
11963
|
+
// #110: the role's ACP permission policy (yolo|escalate|filter), resolved with
|
|
11964
|
+
// the same env-override-then-profile precedence as terminal. The harness protocol
|
|
11965
|
+
// (`envProtocol`/`roleProtocol`/`effectiveProtocol`) is resolved earlier, ahead of
|
|
11966
|
+
// the durable-transcript startup diagnostic (issue #289 review).
|
|
11967
|
+
// `NANO_AGENTIC_PERMISSION` overrides a one-off worker; otherwise the hire profile
|
|
11968
|
+
// decides; else the safe default (yolo). escalate/filter are carried through
|
|
11969
|
+
// verbatim — the acp-executor task enforces yolo and interim-handles the reserved
|
|
11970
|
+
// policies.
|
|
10756
11971
|
const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
|
|
10757
11972
|
const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
|
|
10758
11973
|
|
|
@@ -10767,8 +11982,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
10767
11982
|
// container profile's command STILL pipes plain JSON to an ACP harness — the
|
|
10768
11983
|
// same husk. `effectiveHarnessProtocol` reduces a container's protocol to `pipe`
|
|
10769
11984
|
// so the refusal covers that pipe-only executor too, not just the host (thread
|
|
10770
|
-
// 4118327651).
|
|
10771
|
-
const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
|
|
11985
|
+
// 4118327651). `effectiveProtocol` is computed earlier (issue #289 review).
|
|
10772
11986
|
const startupMismatch = detectProtocolMismatch({ command: profile.command, args: effectiveArgs, protocol: effectiveProtocol });
|
|
10773
11987
|
if (startupMismatch) {
|
|
10774
11988
|
// Non-restartable config failure: under the supervisor this must exit with
|
|
@@ -10894,7 +12108,14 @@ async function workAgent(req, flags, ctx) {
|
|
|
10894
12108
|
// Assemble + normalize the task envelope from headers (defaults) and
|
|
10895
12109
|
// variables (overrides), then resolve any secrets it references.
|
|
10896
12110
|
const envelope = normalizeTaskEnvelope(job.customHeaders ?? {}, job.variables ?? {}, { basePromptOverride });
|
|
10897
|
-
const { resolved, missing, names } = resolveJobSecrets(secretResolver, envelope);
|
|
12111
|
+
const { resolved, missing, names } = await resolveJobSecrets(secretResolver, envelope);
|
|
12112
|
+
// #222: recheck after the awaited secret resolution (stage: secrets) —
|
|
12113
|
+
// resolveJobSecrets AWAITS the gh-cache-miss forced refresh, so a
|
|
12114
|
+
// force-stop / lease loss can win DURING that await. Gate BEFORE the
|
|
12115
|
+
// missing-secret settlement below so an abort that landed mid-resolve
|
|
12116
|
+
// returns WITHOUT settling (which would race the force-stop yield),
|
|
12117
|
+
// mirroring the post-prompt-fetch gate above.
|
|
12118
|
+
if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'secrets', logger })) return;
|
|
10898
12119
|
if (missing.length > 0) {
|
|
10899
12120
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
10900
12121
|
const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
|
|
@@ -10998,7 +12219,21 @@ async function workAgent(req, flags, ctx) {
|
|
|
10998
12219
|
} else if (agentInstanceProducer.retryPending) {
|
|
10999
12220
|
logger.info(`[${jobType}] AgentInstance producer not yet active for external agent job (${aiCorr}) — the initial createAgentInstance did not mint a key in time (transient rejection, no-key response, or bounded-out create), but the create stays armed to retry on the ACP hot path, so a run that recovers can still record a durable transcript (job completion unaffected).`);
|
|
11000
12221
|
} else {
|
|
11001
|
-
|
|
12222
|
+
// #289: report WHICH precondition failed (SDK create, SDK update, ACP
|
|
12223
|
+
// classifier, or not-external) plus the host SDK versions, instead of
|
|
12224
|
+
// merging the causes into one opaque line — so the no-transcript cause is
|
|
12225
|
+
// root-causable from this line alone. A PERMANENT create rejection (a 4xx
|
|
12226
|
+
// the engine will never mint on retry) is NOT a construction-time
|
|
12227
|
+
// precondition failure — activation WAS attempted and permanently
|
|
12228
|
+
// rejected — so report it distinctly rather than as "activation not
|
|
12229
|
+
// attempted: unknown precondition" (issue #289 review).
|
|
12230
|
+
const rejectedStatus = agentInstanceProducer.permanentlyRejected;
|
|
12231
|
+
if (rejectedStatus != null) {
|
|
12232
|
+
logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation attempted but permanently rejected (HTTP ${rejectedStatus}); host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
|
|
12233
|
+
} else {
|
|
12234
|
+
const reason = agentInstanceProducer.unavailableReason || 'unknown precondition';
|
|
12235
|
+
logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: ${reason}; host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
|
|
12236
|
+
}
|
|
11002
12237
|
}
|
|
11003
12238
|
} catch (err) {
|
|
11004
12239
|
logger.warn(`[${jobType}] AgentInstance producer activate() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; continuing without a durable transcript (job completion unaffected).`);
|
|
@@ -11137,15 +12372,38 @@ async function workAgent(req, flags, ctx) {
|
|
|
11137
12372
|
let cwd;
|
|
11138
12373
|
let extraEnv;
|
|
11139
12374
|
let repoToken = null;
|
|
12375
|
+
let repoCredentialSource = null;
|
|
11140
12376
|
if (hasRepo) {
|
|
11141
12377
|
const provider = envelope.repository.provider || 'github';
|
|
11142
12378
|
const authRef = envelope.repository.authRef;
|
|
11143
|
-
|
|
12379
|
+
// Resolve the credential AND its source once: the source gates whether
|
|
12380
|
+
// a later auth failure may fall back to the operator's `gh auth token`
|
|
12381
|
+
// (only a 'gh-cache' token may — never an explicit GITHUB_TOKEN or a
|
|
12382
|
+
// custom authRef; Copilot review on #293).
|
|
12383
|
+
const cred = githubCloneTokenWithSource({ provider, authRef, secretResolver });
|
|
12384
|
+
repoToken = cred.token; // absent → anonymous clone
|
|
12385
|
+
repoCredentialSource = cred.source;
|
|
11144
12386
|
try {
|
|
11145
12387
|
mkdirSync(workerNsDir, { recursive: true });
|
|
11146
12388
|
runDir = mkdtempSync(join(workerNsDir, 'run-'));
|
|
11147
12389
|
liveRunDirs.add(runDir);
|
|
11148
|
-
provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal });
|
|
12390
|
+
provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal, credentialSource: repoCredentialSource });
|
|
12391
|
+
// #293: provisioning's clone auth-retry may have ROTATED the token
|
|
12392
|
+
// (adopting the fresh gh token into the workspace's askpass env).
|
|
12393
|
+
// Adopt that effective credential for the rest of the job —
|
|
12394
|
+
// finalizeGit's push/PR-reconcile and the checkpoint GC/ref-delete
|
|
12395
|
+
// askpass — so they don't keep failing with the stale pre-provision
|
|
12396
|
+
// token (Copilot review on #293).
|
|
12397
|
+
if (provisioned.effectiveToken) repoToken = provisioned.effectiveToken;
|
|
12398
|
+
if (provisioned.credentialSource) repoCredentialSource = provisioned.credentialSource;
|
|
12399
|
+
// #293: when the rotation above swapped a gh-cache credential, the
|
|
12400
|
+
// `resolved` secret map (built pre-provisioning, forwarded to the
|
|
12401
|
+
// harness as `secretEnv`) still holds the STALE token — sync it so an
|
|
12402
|
+
// allowPr harness's own gh/git runs use the credential provisioning
|
|
12403
|
+
// just adopted, not the one the clone proved dead (Copilot review on
|
|
12404
|
+
// #293). No-op for an env/custom identity, an unchanged token, or a
|
|
12405
|
+
// key the job never forwarded.
|
|
12406
|
+
applyRotatedGhCredential({ resolved, provider, authRef, source: repoCredentialSource, token: repoToken });
|
|
11149
12407
|
if (provisioned.baseFetchError) {
|
|
11150
12408
|
logger.warn(`[${jobType}] job ${job.jobKey} base fetch failed (${aiCorr}) — ${oneLineLog(provisioned.baseFetchError)}; base...head diffs may be unavailable`);
|
|
11151
12409
|
}
|
|
@@ -11498,6 +12756,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
11498
12756
|
hasPrBranch: provisioned.hasPrBranch,
|
|
11499
12757
|
envelope,
|
|
11500
12758
|
token: repoToken,
|
|
12759
|
+
credentialSource: repoCredentialSource,
|
|
11501
12760
|
logger,
|
|
11502
12761
|
corr: aiCorr,
|
|
11503
12762
|
keepRuns,
|
|
@@ -11542,6 +12801,17 @@ async function workAgent(req, flags, ctx) {
|
|
|
11542
12801
|
} else if (provisioned) {
|
|
11543
12802
|
gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
|
|
11544
12803
|
}
|
|
12804
|
+
// Propagate a mid-finalize credential rotation (finalizeGit's push
|
|
12805
|
+
// auth-failure recovery re-read a fresh gh token) to every subsystem that
|
|
12806
|
+
// captured the pre-rotation token: the outer `repoToken` (later redaction /
|
|
12807
|
+
// consumers) and the WIP-checkpoint cleanup, whose post-ack ref deletion
|
|
12808
|
+
// would otherwise authenticate with the credential origin already rejected
|
|
12809
|
+
// and leave the remote WIP ref behind until a GC sweep (Copilot review on
|
|
12810
|
+
// #293). No-op when the token was not rotated.
|
|
12811
|
+
if (gitResult?.rotatedToken) {
|
|
12812
|
+
repoToken = gitResult.rotatedToken;
|
|
12813
|
+
checkpointing?.updateCredential?.(gitResult.rotatedToken);
|
|
12814
|
+
}
|
|
11545
12815
|
// #264: the work is durable on the pushed branch, so the WIP ref goes —
|
|
11546
12816
|
// but only once the engine acks job.complete (below): until then a lost
|
|
11547
12817
|
// lease or failed settle redelivers the job, and the next activation must
|
|
@@ -12038,6 +13308,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12038
13308
|
if (finished) return;
|
|
12039
13309
|
finished = true;
|
|
12040
13310
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
13311
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12041
13312
|
if (reaperTimer) clearInterval(reaperTimer);
|
|
12042
13313
|
if (runDirTimer) clearInterval(runDirTimer);
|
|
12043
13314
|
if (agenticPlane) {
|
|
@@ -12068,6 +13339,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12068
13339
|
aborting = true;
|
|
12069
13340
|
draining = true;
|
|
12070
13341
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
13342
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12071
13343
|
logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
|
|
12072
13344
|
// Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
|
|
12073
13345
|
// clears the ownership registry, so we can yield each one afterwards.
|
|
@@ -12113,6 +13385,11 @@ async function workAgent(req, flags, ctx) {
|
|
|
12113
13385
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
12114
13386
|
const n0 = inFlightCount();
|
|
12115
13387
|
logger.info(`Received ${signal} — draining: polling stopped; waiting on ${n0} in-flight job(s) to finish (send SIGTERM to abort).`);
|
|
13388
|
+
// The refresh timer is ref'd (so it fires on cadence even under a busy
|
|
13389
|
+
// loop) and there may be NO in-flight job during a drain — clear it now
|
|
13390
|
+
// so it can't keep the process alive once the drain completes and the
|
|
13391
|
+
// runtime is interrupted.
|
|
13392
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12116
13393
|
// Poll until every in-flight job has settled and released its slot, then
|
|
12117
13394
|
// interrupt the now-idle runtime and exit. Wait INDEFINITELY — a force
|
|
12118
13395
|
// abort (SIGTERM) is the only escape hatch.
|
|
@@ -12508,8 +13785,9 @@ function formatDuration(ms) {
|
|
|
12508
13785
|
return `${d}d${h % 24}h`;
|
|
12509
13786
|
}
|
|
12510
13787
|
|
|
12511
|
-
/** Project a live/stored worker record to a status row (pure w.r.t. `now`).
|
|
12512
|
-
|
|
13788
|
+
/** Project a live/stored worker record to a status row (pure w.r.t. `now`).
|
|
13789
|
+
* `readActivity` is an injectable seam over readWorkerActivity (testing). */
|
|
13790
|
+
function summarizeSupervisorWorker(w, now = Date.now(), { readActivity = readWorkerActivity } = {}) {
|
|
12513
13791
|
const alive = isPidAlive(w.pid);
|
|
12514
13792
|
// Absolute base epoch (ms) the worker started, so an attached console can
|
|
12515
13793
|
// re-age `uptimeMs` locally on each tick (see reageSupervisorStatus) without
|
|
@@ -12523,8 +13801,9 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12523
13801
|
let activity = null; // { state: 'busy'|'idle', jobs: [{ key, type, sinceMs, sinceEpochMs }] }
|
|
12524
13802
|
let engine = null; // job-polling engine base URL this worker reported (#99)
|
|
12525
13803
|
let agentic = null; // { status, mode, url, discovered } agentic-channel state (#99)
|
|
13804
|
+
let ghAuth = null; // #293: { present, fingerprint, scopes, since, ageMs, lastCheckAgoMs } — never the token
|
|
12526
13805
|
if (alive) {
|
|
12527
|
-
const act =
|
|
13806
|
+
const act = readActivity(w.id);
|
|
12528
13807
|
if (act && act.pid === w.pid) {
|
|
12529
13808
|
const jobs = Array.isArray(act.jobs)
|
|
12530
13809
|
? act.jobs
|
|
@@ -12556,6 +13835,24 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12556
13835
|
// stale incarnation can't show a dead worker as connected to a hub.
|
|
12557
13836
|
engine = typeof act.engine === 'string' && act.engine ? act.engine : null;
|
|
12558
13837
|
agentic = act.agentic && typeof act.agentic === 'object' ? act.agentic : null;
|
|
13838
|
+
// #293: the gh token identity rides the same marker. Re-age the ticking
|
|
13839
|
+
// durations from the stamped `since`/marker `updatedAt` so a console that
|
|
13840
|
+
// re-ages locally (and a status snapshot read seconds later) shows a
|
|
13841
|
+
// current age, not the write-time one. Never carries the token value.
|
|
13842
|
+
if (act.ghAuth && typeof act.ghAuth === 'object') {
|
|
13843
|
+
const g = act.ghAuth;
|
|
13844
|
+
const ageMs = Number.isFinite(g.since) ? Math.max(0, now - g.since) : (Number.isFinite(g.ageMs) ? g.ageMs : null);
|
|
13845
|
+
const lastCheckAgoMs = Number.isFinite(act.updatedAt) && Number.isFinite(g.lastCheckAgoMs)
|
|
13846
|
+
? Math.max(0, now - act.updatedAt + g.lastCheckAgoMs)
|
|
13847
|
+
: (Number.isFinite(g.lastCheckAgoMs) ? g.lastCheckAgoMs : null);
|
|
13848
|
+
// Coerce the identity strings at the (untrusted) marker boundary: a
|
|
13849
|
+
// malformed/tampered/older marker could carry a non-string `scopes` or
|
|
13850
|
+
// `fingerprint` (a number, array, object), and a downstream string op
|
|
13851
|
+
// (e.g. `scopes.split(',')` in supervisorGhAuthCell) would THROW and
|
|
13852
|
+
// crash `supervisor status` (Copilot review on #294). Keep only real
|
|
13853
|
+
// strings; anything else degrades to null (rendered as `?`/unknown).
|
|
13854
|
+
ghAuth = { present: g.present === true, fingerprint: typeof g.fingerprint === 'string' ? g.fingerprint : null, scopes: typeof g.scopes === 'string' ? g.scopes : null, since: Number.isFinite(g.since) ? g.since : null, ageMs, lastCheckAgoMs };
|
|
13855
|
+
}
|
|
12559
13856
|
}
|
|
12560
13857
|
// No marker (or a stale-pid one): leave activity null → rendered as unknown.
|
|
12561
13858
|
}
|
|
@@ -12580,6 +13877,7 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12580
13877
|
activity,
|
|
12581
13878
|
engine,
|
|
12582
13879
|
agentic,
|
|
13880
|
+
ghAuth,
|
|
12583
13881
|
};
|
|
12584
13882
|
}
|
|
12585
13883
|
|
|
@@ -12614,6 +13912,16 @@ function supervisorStatusSignature(workers) {
|
|
|
12614
13912
|
// change is a real transition that must repaint attached consoles (#99).
|
|
12615
13913
|
w.engine ?? '',
|
|
12616
13914
|
w.agentic ? (w.agentic.status ?? '') : null,
|
|
13915
|
+
// #293: the gh credential IDENTITY — presence, fingerprint, scopes, and the
|
|
13916
|
+
// token's `since` epoch. A token rotation / scope change / logout on an
|
|
13917
|
+
// otherwise-idle worker is a real transition the attached console must
|
|
13918
|
+
// repaint for (the GH AUTH cell changes), so fold it into the signature.
|
|
13919
|
+
// Deliberately EXCLUDES the ticking durations (ageMs, lastCheckAgoMs) —
|
|
13920
|
+
// those merely elapse and are re-aged locally (see reageSupervisorStatus),
|
|
13921
|
+
// so including them would make every clock tick a spurious "change".
|
|
13922
|
+
w.ghAuth
|
|
13923
|
+
? `${w.ghAuth.present ? 1 : 0}\u0000${w.ghAuth.fingerprint ?? ''}\u0000${w.ghAuth.scopes ?? ''}\u0000${w.ghAuth.since ?? ''}`
|
|
13924
|
+
: null,
|
|
12617
13925
|
]),
|
|
12618
13926
|
);
|
|
12619
13927
|
}
|
|
@@ -12676,6 +13984,32 @@ function supervisorAgenticCell(w) {
|
|
|
12676
13984
|
return String(w.agentic.status);
|
|
12677
13985
|
}
|
|
12678
13986
|
|
|
13987
|
+
/**
|
|
13988
|
+
* GH AUTH cell (#293): the cached gh token's age + scopes, so "this worker is
|
|
13989
|
+
* holding a stale token" is a one-line check instead of a manual
|
|
13990
|
+
* /proc/<pid>/environ vs `gh auth token` comparison. `-` for a down/stopping
|
|
13991
|
+
* worker, `?` for a live worker not (yet) reporting or on an older build whose
|
|
13992
|
+
* marker predates the field, `none` when gh is missing/not logged in. The
|
|
13993
|
+
* fingerprint is available in the status JSON (and the worker log) when two
|
|
13994
|
+
* workers need to be told apart; the table keeps the cell narrow (age + scopes).
|
|
13995
|
+
*/
|
|
13996
|
+
function supervisorGhAuthCell(w) {
|
|
13997
|
+
if (w.state !== 'running') return '-';
|
|
13998
|
+
if (!w.activity) return '?'; // alive but not reporting
|
|
13999
|
+
const g = w.ghAuth;
|
|
14000
|
+
if (!g) return '?'; // marker predates the ghAuth field
|
|
14001
|
+
if (!g.present) return 'none';
|
|
14002
|
+
const age = g.ageMs != null ? formatDuration(g.ageMs) : '?';
|
|
14003
|
+
// Defensive against a malformed activity marker (untrusted JSON): only call
|
|
14004
|
+
// string ops on a real string, so a non-string `scopes` (a corrupt/older/
|
|
14005
|
+
// tampered marker) renders `?` instead of throwing and crashing the whole
|
|
14006
|
+
// status table (Copilot review on #294). statusFromActivity already coerces at
|
|
14007
|
+
// the marker boundary; this is the belt-and-braces at the render edge, which
|
|
14008
|
+
// also runs on frames built elsewhere (e.g. reageSupervisorStatus passthrough).
|
|
14009
|
+
const scopes = typeof g.scopes === 'string' && g.scopes ? g.scopes.split(',').map((s) => s.trim()).filter(Boolean).join(',') : '?';
|
|
14010
|
+
return `${age} [${scopes}]`;
|
|
14011
|
+
}
|
|
14012
|
+
|
|
12679
14013
|
/**
|
|
12680
14014
|
* Re-age a supervisor status snapshot to `now`, recomputing the ticking
|
|
12681
14015
|
* durations (`uptimeMs`, per-job `sinceMs`) from the absolute base epochs the
|
|
@@ -12718,7 +14052,17 @@ function reageSupervisorStatus(status, now = Date.now()) {
|
|
|
12718
14052
|
const state = jobs.some((j) => j && typeof j === 'object' && !j.settlementPending) ? 'busy' : 'idle';
|
|
12719
14053
|
activity = { ...activity, jobs, state };
|
|
12720
14054
|
}
|
|
12721
|
-
|
|
14055
|
+
// #293: re-age the gh credential's AGE from its absolute `since` epoch, the
|
|
14056
|
+
// same way uptime/job-age tick locally — otherwise an attached console that
|
|
14057
|
+
// re-ages the cached snapshot on every tick (never rebuilding it via
|
|
14058
|
+
// summarizeSupervisorWorker) would freeze the GH AUTH cell's age at its
|
|
14059
|
+
// snapshot-time value. `lastCheckAgoMs` is RELATIVE (no absolute base rides
|
|
14060
|
+
// the frame), so it cannot be re-aged here and is left as-is.
|
|
14061
|
+
let ghAuth = w.ghAuth;
|
|
14062
|
+
if (ghAuth && typeof ghAuth === 'object' && Number.isFinite(ghAuth.since)) {
|
|
14063
|
+
ghAuth = { ...ghAuth, ageMs: Math.max(0, now - ghAuth.since) };
|
|
14064
|
+
}
|
|
14065
|
+
return { ...w, uptimeMs, activity, ghAuth };
|
|
12722
14066
|
}),
|
|
12723
14067
|
};
|
|
12724
14068
|
}
|
|
@@ -12878,6 +14222,7 @@ function formatSupervisorStatus(status) {
|
|
|
12878
14222
|
state: String(w.state) === 'down' && w.configFailed === true ? 'down (config)' : String(w.state),
|
|
12879
14223
|
engine: supervisorEngineCell(w),
|
|
12880
14224
|
agentic: supervisorAgenticCell(w),
|
|
14225
|
+
ghAuth: supervisorGhAuthCell(w),
|
|
12881
14226
|
job: supervisorJobCell(w),
|
|
12882
14227
|
pid: w.pid ? String(w.pid) : '-',
|
|
12883
14228
|
restarts: String(w.restarts),
|
|
@@ -12887,8 +14232,8 @@ function formatSupervisorStatus(status) {
|
|
|
12887
14232
|
// ENGINE + AGENTIC sit early (just after STATE) so the pinned live view's
|
|
12888
14233
|
// width clamp (which trims from the right) drops the least-critical columns
|
|
12889
14234
|
// (LAST EXIT, UPTIME) first and keeps the visibility diagnostics visible.
|
|
12890
|
-
const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
|
|
12891
|
-
const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'job', 'pid', 'restarts', 'uptime', 'last'];
|
|
14235
|
+
const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', ghAuth: 'GH AUTH', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
|
|
14236
|
+
const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'ghAuth', 'job', 'pid', 'restarts', 'uptime', 'last'];
|
|
12892
14237
|
const width = {};
|
|
12893
14238
|
for (const c of cols) width[c] = Math.max(head[c].length, ...rows.map((r) => r[c].length));
|
|
12894
14239
|
const fmt = (r) => ' ' + cols.map((c) => r[c].padEnd(width[c])).join(' ');
|
|
@@ -13800,6 +15145,40 @@ async function runSupervisorDaemon() {
|
|
|
13800
15145
|
attachClients.add(sock);
|
|
13801
15146
|
sock.write(encodeFrame(statusFrame(false)));
|
|
13802
15147
|
break;
|
|
15148
|
+
case 'reload-credentials': {
|
|
15149
|
+
// #293: tell every worker to re-read its gh auth token (SIGUSR1,
|
|
15150
|
+
// async — the still-valid old token is served until the new one
|
|
15151
|
+
// atomically lands) WITHOUT draining in-flight jobs — the operator
|
|
15152
|
+
// re-logged in and wants
|
|
15153
|
+
// the fleet to pick up the fresh credential now, not after a restart.
|
|
15154
|
+
// Windows cannot deliver SIGUSR1 (Node terminates the process), so
|
|
15155
|
+
// refuse there; the TTL refresh still adopts a rotated token.
|
|
15156
|
+
if (osPlatform() === 'win32') {
|
|
15157
|
+
sock.write(encodeFrame({ ok: false, error: 'reload-credentials is not supported on Windows (it relies on SIGUSR1, which Windows cannot deliver) — the workers\' TTL refresh still picks up a new token, or use `supervisor restart`', final: true }));
|
|
15158
|
+
break;
|
|
15159
|
+
}
|
|
15160
|
+
const rawCreds = Array.isArray(req.targets)
|
|
15161
|
+
? req.targets.flatMap((t) => resolveTargets(t))
|
|
15162
|
+
: resolveTargets(req.target);
|
|
15163
|
+
const credIds = [...new Set(rawCreds)];
|
|
15164
|
+
const signaled = [];
|
|
15165
|
+
const failed = [];
|
|
15166
|
+
for (const id of credIds) {
|
|
15167
|
+
const w = workers.get(id);
|
|
15168
|
+
const alive = !!(w && isPidAlive(w.pid));
|
|
15169
|
+
// #293: only signal a worker whose SIGUSR1 (reload) listener is
|
|
15170
|
+
// PROVEN installed (reloadCredentialsReady) — otherwise the signal
|
|
15171
|
+
// hits Node's default action (start the inspector) and the daemon
|
|
15172
|
+
// would wrongly record "reloaded". Skipped workers land in `failed`
|
|
15173
|
+
// so `ok` stays honest; they pick up the fresh token at their own
|
|
15174
|
+
// startup prime.
|
|
15175
|
+
if (!reloadCredentialsReady(w, alive ? readWorkerActivity(id) : null, alive)) { failed.push(id); continue; }
|
|
15176
|
+
try { process.kill(w.pid, 'SIGUSR1'); signaled.push(id); }
|
|
15177
|
+
catch { failed.push(id); }
|
|
15178
|
+
}
|
|
15179
|
+
sock.write(encodeFrame({ ok: failed.length === 0, type: 'credentials-reloaded', signaled, failed, final: true }));
|
|
15180
|
+
break;
|
|
15181
|
+
}
|
|
13803
15182
|
case 'stop': {
|
|
13804
15183
|
// #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
|
|
13805
15184
|
// jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
|
|
@@ -14305,6 +15684,45 @@ async function supervisorReloadCmd(req) {
|
|
|
14305
15684
|
if (outcome === 'error') process.exit(1);
|
|
14306
15685
|
}
|
|
14307
15686
|
|
|
15687
|
+
/**
|
|
15688
|
+
* Re-read every supervised worker's cached gh auth token WITHOUT draining
|
|
15689
|
+
* in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
|
|
15690
|
+
* worker re-reads `gh auth token` NON-DESTRUCTIVELY (a forced async re-read that
|
|
15691
|
+
* swaps the cache atomically when the new value lands — the still-valid old
|
|
15692
|
+
* token is served meanwhile, never dropped), and logs the new fingerprint/
|
|
15693
|
+
* scopes. Use after a `gh auth login`/`refresh` so the fleet picks up the fresh
|
|
15694
|
+
* credential immediately instead of waiting for the TTL refresh or a restart.
|
|
15695
|
+
*/
|
|
15696
|
+
async function supervisorReloadCredentialsCmd(req) {
|
|
15697
|
+
const logger = getLogger();
|
|
15698
|
+
const target = req.positional[1] || 'all';
|
|
15699
|
+
const running = await liveSupervisor();
|
|
15700
|
+
if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
15701
|
+
const socketPath = running.socket || getSupervisorSocketPath();
|
|
15702
|
+
let res;
|
|
15703
|
+
try {
|
|
15704
|
+
res = await supervisorRequest({ op: 'reload-credentials', target }, { socketPath });
|
|
15705
|
+
} catch (err) {
|
|
15706
|
+
logger.error(`Could not reach the supervisor control socket to reload credentials: ${err?.message || err}`);
|
|
15707
|
+
process.exit(1);
|
|
15708
|
+
}
|
|
15709
|
+
// A structured credentials-reloaded result carries `signaled`/`failed` arrays
|
|
15710
|
+
// even on a PARTIAL failure (daemon returns `ok:false` with no `error` when
|
|
15711
|
+
// some workers were down). Report those lists BEFORE the generic failure guard
|
|
15712
|
+
// — exactly as the rolling-reload client does — so an operator can see which
|
|
15713
|
+
// workers picked up the new token and which are down, instead of a bare
|
|
15714
|
+
// "reload-credentials failed". The generic guard then only catches a truly
|
|
15715
|
+
// unstructured failure (unreachable socket, the Windows refusal with `error`).
|
|
15716
|
+
if (!res) { logger.error('reload-credentials failed'); process.exit(1); }
|
|
15717
|
+
const hasStructured = Array.isArray(res.signaled) || Array.isArray(res.failed);
|
|
15718
|
+
if (res.ok === false && !hasStructured) { logger.error(res.error || 'reload-credentials failed'); process.exit(1); }
|
|
15719
|
+
const signaled = Array.isArray(res.signaled) ? res.signaled : [];
|
|
15720
|
+
const failed = Array.isArray(res.failed) ? res.failed : [];
|
|
15721
|
+
if (signaled.length > 0) logger.info(`Told ${signaled.length} worker(s) to re-read their gh auth token: ${signaled.join(', ')}. Each re-reads \`gh auth token\` (non-destructively — the still-valid old token is served until the new one lands) and logs the new fingerprint/scopes.`);
|
|
15722
|
+
else logger.warn('No running workers matched — nothing to reload.');
|
|
15723
|
+
if (failed.length > 0) { logger.error(`Could not signal: ${failed.join(', ')} (down or gone).`); process.exit(1); }
|
|
15724
|
+
}
|
|
15725
|
+
|
|
14308
15726
|
/**
|
|
14309
15727
|
* Count the in-flight jobs across a fleet snapshot (array of
|
|
14310
15728
|
* `summarizeSupervisorWorker` results) — the number an operator is waiting on
|
|
@@ -14515,6 +15933,10 @@ async function attachSupervisorConsole(state) {
|
|
|
14515
15933
|
out(`• removed: ${frame.removed.join(', ') || '(none matched)'}.`);
|
|
14516
15934
|
} else if (frame.type === 'restarted') {
|
|
14517
15935
|
out(`• restarted: ${frame.restarted.join(', ') || '(none matched)'}.`);
|
|
15936
|
+
} else if (frame.type === 'credentials-reloaded') {
|
|
15937
|
+
const sig = Array.isArray(frame.signaled) ? frame.signaled : [];
|
|
15938
|
+
const fail = Array.isArray(frame.failed) ? frame.failed : [];
|
|
15939
|
+
out(`• credentials reload signaled: ${sig.join(', ') || '(none matched)'}${fail.length ? ` — could not signal: ${fail.join(', ')}` : ''}.`);
|
|
14518
15940
|
} else if (frame.ok === false) {
|
|
14519
15941
|
out(`! ${frame.error}`);
|
|
14520
15942
|
}
|
|
@@ -14560,10 +15982,15 @@ async function attachSupervisorConsole(state) {
|
|
|
14560
15982
|
case '': break;
|
|
14561
15983
|
case 'help':
|
|
14562
15984
|
out('Commands: status | add <profile> [--name <worker>] [--instances <n>] [work flags] |');
|
|
14563
|
-
out(' remove <id|profile|all> | restart <id|profile|all> |');
|
|
15985
|
+
out(' remove <id|profile|all> | restart <id|profile|all> | reload-credentials [id|profile|all] |');
|
|
14564
15986
|
out(' logs [id] | detach | stop | help');
|
|
14565
15987
|
break;
|
|
14566
15988
|
case 'status': sock.write(encodeFrame({ op: 'status' })); break;
|
|
15989
|
+
case 'reload-credentials': {
|
|
15990
|
+
const target = parts.shift() || 'all';
|
|
15991
|
+
sock.write(encodeFrame({ op: 'reload-credentials', target }));
|
|
15992
|
+
break;
|
|
15993
|
+
}
|
|
14567
15994
|
case 'add': {
|
|
14568
15995
|
const profile = parts.shift();
|
|
14569
15996
|
if (!profile) { out('usage: add <profile> [--name <worker>] [--instances <n>] [work flags]'); break; }
|
|
@@ -15109,6 +16536,9 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
15109
16536
|
case 'reload':
|
|
15110
16537
|
await supervisorReloadCmd(req);
|
|
15111
16538
|
return;
|
|
16539
|
+
case 'reload-credentials':
|
|
16540
|
+
await supervisorReloadCredentialsCmd(req);
|
|
16541
|
+
return;
|
|
15112
16542
|
case 'stop':
|
|
15113
16543
|
await supervisorStopCmd(coerceBool(flags?.force, false));
|
|
15114
16544
|
return;
|
|
@@ -15117,7 +16547,7 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
15117
16547
|
supervisorLogsCmd(req);
|
|
15118
16548
|
return;
|
|
15119
16549
|
default:
|
|
15120
|
-
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach`);
|
|
16550
|
+
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
|
|
15121
16551
|
process.exit(1);
|
|
15122
16552
|
}
|
|
15123
16553
|
}
|
|
@@ -17922,8 +19352,30 @@ export {
|
|
|
17922
19352
|
namespaceHasLiveHarness,
|
|
17923
19353
|
authUrl,
|
|
17924
19354
|
githubCloneToken,
|
|
19355
|
+
githubCloneTokenWithSource,
|
|
17925
19356
|
ghAuthTokenFromCli,
|
|
17926
19357
|
primeGhAuthToken,
|
|
19358
|
+
refreshGhAuthToken,
|
|
19359
|
+
refreshGhAuthTokenAsync,
|
|
19360
|
+
invalidateGhAuthToken,
|
|
19361
|
+
recoverGhTokenAfterAuthFailure,
|
|
19362
|
+
setGhAuthTokenReader,
|
|
19363
|
+
resetGhAuthTokenReader,
|
|
19364
|
+
classifyGhAuthRead,
|
|
19365
|
+
GH_AUTH_READ_FAILED,
|
|
19366
|
+
adoptGhAuthToken,
|
|
19367
|
+
adoptGhAuthTokenAsync,
|
|
19368
|
+
readGhAuthTokenAsync,
|
|
19369
|
+
ghCredentialEnvName,
|
|
19370
|
+
applyRotatedGhCredential,
|
|
19371
|
+
ghAuthTokenInfo,
|
|
19372
|
+
describeGhAuthToken,
|
|
19373
|
+
ghTokenFingerprint,
|
|
19374
|
+
ghTokenScopes,
|
|
19375
|
+
ghTokenScopesAsync,
|
|
19376
|
+
isGitAuthFailure,
|
|
19377
|
+
startGhAuthTokenRefresh,
|
|
19378
|
+
GH_AUTH_TOKEN_TTL_MS,
|
|
17927
19379
|
ghAuthEnv,
|
|
17928
19380
|
redactToken,
|
|
17929
19381
|
agentRunsRoot,
|
|
@@ -18004,10 +19456,12 @@ export {
|
|
|
18004
19456
|
countSupervisorInFlight,
|
|
18005
19457
|
supervisorEngineCell,
|
|
18006
19458
|
supervisorAgenticCell,
|
|
19459
|
+
supervisorGhAuthCell,
|
|
18007
19460
|
agenticStateForTarget,
|
|
18008
19461
|
normalizeAgenticMessage,
|
|
18009
19462
|
buildActivityPayload,
|
|
18010
19463
|
activityMarkerReadyFor,
|
|
19464
|
+
reloadCredentialsReady,
|
|
18011
19465
|
waitForChildExit,
|
|
18012
19466
|
supervisorWorkerActivityFile,
|
|
18013
19467
|
SETTLEMENT_PENDING_GHOST_TTL_MS,
|
|
@@ -18019,6 +19473,7 @@ export {
|
|
|
18019
19473
|
supervisorStartCmd,
|
|
18020
19474
|
supervisorAddCmd,
|
|
18021
19475
|
supervisorReloadCmd,
|
|
19476
|
+
supervisorReloadCredentialsCmd,
|
|
18022
19477
|
workforceReloadCmd,
|
|
18023
19478
|
runningSupervisor,
|
|
18024
19479
|
readSupervisorState,
|
|
@@ -18127,6 +19582,7 @@ export const metadata = {
|
|
|
18127
19582
|
{ command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
|
|
18128
19583
|
{ command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
|
|
18129
19584
|
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
|
|
19585
|
+
{ command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
|
|
18130
19586
|
{ command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
|
|
18131
19587
|
{ command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
|
|
18132
19588
|
{ command: 'c8ctl nano workforce add qwen --instances 2 --roles pr-review,feature', description: "Add an entry mapped to explicit job types (<rank>:pr-review, <rank>:feature, where <rank> is the qwen hire's rank at start) — does not mutate the hired profile" },
|
|
@@ -18367,7 +19823,7 @@ function printUsage() {
|
|
|
18367
19823
|
console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--terminal pty|pipe] [--protocol pipe|acp] [--permission yolo|escalate|filter] [--env NAME=VALUE ...] [--list]');
|
|
18368
19824
|
console.log(' c8ctl nano assign <profileName> <cap[,cap...]> [--name <n>] [--capabilities <a,b>]');
|
|
18369
19825
|
console.log(' c8ctl nano work <profileName> [--auto [--auto-scope <p>]] [--arg <switch> ...] [--recovery-window <ms>] [--idle-timeout <ms>] [--job-timeout <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
|
|
18370
|
-
console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach] ... (manage many workers from one terminal)');
|
|
19826
|
+
console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach] ... (manage many workers from one terminal)');
|
|
18371
19827
|
console.log(' c8ctl nano workforce [add|remove|list|start|status|stop|reload] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
|
|
18372
19828
|
console.log('');
|
|
18373
19829
|
console.log('Subcommands:');
|