c8ctl-plugin-nano 1.70.2 → 1.70.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -2
- package/agent-checkpoint.mjs +243 -6
- package/agent-instance.mjs +180 -22
- package/c8ctl-plugin.js +2313 -121
- package/package.json +8 -8
- package/supervisor.dist.js +11 -11
package/c8ctl-plugin.js
CHANGED
|
@@ -90,11 +90,115 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
|
|
|
90
90
|
// so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
|
|
91
91
|
// a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
|
|
92
92
|
import { resolveEffectiveEnvelope } from './agent-resume.mjs';
|
|
93
|
-
import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
|
|
93
|
+
import { checkpointConfig, checkpointEligibility, checkpointRef, recoveredCheckpointRef, predecessorCheckpointRef, resumeRefIsExplicit, pushNamedRef, observeRemoteRefSha, makeOrphanProbe, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
|
|
94
94
|
|
|
95
95
|
const requireFromHere = createRequire(import.meta.url);
|
|
96
96
|
const pluginDir = dirname(fileURLToPath(import.meta.url));
|
|
97
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Build the ordered list of `require` resolvers used to read the HOST c8ctl / Camunda
|
|
100
|
+
* SDK package manifests (issue #289 review). The host SDK lives in the host **c8ctl
|
|
101
|
+
* CLI's** dependency graph, NOT this plugin's. In the managed layout the plugin loads
|
|
102
|
+
* from `<data>/plugins/node_modules`, a SIBLING of `@camunda8/cli`; in the global
|
|
103
|
+
* layout the SDK is nested BENEATH the CLI package. Node resolution never searches a
|
|
104
|
+
* sibling package's `node_modules`, so a probe rooted only at this module commonly
|
|
105
|
+
* reads `null` for both versions — exactly when it needs to name the old host SDK.
|
|
106
|
+
* Resolve from the host graph instead:
|
|
107
|
+
* 1. a `require` rooted at the real c8ctl entry (`C8CTL_NANO_ENTRY` / `process.argv[1]`),
|
|
108
|
+
* canonicalized through symlinks — the CLI and its nested SDK live in THIS graph;
|
|
109
|
+
* 2. a `require` rooted at the resolved `@camunda8/cli` package itself, so an SDK
|
|
110
|
+
* nested beneath the CLI (not hoisted to the entry root) still resolves;
|
|
111
|
+
* 3. this module's own graph as a last-resort fallback (works in a hoisted/global
|
|
112
|
+
* layout where the SDK is visible from here).
|
|
113
|
+
* Every step is guarded: an unresolvable root is skipped, never thrown.
|
|
114
|
+
*
|
|
115
|
+
* The entry is `realpathSync`'d first because a globally installed CLI is launched
|
|
116
|
+
* through a launcher symlink (e.g. `/usr/local/bin/c8ctl` → `.../lib/node_modules/@camunda8/cli/bin/...`),
|
|
117
|
+
* which Node preserves verbatim in `process.argv[1]`. Rooting `createRequire` at the
|
|
118
|
+
* raw symlink would search `/usr/local/bin/node_modules` and miss the nested host SDK
|
|
119
|
+
* entirely — exactly the global layout this probe exists to diagnose.
|
|
120
|
+
*/
|
|
121
|
+
export function buildHostSdkResolvers(entryArg) {
|
|
122
|
+
const resolvers = [];
|
|
123
|
+
const seen = new Set();
|
|
124
|
+
const push = (rq) => {
|
|
125
|
+
if (typeof rq === 'function' && !seen.has(rq)) {
|
|
126
|
+
seen.add(rq);
|
|
127
|
+
resolvers.push(rq);
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
const rawEntry = entryArg !== undefined ? entryArg : (process.env.C8CTL_NANO_ENTRY || process.argv[1]);
|
|
131
|
+
let entry = rawEntry;
|
|
132
|
+
if (entry) {
|
|
133
|
+
try {
|
|
134
|
+
entry = realpathSync(entry);
|
|
135
|
+
} catch {
|
|
136
|
+
/* not a real path yet (or already canonical) — fall back to the raw entry */
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
let entryRequire = null;
|
|
140
|
+
if (entry) {
|
|
141
|
+
try {
|
|
142
|
+
entryRequire = createRequire(entry);
|
|
143
|
+
push(entryRequire);
|
|
144
|
+
} catch {
|
|
145
|
+
/* unresolvable entry — skip */
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
for (const rq of [entryRequire, requireFromHere]) {
|
|
149
|
+
if (!rq) continue;
|
|
150
|
+
try {
|
|
151
|
+
const cliPkg = rq.resolve('@camunda8/cli/package.json');
|
|
152
|
+
push(createRequire(cliPkg));
|
|
153
|
+
} catch {
|
|
154
|
+
/* CLI not resolvable from this root — skip */
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
push(requireFromHere);
|
|
158
|
+
return resolvers;
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Best-effort host c8ctl / Camunda SDK versions (issue #289). The host SDK lives in the
|
|
163
|
+
* host c8ctl CLI's dependency graph, which this plugin's (sibling) `node_modules` cannot
|
|
164
|
+
* reach, so the probe walks an ordered chain of resolvers rooted at the host CLI entry /
|
|
165
|
+
* package first and this module only as a fallback (see `buildHostSdkResolvers`) — the
|
|
166
|
+
* first resolver to yield a version wins. Every read is guarded: an unresolvable package
|
|
167
|
+
* (hoisting layout, bundling) simply yields null rather than throwing. Tests may inject a
|
|
168
|
+
* single `requireFn` (used as the sole resolver) or an explicit `resolvers` array.
|
|
169
|
+
*/
|
|
170
|
+
export function probeHostSdkInfo(requireFn, resolvers) {
|
|
171
|
+
const chain = Array.isArray(resolvers) && resolvers.length > 0
|
|
172
|
+
? resolvers
|
|
173
|
+
: requireFn
|
|
174
|
+
? [requireFn]
|
|
175
|
+
: buildHostSdkResolvers();
|
|
176
|
+
const read = (spec) => {
|
|
177
|
+
for (const rq of chain) {
|
|
178
|
+
try {
|
|
179
|
+
const v = rq(`${spec}/package.json`)?.version;
|
|
180
|
+
if (typeof v === 'string' && v.trim() !== '') return String(v);
|
|
181
|
+
} catch {
|
|
182
|
+
/* try the next resolver in the chain */
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return null;
|
|
186
|
+
};
|
|
187
|
+
return {
|
|
188
|
+
cliVersion: read('@camunda8/cli'),
|
|
189
|
+
sdkVersion: read('@camunda8/orchestration-cluster-api'),
|
|
190
|
+
};
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** Render host SDK versions for a log line (issue #289). Never throws. */
|
|
194
|
+
export function formatHostSdkInfo(info) {
|
|
195
|
+
const cli = info?.cliVersion ? `@camunda8/cli ${info.cliVersion}` : '@camunda8/cli (version unknown)';
|
|
196
|
+
const sdk = info?.sdkVersion
|
|
197
|
+
? `@camunda8/orchestration-cluster-api ${info.sdkVersion}`
|
|
198
|
+
: '@camunda8/orchestration-cluster-api (version unknown)';
|
|
199
|
+
return `${cli}, ${sdk}`;
|
|
200
|
+
}
|
|
201
|
+
|
|
98
202
|
/**
|
|
99
203
|
* Read the bundled-binary marker (nanobpmn-binary.json) written by the upstream
|
|
100
204
|
* release pipeline. Records which nanobpmn build the shipped binaries came from.
|
|
@@ -156,6 +260,24 @@ const READINESS_TIMEOUT_MS = 60_000;
|
|
|
156
260
|
const READINESS_POLL_MS = 500;
|
|
157
261
|
const HEALTH_TIMEOUT_MS = 1_500;
|
|
158
262
|
const STOP_GRACE_MS = 8_000;
|
|
263
|
+
// #296: a `stop --force` SIGTERMs a `nano work` child, which (on an interrupted
|
|
264
|
+
// agent job) flushes a last WIP checkpoint and promotes it to a durable recovery
|
|
265
|
+
// ref (#295) before exiting. That flush+promote must fit inside the window the
|
|
266
|
+
// supervisor allows before its SIGKILL backstop, or the recovery ref is never
|
|
267
|
+
// written. This is the bounded budget the worker gives that force-stop recovery,
|
|
268
|
+
// and the EXTRA grace `forceStopWorker` waits beyond STOP_GRACE_MS before SIGKILL
|
|
269
|
+
// (a child that exits sooner is reaped immediately — this only lengthens the
|
|
270
|
+
// SIGKILL backstop for a worker that is actively preserving recovery).
|
|
271
|
+
const FORCE_STOP_RECOVERY_MS = 20_000;
|
|
272
|
+
// #296: the backstop deadline an OUTER daemon-stop path (the socket-unreachable
|
|
273
|
+
// `stop --force` fallback and `stopSupervisorProcess`) waits for the DAEMON — and
|
|
274
|
+
// its whole process group, which includes the workers — to exit before SIGKILLing
|
|
275
|
+
// the group. It must exceed what the daemon itself allows a worker before ITS
|
|
276
|
+
// SIGKILL backstop (`forceStopWorker` waits STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS),
|
|
277
|
+
// plus a small margin for the daemon to observe the worker's exit and shut down.
|
|
278
|
+
// Otherwise the outer path SIGKILLs the group ~18s early and kills a worker that is
|
|
279
|
+
// still flushing + promoting its durable recovery ref (#295).
|
|
280
|
+
const FORCE_STOP_DAEMON_GRACE_MS = STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS + 2_000;
|
|
159
281
|
// #202: the signal a graceful `supervisor stop`/`workforce stop` sends each
|
|
160
282
|
// `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
|
|
161
283
|
// then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
|
|
@@ -352,6 +474,22 @@ function activityMarkerReadyFor(act, pid) {
|
|
|
352
474
|
return !!(act && act.pid === pid && Number.isFinite(act.readyAt));
|
|
353
475
|
}
|
|
354
476
|
|
|
477
|
+
/**
|
|
478
|
+
* Whether a worker should receive a `reload-credentials` SIGUSR1 this round.
|
|
479
|
+
* All three must hold (#293): the worker exists, its PID is alive, AND its
|
|
480
|
+
* activity marker proves its SIGUSR1 reload listener is installed (a finite
|
|
481
|
+
* `readyAt` + matching pid, via activityMarkerReadyFor). workAgent installs that
|
|
482
|
+
* listener only once its activation loop is up, which is the same handshake that
|
|
483
|
+
* stamps `readyAt`. Signalling a freshly spawned worker BEFORE it installs the
|
|
484
|
+
* handler delivers SIGUSR1 to Node's DEFAULT action — starting the inspector
|
|
485
|
+
* instead of reloading — so a not-yet-ready worker is skipped (it re-reads the
|
|
486
|
+
* fresh token at its own startup prime anyway). Pure so it can be unit-tested.
|
|
487
|
+
*/
|
|
488
|
+
function reloadCredentialsReady(worker, activity, pidAlive) {
|
|
489
|
+
if (!worker || !pidAlive) return false;
|
|
490
|
+
return activityMarkerReadyFor(activity, worker.pid);
|
|
491
|
+
}
|
|
492
|
+
|
|
355
493
|
/**
|
|
356
494
|
* Deterministic control-socket path shared by the daemon and every client.
|
|
357
495
|
* Derived from a hash of the (possibly overridden) state home so distinct
|
|
@@ -2757,6 +2895,19 @@ const RESERVED_RESULT_KEYS = new Set([
|
|
|
2757
2895
|
// push-checkpoint / points world-restore at an arbitrary SHA, so reserve it
|
|
2758
2896
|
// alongside the other host-owned git-result keys (thread 4117276817).
|
|
2759
2897
|
'worldMarker',
|
|
2898
|
+
// `resumeFromCheckpoint` is the host/process-owned resume pointer (issue #295):
|
|
2899
|
+
// the plugin writes it at the PROCESS scope so a re-entered (new element
|
|
2900
|
+
// instance) activation can find the predecessor's WIP checkpoint. When it is a
|
|
2901
|
+
// bare STRING it is trusted as an EXPLICIT, occurrence-authoritative ref that
|
|
2902
|
+
// BYPASSES both the elementId correlation AND the ambiguity guard
|
|
2903
|
+
// (resumeRefIsExplicit). Because a successful agent result is merged into the
|
|
2904
|
+
// process variables, an unreserved `resumeFromCheckpoint` lets an untrusted
|
|
2905
|
+
// harness emit a bare-string ref that a later activation then trusts as
|
|
2906
|
+
// occurrence-authoritative — restoring an ARBITRARY WIP ref from the repository
|
|
2907
|
+
// with no correlation or guard. Reserve it so the agent result can never inject
|
|
2908
|
+
// or shadow the host/process-owned pointer (Copilot review on #296, thread
|
|
2909
|
+
// 4243776734).
|
|
2910
|
+
'resumeFromCheckpoint',
|
|
2760
2911
|
]);
|
|
2761
2912
|
|
|
2762
2913
|
// Parse `text` as a JSON object, returning it only when it is a plain object.
|
|
@@ -4451,10 +4602,28 @@ function makeSecretResolver(kind) {
|
|
|
4451
4602
|
return null; // unknown → caller reports a clear error
|
|
4452
4603
|
}
|
|
4453
4604
|
|
|
4605
|
+
// Force a fresh `gh auth token` read, bypassing the TTL cache, and
|
|
4606
|
+
// return the token (or null). Injectable into resolveJobSecrets so a default
|
|
4607
|
+
// github gh-cache MISS re-reads at job-resolve time — catching a `gh auth login`
|
|
4608
|
+
// the worker-lifetime refresh timer has not re-primed yet — instead of trusting
|
|
4609
|
+
// provisionRepo's (non-universal) auth-failure recovery (Copilot review on #294).
|
|
4610
|
+
// ASYNC (spawn, never spawnSync): resolveJobSecrets runs on the activated-job
|
|
4611
|
+
// path after recordJobStart, so a SYNCHRONOUS `gh auth token` + scopes probe
|
|
4612
|
+
// here would block the shared event loop (and the lock-extension heartbeat) for
|
|
4613
|
+
// up to ~15s — the very hot-path stall the async refresh exists to prevent
|
|
4614
|
+
// (Copilot review on #294). refreshGhAuthTokenAsync also defers to any refresh
|
|
4615
|
+
// already in flight instead of starting a second concurrent read.
|
|
4616
|
+
async function defaultForceRefreshGhAuthToken() {
|
|
4617
|
+
const r = await refreshGhAuthTokenAsync({ force: true });
|
|
4618
|
+
return (r && r.token) || null;
|
|
4619
|
+
}
|
|
4620
|
+
|
|
4454
4621
|
// Resolve the names a job needs (setup.secretRefs, plus the repo/PR credential
|
|
4455
4622
|
// when allowPr). Returns resolved values + a list of names that were missing so
|
|
4456
|
-
// the caller can fail the job with a clear provisioning error.
|
|
4457
|
-
|
|
4623
|
+
// the caller can fail the job with a clear provisioning error. ASYNC: the
|
|
4624
|
+
// gh-cache-miss force-refresh awaits a spawned (non-blocking) gh read, so the
|
|
4625
|
+
// caller must `await` this (the workAgent job runner already is async).
|
|
4626
|
+
async function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCli, forceRefreshGhAuthToken = defaultForceRefreshGhAuthToken } = {}) {
|
|
4458
4627
|
const names = new Set();
|
|
4459
4628
|
for (const n of envelope.setup?.secretRefs || []) if (n) names.add(n);
|
|
4460
4629
|
const resolved = {};
|
|
@@ -4466,7 +4635,10 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
|
|
|
4466
4635
|
}
|
|
4467
4636
|
// The github clone/push credential is resolved with a gh-CLI fallback so a
|
|
4468
4637
|
// default GITHUB_TOKEN isn't reported "missing" merely because it isn't in the
|
|
4469
|
-
// env when `gh auth login` provides it. A
|
|
4638
|
+
// env when `gh auth login` provides it. A default github credential whose gh
|
|
4639
|
+
// cache is a MISS is force-re-read here (bypassing the TTL cache); it is
|
|
4640
|
+
// reported missing only when that forced read ALSO finds no token. A custom
|
|
4641
|
+
// authRef stays strict.
|
|
4470
4642
|
if (envelope.task?.allowPr) {
|
|
4471
4643
|
const provider = envelope.repository?.provider || 'github';
|
|
4472
4644
|
const authRef = envelope.repository?.authRef;
|
|
@@ -4476,16 +4648,35 @@ function resolveJobSecrets(resolver, envelope, { ghAuthToken = ghAuthTokenFromCl
|
|
|
4476
4648
|
// so provisioning sheds rather than silently borrowing the default/gh token.
|
|
4477
4649
|
if (!missing.includes('repository.authRef')) missing.push('repository.authRef');
|
|
4478
4650
|
} else {
|
|
4479
|
-
const ghAuthRef =
|
|
4480
|
-
? ref.name
|
|
4481
|
-
: (provider === 'github' ? 'GITHUB_TOKEN' : undefined);
|
|
4651
|
+
const ghAuthRef = ghCredentialEnvName({ provider, authRef });
|
|
4482
4652
|
if (ghAuthRef) {
|
|
4483
4653
|
names.add(ghAuthRef);
|
|
4484
|
-
const token =
|
|
4654
|
+
const { token, source } = githubCloneTokenWithSource({ provider, authRef, secretResolver: resolver, ghAuthToken });
|
|
4485
4655
|
const missingIdx = missing.indexOf(ghAuthRef);
|
|
4486
4656
|
if (token) {
|
|
4487
4657
|
resolved[ghAuthRef] = token;
|
|
4488
4658
|
if (missingIdx !== -1) missing.splice(missingIdx, 1);
|
|
4659
|
+
} else if (source === 'gh-cache') {
|
|
4660
|
+
// A DEFAULT github credential whose gh-CLI cache is a MISS (null).
|
|
4661
|
+
// provisionRepo's null → fresh auth-failure recovery is NOT universal:
|
|
4662
|
+
// container jobs never call provisionRepo, and an anonymous clone of a
|
|
4663
|
+
// PUBLIC repo SUCCEEDS (no auth failure → no forced re-read), so the
|
|
4664
|
+
// harness would otherwise reach an `allowPr` job with GITHUB_TOKEN
|
|
4665
|
+
// ABSENT instead of failing early. FORCE a fresh `gh auth token` read
|
|
4666
|
+
// HERE (catching a `gh auth login` the worker-lifetime refresh timer
|
|
4667
|
+
// has not re-primed yet): populate + drop from `missing` only if a
|
|
4668
|
+
// token now exists; otherwise KEEP it missing so the job fails with a
|
|
4669
|
+
// clear provisioning error rather than silently handing the harness no
|
|
4670
|
+
// credential (Copilot review on #294). Priming the cache here also
|
|
4671
|
+
// feeds provisioning's own re-resolve the same fresh token. A
|
|
4672
|
+
// custom/invalid authRef reports source 'none' and still fails strict.
|
|
4673
|
+
const fresh = await forceRefreshGhAuthToken();
|
|
4674
|
+
if (fresh) {
|
|
4675
|
+
resolved[ghAuthRef] = fresh;
|
|
4676
|
+
if (missingIdx !== -1) missing.splice(missingIdx, 1);
|
|
4677
|
+
} else if (missingIdx === -1) {
|
|
4678
|
+
missing.push(ghAuthRef);
|
|
4679
|
+
}
|
|
4489
4680
|
} else if (missingIdx === -1) {
|
|
4490
4681
|
missing.push(ghAuthRef);
|
|
4491
4682
|
}
|
|
@@ -4779,42 +4970,690 @@ function credArgs() {
|
|
|
4779
4970
|
// a trimmed token, or null when gh is missing / not logged in. The token is fed to
|
|
4780
4971
|
// git via GIT_ASKPASS only (never argv/URL/helper), preserving the ephemeral-token
|
|
4781
4972
|
// guarantee.
|
|
4782
|
-
// Memoized for the process lifetime: this is a synchronous spawnSync (up to a
|
|
4783
|
-
// 10s timeout) that can be reached per job, so consult the CLI at most once per
|
|
4784
|
-
// worker run rather than blocking a handler on every job. A sentinel
|
|
4785
|
-
// distinguishes "not yet computed" from
|
|
4786
|
-
// a cached null (gh missing / not logged in).
|
|
4787
4973
|
//
|
|
4788
|
-
//
|
|
4789
|
-
//
|
|
4790
|
-
//
|
|
4791
|
-
//
|
|
4792
|
-
//
|
|
4793
|
-
//
|
|
4974
|
+
// Caching (issue #293): the lookup is a synchronous spawnSync (up to a 10s
|
|
4975
|
+
// timeout), so it cannot run on the per-job hot path — but a PROCESS-LIFETIME
|
|
4976
|
+
// memo goes stale the moment the operator refreshes the gh login (re-login, a new
|
|
4977
|
+
// scope, a rotation), and nothing told the worker (or the operator) it was now
|
|
4978
|
+
// handing every job the OLD token. The cache is therefore a RECORD
|
|
4979
|
+
// (`{ token, fingerprint, scopes, since, lastCheck }`) with a TTL:
|
|
4980
|
+
// - `ghAuthTokenFromCli()` (the per-job hot path) serves the PRIMED cache
|
|
4981
|
+
// record VERBATIM — even once `lastCheck` is older than
|
|
4982
|
+
// GH_AUTH_TOKEN_TTL_MS — and NEVER triggers a synchronous TTL re-read, so a
|
|
4983
|
+
// job never pays `gh auth token`'s spawn cost (Copilot review on #293). The
|
|
4984
|
+
// sole synchronous read is for a still-UNSET cache (a worker that never
|
|
4985
|
+
// primed), and even that defers to any async refresh already in flight. The
|
|
4986
|
+
// off-path refreshers below keep a primed record current.
|
|
4987
|
+
// - `nano work` PRIMES at startup (`primeGhAuthToken()`) AND re-primes on a
|
|
4988
|
+
// timer (`startGhAuthTokenRefresh`) so the TTL re-read happens OFF the job
|
|
4989
|
+
// path. The timer's refresh is ASYNC (spawn, never spawnSync) and DEFERRED
|
|
4990
|
+
// while a job is active, so it can never block the event loop / lock
|
|
4991
|
+
// heartbeat (Copilot review on #293) — a refreshed login is therefore
|
|
4992
|
+
// adopted within one TTL only when the worker is idle at a tick. Under
|
|
4993
|
+
// sustained back-to-back load the defer is BOUNDED by
|
|
4994
|
+
// GH_AUTH_TOKEN_MAX_DEFER_MS, after which the tick refreshes DESPITE an
|
|
4995
|
+
// active job (still async) to cap staleness at a few intervals. When the
|
|
4996
|
+
// identity changes it republishes the activity marker via an onChange
|
|
4997
|
+
// callback.
|
|
4998
|
+
// - A git/gh auth FAILURE (clone/push/PR-reconcile — see isGitAuthFailure)
|
|
4999
|
+
// forces one re-read (`refreshGhAuthToken({ force, budgetMs })`, bounded by
|
|
5000
|
+
// the caller's remaining lease budget so the synchronous re-read can't
|
|
5001
|
+
// overrun the lock heartbeat) and a single retry, so a rotated token
|
|
5002
|
+
// self-heals instead of failing the job. The re-read is
|
|
5003
|
+
// NON-destructive (it never invalidates the cache first), so a transient
|
|
5004
|
+
// re-read failure keeps the still-valid cached credential.
|
|
5005
|
+
// - `primeGhAuthToken({ logger })` logs the token's FINGERPRINT (a short sha256
|
|
5006
|
+
// prefix — never the value) and its OAuth scopes at startup and whenever the
|
|
5007
|
+
// token changes, so "this worker has a stale token" is a one-line check; the
|
|
5008
|
+
// same record rides the supervisor activity marker (`ghAuthTokenInfo()`) into
|
|
5009
|
+
// `supervisor status`, and `supervisor reload-credentials` re-reads it
|
|
5010
|
+
// fleet-wide (async; the old token is served until the new one lands)
|
|
5011
|
+
// without restarting workers.
|
|
5012
|
+
// A sentinel distinguishes "not yet computed" from a cached null (gh missing /
|
|
5013
|
+
// not logged in).
|
|
4794
5014
|
const GH_AUTH_TOKEN_UNSET = Symbol('gh-auth-token-unset');
|
|
5015
|
+
// How long a cached gh token is trusted before a re-read. 5 minutes bounds the
|
|
5016
|
+
// staleness window after a `gh auth login`/`refresh` while keeping `gh auth
|
|
5017
|
+
// token` (a cheap keychain read) off the per-job path. Override with
|
|
5018
|
+
// NANO_GH_TOKEN_TTL_MS (0 = re-read on every lookup, honouring the spawn cost).
|
|
5019
|
+
const GH_AUTH_TOKEN_TTL_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_TTL_MS, 5 * 60_000));
|
|
5020
|
+
// Upper bound on how long the maintenance timer may KEEP DEFERRING its refresh
|
|
5021
|
+
// while jobs run back-to-back. The tick normally defers while a job is active
|
|
5022
|
+
// (`isIdle`) so it never even spawns `gh` mid-job — but that defer is only an
|
|
5023
|
+
// optimization, because the refresh is async (spawn, never spawnSync) and so
|
|
5024
|
+
// cannot block the event loop / lock heartbeat even when it DOES run mid-job.
|
|
5025
|
+
// Under SUSTAINED saturation the worker is never idle at a tick, so an unbounded
|
|
5026
|
+
// defer would leave an expired, non-null cache reaching every subsequent job for
|
|
5027
|
+
// the worker's whole busy lifetime — defeating issue #293's "keep the token
|
|
5028
|
+
// fresh for the worker's lifetime" goal (Copilot review on #294). Bound it: once
|
|
5029
|
+
// a PRIMED cache is stale beyond this window, the tick refreshes DESPITE an
|
|
5030
|
+
// active job (still async/non-blocking), capping staleness at a few intervals
|
|
5031
|
+
// instead of the busy lifetime. Derived from the TTL (4 intervals) and overridable
|
|
5032
|
+
// via NANO_GH_TOKEN_MAX_DEFER_MS. Priming an UNSET cache is NEVER forced mid-job
|
|
5033
|
+
// (the defer there still holds), so a worker that starts logged-out stays anonymous
|
|
5034
|
+
// until it goes idle, exactly as before.
|
|
5035
|
+
const GH_AUTH_TOKEN_MAX_DEFER_MS = Math.max(0, coerceInt(process.env.NANO_GH_TOKEN_MAX_DEFER_MS, 4 * GH_AUTH_TOKEN_TTL_MS));
|
|
5036
|
+
// The gh subprocesses below must never block on an interactive prompt inside a
|
|
5037
|
+
// worker (a missing/expired login would otherwise hang the event loop for the
|
|
5038
|
+
// full timeout). GH_PROMPT_DISABLED fails fast instead; GH_NO_UPDATE_NOTIFIER
|
|
5039
|
+
// keeps gh's upgrade check out of the captured output.
|
|
5040
|
+
const GH_NONINTERACTIVE_ENV = () => ({ ...process.env, GH_PROMPT_DISABLED: '1', GH_NO_UPDATE_NOTIFIER: '1' });
|
|
4795
5041
|
let ghAuthTokenCache = GH_AUTH_TOKEN_UNSET;
|
|
4796
|
-
|
|
4797
|
-
|
|
4798
|
-
|
|
5042
|
+
// A Promise while an ASYNC gh-token refresh (the periodic timer tick or the
|
|
5043
|
+
// `reload-credentials` SIGUSR1 handler) is actually spawning `gh` subprocesses;
|
|
5044
|
+
// null otherwise. This is the serialization point that keeps a refresh off the
|
|
5045
|
+
// job path (Copilot review on #293): while it is set, the SYNCHRONOUS per-job
|
|
5046
|
+
// lookup (`ghAuthTokenFromCli`) must NOT start its own blocking `spawnSync`
|
|
5047
|
+
// re-read — it hands out the currently-cached token and lets the in-flight
|
|
5048
|
+
// async refresh update the cache, so an async refresh overlapping a freshly
|
|
5049
|
+
// activated job can never degrade into a 15s event-loop/lease-heartbeat stall.
|
|
5050
|
+
// It also prevents the timer and the SIGUSR1 handler from racing two concurrent
|
|
5051
|
+
// reads: a second async refresh awaits the first instead of spawning again.
|
|
5052
|
+
let ghAuthRefreshInFlight = null;
|
|
5053
|
+
// Monotonic generation counter bumped on EVERY write to `ghAuthTokenCache`
|
|
5054
|
+
// (`writeGhAuthCache` below is the single mutation point). It fences a stale
|
|
5055
|
+
// async refresh against a newer forced recovery (Copilot review on #293): the
|
|
5056
|
+
// synchronous clone/push auth-failure recovery (a forced `refreshGhAuthToken({
|
|
5057
|
+
// force })`) CANNOT await an already-running async timer
|
|
5058
|
+
// refresh (sync code can't await the `ghAuthRefreshInFlight` promise). So an
|
|
5059
|
+
// async refresh can read the OLD token, yield across its scopes probe, the sync
|
|
5060
|
+
// recovery can adopt the ROTATED token in the meantime, and then the async
|
|
5061
|
+
// refresh's late `adoptGhAuthTokenAsync` would overwrite the cache with its
|
|
5062
|
+
// now-stale read — resurrecting the revoked credential. `refreshGhAuthTokenAsync`
|
|
5063
|
+
// snapshots this generation BEFORE it starts reading and `adoptGhAuthTokenAsync`
|
|
5064
|
+
// refuses to commit when the generation has advanced (a newer definitive write
|
|
5065
|
+
// landed), so an older read can never clobber a newer forced recovery.
|
|
5066
|
+
let ghAuthCacheGeneration = 0;
|
|
5067
|
+
// The single mutation point for `ghAuthTokenCache`. Every write (adopt, preserve,
|
|
5068
|
+
// invalidate/reset) goes through here so the generation fence above sees them all.
|
|
5069
|
+
function writeGhAuthCache(record) {
|
|
5070
|
+
ghAuthTokenCache = record;
|
|
5071
|
+
ghAuthCacheGeneration++;
|
|
5072
|
+
return record;
|
|
5073
|
+
}
|
|
5074
|
+
// Test-only seam: the `gh auth token` reader `refreshGhAuthToken` consults when
|
|
5075
|
+
// no per-call `run` is injected. Production leaves this unset (the real
|
|
5076
|
+
// spawnSync of the gh CLI); a test installs a stub so the clone/push
|
|
5077
|
+
// invalidation-and-retry paths — which refresh through the MODULE cache, not a
|
|
5078
|
+
// per-call `run` — can rotate the token deterministically. Always a function
|
|
5079
|
+
// returning a trimmed token-or-null; reset with `resetGhAuthTokenReader()`.
|
|
5080
|
+
let ghAuthTokenReader = null;
|
|
5081
|
+
function setGhAuthTokenReader(fn) { ghAuthTokenReader = typeof fn === 'function' ? fn : null; }
|
|
5082
|
+
function resetGhAuthTokenReader() { ghAuthTokenReader = null; }
|
|
5083
|
+
|
|
5084
|
+
// Sentinel distinguishing a TRANSIENT `gh auth token` read failure from a
|
|
5085
|
+
// definitive logout (Copilot review on #293). The readers used to collapse a
|
|
5086
|
+
// timeout / spawn error / unrecognised nonzero exit into `null`, which the
|
|
5087
|
+
// refresh path then adopted as a real "logged out" change — discarding a
|
|
5088
|
+
// previously usable cached token, stamping the cache fresh with `null`, and
|
|
5089
|
+
// dropping the worker (and its clone-auth recovery) to anonymous until the next
|
|
5090
|
+
// interval. A read that returns this sentinel instead means "could not
|
|
5091
|
+
// determine"; `adoptGhAuthToken*` preserves the prior credential rather than
|
|
5092
|
+
// wiping it, and only a DEFINITIVE result (a real token, or a confirmed logout)
|
|
5093
|
+
// ever changes the cache.
|
|
5094
|
+
const GH_AUTH_READ_FAILED = Symbol('gh-auth-read-failed');
|
|
5095
|
+
// gh's not-logged-in signatures ("no oauth token found for github.com",
|
|
5096
|
+
// "You are not logged into any GitHub hosts"). Matched on stderr ONLY for a
|
|
5097
|
+
// nonzero exit, so an unrelated nonzero failure (keychain error, transient
|
|
5098
|
+
// network) stays a transient read failure rather than a spurious logout.
|
|
5099
|
+
const GH_LOGGED_OUT_RE = /no oauth token|no token found|not logged in/i;
|
|
5100
|
+
// Map one completed (or failed) `gh auth token` invocation to:
|
|
5101
|
+
// - a trimmed token string → logged in
|
|
5102
|
+
// - null → the read DEFINITIVELY established no login (clean exit with empty
|
|
5103
|
+
// output, or a nonzero exit whose stderr carries gh's logged-out message)
|
|
5104
|
+
// - GH_AUTH_READ_FAILED → the read could not be completed (spawn error,
|
|
5105
|
+
// timeout — spawnSync sets `.error` — or a nonzero exit WITHOUT a
|
|
5106
|
+
// logged-out signal), so the caller must preserve any prior cached token.
|
|
5107
|
+
function classifyGhAuthRead({ error = null, status = null, stdout = '', stderr = '' } = {}) {
|
|
5108
|
+
if (error) return GH_AUTH_READ_FAILED;
|
|
5109
|
+
if (status === 0) return (String(stdout ?? '').trim()) || null;
|
|
5110
|
+
if (GH_LOGGED_OUT_RE.test(String(stderr ?? ''))) return null;
|
|
5111
|
+
return GH_AUTH_READ_FAILED;
|
|
5112
|
+
}
|
|
5113
|
+
|
|
5114
|
+
function readGhAuthTokenDefault(timeoutMs = 10_000) {
|
|
5115
|
+
let r;
|
|
5116
|
+
try {
|
|
5117
|
+
r = spawnSync('gh', ['auth', 'token'], { encoding: 'utf8', timeout: Math.max(1, timeoutMs), env: GH_NONINTERACTIVE_ENV() });
|
|
5118
|
+
} catch {
|
|
5119
|
+
return GH_AUTH_READ_FAILED;
|
|
5120
|
+
}
|
|
5121
|
+
return classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
|
|
5122
|
+
}
|
|
5123
|
+
|
|
5124
|
+
// A short, non-reversible identity for a token so logs/status can show WHEN the
|
|
5125
|
+
// credential changed (and that two workers hold the SAME one) without ever
|
|
5126
|
+
// printing a usable fragment of the secret. 12 hex chars of sha256.
|
|
5127
|
+
function ghTokenFingerprint(token) {
|
|
5128
|
+
if (!token) return null;
|
|
5129
|
+
try { return createHash('sha256').update(String(token)).digest('hex').slice(0, 12); }
|
|
5130
|
+
catch { return null; }
|
|
5131
|
+
}
|
|
5132
|
+
|
|
5133
|
+
// Read the token's OAuth scopes from the live `X-OAuth-Scopes` response header
|
|
5134
|
+
// (the issue's own diagnostic). Best-effort and bounded: a single lightweight
|
|
5135
|
+
// api.github.com ping, failure → null (scopes simply go unreported). The token
|
|
5136
|
+
// rides the Authorization HEADER — never argv — so it can't leak via `ps`.
|
|
5137
|
+
function ghTokenScopes(token, { timeoutMs = 5_000, run = spawnSync } = {}) {
|
|
5138
|
+
if (!token) return null;
|
|
4799
5139
|
try {
|
|
4800
|
-
const r =
|
|
4801
|
-
const
|
|
4802
|
-
|
|
5140
|
+
const r = run('gh', ['api', '-i', '--include', 'user'], { encoding: 'utf8', timeout: timeoutMs, env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
|
|
5141
|
+
const out = `${r.stdout || ''}\n${r.stderr || ''}`;
|
|
5142
|
+
const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
|
|
5143
|
+
return m ? m[1].trim() : null;
|
|
4803
5144
|
} catch {
|
|
5145
|
+
return null;
|
|
5146
|
+
}
|
|
5147
|
+
}
|
|
5148
|
+
|
|
5149
|
+
// Fold one (re)read result into the cache record. Shared by the sync and async
|
|
5150
|
+
// refresh paths so both maintain identical cache semantics. Returns
|
|
5151
|
+
// { token, changed, snapshotChanged } — `changed` is true when the token VALUE
|
|
5152
|
+
// differs from what the cache previously held (including null → token and token
|
|
5153
|
+
// → null), so a caller doing credential-rotation recovery fires exactly when the
|
|
5154
|
+
// credential the worker hands out actually rotates. `snapshotChanged` is the
|
|
5155
|
+
// broader "did the OBSERVABLE snapshot move" flag: `changed` OR a same-token
|
|
5156
|
+
// scopes self-repair (a prior null scopes reading re-probed to a real value) —
|
|
5157
|
+
// both of which change what `supervisor status`/the activity marker should show,
|
|
5158
|
+
// so the log + republish paths gate on IT, not on `changed` (Copilot review on
|
|
5159
|
+
// #293). `scopes` is read only when the token changed, on the first populate, or
|
|
5160
|
+
// to re-probe a cached null reading.
|
|
5161
|
+
function adoptGhAuthToken(token, { scopesProbe = ghTokenScopes } = {}) {
|
|
5162
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5163
|
+
const now = Date.now();
|
|
5164
|
+
if (token === GH_AUTH_READ_FAILED) {
|
|
5165
|
+
// Transient read failure: never discard a usable cached credential. Preserve
|
|
5166
|
+
// the prior record but bump `lastCheck` so the per-job hot path keeps serving
|
|
5167
|
+
// it via the TTL short-circuit (no blocking re-read on every lookup); the
|
|
5168
|
+
// off-path refresh timer retries on its own cadence. With no usable prior
|
|
5169
|
+
// token, fall back to anonymous (null) as before.
|
|
5170
|
+
if (prev && prev.token != null) {
|
|
5171
|
+
writeGhAuthCache({ ...prev, lastCheck: now });
|
|
5172
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5173
|
+
}
|
|
4804
5174
|
token = null;
|
|
4805
5175
|
}
|
|
4806
|
-
|
|
4807
|
-
|
|
5176
|
+
const changed = !prev || prev.token !== token;
|
|
5177
|
+
// On a CHANGE re-read the scopes; otherwise carry the previous reading forward
|
|
5178
|
+
// (a TTL re-read that finds the same token must not pay the scopes ping, and
|
|
5179
|
+
// must not silently DROP the scopes it already reported). BUT a prior probe
|
|
5180
|
+
// that failed transiently stored `scopes: null` — carrying that null forward
|
|
5181
|
+
// would leave the GH AUTH visibility unrepaired until the token rotates or the
|
|
5182
|
+
// worker restarts (Copilot review on #293). Re-probe when the cached scope
|
|
5183
|
+
// value is MISSING (null) while a token is present; reuse only a SUCCESSFUL
|
|
5184
|
+
// (non-null) prior reading.
|
|
5185
|
+
const scopes = token ? ((changed || !(prev && prev.scopes)) ? scopesProbe(token) : prev.scopes) : null;
|
|
5186
|
+
// `snapshotChanged` is true when the OBSERVABLE identity (the token-free
|
|
5187
|
+
// fingerprint + scopes that ghAuthTokenInfo/describeGhAuthToken surface) moved
|
|
5188
|
+
// — not only when the token VALUE rotated. A scopes self-repair (a prior
|
|
5189
|
+
// transient probe cached `scopes:null`, now re-probed to a real value) leaves
|
|
5190
|
+
// the token unchanged yet DOES change what `supervisor status` should show, so
|
|
5191
|
+
// the republish/log path must fire on it too; gating republishing on `changed`
|
|
5192
|
+
// alone leaves an idle worker reporting `?` scopes indefinitely (Copilot review
|
|
5193
|
+
// on #293).
|
|
5194
|
+
const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
|
|
5195
|
+
writeGhAuthCache({
|
|
5196
|
+
token,
|
|
5197
|
+
fingerprint: ghTokenFingerprint(token),
|
|
5198
|
+
scopes: scopes ?? null,
|
|
5199
|
+
// `since` tracks the TOKEN's age (when THIS value was first seen) so status
|
|
5200
|
+
// can answer "how stale is the credential"; `lastCheck` tracks the CACHE's
|
|
5201
|
+
// age (when we last asked gh) so the TTL is honoured.
|
|
5202
|
+
since: prev && prev.token === token && token != null ? prev.since : now,
|
|
5203
|
+
lastCheck: now,
|
|
5204
|
+
});
|
|
5205
|
+
return { token, changed, snapshotChanged };
|
|
5206
|
+
}
|
|
5207
|
+
|
|
5208
|
+
// The async twin of adoptGhAuthToken: identical cache semantics, but the scopes
|
|
5209
|
+
// probe is awaited so it may return a Promise (the async ghTokenScopesAsync).
|
|
5210
|
+
// `generation` (when provided by refreshGhAuthTokenAsync) is the cache generation
|
|
5211
|
+
// snapshotted BEFORE this refresh started reading. If it no longer matches at
|
|
5212
|
+
// commit time, a NEWER definitive write (a forced sync clone/push recovery, an
|
|
5213
|
+
// invalidate, or another refresh) landed while we were reading/probing — so this
|
|
5214
|
+
// read is stale and MUST NOT be committed, or it would resurrect the superseded
|
|
5215
|
+
// token (Copilot review on #293). We leave the newer cache in place and report
|
|
5216
|
+
// it as superseded.
|
|
5217
|
+
async function adoptGhAuthTokenAsync(token, { scopesProbe, generation = null } = {}) {
|
|
5218
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5219
|
+
const now = Date.now();
|
|
5220
|
+
const superseded = () => {
|
|
5221
|
+
const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5222
|
+
return { token: c ? c.token : null, changed: false, snapshotChanged: false, superseded: true };
|
|
5223
|
+
};
|
|
5224
|
+
if (token === GH_AUTH_READ_FAILED) {
|
|
5225
|
+
// Transient read failure (see adoptGhAuthToken): preserve the prior token and
|
|
5226
|
+
// bump `lastCheck` so the hot path keeps serving it; anonymous only with no prior.
|
|
5227
|
+
if (prev && prev.token != null) {
|
|
5228
|
+
if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
|
|
5229
|
+
writeGhAuthCache({ ...prev, lastCheck: now });
|
|
5230
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5231
|
+
}
|
|
5232
|
+
token = null;
|
|
5233
|
+
}
|
|
5234
|
+
const changed = !prev || prev.token !== token;
|
|
5235
|
+
// Same re-probe-on-missing rule as the sync twin: a same-token refresh reuses a
|
|
5236
|
+
// successful prior scope reading but RE-PROBES when the cached value is null
|
|
5237
|
+
// (a prior transient probe failure), so the GH AUTH visibility self-repairs.
|
|
5238
|
+
const scopes = token ? ((changed || !(prev && prev.scopes)) ? await scopesProbe(token) : prev.scopes) : null;
|
|
5239
|
+
// See adoptGhAuthToken: publish when the observable snapshot (fingerprint +
|
|
5240
|
+
// scopes) moved, not only when the token value rotated, so a same-token scopes
|
|
5241
|
+
// self-repair still republishes the activity marker (Copilot review on #293).
|
|
5242
|
+
const snapshotChanged = changed || (!!prev && (prev.scopes ?? null) !== (scopes ?? null));
|
|
5243
|
+
// Generation fence: re-check AFTER the (awaited) scopes probe, the last point a
|
|
5244
|
+
// forced recovery could have slipped a newer credential in underneath us.
|
|
5245
|
+
if (generation != null && generation !== ghAuthCacheGeneration) return superseded();
|
|
5246
|
+
writeGhAuthCache({
|
|
5247
|
+
token,
|
|
5248
|
+
fingerprint: ghTokenFingerprint(token),
|
|
5249
|
+
scopes: scopes ?? null,
|
|
5250
|
+
since: prev && prev.token === token && token != null ? prev.since : now,
|
|
5251
|
+
lastCheck: now,
|
|
5252
|
+
});
|
|
5253
|
+
return { token, changed, snapshotChanged };
|
|
5254
|
+
}
|
|
5255
|
+
|
|
5256
|
+
// One (re)read of `gh auth token`, folding the result into the cache record.
|
|
5257
|
+
// Returns { token, changed } (see adoptGhAuthToken). SYNCHRONOUS (spawnSync) —
|
|
5258
|
+
// safe off the job path (the startup prime and the clone/push auth-failure
|
|
5259
|
+
// recovery, where the caller is already blocked waiting on git) but it blocks
|
|
5260
|
+
// the event loop while it runs, so the periodic refresh AND the SIGUSR1
|
|
5261
|
+
// reload-credentials handler use refreshGhAuthTokenAsync instead.
|
|
5262
|
+
function refreshGhAuthToken({ force = false, run = null, scopesProbe, budgetMs = null } = {}) {
|
|
5263
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5264
|
+
const now = Date.now();
|
|
5265
|
+
if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
|
|
5266
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5267
|
+
}
|
|
5268
|
+
// When a caller on the ACTIVE-JOB path (the clone/push auth-failure recovery)
|
|
5269
|
+
// threads its remaining lease budget, bound BOTH synchronous gh subprocesses
|
|
5270
|
+
// (the `gh auth token` read AND the scopes probe) to it so this forced
|
|
5271
|
+
// spawnSync re-read can never block the event loop / lock heartbeat past the
|
|
5272
|
+
// caller's lease-safe window — the finalizeGit/clone git ops are already
|
|
5273
|
+
// netDeadline-bounded, and this read must participate in the same budget
|
|
5274
|
+
// (Copilot review on #294). Absent a budget (startup prime, UNSET-cache hot
|
|
5275
|
+
// path, unit tests) the historical 10s read / 5s scopes caps stand.
|
|
5276
|
+
const haveBudget = Number.isFinite(budgetMs) && budgetMs > 0;
|
|
5277
|
+
const deadline = haveBudget ? now + budgetMs : null;
|
|
5278
|
+
const readTimeout = haveBudget ? Math.max(1, Math.min(10_000, budgetMs)) : 10_000;
|
|
5279
|
+
let token = null;
|
|
5280
|
+
if (run) {
|
|
5281
|
+
// Explicit per-call spawn injection (unit tests of this function in isolation).
|
|
5282
|
+
try {
|
|
5283
|
+
const r = run('gh', ['auth', 'token'], { encoding: 'utf8', timeout: readTimeout, env: GH_NONINTERACTIVE_ENV() });
|
|
5284
|
+
token = classifyGhAuthRead({ error: r.error, status: r.status, stdout: r.stdout, stderr: r.stderr });
|
|
5285
|
+
} catch {
|
|
5286
|
+
token = GH_AUTH_READ_FAILED;
|
|
5287
|
+
}
|
|
5288
|
+
} else {
|
|
5289
|
+
// The module reader (production: the real gh CLI; tests: setGhAuthTokenReader).
|
|
5290
|
+
token = (ghAuthTokenReader || readGhAuthTokenDefault)(readTimeout);
|
|
5291
|
+
}
|
|
5292
|
+
// The scopes probe gets whatever of the budget the read left (still capped at
|
|
5293
|
+
// 5s); a read that consumed the whole budget leaves it ~1ms so the probe fails
|
|
5294
|
+
// fast to null (scopes are best-effort) rather than overrunning the lease.
|
|
5295
|
+
const scopesTimeout = haveBudget ? Math.max(1, Math.min(5_000, deadline - Date.now())) : 5_000;
|
|
5296
|
+
// The scopes probe defaults to the live `gh api -i user` ping — but when a
|
|
5297
|
+
// test installed a token READER (hermetic, no network) and did not inject a
|
|
5298
|
+
// scopesProbe, skip the ping so a stubbed refresh never hits the network.
|
|
5299
|
+
const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((t) => ghTokenScopes(t, { timeoutMs: scopesTimeout })));
|
|
5300
|
+
return adoptGhAuthToken(token, { scopesProbe: probe });
|
|
5301
|
+
}
|
|
5302
|
+
|
|
5303
|
+
// Async `gh auth token` read (spawn, never spawnSync) for the periodic refresh
|
|
5304
|
+
// path — the whole point of the timer is that it must NOT block the worker's
|
|
5305
|
+
// event loop (and with it the lock heartbeat) while a job runs. Same 10s bound
|
|
5306
|
+
// and non-interactive env as the sync reader; resolves a trimmed token-or-null.
|
|
5307
|
+
// `_spawn` is a test-only seam (production uses node:child_process.spawn).
|
|
5308
|
+
function readGhAuthTokenAsync({ _spawn = spawn } = {}) {
|
|
5309
|
+
return new Promise((resolve) => {
|
|
5310
|
+
let child;
|
|
5311
|
+
try {
|
|
5312
|
+
child = _spawn('gh', ['auth', 'token'], { env: GH_NONINTERACTIVE_ENV() });
|
|
5313
|
+
} catch {
|
|
5314
|
+
resolve(GH_AUTH_READ_FAILED);
|
|
5315
|
+
return;
|
|
5316
|
+
}
|
|
5317
|
+
let out = '';
|
|
5318
|
+
let err = '';
|
|
5319
|
+
let settled = false;
|
|
5320
|
+
const finish = (token) => {
|
|
5321
|
+
if (settled) return;
|
|
5322
|
+
settled = true;
|
|
5323
|
+
clearTimeout(timer);
|
|
5324
|
+
resolve(token);
|
|
5325
|
+
};
|
|
5326
|
+
// Same 10s bound as the sync reader. On expiry SIGTERM the child (matching
|
|
5327
|
+
// spawnSync's timeout semantics) and resolve the transient sentinel — a
|
|
5328
|
+
// wedged gh must never pin the refresh chain, NOR masquerade as a logout.
|
|
5329
|
+
const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(GH_AUTH_READ_FAILED); }, 10_000);
|
|
5330
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
5331
|
+
child.stdout?.on('data', (d) => { out += d; });
|
|
5332
|
+
// stderr is now CAPTURED (not just drained) so classifyGhAuthRead can tell a
|
|
5333
|
+
// definitive logout (gh's "no oauth token found" message) from a transient
|
|
5334
|
+
// nonzero failure that must preserve the cached credential.
|
|
5335
|
+
child.stderr?.on('data', (d) => { err += d; });
|
|
5336
|
+
child.on('error', () => finish(GH_AUTH_READ_FAILED));
|
|
5337
|
+
child.on('close', (code) => finish(classifyGhAuthRead({ status: code, stdout: out, stderr: err })));
|
|
5338
|
+
});
|
|
5339
|
+
}
|
|
5340
|
+
|
|
5341
|
+
// Async scopes probe mirroring ghTokenScopes (same 5s default bound, same
|
|
5342
|
+
// header parse, same GH_TOKEN-in-env delivery so the token never touches argv).
|
|
5343
|
+
// `_spawn` is a test-only seam.
|
|
5344
|
+
function ghTokenScopesAsync(token, { timeoutMs = 5_000, _spawn = spawn } = {}) {
|
|
5345
|
+
if (!token) return Promise.resolve(null);
|
|
5346
|
+
return new Promise((resolve) => {
|
|
5347
|
+
let child;
|
|
5348
|
+
try {
|
|
5349
|
+
child = _spawn('gh', ['api', '-i', '--include', 'user'], { env: { ...GH_NONINTERACTIVE_ENV(), GH_TOKEN: token } });
|
|
5350
|
+
} catch {
|
|
5351
|
+
resolve(null);
|
|
5352
|
+
return;
|
|
5353
|
+
}
|
|
5354
|
+
let out = '';
|
|
5355
|
+
let settled = false;
|
|
5356
|
+
const finish = (scopes) => {
|
|
5357
|
+
if (settled) return;
|
|
5358
|
+
settled = true;
|
|
5359
|
+
clearTimeout(timer);
|
|
5360
|
+
resolve(scopes);
|
|
5361
|
+
};
|
|
5362
|
+
const timer = setTimeout(() => { try { child.kill('SIGTERM'); } catch { /* best effort */ } finish(null); }, timeoutMs);
|
|
5363
|
+
if (typeof timer.unref === 'function') timer.unref();
|
|
5364
|
+
child.stdout?.on('data', (d) => { out += d; });
|
|
5365
|
+
child.stderr?.on('data', (d) => { out += d; });
|
|
5366
|
+
child.on('error', () => finish(null));
|
|
5367
|
+
child.on('close', () => {
|
|
5368
|
+
const m = out.match(/^x-oauth-scopes:\s*(.+)$/im);
|
|
5369
|
+
finish(m ? m[1].trim() : null);
|
|
5370
|
+
});
|
|
5371
|
+
});
|
|
5372
|
+
}
|
|
5373
|
+
|
|
5374
|
+
// The async twin of refreshGhAuthToken for the periodic timer: same TTL
|
|
5375
|
+
// short-circuit and the same cache record, but the gh subprocesses are spawned
|
|
5376
|
+
// (never spawnSync) so a refresh that DOES run while a job is active yields the
|
|
5377
|
+
// event loop instead of blocking the lock heartbeat (Copilot review on #293).
|
|
5378
|
+
// When a test installed a token READER (hermetic, no network) and did not
|
|
5379
|
+
// inject a scopesProbe, the scopes ping is skipped — mirroring the sync path.
|
|
5380
|
+
async function refreshGhAuthTokenAsync({ force = false, scopesProbe, _spawn } = {}) {
|
|
5381
|
+
const prev = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5382
|
+
const now = Date.now();
|
|
5383
|
+
if (!force && prev && now - prev.lastCheck < GH_AUTH_TOKEN_TTL_MS) {
|
|
5384
|
+
return { token: prev.token, changed: false, snapshotChanged: false };
|
|
5385
|
+
}
|
|
5386
|
+
// Serialize against any refresh already spawning (another timer tick, or the
|
|
5387
|
+
// reload-credentials SIGUSR1 handler): await IT and report its (already
|
|
5388
|
+
// applied) cache rather than starting a second concurrent `gh` read. This is
|
|
5389
|
+
// what lets the synchronous per-job path safely defer to an in-flight async
|
|
5390
|
+
// refresh instead of blocking (Copilot review on #293).
|
|
5391
|
+
// LOOP (not a one-shot `if`): re-check the marker after EVERY await so forced
|
|
5392
|
+
// refreshes stay serialized. A plain `if` let multiple forced callers all await
|
|
5393
|
+
// the SAME in-flight promise and then fall through together, each overwriting
|
|
5394
|
+
// ghAuthRefreshInFlight with its own read; the first to settle then clears the
|
|
5395
|
+
// marker (its `finally` nulls it) while another forced read is still active, so
|
|
5396
|
+
// the hot path (ghAuthTokenFromCli) sees no marker and resumes a blocking
|
|
5397
|
+
// synchronous re-read — the very stall the marker exists to prevent. Re-checking
|
|
5398
|
+
// in a loop means a forced caller that finds a NEW refresh in flight (started by
|
|
5399
|
+
// another forced caller that fell through first) awaits THAT one too, so at most
|
|
5400
|
+
// one forced read runs at a time and the marker is never cleared out from under
|
|
5401
|
+
// an active read (Copilot review on #293).
|
|
5402
|
+
while (ghAuthRefreshInFlight) {
|
|
5403
|
+
try { await ghAuthRefreshInFlight; } catch { /* swallowed — the owner logs/handles it */ }
|
|
5404
|
+
// A NON-forced caller (a periodic timer tick) is happy with whatever the
|
|
5405
|
+
// in-flight refresh just applied. A FORCED caller (the reload-credentials
|
|
5406
|
+
// SIGUSR1 handler) must NOT be: the in-flight read may have captured the OLD
|
|
5407
|
+
// token microseconds before the operator's `gh auth refresh`/`login` landed,
|
|
5408
|
+
// so returning its result would report "reloaded" while leaving the stale
|
|
5409
|
+
// credential cached until the next TTL. Loop back: once no refresh is in
|
|
5410
|
+
// flight, fall through to start a fresh forced read AFTER the signal.
|
|
5411
|
+
if (!force) {
|
|
5412
|
+
const c = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5413
|
+
return { token: c ? c.token : null, changed: false, snapshotChanged: false };
|
|
5414
|
+
}
|
|
5415
|
+
}
|
|
5416
|
+
// Honor the module reader seam when a test installed one (mirroring the sync
|
|
5417
|
+
// path) so a stubbed refresh never spawns; production (reader null) spawns.
|
|
5418
|
+
// Snapshot the cache generation BEFORE the read begins: if a forced sync
|
|
5419
|
+
// recovery commits a newer credential while we read/probe, adoptGhAuthTokenAsync
|
|
5420
|
+
// sees the generation advance and refuses to clobber it (Copilot review on #293).
|
|
5421
|
+
const startGen = ghAuthCacheGeneration;
|
|
5422
|
+
const run = (async () => {
|
|
5423
|
+
const token = ghAuthTokenReader ? ((ghAuthTokenReader() || null)) : await readGhAuthTokenAsync({ _spawn });
|
|
5424
|
+
const probe = scopesProbe ?? (ghAuthTokenReader ? (() => null) : ((tok) => ghTokenScopesAsync(tok, { _spawn })));
|
|
5425
|
+
return adoptGhAuthTokenAsync(token, { scopesProbe: probe, generation: startGen });
|
|
5426
|
+
})();
|
|
5427
|
+
// Publish the in-flight marker BEFORE the first await inside `run` has a chance
|
|
5428
|
+
// to yield, so a job activated on the very next tick already sees it. The
|
|
5429
|
+
// marker never rejects (so a concurrent awaiter can't throw on it); the real
|
|
5430
|
+
// result/rejection is surfaced to this caller below.
|
|
5431
|
+
ghAuthRefreshInFlight = run.then(() => {}, () => {});
|
|
5432
|
+
try {
|
|
5433
|
+
return await run;
|
|
5434
|
+
} finally {
|
|
5435
|
+
ghAuthRefreshInFlight = null;
|
|
5436
|
+
}
|
|
4808
5437
|
}
|
|
4809
5438
|
|
|
4810
|
-
|
|
4811
|
-
//
|
|
4812
|
-
//
|
|
4813
|
-
|
|
4814
|
-
|
|
5439
|
+
function ghAuthTokenFromCli() {
|
|
5440
|
+
// If an async refresh is already spawning (a timer tick or the
|
|
5441
|
+
// reload-credentials SIGUSR1 handler), do NOT start a blocking synchronous
|
|
5442
|
+
// re-read on this per-job path — that is exactly the 15s event-loop/lease
|
|
5443
|
+
// stall the async refresh exists to avoid (Copilot review on #293). Hand out
|
|
5444
|
+
// the currently-cached token; the in-flight refresh updates the cache for the
|
|
5445
|
+
// next lookup, and the clone/push auth-failure recovery still force-re-reads
|
|
5446
|
+
// if the handed-out token turns out stale. A not-yet-primed cache (UNSET)
|
|
5447
|
+
// yields null — the same anonymous fallback the refresh would briefly produce.
|
|
5448
|
+
if (ghAuthRefreshInFlight) {
|
|
5449
|
+
return ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache.token;
|
|
5450
|
+
}
|
|
5451
|
+
if (ghAuthTokenCache === GH_AUTH_TOKEN_UNSET) return refreshGhAuthToken({ force: true }).token;
|
|
5452
|
+
// A PRIMED cache is served verbatim — NEVER a synchronous TTL-triggered
|
|
5453
|
+
// re-read on this per-job hot path (Copilot review on #293). The async timer
|
|
5454
|
+
// DEFERS a tick while a job is active, so a skipped tick can leave the cache
|
|
5455
|
+
// past its TTL when the next job arrives; a `refreshGhAuthToken()` here would
|
|
5456
|
+
// then run `spawnSync('gh auth token')` + the synchronous scopes probe inline,
|
|
5457
|
+
// blocking the event loop (and the lock heartbeat) for up to 15s. Startup
|
|
5458
|
+
// priming, the async maintenance timer, and the clone/push auth-failure
|
|
5459
|
+
// recovery (which force-re-reads if the handed-out token turns out stale) own
|
|
5460
|
+
// every refresh; this path only hands out what they populated.
|
|
5461
|
+
return ghAuthTokenCache.token;
|
|
5462
|
+
}
|
|
5463
|
+
|
|
5464
|
+
// Drop the cached gh token so the next lookup re-reads the CLI. A cache-reset
|
|
5465
|
+
// primitive — exported and used by the tests to reset state between cases. NO
|
|
5466
|
+
// production path calls it anymore: both the git clone/push auth-failure
|
|
5467
|
+
// recovery (recoverGhTokenAfterAuthFailure) and the supervisor's
|
|
5468
|
+
// `reload-credentials` broadcast (SIGUSR1) were made deliberately
|
|
5469
|
+
// NON-destructive (a forced re-read that preserves/atomically-swaps the cached
|
|
5470
|
+
// record), so neither drops the cache first (Copilot review on #294).
|
|
5471
|
+
function invalidateGhAuthToken() {
|
|
5472
|
+
writeGhAuthCache(GH_AUTH_TOKEN_UNSET);
|
|
5473
|
+
}
|
|
5474
|
+
|
|
5475
|
+
// Recover from a git/gh auth failure on a gh-cache-derived credential: force a
|
|
5476
|
+
// fresh `gh auth token` read and return it IFF it differs from the credential
|
|
5477
|
+
// that just FAILED (`failingToken`), otherwise null.
|
|
5478
|
+
//
|
|
5479
|
+
// Deliberately does NOT `invalidateGhAuthToken()` first (Copilot review on
|
|
5480
|
+
// #294). An invalidate-before-reread blows the cached record away, so if the
|
|
5481
|
+
// forced re-read then fails TRANSIENTLY (gh momentarily unavailable), there is
|
|
5482
|
+
// no prior record left for `adoptGhAuthToken` to preserve — it falls back to
|
|
5483
|
+
// `{token:null}` and the worker is left "logged out", ERASING a still-usable —
|
|
5484
|
+
// and possibly NEWER (a `reload-credentials` SIGUSR1 swap that landed mid-job)
|
|
5485
|
+
// — cached credential. `refreshGhAuthToken({ force })` already re-reads past the
|
|
5486
|
+
// TTL, and `adoptGhAuthToken` PRESERVES the prior cached token on a transient
|
|
5487
|
+
// read failure, so a transient hiccup keeps the good token (and we even retry
|
|
5488
|
+
// with a newer cached one) instead of discarding it. This mirrors the SIGUSR1
|
|
5489
|
+
// reload-credentials handler, which is likewise deliberately NON-destructive.
|
|
5490
|
+
//
|
|
5491
|
+
// The retry is gated on `fresh !== failingToken` — NOT on the re-read's
|
|
5492
|
+
// `changed` flag (which compares against the CACHE): when the cache already
|
|
5493
|
+
// holds the newer token (the `reload-credentials` swap case), a forced re-read
|
|
5494
|
+
// returns it with `changed:false`, yet that newer token is exactly the
|
|
5495
|
+
// credential we want to retry with. `fresh !== failingToken` also covers the
|
|
5496
|
+
// genuine-denial case (the re-read returns the SAME failing token → null → no
|
|
5497
|
+
// retry) and the null→fresh recovery (a logged-out worker re-reads a real token
|
|
5498
|
+
// after `gh auth login`).
|
|
5499
|
+
// `budgetMs` (threaded by the clone/push recovery call sites from their
|
|
5500
|
+
// remaining lease budget — provTimeoutMs()/netTimeoutMs()) bounds the forced
|
|
5501
|
+
// synchronous `gh auth token` + scopes probe so this on-the-active-job-path
|
|
5502
|
+
// re-read participates in the caller's lease-safe window instead of blocking the
|
|
5503
|
+
// event loop / lock heartbeat for the full 15s (Copilot review on #294).
|
|
5504
|
+
function recoverGhTokenAfterAuthFailure(failingToken, { budgetMs = null } = {}) {
|
|
5505
|
+
const { token: fresh } = refreshGhAuthToken({ force: true, budgetMs });
|
|
5506
|
+
return fresh && fresh !== failingToken ? fresh : null;
|
|
5507
|
+
}
|
|
5508
|
+
|
|
5509
|
+
// A snapshot of the cached token's observable identity for logs and the
|
|
5510
|
+
// supervisor activity marker. Never carries the token value. `ageMs` is the
|
|
5511
|
+
// TOKEN's age (since this value was first seen); `lastCheckAgoMs` is how long
|
|
5512
|
+
// ago gh was last consulted (TTL freshness). Returns null when the cache has
|
|
5513
|
+
// never been populated (a worker that has not primed yet).
|
|
5514
|
+
function ghAuthTokenInfo(now = Date.now()) {
|
|
5515
|
+
const c = ghAuthTokenCache;
|
|
5516
|
+
if (c === GH_AUTH_TOKEN_UNSET) return null;
|
|
5517
|
+
return {
|
|
5518
|
+
present: c.token != null,
|
|
5519
|
+
fingerprint: c.fingerprint ?? null,
|
|
5520
|
+
scopes: c.scopes ?? null,
|
|
5521
|
+
since: c.since ?? null,
|
|
5522
|
+
ageMs: Number.isFinite(c.since) ? Math.max(0, now - c.since) : null,
|
|
5523
|
+
lastCheckAgoMs: Number.isFinite(c.lastCheck) ? Math.max(0, now - c.lastCheck) : null,
|
|
5524
|
+
};
|
|
5525
|
+
}
|
|
5526
|
+
|
|
5527
|
+
// One-line, token-free description for logs: `gho_…` →
|
|
5528
|
+
// "gh auth token: fingerprint ab12cd34ef56 (scopes: gist, read:org, repo; age 2m)".
|
|
5529
|
+
function describeGhAuthToken(info) {
|
|
5530
|
+
if (!info) return 'gh auth token: (not yet read)';
|
|
5531
|
+
if (!info.present) return 'gh auth token: none (gh missing or not logged in)';
|
|
5532
|
+
const bits = [];
|
|
5533
|
+
bits.push(`fingerprint ${info.fingerprint || 'unknown'}`);
|
|
5534
|
+
bits.push(info.scopes ? `scopes: ${info.scopes}` : 'scopes: unknown');
|
|
5535
|
+
if (info.ageMs != null) bits.push(`age ${formatDuration(info.ageMs)}`);
|
|
5536
|
+
return `gh auth token: ${bits.join('; ')}`;
|
|
5537
|
+
}
|
|
5538
|
+
|
|
5539
|
+
// Warm the gh-token cache, off the job-handling path. Safe to call any number of
|
|
5540
|
+
// times: within the TTL it is a no-op cache hit; past it (or after an
|
|
5541
|
+
// invalidation) it re-reads the CLI. With a `logger` it reports the token's
|
|
5542
|
+
// fingerprint + scopes on the FIRST populate and whenever the observable
|
|
5543
|
+
// snapshot moves — a rotated token OR a scopes self-repair (a prior null reading
|
|
5544
|
+
// re-probed to a real value) — the "which credential is this worker holding?"
|
|
5545
|
+
// line from issue #293. Returns true once the cache is populated.
|
|
5546
|
+
function primeGhAuthToken({ logger = null } = {}) {
|
|
5547
|
+
const { snapshotChanged } = refreshGhAuthToken();
|
|
5548
|
+
if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
|
|
5549
|
+
logger.info(describeGhAuthToken(ghAuthTokenInfo()));
|
|
5550
|
+
}
|
|
4815
5551
|
return ghAuthTokenCache !== GH_AUTH_TOKEN_UNSET;
|
|
4816
5552
|
}
|
|
4817
5553
|
|
|
5554
|
+
// Keep the cache fresh for the worker's lifetime: re-prime on a TTL cadence so
|
|
5555
|
+
// a `gh auth login`/`refresh` the operator runs while the fleet is up is adopted
|
|
5556
|
+
// within one interval — WITHOUT ever blocking the job path (Copilot review on
|
|
5557
|
+
// #293). Two guards make that true:
|
|
5558
|
+
// 1. The refresh itself is ASYNC (refreshGhAuthTokenAsync — spawn, never
|
|
5559
|
+
// spawnSync), so even when a tick fires mid-job the gh subprocesses yield
|
|
5560
|
+
// the event loop instead of freezing the lock heartbeat for up to 15s.
|
|
5561
|
+
// 2. `isIdle` (optional) lets the worker DEFER a tick while a job is active:
|
|
5562
|
+
// the refresh is then simply adopted by the next idle tick (staleness is
|
|
5563
|
+
// bounded by the job length + one interval, and the sync per-job paths —
|
|
5564
|
+
// TTL lookup, auth-failure recovery — remain as the safety net). That defer
|
|
5565
|
+
// is itself BOUNDED (`maxStaleMs`): once a PRIMED cache is stale beyond the
|
|
5566
|
+
// bound, the tick refreshes DESPITE an active job (still async/non-blocking),
|
|
5567
|
+
// so sustained back-to-back jobs can't keep deferring the refresh for the
|
|
5568
|
+
// worker's whole busy lifetime and serve an expired non-null cache to every
|
|
5569
|
+
// job (Copilot review on #294). An UNSET cache is never primed mid-job — the
|
|
5570
|
+
// defer there still holds — so a logged-out worker stays anonymous until idle.
|
|
5571
|
+
// Ticks never overlap (a slow gh can't stack refreshes), a tick failure is
|
|
5572
|
+
// swallowed (the next tick retries), and the change log fires from here so a
|
|
5573
|
+
// rotation is visible even when no job runs. `onChange` (optional) fires on a
|
|
5574
|
+
// genuine observable-snapshot change — a rotated token OR a scopes self-repair
|
|
5575
|
+
// (a prior null scopes reading re-probed to a real value), both of which move
|
|
5576
|
+
// what `supervisor status` should show — so the caller can republish the
|
|
5577
|
+
// supervisor activity marker; an idle worker otherwise keeps reporting the
|
|
5578
|
+
// pre-refresh fingerprint/scopes indefinitely (Copilot review on #293).
|
|
5579
|
+
//
|
|
5580
|
+
// The timer is UNREF'd (like the other maintenance/watchdog intervals): this is
|
|
5581
|
+
// a background credential refresh, NOT a serving handle, so it must never be the
|
|
5582
|
+
// thing that keeps the process alive. `unref()` does not starve the cadence —
|
|
5583
|
+
// contrary to a common misconception, it does not deprioritise or defer the
|
|
5584
|
+
// timer; it only drops the timer's claim on the event loop's "keep the process
|
|
5585
|
+
// alive" accounting. A busy worker's poll/heartbeat fibers keep the loop running,
|
|
5586
|
+
// so an unref'd interval still fires exactly on cadence. The one case where
|
|
5587
|
+
// unref matters is the dangerous one: if the worker's supervisor runtime has
|
|
5588
|
+
// unexpectedly terminated (its serving handles gone), a REF'd refresh timer
|
|
5589
|
+
// would keep the PID alive forever, masking the dead worker so the daemon never
|
|
5590
|
+
// observes the exit and never restarts the non-serving worker — whereas an
|
|
5591
|
+
// unref'd timer lets the now-idle process exit and be restarted. (The `--auto`
|
|
5592
|
+
// poll timer is deliberately the opposite — ref'd — because it IS a serving
|
|
5593
|
+
// handle that must keep a 0-poller worker alive to retry.) The returned handle
|
|
5594
|
+
// carries a `stop()` (clearInterval) so tests can halt it deterministically.
|
|
5595
|
+
function startGhAuthTokenRefresh({ logger = null, intervalMs = GH_AUTH_TOKEN_TTL_MS, isIdle = null, onChange = null, maxStaleMs = GH_AUTH_TOKEN_MAX_DEFER_MS, _spawn } = {}) {
|
|
5596
|
+
// Clamp to a 1s floor so a test can pass a tiny interval; an explicit 0 is
|
|
5597
|
+
// honoured (→ the floor), not treated as "use the default".
|
|
5598
|
+
const raw = Number.isFinite(Number(intervalMs)) ? Number(intervalMs) : GH_AUTH_TOKEN_TTL_MS;
|
|
5599
|
+
const every = Math.max(1000, raw || 1000);
|
|
5600
|
+
// The bounded-defer ceiling: a non-finite/negative value disables the bound
|
|
5601
|
+
// (pure defer-while-busy, the pre-#294 behaviour).
|
|
5602
|
+
const staleBound = Number.isFinite(Number(maxStaleMs)) && Number(maxStaleMs) >= 0 ? Number(maxStaleMs) : Infinity;
|
|
5603
|
+
let inFlight = false;
|
|
5604
|
+
const tick = async () => {
|
|
5605
|
+
if (inFlight) return; // a slow gh must not stack overlapping refreshes
|
|
5606
|
+
if (isIdle && !isIdle()) {
|
|
5607
|
+
// A job is active — normally defer so we don't even spawn gh mid-job. But
|
|
5608
|
+
// bound that defer: a PRIMED cache stale beyond `staleBound` must refresh
|
|
5609
|
+
// anyway (still async/non-blocking) rather than reach every subsequent job
|
|
5610
|
+
// for the worker's whole busy lifetime under sustained saturation (Copilot
|
|
5611
|
+
// review on #294). An UNSET cache is never primed mid-job (no bound applies),
|
|
5612
|
+
// so a logged-out worker stays anonymous until it goes idle, as before.
|
|
5613
|
+
const cached = ghAuthTokenCache === GH_AUTH_TOKEN_UNSET ? null : ghAuthTokenCache;
|
|
5614
|
+
const staleBeyondBound = cached && (Date.now() - cached.lastCheck) >= staleBound;
|
|
5615
|
+
if (!staleBeyondBound) return; // defer to the next idle tick
|
|
5616
|
+
}
|
|
5617
|
+
inFlight = true;
|
|
5618
|
+
try {
|
|
5619
|
+
const { snapshotChanged } = await refreshGhAuthTokenAsync({ _spawn });
|
|
5620
|
+
if (logger && (snapshotChanged || ghAuthTokenCache === GH_AUTH_TOKEN_UNSET)) {
|
|
5621
|
+
logger.info(describeGhAuthToken(ghAuthTokenInfo()));
|
|
5622
|
+
}
|
|
5623
|
+
if (snapshotChanged && onChange) {
|
|
5624
|
+
try { onChange(); } catch { /* best effort — republishing is advisory */ }
|
|
5625
|
+
}
|
|
5626
|
+
} catch {
|
|
5627
|
+
// A failed tick (gh wedged, spawn error) must never crash the worker or
|
|
5628
|
+
// surface as an unhandled rejection — the next tick retries.
|
|
5629
|
+
} finally {
|
|
5630
|
+
inFlight = false;
|
|
5631
|
+
}
|
|
5632
|
+
};
|
|
5633
|
+
const t = setInterval(() => { tick(); }, every);
|
|
5634
|
+
// Unref so a dead worker runtime (serving handles gone) can exit and be
|
|
5635
|
+
// restarted — this maintenance refresh must not keep a non-serving PID alive
|
|
5636
|
+
// (see the header comment). Guarded for fake-timer test harnesses.
|
|
5637
|
+
if (typeof t.unref === 'function') t.unref();
|
|
5638
|
+
t.stop = () => clearInterval(t);
|
|
5639
|
+
return t;
|
|
5640
|
+
}
|
|
5641
|
+
|
|
5642
|
+
// Classify a git/gh failure as an AUTH (or auth-scope) failure — the cases where
|
|
5643
|
+
// the cached credential, not the request, is the problem: a 401/403, git's
|
|
5644
|
+
// credential prompts on an anonymous/rejected clone, or GitHub's OAuth-scope
|
|
5645
|
+
// refusal ("refusing to allow an OAuth App to create or update workflow …
|
|
5646
|
+
// without `workflow` scope" — the exact escalation from issue #293). Matched on
|
|
5647
|
+
// the ALREADY-REDACTED detail string (gitErrorDetail/describeGitFailure output),
|
|
5648
|
+
// so the token value is never inspected here. Deliberately conservative: a
|
|
5649
|
+
// non-fast-forward reject, a missing repo (404) or a network timeout are NOT
|
|
5650
|
+
// auth failures and must not trigger a token drop + retry.
|
|
5651
|
+
function isGitAuthFailure(detail) {
|
|
5652
|
+
const s = String(detail ?? '');
|
|
5653
|
+
if (!s) return false;
|
|
5654
|
+
return /HTTP\s*40[13]\b|\b40[13]\s*\((Unauthorized|Forbidden)\)|returned error:\s*40[13]\b|could not read (Username|Password)|Authentication failed|authentication required|Permission to .+ denied|refusing to allow an OAuth App|without .?workflow.? scope|Bad credentials|Resource not accessible by (integration|personal access token)/i.test(s);
|
|
5655
|
+
}
|
|
5656
|
+
|
|
4818
5657
|
// Read the operator's own git identity from their GLOBAL config, using the real
|
|
4819
5658
|
// host environment (process.env) rather than a job's sanitized gitEnv — so it
|
|
4820
5659
|
// resolves even on the anonymous clone path, where gitEnv points
|
|
@@ -4942,6 +5781,82 @@ function normalizeAuthRef(authRef) {
|
|
|
4942
5781
|
return { kind: 'custom', name: trimmed };
|
|
4943
5782
|
}
|
|
4944
5783
|
|
|
5784
|
+
// The env-var NAME the github clone/push credential is forwarded to the harness
|
|
5785
|
+
// under. Single source of truth so `resolveJobSecrets` (which builds the
|
|
5786
|
+
// `resolved` secret map) and the clone-recovery rotation glue (which keeps that
|
|
5787
|
+
// map in sync when the token rotates) can NEVER drift on the key: a custom
|
|
5788
|
+
// authRef uses its own name; the default github credential uses GITHUB_TOKEN.
|
|
5789
|
+
// Returns null when no credential is forwarded (a non-github provider with no
|
|
5790
|
+
// custom ref, or a blank/invalid authRef).
|
|
5791
|
+
function ghCredentialEnvName({ provider, authRef }) {
|
|
5792
|
+
const prov = provider || 'github';
|
|
5793
|
+
const ref = normalizeAuthRef(authRef);
|
|
5794
|
+
if (ref.kind === 'invalid') return null;
|
|
5795
|
+
if (ref.kind === 'custom') return ref.name;
|
|
5796
|
+
return prov === 'github' ? 'GITHUB_TOKEN' : null;
|
|
5797
|
+
}
|
|
5798
|
+
|
|
5799
|
+
// When clone recovery ROTATES the cache-derived credential during provisioning
|
|
5800
|
+
// (a stale gh token re-read after an auth failure), the `resolved` secret map —
|
|
5801
|
+
// built BEFORE provisioning and later forwarded to the harness as `secretEnv` —
|
|
5802
|
+
// still holds the STALE token, so an allowPr harness's own `gh`/`git` runs would
|
|
5803
|
+
// use a credential the clone already proved dead (Copilot review on #293). Sync
|
|
5804
|
+
// the map so it hands the harness the SAME credential provisioning just adopted.
|
|
5805
|
+
// Guard rails: only a 'gh-cache'-sourced credential may be swapped (an explicit
|
|
5806
|
+
// `env`/`custom` identity is deliberate and never silently replaced), only when
|
|
5807
|
+
// a token actually exists, and ONLY when that key was already forwarded (never
|
|
5808
|
+
// inject a brand-new secret the job did not request). Pure: mutates and returns
|
|
5809
|
+
// `resolved`.
|
|
5810
|
+
function applyRotatedGhCredential({ resolved, provider, authRef, source, token }) {
|
|
5811
|
+
if (!resolved || source !== 'gh-cache' || !token) return resolved;
|
|
5812
|
+
const key = ghCredentialEnvName({ provider, authRef });
|
|
5813
|
+
if (key && Object.prototype.hasOwnProperty.call(resolved, key)) resolved[key] = token;
|
|
5814
|
+
return resolved;
|
|
5815
|
+
}
|
|
5816
|
+
|
|
5817
|
+
// Resolve the github clone/push credential AND record where it came from. The
|
|
5818
|
+
// default credential (env GITHUB_TOKEN) falls back to the gh CLI's stored token
|
|
5819
|
+
// so `gh auth login` setups work without exporting GITHUB_TOKEN. A custom
|
|
5820
|
+
// `authRef` is honored strictly (env/secret resolver only, no gh fallback) so a
|
|
5821
|
+
// misconfigured named secret surfaces as missing rather than silently borrowing
|
|
5822
|
+
// the operator's gh login. An authRef that is present but blank is a
|
|
5823
|
+
// misconfiguration and yields no token (never the gh fallback). `ghAuthToken`
|
|
5824
|
+
// is injectable for testing.
|
|
5825
|
+
// The returned `source` matters downstream: the #293 auth-failure recovery
|
|
5826
|
+
// (force-re-read `gh auth token`, retry once — NON-destructive, never drops the
|
|
5827
|
+
// cache first) is only legitimate when
|
|
5828
|
+
// the token CAME from that cache ('gh-cache'). Retrying an explicit
|
|
5829
|
+
// GITHUB_TOKEN ('env') or a custom authRef ('custom') failure with the
|
|
5830
|
+
// operator's unrelated gh login would silently swap the configured identity
|
|
5831
|
+
// mid-job (Copilot review on #293), so those sources never take the fallback.
|
|
5832
|
+
// A DEFAULT github lookup that found NO cached gh token ALSO reports
|
|
5833
|
+
// 'gh-cache' (with `token: null`) rather than 'none': it is still the gh-cache
|
|
5834
|
+
// credential path, merely a cache MISS. Tagging it 'gh-cache' keeps the
|
|
5835
|
+
// clone/push auth-failure recovery ELIGIBLE so a worker that cached
|
|
5836
|
+
// "logged out" can still self-heal (a forced re-read recovers null → fresh
|
|
5837
|
+
// token after the operator runs `gh auth login`) instead of failing the job
|
|
5838
|
+
// anonymous and waiting for the periodic TTL refresh (Copilot review on #293).
|
|
5839
|
+
// 'none' is reserved for a non-github provider and an invalid/blank authRef —
|
|
5840
|
+
// neither may borrow the gh login.
|
|
5841
|
+
// Returns { token, source: 'env' | 'custom' | 'gh-cache' | 'none' }.
|
|
5842
|
+
function githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
|
|
5843
|
+
const prov = provider || 'github';
|
|
5844
|
+
const ref = normalizeAuthRef(authRef);
|
|
5845
|
+
if (ref.kind === 'invalid') return { token: null, source: 'none' };
|
|
5846
|
+
const usesDefault = prov === 'github' && ref.kind === 'default';
|
|
5847
|
+
const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
|
|
5848
|
+
const resolved = name ? (secretResolver.resolve(name) || null) : null;
|
|
5849
|
+
if (resolved) return { token: resolved, source: ref.kind === 'custom' ? 'custom' : 'env' };
|
|
5850
|
+
if (!resolved && usesDefault) {
|
|
5851
|
+
const token = ghAuthToken() || null;
|
|
5852
|
+
// A hit AND a miss both report 'gh-cache' (see the header): the miss keeps
|
|
5853
|
+
// the auth-failure recovery eligible so a post-`gh auth login` forced
|
|
5854
|
+
// re-read can still recover this job.
|
|
5855
|
+
return { token, source: 'gh-cache' };
|
|
5856
|
+
}
|
|
5857
|
+
return { token: null, source: 'none' };
|
|
5858
|
+
}
|
|
5859
|
+
|
|
4945
5860
|
// Resolve the github clone/push credential. The default credential (env
|
|
4946
5861
|
// GITHUB_TOKEN) falls back to the gh CLI's stored token so `gh auth login`
|
|
4947
5862
|
// setups work without exporting GITHUB_TOKEN. A custom `authRef` is honored
|
|
@@ -4949,16 +5864,10 @@ function normalizeAuthRef(authRef) {
|
|
|
4949
5864
|
// secret surfaces as missing rather than silently borrowing the operator's gh
|
|
4950
5865
|
// login. An authRef that is present but blank is a misconfiguration and yields no
|
|
4951
5866
|
// token (never the gh fallback). `ghAuthToken` is injectable for testing.
|
|
4952
|
-
// Returns a token or null.
|
|
5867
|
+
// Returns a token or null. (Callers that need the credential's SOURCE — e.g. to
|
|
5868
|
+
// gate the #293 gh-token auth-failure retry — use githubCloneTokenWithSource.)
|
|
4953
5869
|
function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghAuthTokenFromCli }) {
|
|
4954
|
-
|
|
4955
|
-
const ref = normalizeAuthRef(authRef);
|
|
4956
|
-
if (ref.kind === 'invalid') return null;
|
|
4957
|
-
const usesDefault = prov === 'github' && ref.kind === 'default';
|
|
4958
|
-
const name = ref.kind === 'custom' ? ref.name : (prov === 'github' ? 'GITHUB_TOKEN' : null);
|
|
4959
|
-
let token = name ? (secretResolver.resolve(name) || null) : null;
|
|
4960
|
-
if (!token && usesDefault) token = ghAuthToken() || null;
|
|
4961
|
-
return token;
|
|
5870
|
+
return githubCloneTokenWithSource({ provider, authRef, secretResolver, ghAuthToken }).token;
|
|
4962
5871
|
}
|
|
4963
5872
|
|
|
4964
5873
|
// Clone repo into <runDir>/workspace and check out / create the working branch.
|
|
@@ -4973,11 +5882,11 @@ function githubCloneToken({ provider, authRef, secretResolver, ghAuthToken = ghA
|
|
|
4973
5882
|
// resume sides agree on which no-create push targets are recoverable (#270).
|
|
4974
5883
|
const CONVENTIONAL_BASE_BRANCHES = new Set(['main', 'master']);
|
|
4975
5884
|
|
|
4976
|
-
function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, _runGit = null }) {
|
|
5885
|
+
function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, logger = null, corr = '', abortSignal = null, credentialSource = null, _runGit = null }) {
|
|
4977
5886
|
// Test-only seam mirroring finalizeGit: route every git call through `runGitFn` so a
|
|
4978
5887
|
// deterministic test can observe the per-call timeouts drawn from the shared
|
|
4979
5888
|
// provisioning budget below. Defaults to the module `runGit` (production unchanged).
|
|
4980
|
-
|
|
5889
|
+
let runGitFn = _runGit || runGit;
|
|
4981
5890
|
// #222: the git ops below are blocking spawnSync calls that cannot observe an
|
|
4982
5891
|
// AbortSignal mid-call, so provisioning is made signal-aware by rechecking
|
|
4983
5892
|
// BETWEEN operations: if a lock-loss race won, bail before the NEXT side effect
|
|
@@ -5003,7 +5912,9 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5003
5912
|
// already-lost lease without starting any repository side effect.
|
|
5004
5913
|
throwIfAborted('entry');
|
|
5005
5914
|
const askpass = writeAskpass(runDir, token);
|
|
5006
|
-
const
|
|
5915
|
+
// `let` (not const): the #293 clone auth-failure retry swaps in a FRESH token
|
|
5916
|
+
// (rewritten askpass + GIT_TOKEN) and re-runs the clone with an updated env.
|
|
5917
|
+
let gitEnv = {
|
|
5007
5918
|
...process.env,
|
|
5008
5919
|
GIT_TERMINAL_PROMPT: '0',
|
|
5009
5920
|
GIT_CONFIG_NOSYSTEM: '1',
|
|
@@ -5091,8 +6002,80 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5091
6002
|
const detailNote = detail && detail !== 'unknown error' ? ` — last git output: ${detail}` : '';
|
|
5092
6003
|
throw new ProvisionError(`git clone timed out after ${effectiveTimeoutMs}ms — the repo may be too large, or the network stalled; raise the timeout (repository.cloneTimeoutMs in the envelope, or the worker's --clone-timeout flag) or scope the clone with filter/singleBranch/depth${detailNote}`);
|
|
5093
6004
|
}
|
|
5094
|
-
|
|
6005
|
+
const detail = gitErrorDetail(clone, token);
|
|
6006
|
+
// #293: an AUTH failure here usually means the cached gh token went stale
|
|
6007
|
+
// (the operator re-logged in / rotated it after this worker primed). Force a
|
|
6008
|
+
// fresh re-read and retry the clone ONCE with it before reporting failure —
|
|
6009
|
+
// a rotated credential self-heals instead of shedding the job. Only when the
|
|
6010
|
+
// re-read yields a token DIFFERENT from the one that failed (a same-token
|
|
6011
|
+
// retry can't fix a genuine permission denial). The recovery is
|
|
6012
|
+
// NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
|
|
6013
|
+
// the cache first, so a transiently-failing re-read can't erase a still-valid
|
|
6014
|
+
// — possibly newer — cached credential (Copilot review on #294). The retry
|
|
6015
|
+
// re-runs the whole clone with the new token threaded through the askpass
|
|
6016
|
+
// env, re-anchored to the remaining provisioning budget.
|
|
6017
|
+
// GATED on the credential's SOURCE (Copilot review on #293): this fallback
|
|
6018
|
+
// replaces the failed token with the operator's `gh auth token`, so it is
|
|
6019
|
+
// only legitimate when the failed token CAME from that same gh cache
|
|
6020
|
+
// ('gh-cache'). An explicit GITHUB_TOKEN ('env') or custom authRef
|
|
6021
|
+
// ('custom') is a deliberately configured identity — silently swapping it
|
|
6022
|
+
// for the operator's gh login would violate githubCloneToken's documented
|
|
6023
|
+
// precedence and could act as a different identity, so those sources fail
|
|
6024
|
+
// without the gh retry.
|
|
6025
|
+
// The `token` guard is deliberately ABSENT (Copilot review on #293): a
|
|
6026
|
+
// default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
|
|
6027
|
+
// a worker that cached "logged out" recovers null → fresh token after the
|
|
6028
|
+
// operator runs `gh auth login` instead of failing the job anonymous. That
|
|
6029
|
+
// recovery must REBUILD the credential plumbing the anonymous path omitted
|
|
6030
|
+
// (the askpass helper, GIT_ASKPASS/GIT_TOKEN, and the username-embedded
|
|
6031
|
+
// remote) — a bare GIT_TOKEN swap leaves the clone unauthenticated.
|
|
6032
|
+
if (credentialSource === 'gh-cache' && isGitAuthFailure(detail)) {
|
|
6033
|
+
const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: provTimeoutMs() });
|
|
6034
|
+
if (fresh) {
|
|
6035
|
+
logger?.info?.(`git clone auth failed${cs} — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the clone once`);
|
|
6036
|
+
throwIfAborted('clone-auth-retry');
|
|
6037
|
+
// Rebuild the credential env for the fresh token. On the null→fresh
|
|
6038
|
+
// recovery the original env was the ANONYMOUS one (no askpass helper,
|
|
6039
|
+
// GIT_CONFIG_GLOBAL neutralized), so re-mint the askpass helper and
|
|
6040
|
+
// restore global config; on a stale-token rotation this is a no-op
|
|
6041
|
+
// re-point of the same GIT_TOKEN.
|
|
6042
|
+
const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
|
|
6043
|
+
const retryAskpass = writeAskpass(runDir, fresh);
|
|
6044
|
+
if (retryAskpass) {
|
|
6045
|
+
retryEnv.GIT_ASKPASS = retryAskpass;
|
|
6046
|
+
// The anonymous path set GIT_CONFIG_GLOBAL=/dev/null so a no-token
|
|
6047
|
+
// clone can't inherit operator credentials; the fresh token is ours,
|
|
6048
|
+
// so global config (e.g. http.proxy) may apply again.
|
|
6049
|
+
delete retryEnv.GIT_CONFIG_GLOBAL;
|
|
6050
|
+
}
|
|
6051
|
+
// Rebuild the clone argv so the remote carries the fresh credential's
|
|
6052
|
+
// username (authUrl embeds x-access-token only when a token is present);
|
|
6053
|
+
// the anonymous remote had no username, so git would never consult the
|
|
6054
|
+
// askpass helper for it. `workspaceDir` stays last; the remote is the
|
|
6055
|
+
// argument immediately before it (cloneArgs.push(remote, workspaceDir)).
|
|
6056
|
+
const retryCloneArgs = cloneArgs.slice();
|
|
6057
|
+
retryCloneArgs[retryCloneArgs.length - 2] = authUrl(repo.url, repo.provider || 'github', true);
|
|
6058
|
+
const retry = runGitFn(retryCloneArgs, { env: retryEnv, timeoutMs: provTimeoutMs() });
|
|
6059
|
+
if (retry.status === 0) {
|
|
6060
|
+
// Adopt the fresh token for the REST of provisioning + the returned
|
|
6061
|
+
// gitEnv (finalizeGit pushes with it) so the whole job runs on the
|
|
6062
|
+
// rotated credential.
|
|
6063
|
+
token = fresh;
|
|
6064
|
+
gitEnv = retryEnv;
|
|
6065
|
+
return provisionRepoAfterClone();
|
|
6066
|
+
}
|
|
6067
|
+
throw new ProvisionError(`git clone failed after a fresh gh token retry: ${gitErrorDetail(retry, fresh)}`);
|
|
6068
|
+
}
|
|
6069
|
+
}
|
|
6070
|
+
throw new ProvisionError(`git clone failed: ${detail}`);
|
|
5095
6071
|
}
|
|
6072
|
+
return provisionRepoAfterClone();
|
|
6073
|
+
|
|
6074
|
+
// Everything after a successful clone — hoisted into a closure so the
|
|
6075
|
+
// auth-retry path above re-enters it without duplicating the body. The clone
|
|
6076
|
+
// result itself is not read below (only its success mattered); the workspace
|
|
6077
|
+
// on disk is the real output.
|
|
6078
|
+
function provisionRepoAfterClone() {
|
|
5096
6079
|
|
|
5097
6080
|
// #222: the clone (the dominant blocking op and the first repo side effect) is
|
|
5098
6081
|
// done. Recheck the abort signal before the remaining side effects/network ops
|
|
@@ -5626,7 +6609,16 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5626
6609
|
// the authoritative result so eligibility (auto checkpointing) doesn't
|
|
5627
6610
|
// under-detect an authenticated clone.
|
|
5628
6611
|
const authenticated = !!token || isAuthenticatedRemote(repo.url);
|
|
5629
|
-
|
|
6612
|
+
// `effectiveToken` is the credential the workspace ACTUALLY runs on after any
|
|
6613
|
+
// #293 auth-failure retry rotated it mid-provisioning (the local `token` is
|
|
6614
|
+
// reassigned on adoption). The caller (workAgent) must use THIS — not its own
|
|
6615
|
+
// pre-provisioning resolution — for the post-provisioning credential users
|
|
6616
|
+
// (finalizeGit's PR reconcile/push, the checkpoint GC/ref-delete askpass), or
|
|
6617
|
+
// a recovered clone would keep failing with the stale credential (Copilot
|
|
6618
|
+
// review on #293). `credentialSource` is echoed back so finalizeGit can gate
|
|
6619
|
+
// its own push-retry on the same source rule.
|
|
6620
|
+
return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, effectiveToken: token || null, credentialSource, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
|
|
6621
|
+
} // end provisionRepoAfterClone
|
|
5630
6622
|
}
|
|
5631
6623
|
|
|
5632
6624
|
// Look up a PR for this branch (2a does NOT open it — the harness does, driven
|
|
@@ -5789,7 +6781,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5789
6781
|
...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
|
|
5790
6782
|
...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
|
|
5791
6783
|
};
|
|
5792
|
-
|
|
6784
|
+
// #296 (F9): the checkpoint-path git runner must stay USABLE after the harness
|
|
6785
|
+
// abort. The force-stop RECOVERY path (flush + preserveRecovery, promoting the
|
|
6786
|
+
// durable recovery ref #295) runs only once `result.aborted` is observed — i.e.
|
|
6787
|
+
// AFTER the job's `abortSignal` has already fired. Binding this runner to
|
|
6788
|
+
// `abortSignal` (the round-4 F8 attempt) made `createGitRunner` fail every
|
|
6789
|
+
// recovery command (`[aborted]`), so the abort flush could not push its snapshot
|
|
6790
|
+
// and preserveRecovery could not even observe/create the recovery ref — recovery
|
|
6791
|
+
// was self-defeating. Instead drive it from a DEDICATED recovery-deadline
|
|
6792
|
+
// controller that is NEVER aborted by the harness stop, only when the recovery
|
|
6793
|
+
// budget (FORCE_STOP_RECOVERY_MS) expires. That still delivers F8's goal — an
|
|
6794
|
+
// in-flight push is torn down at the deadline so the chain settles and `close()`
|
|
6795
|
+
// drains inside the supervisor's SIGKILL window — without killing the recovery
|
|
6796
|
+
// before it can write the ref.
|
|
6797
|
+
const recoveryController = new AbortController();
|
|
6798
|
+
const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: recoveryController.signal });
|
|
5793
6799
|
// The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
|
|
5794
6800
|
// if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
|
|
5795
6801
|
// git child is killed and later ops fail fast, so setup unwinds instead of holding
|
|
@@ -5800,6 +6806,40 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5800
6806
|
// which time the workspace (and its askpass helper) has been reaped.
|
|
5801
6807
|
const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
|
|
5802
6808
|
|
|
6809
|
+
// Bound every checkpoint-path SDK call: a timed-out call rejects; a late settle is
|
|
6810
|
+
// swallowed so it can never surface as an unhandled rejection. Hoisted here (ahead
|
|
6811
|
+
// of the GC block, which reuses them) so the #295 ambiguity guard below is bounded.
|
|
6812
|
+
const CP_SDK_TIMEOUT_MS = cfg.sdkTimeoutMs;
|
|
6813
|
+
const withDeadline = (p, ms, label, signal = null) => {
|
|
6814
|
+
let timer;
|
|
6815
|
+
let onAbort = null;
|
|
6816
|
+
// #296 (review): a bare Promise.race REJECTS on timeout but leaves the SDK
|
|
6817
|
+
// mutation in flight — a timed-out `resumeFromCheckpoint` write could then
|
|
6818
|
+
// complete AFTER a retry already wrote a newer pointer and overwrite it with
|
|
6819
|
+
// the stale ref. The SDK returns a CancelablePromise (toCancelable): cancel it
|
|
6820
|
+
// on timeout so the underlying fetch is aborted and the late settle can never
|
|
6821
|
+
// land. `p.cancel?.()` is a no-op for a plain (non-cancelable) promise, and a
|
|
6822
|
+
// cancelled SDK call rejects with CancelSdkError, which the `.catch(() => {})`
|
|
6823
|
+
// below swallows.
|
|
6824
|
+
// #296 (review r4245146247): the SAME stale-write hazard applies to the job's
|
|
6825
|
+
// ABORT (lease loss), not just the per-call timeout. An optional `signal` lets a
|
|
6826
|
+
// setup-path side-effect write be cancelled the instant the job loses its lease,
|
|
6827
|
+
// so a write left pending after a replacement activation published a newer
|
|
6828
|
+
// pointer is torn down and its late settle can never overwrite that newer value.
|
|
6829
|
+
const race = new Promise((_r, reject) => {
|
|
6830
|
+
timer = setTimeout(() => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} timed out after ${ms}ms`)); }, ms);
|
|
6831
|
+
timer?.unref?.();
|
|
6832
|
+
if (signal) {
|
|
6833
|
+
if (signal.aborted === true) { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); return; }
|
|
6834
|
+
onAbort = () => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); };
|
|
6835
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
6836
|
+
}
|
|
6837
|
+
});
|
|
6838
|
+
Promise.resolve(p).catch(() => {});
|
|
6839
|
+
const cleanup = () => { clearTimeout(timer); if (signal && onAbort) { try { signal.removeEventListener('abort', onAbort); } catch { /* best effort */ } } };
|
|
6840
|
+
return Promise.race([Promise.resolve(p).finally(cleanup), race]);
|
|
6841
|
+
};
|
|
6842
|
+
|
|
5803
6843
|
let prior = null;
|
|
5804
6844
|
let restored = null;
|
|
5805
6845
|
let startupFetchFailed = false;
|
|
@@ -5809,6 +6849,93 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5809
6849
|
startupFetchFailed = true;
|
|
5810
6850
|
log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
|
|
5811
6851
|
}
|
|
6852
|
+
// #295: nothing at this run's deterministic ref, but a human may have answered an
|
|
6853
|
+
// SLA escalation and re-entered the task as a NEW element instance. The predecessor
|
|
6854
|
+
// recorded its ref in a process-scoped `resumeFromCheckpoint` variable (survives
|
|
6855
|
+
// re-entry); adopt it under THIS run's key so the restore/ownership/discard paths
|
|
6856
|
+
// below operate unchanged and no GC can reclaim it between activations.
|
|
6857
|
+
if (!prior && !startupFetchFailed) {
|
|
6858
|
+
// Correlate the shared process-scoped slot to THIS task. Copilot review on #296
|
|
6859
|
+
// (r4244212483): a SELF-RECORDED (object) predecessor is correlated only by the
|
|
6860
|
+
// STATIC elementId, which is NOT unique under parallel multi-instance of one
|
|
6861
|
+
// agent-task element (N occurrences share the elementId AND the one process-scoped
|
|
6862
|
+
// resume slot). Counting ACTIVE occurrences cannot establish occurrence identity
|
|
6863
|
+
// (index lag can hide a live sibling; a terminal sibling drops out of the ACTIVE
|
|
6864
|
+
// result yet may have been the slot's last writer), so NO engine query makes a
|
|
6865
|
+
// self-record adoption provably safe. We therefore adopt ONLY an EXPLICIT,
|
|
6866
|
+
// occurrence-authoritative ref — a bare STRING the process delivers naming THIS
|
|
6867
|
+
// occurrence's exact predecessor (nano-workforce#891, option 2). A self-recorded
|
|
6868
|
+
// object is NEVER auto-adopted (fail closed to a clean start); until #891 delivers
|
|
6869
|
+
// the explicit ref, a resume is transcript-only rather than risking a sibling's
|
|
6870
|
+
// state. `predecessorCheckpointRef` still parses/locates the candidate ref; the
|
|
6871
|
+
// explicit check below is what authorizes adopting it.
|
|
6872
|
+
const predRef = predecessorCheckpointRef(job?.variables, ref, job?.elementId ?? null);
|
|
6873
|
+
if (predRef) {
|
|
6874
|
+
const adoptAllowed = resumeRefIsExplicit(job?.variables);
|
|
6875
|
+
if (!adoptAllowed) {
|
|
6876
|
+
log.warn(`not adopting predecessor WIP checkpoint ${predRef} — it is an elementId-correlated self-record, which cannot be proven to name THIS occurrence (parallel multi-instance of one agent-task element shares the elementId and the process-scoped resume slot); starting clean. Resume is transcript-only until the process delivers an explicit occurrence-authoritative ref (nano-workforce#891).`);
|
|
6877
|
+
}
|
|
6878
|
+
let predCp = null;
|
|
6879
|
+
if (adoptAllowed) {
|
|
6880
|
+
try { predCp = await fetchCp({ git: startupGit, ref: predRef }); } catch (err) { log.debug(`could not fetch predecessor WIP checkpoint ${predRef} — ${oneLineLog(err?.message || err)}.`); }
|
|
6881
|
+
}
|
|
6882
|
+
if (predCp?.sha) {
|
|
6883
|
+
// #296 (review): after ANY adoption the process-scoped slot still names the
|
|
6884
|
+
// stale PREDECESSOR ref. If this resumed activation then succeeds WITHOUT
|
|
6885
|
+
// taking another checkpoint (a short run before the safety timer), its adopted
|
|
6886
|
+
// WIP ref is deleted on ack while the slot keeps pointing at the predecessor —
|
|
6887
|
+
// a later activation would re-restore that old snapshot. Advance the slot to
|
|
6888
|
+
// THIS run's key (with the sha we actually adopted) so the pointer tracks the
|
|
6889
|
+
// current owner. Factored into one helper so EVERY adoption branch advances it
|
|
6890
|
+
// (the create-only win AND the lost-race re-read, whose race winner's own
|
|
6891
|
+
// best-effort advance may have failed — this makes the advance idempotent
|
|
6892
|
+
// across the race rather than reliant on the winner). Best-effort; a failure
|
|
6893
|
+
// never blocks setup.
|
|
6894
|
+
const advanceResumePointer = async (adoptedSha) => {
|
|
6895
|
+
if (!adoptedSha || typeof camunda?.createElementInstanceVariables !== 'function' || !job?.processInstanceKey) return;
|
|
6896
|
+
// #296 (review r4245146247): recheck the job signal before this process-scoped
|
|
6897
|
+
// side effect. The startup git work above (fetch/push) may have straddled a
|
|
6898
|
+
// lease loss; once aborted, a replacement activation owns this pointer slot and
|
|
6899
|
+
// may already have published a newer value, so writing our (now stale) ref here
|
|
6900
|
+
// would overwrite it. Skip when aborted, and bind the SDK call to abortSignal so
|
|
6901
|
+
// a lease loss DURING the write cancels it before its late settle can land.
|
|
6902
|
+
if (abortSignal?.aborted === true) return;
|
|
6903
|
+
try {
|
|
6904
|
+
await withDeadline(camunda.createElementInstanceVariables({
|
|
6905
|
+
elementInstanceKey: String(job.processInstanceKey),
|
|
6906
|
+
variables: { resumeFromCheckpoint: { ref, sha: adoptedSha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch: provisioned.workingBranch || null, at: Date.now(), adoptedFrom: predRef } },
|
|
6907
|
+
local: true,
|
|
6908
|
+
}), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', abortSignal || undefined);
|
|
6909
|
+
} catch (err) { log.debug(`could not advance the resumeFromCheckpoint pointer to ${ref} — ${oneLineLog(err?.message || err)}.`); }
|
|
6910
|
+
};
|
|
6911
|
+
// Create-only adoption (default expectSha=''): only write our key while it
|
|
6912
|
+
// is ABSENT, so a concurrent same-key retry that already adopted + advanced
|
|
6913
|
+
// the ref is never clobbered (Copilot review on #296).
|
|
6914
|
+
const copied = await pushNamedRef({ git: startupGit, ref, sha: predCp.sha });
|
|
6915
|
+
if (copied?.ok) {
|
|
6916
|
+
try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = { ...predCp, ref }; }
|
|
6917
|
+
if (!prior) prior = { ...predCp, ref };
|
|
6918
|
+
logger.info?.(`${prefix} (${corr}): adopted predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} under this run's key ${ref} (SLA-escalation resume); restoring it below.`);
|
|
6919
|
+
await advanceResumePointer(predCp.sha);
|
|
6920
|
+
} else if (copied?.staleLease) {
|
|
6921
|
+
// A concurrent activation populated this key first — adopt what IS there
|
|
6922
|
+
// rather than force over it; the ref now holds the race winner's state.
|
|
6923
|
+
try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = null; }
|
|
6924
|
+
if (prior) {
|
|
6925
|
+
logger.info?.(`${prefix} (${corr}): a concurrent activation already populated ${ref}; adopting its checkpoint ${prior.sha ? '@' + String(prior.sha).slice(0, 12) : ''} (SLA-escalation resume); restoring it below.`);
|
|
6926
|
+
// Advance the pointer to the re-read sha too: the race winner's own
|
|
6927
|
+
// best-effort advance may have failed, which would leave the slot naming
|
|
6928
|
+
// the bare predecessor and risk a later re-adoption of stale work.
|
|
6929
|
+
await advanceResumePointer(prior.sha);
|
|
6930
|
+
} else {
|
|
6931
|
+
log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but ${ref} was concurrently moved and could not be re-read — the predecessor ref is retained for explicit recovery.`);
|
|
6932
|
+
}
|
|
6933
|
+
} else {
|
|
6934
|
+
log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but could not copy it to ${ref} — ${copied?.error || 'unknown'}; the predecessor ref is retained for explicit recovery.`);
|
|
6935
|
+
}
|
|
6936
|
+
}
|
|
6937
|
+
}
|
|
6938
|
+
}
|
|
5812
6939
|
if (prior) {
|
|
5813
6940
|
const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
|
|
5814
6941
|
if (r?.restored) {
|
|
@@ -5836,12 +6963,12 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5836
6963
|
|
|
5837
6964
|
// Orphan GC for this remote, throttled per worker process, in the background.
|
|
5838
6965
|
let gc = Promise.resolve();
|
|
6966
|
+
// #296: the GC sweep's abort controller, hoisted so the force-stop recovery
|
|
6967
|
+
// DEADLINE (abortRecovery) can tear the (up to 120s) sweep down too — otherwise
|
|
6968
|
+
// the single shared force-stop budget would bound only the recovery git, and a
|
|
6969
|
+
// still-running GC could hold close()'s drain past the supervisor's SIGKILL window.
|
|
6970
|
+
let gcAbort = null;
|
|
5839
6971
|
const gcKey = provisioned.remote || provisioned.workspaceDir;
|
|
5840
|
-
// Bound every checkpoint-path SDK call: the GC sweep and the variable write are
|
|
5841
|
-
// awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
|
|
5842
|
-
// call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
|
|
5843
|
-
// late settle is swallowed so it can never surface as an unhandled rejection.
|
|
5844
|
-
const CP_SDK_TIMEOUT_MS = 10000;
|
|
5845
6972
|
// A single overall bound on the whole GC sweep. Individual ops are already
|
|
5846
6973
|
// bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
|
|
5847
6974
|
// but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
|
|
@@ -5852,23 +6979,27 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5852
6979
|
// reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
|
|
5853
6980
|
// 4099558206 / 4099762442).
|
|
5854
6981
|
const GC_OVERALL_TIMEOUT_MS = 120000;
|
|
5855
|
-
const withDeadline = (p, ms, label) => {
|
|
5856
|
-
let timer;
|
|
5857
|
-
const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
|
|
5858
|
-
Promise.resolve(p).catch(() => {});
|
|
5859
|
-
return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
|
|
5860
|
-
};
|
|
5861
6982
|
if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5865
|
-
|
|
6983
|
+
// #295: a nano-managed ref (WIP or durable recovery) is an orphan only when its
|
|
6984
|
+
// element is terminal AND its PROCESS instance is finished too. An element
|
|
6985
|
+
// cancelled by an SLA boundary timer, inside a process that is still active (human
|
|
6986
|
+
// escalation pending), must NOT be reclaimed — the TTL is the final backstop.
|
|
6987
|
+
// `makeOrphanProbe` encodes that; it degrades to the legacy element-only check when
|
|
6988
|
+
// the host SDK exposes no `getProcessInstance`. The sweep applies this to BOTH the
|
|
6989
|
+
// `refs/nano/wip/*` and `refs/nano/recovered/*` namespaces so recovery refs cannot
|
|
6990
|
+
// grow without bound once their escalation is resolved.
|
|
6991
|
+
const isTerminal = makeOrphanProbe({
|
|
6992
|
+
getElementInstance: typeof camunda?.getElementInstance === 'function'
|
|
6993
|
+
? (key) => withDeadline(
|
|
5866
6994
|
camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
|
|
5867
|
-
CP_SDK_TIMEOUT_MS, 'getElementInstance')
|
|
5868
|
-
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
|
|
6995
|
+
CP_SDK_TIMEOUT_MS, 'getElementInstance')
|
|
6996
|
+
: undefined,
|
|
6997
|
+
getProcessInstance: typeof camunda?.getProcessInstance === 'function'
|
|
6998
|
+
? (pik) => withDeadline(
|
|
6999
|
+
camunda.getProcessInstance({ processInstanceKey: String(pik) }, { consistency: { waitUpToMs: 0 } }),
|
|
7000
|
+
CP_SDK_TIMEOUT_MS, 'getProcessInstance')
|
|
7001
|
+
: null,
|
|
7002
|
+
});
|
|
5872
7003
|
// Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
|
|
5873
7004
|
// job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
|
|
5874
7005
|
// push-deletes stale ones, while the workspace is concurrently driven by the harness
|
|
@@ -5878,6 +7009,7 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5878
7009
|
// decouples them; its dir is reaped when the sweep settles. On the overall deadline
|
|
5879
7010
|
// we abort the sweep's cancellable git so the promise unwinds instead of hanging.
|
|
5880
7011
|
const sweepAbort = new AbortController();
|
|
7012
|
+
gcAbort = sweepAbort;
|
|
5881
7013
|
let sweepDir = null;
|
|
5882
7014
|
try {
|
|
5883
7015
|
if (!originUrl) throw new Error('workspace had no origin URL');
|
|
@@ -5901,10 +7033,10 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5901
7033
|
// the sweep's git is still live. The deadline only bounds it by aborting the git.
|
|
5902
7034
|
gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
|
|
5903
7035
|
.then((r) => {
|
|
5904
|
-
if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned
|
|
5905
|
-
else if (sweepAbort.signal.aborted) log.debug(`
|
|
5906
|
-
else if (r?.error) log.debug(`
|
|
5907
|
-
}, (err) => log.debug(`
|
|
7036
|
+
if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned checkpoint ref(s): ${r.deleted.map((d) => `${d.ref}@${String(d.sha || '') || '?'} [${d.why}]`).join(', ')}.`);
|
|
7037
|
+
else if (sweepAbort.signal.aborted) log.debug(`checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
|
|
7038
|
+
else if (r?.error) log.debug(`checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
|
|
7039
|
+
}, (err) => log.debug(`checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
|
|
5908
7040
|
.finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
|
|
5909
7041
|
} catch (err) {
|
|
5910
7042
|
if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
|
|
@@ -5918,17 +7050,80 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5918
7050
|
// delete against it so a stale activation cannot delete a newer owner's WIP.
|
|
5919
7051
|
let ownedSha = prior?.sha || '';
|
|
5920
7052
|
let varWarned = false;
|
|
7053
|
+
let resumeVarWarned = false;
|
|
7054
|
+
// #296 (review): the in-flight ABORT-path metadata write (`setVar` fired
|
|
7055
|
+
// fire-and-forget on a force-stop flush, below). The force-stop recovery path
|
|
7056
|
+
// later writes the durable recovery ref to the SAME process-scoped
|
|
7057
|
+
// `resumeFromCheckpoint` slot. If this earlier WIP write is still in flight and
|
|
7058
|
+
// lands AFTER the recovery-pointer write, it overwrites the recovery pointer with
|
|
7059
|
+
// a stale WIP ref — and once the WIP TTL expires, auto-resume can no longer
|
|
7060
|
+
// discover the still-existing recovery ref. Track the promise so the recovery
|
|
7061
|
+
// path can AWAIT it (bounded) before writing the recovery pointer, guaranteeing
|
|
7062
|
+
// the recovery write lands last. The WIP write still lands on its own when
|
|
7063
|
+
// promotion fails (the fallback is retained); we only serialize the ordering.
|
|
7064
|
+
let abortMetaWrite = null;
|
|
5921
7065
|
const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
|
|
5922
|
-
?
|
|
5923
|
-
|
|
5924
|
-
|
|
7066
|
+
? (res, signal = null) => {
|
|
7067
|
+
// #296 (review r4245278256): bind these shared process-scoped writes to the
|
|
7068
|
+
// job abort signal for NON-`abort` checkpoints (periodic/`failed`). A periodic
|
|
7069
|
+
// or failed checkpoint can be AWAITING this mutation (`await setVar(res).promise`
|
|
7070
|
+
// below) when heartbeat lease loss interrupts the raw runner and releases the
|
|
7071
|
+
// slot; a replacement activation may then publish a newer `resumeFromCheckpoint`
|
|
7072
|
+
// pointer before this stale call completes and overwrites it — and the later
|
|
7073
|
+
// post-flush signal recheck cannot undo a write that already landed. Passing
|
|
7074
|
+
// `abortSignal` makes `withDeadline` cancel the in-flight SDK call the instant
|
|
7075
|
+
// the lease is lost, so the stale write is torn down and can never land late.
|
|
7076
|
+
// The `abort`-reason path deliberately passes NO signal: that write is owned by
|
|
7077
|
+
// the dedicated `settleAbortMetaWrite` flow (recovery-deadline governed), which
|
|
7078
|
+
// must still fire AFTER the job abort during the force-stop recovery window.
|
|
7079
|
+
// #296: run the element-scope and process-scope writes CONCURRENTLY, not
|
|
7080
|
+
// sequentially. Each carries its own CP_SDK_TIMEOUT_MS deadline, so a
|
|
7081
|
+
// sequential pair could stall for 2× the timeout (20s) — enough to exhaust
|
|
7082
|
+
// the whole FORCE_STOP_RECOVERY_MS budget during a force-stop flush and
|
|
7083
|
+
// starve the recovery git promotion. Racing them caps the worst case at a
|
|
7084
|
+
// single timeout, and each has an independent try/catch so one failing (or
|
|
7085
|
+
// being absent, e.g. no processInstanceKey) never blocks the other.
|
|
7086
|
+
// #296 (review, r4244281967): RETAIN the underlying CancelablePromise(s) so
|
|
7087
|
+
// the abort path can CANCEL them when the recovery deadline fires — not just
|
|
7088
|
+
// stop awaiting. Otherwise a fire-and-forget abort write whose own SDK timeout
|
|
7089
|
+
// outlasts the recovery window stays in flight after the worker yields and can
|
|
7090
|
+
// land LATE over a newer activation's `resumeFromCheckpoint`. The SDK returns a
|
|
7091
|
+
// CancelablePromise (toCancelable); `.cancel?.()` is a no-op for a plain promise
|
|
7092
|
+
// (the test stubs), so capturing the raw return is enough.
|
|
7093
|
+
const rawWrites = [
|
|
7094
|
+
camunda.createElementInstanceVariables({
|
|
5925
7095
|
elementInstanceKey: String(job.elementInstanceKey),
|
|
5926
7096
|
variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
|
|
5927
7097
|
local: true,
|
|
5928
|
-
}),
|
|
5929
|
-
|
|
5930
|
-
|
|
7098
|
+
}),
|
|
7099
|
+
];
|
|
7100
|
+
const writes = [
|
|
7101
|
+
withDeadline(rawWrites[0], CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
|
|
7102
|
+
if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
|
|
7103
|
+
}),
|
|
7104
|
+
];
|
|
7105
|
+
// #295: also record the ref at the PROCESS instance scope (local on the root
|
|
7106
|
+
// scope key = the process instance key). Unlike the local `agentCheckpoint`
|
|
7107
|
+
// above, this survives the element re-entry an SLA escalation triggers, so a
|
|
7108
|
+
// resumed run (a NEW element instance) can find and adopt its predecessor's
|
|
7109
|
+
// checkpoint via `resumeFromCheckpoint`. Best-effort and independent of the
|
|
7110
|
+
// local write; a failure never blocks the push.
|
|
7111
|
+
if (job?.processInstanceKey) {
|
|
7112
|
+
const rawProcess = camunda.createElementInstanceVariables({
|
|
7113
|
+
elementInstanceKey: String(job.processInstanceKey),
|
|
7114
|
+
variables: { resumeFromCheckpoint: { ref, sha: res.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch, at: res.at } },
|
|
7115
|
+
local: true,
|
|
7116
|
+
});
|
|
7117
|
+
rawWrites.push(rawProcess);
|
|
7118
|
+
writes.push(withDeadline(rawProcess, CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
|
|
7119
|
+
if (!resumeVarWarned) { resumeVarWarned = true; log.warn(`could not set the resumeFromCheckpoint variable — ${oneLineLog(err?.message || err)}; a resumed run may not auto-find ${ref}.`); }
|
|
7120
|
+
}));
|
|
5931
7121
|
}
|
|
7122
|
+
const promise = Promise.all(writes);
|
|
7123
|
+
// Cancel every in-flight underlying write (best-effort; no-op for plain
|
|
7124
|
+
// promises). Used by the abort path when the recovery deadline fires.
|
|
7125
|
+
const cancel = () => { for (const w of rawWrites) { try { w?.cancel?.(); } catch { /* best effort */ } } };
|
|
7126
|
+
return { promise, cancel };
|
|
5932
7127
|
}
|
|
5933
7128
|
: null;
|
|
5934
7129
|
|
|
@@ -5968,7 +7163,35 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5968
7163
|
onCheckpoint: async (res) => {
|
|
5969
7164
|
ownedSha = res.sha;
|
|
5970
7165
|
log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
|
|
5971
|
-
if (setVar)
|
|
7166
|
+
if (!setVar) return;
|
|
7167
|
+
// #296: `ownedSha` (above) is the only state preserveRecovery() needs from
|
|
7168
|
+
// this callback, and it is already set. Only a FORCE-STOP (`abort`) flush
|
|
7169
|
+
// fires the metadata writes off the serialized flush/promote chain: that path
|
|
7170
|
+
// is bounded by FORCE_STOP_RECOVERY_MS and the worker returns early (no
|
|
7171
|
+
// settle), so awaiting a hung engine there could burn the whole recovery
|
|
7172
|
+
// budget before the recovery git promotion runs — and the durable recovery
|
|
7173
|
+
// ref, not these vars, is the durable artifact of a force-stopped run.
|
|
7174
|
+
// A `failed` flush is DIFFERENT: the job settles (settleJob.fail) right after
|
|
7175
|
+
// the flush returns, and a retry/reactivation can read variables before a
|
|
7176
|
+
// fire-and-forget `resumeFromCheckpoint` write is durable — worse, a delayed
|
|
7177
|
+
// process-scoped write could overwrite a NEWER activation's pointer. So the
|
|
7178
|
+
// failed path (and every normal checkpoint) AWAITS the write before
|
|
7179
|
+
// settlement, keeping the predecessor pointer durable before the job settles.
|
|
7180
|
+
// #296 (review r4245278256): and it passes `abortSignal`, so if heartbeat lease
|
|
7181
|
+
// loss fires DURING this awaited write the in-flight SDK call is cancelled — a
|
|
7182
|
+
// replacement activation then owns the slot and this run's now-stale pointer can
|
|
7183
|
+
// never land late over its newer value.
|
|
7184
|
+
// #296 (review): on the abort path, CAPTURE the fire-and-forget write (promise +
|
|
7185
|
+
// cancel) so the force-stop recovery path can await it before writing the
|
|
7186
|
+
// recovery pointer to the same slot — otherwise a slow WIP write lands after and
|
|
7187
|
+
// clobbers the recovery ref. `.promise.catch(() => {})` keeps it fire-and-forget
|
|
7188
|
+
// for the flush; the settled/rejected result is irrelevant to the caller. The
|
|
7189
|
+
// retained `cancel` lets settleAbortMetaWrite tear down the in-flight SDK call
|
|
7190
|
+
// when the recovery deadline fires (r4244281967), so it can never land late. This
|
|
7191
|
+
// path deliberately passes NO abortSignal to setVar: the abort write is owned by
|
|
7192
|
+
// settleAbortMetaWrite's recovery-deadline flow and must outlive the job abort.
|
|
7193
|
+
if (res.reason === 'abort') { const w = setVar(res); abortMetaWrite = { promise: w.promise.catch(() => {}), cancel: w.cancel }; return; }
|
|
7194
|
+
await setVar(res, abortSignal || undefined).promise;
|
|
5972
7195
|
},
|
|
5973
7196
|
});
|
|
5974
7197
|
return {
|
|
@@ -5978,6 +7201,176 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5978
7201
|
restored,
|
|
5979
7202
|
priorNotRestored,
|
|
5980
7203
|
flushTimeoutMs: cfg.flushTimeoutMs,
|
|
7204
|
+
// #296 (review): the git SIGKILL grace window, surfaced so the force-stop path
|
|
7205
|
+
// can reserve a real drain window for close() instead of the 1ms an expired
|
|
7206
|
+
// shared deadline would otherwise hand it (see the force-stop block).
|
|
7207
|
+
gitKillGraceMs: cfg.gitKillGraceMs,
|
|
7208
|
+
// #296 (F9): abort the recovery-deadline controller — tears down an in-flight
|
|
7209
|
+
// checkpoint/recovery git push so `stop()`/`close()` settle promptly. Driven by
|
|
7210
|
+
// the force-stop path once the recovery budget expires (NOT by the job's
|
|
7211
|
+
// abortSignal, which fires BEFORE recovery runs and would fail every recovery
|
|
7212
|
+
// command). Also aborts the background GC sweep so the ONE shared force-stop
|
|
7213
|
+
// deadline bounds recovery AND cleanup together — otherwise a still-running
|
|
7214
|
+
// (up to 120s) sweep could hold close()'s drain past the supervisor's SIGKILL
|
|
7215
|
+
// window even after recovery settled. Idempotent / best-effort; a no-op once
|
|
7216
|
+
// already aborted.
|
|
7217
|
+
abortRecovery() {
|
|
7218
|
+
try { recoveryController.abort(); } catch { /* best effort */ }
|
|
7219
|
+
try { gcAbort?.abort(); } catch { /* best effort */ }
|
|
7220
|
+
},
|
|
7221
|
+
// Adopt a credential the job ROTATED after this checkpointer was created —
|
|
7222
|
+
// e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
|
|
7223
|
+
// (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
|
|
7224
|
+
// its own askpass purely from `token`, so re-binding `token` here is enough
|
|
7225
|
+
// to make the cleanup push authenticate with the live credential instead of
|
|
7226
|
+
// the one the remote already rejected — otherwise the WIP ref lingers until a
|
|
7227
|
+
// later GC (Copilot review on #293). Ignored when the token is unchanged or
|
|
7228
|
+
// falsy; the live checkpointer has already stopped by the time this is called.
|
|
7229
|
+
updateCredential(nextToken) {
|
|
7230
|
+
if (nextToken && nextToken !== token) token = nextToken;
|
|
7231
|
+
},
|
|
7232
|
+
// #296 (review): await the in-flight ABORT-path metadata write (the WIP
|
|
7233
|
+
// `resumeFromCheckpoint` write fired fire-and-forget on a force-stop flush),
|
|
7234
|
+
// bounded, so the force-stop recovery path can guarantee ITS recovery-pointer
|
|
7235
|
+
// write to the same slot lands LAST (a slow WIP write must not overwrite the
|
|
7236
|
+
// durable recovery ref). Best-effort, never throws; resolves immediately when
|
|
7237
|
+
// no abort write is in flight. The caller bounds the wait inside the shared
|
|
7238
|
+
// force-stop budget, so a hung engine cannot stall the force-stop exit.
|
|
7239
|
+
// #296 (review, r4244116341): accept the recovery-deadline `signal` so this SDK
|
|
7240
|
+
// await is cancelled by the SAME deadline that tears down the recovery git — the
|
|
7241
|
+
// recover chain must SETTLE (not dangle) when the budget expires, so a stale
|
|
7242
|
+
// continuation can never write the old recovery pointer over a reactivated
|
|
7243
|
+
// worker's newer one.
|
|
7244
|
+
// #296 (review, r4244281967): stopping the AWAIT is not enough — the underlying
|
|
7245
|
+
// `createElementInstanceVariables` mutation would otherwise stay in flight past
|
|
7246
|
+
// the deadline and could land LATE over a newer activation's `resumeFromCheckpoint`.
|
|
7247
|
+
// So when `signal` aborts we also CANCEL the retained CancelablePromise(s)
|
|
7248
|
+
// (`abortMetaWrite.cancel()`), aborting the fetch so the late settle can never land.
|
|
7249
|
+
async settleAbortMetaWrite(signal = null) {
|
|
7250
|
+
const w = abortMetaWrite;
|
|
7251
|
+
if (!w) return;
|
|
7252
|
+
if (signal?.aborted) { try { w.cancel?.(); } catch { /* best effort */ } return; }
|
|
7253
|
+
if (!signal) { try { await w.promise; } catch { /* best effort */ } return; }
|
|
7254
|
+
let onAbort = null;
|
|
7255
|
+
try {
|
|
7256
|
+
await Promise.race([
|
|
7257
|
+
w.promise.catch(() => {}),
|
|
7258
|
+
new Promise((res) => { onAbort = () => { try { w.cancel?.(); } catch { /* best effort */ } res(); }; signal.addEventListener('abort', onAbort, { once: true }); }),
|
|
7259
|
+
]);
|
|
7260
|
+
} catch { /* best effort */ } finally { if (onAbort) signal.removeEventListener('abort', onAbort); }
|
|
7261
|
+
},
|
|
7262
|
+
// #295: promote the latest snapshot to a durable recovery ref on an SLA
|
|
7263
|
+
// cancellation or a force-stop. `refs/nano/recovered/<elementInstanceKey>`
|
|
7264
|
+
// lives OUTSIDE the `refs/nano/wip/*` namespace, so it survives the
|
|
7265
|
+
// per-checkpoint WIP cleanup (`discardAfterAck`, which only deletes WIP refs)
|
|
7266
|
+
// and workspace reaping — long enough for the human escalation to be answered
|
|
7267
|
+
// and the work recovered. It is NOT exempt from the orphan GC, though: the
|
|
7268
|
+
// sweep reclaims it under the same process-terminal / 7d-TTL policy as a WIP
|
|
7269
|
+
// ref (#296), sparing it only while its process is still ACTIVE. Returns
|
|
7270
|
+
// `{ ref, sha }` (surfaced in the job outcome) or
|
|
7271
|
+
// null when there is nothing durable to preserve / the push failed. Best-effort;
|
|
7272
|
+
// never throws. The live WIP ref is kept too (this is an additional durable copy).
|
|
7273
|
+
async preserveRecovery(reason = 'recovery') {
|
|
7274
|
+
// Copilot review on #295: the caller invokes this right after a BOUNDED
|
|
7275
|
+
// `flush()`, which can RETURN while its forced checkpoint is still in flight
|
|
7276
|
+
// — and `ownedSha` is only updated later, by `onCheckpoint`, once that
|
|
7277
|
+
// checkpoint's push resolves. Reading `ownedSha` immediately could therefore
|
|
7278
|
+
// see `null`/the pre-flush SHA and promote an older snapshot (or nothing).
|
|
7279
|
+
// DRAIN the checkpoint chain first (`stop()` resolves once the in-flight
|
|
7280
|
+
// chain settles, letting `onCheckpoint` record the latest SHA) so the SHA
|
|
7281
|
+
// selected below is the newest snapshot this run actually pushed. This
|
|
7282
|
+
// mirrors the success path, which already `stop()`s before finalizeGit.
|
|
7283
|
+
try { await checkpointer.stop(); } catch { /* best effort — fall through to the last recorded ownedSha */ }
|
|
7284
|
+
if (!ownedSha) return null;
|
|
7285
|
+
const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
|
|
7286
|
+
if (!recRef) return null;
|
|
7287
|
+
try {
|
|
7288
|
+
// Fence the recovery promotion against the sha we OBSERVE on the ref now,
|
|
7289
|
+
// so a slower/stale activation that observed an older/absent value loses
|
|
7290
|
+
// the lease once a newer run moved it, instead of blind-forcing over
|
|
7291
|
+
// whatever is current (Copilot review on #296).
|
|
7292
|
+
const prevRecSha = await observeRemoteRefSha({ git, ref: recRef });
|
|
7293
|
+
// #296 (F7/F10): a bare CAS against the just-observed value is NOT ownership
|
|
7294
|
+
// — a superseded activation can observe the newer run's `shaB` here and
|
|
7295
|
+
// still overwrite it with its older `ownedShaA`. Mirror the WIP path's
|
|
7296
|
+
// `Nano-Checkpoint-Run` ownership check: only overwrite an EXISTING recovery
|
|
7297
|
+
// ref when it is POSITIVELY identified as either ours (a re-preservation by
|
|
7298
|
+
// this same run) or the run we explicitly superseded (`prior?.runId`). An
|
|
7299
|
+
// absent ref is a create-only write. FAIL CLOSED: if the owner cannot be
|
|
7300
|
+
// positively identified — the ownership fetch THREW/failed, or the ref
|
|
7301
|
+
// carries no `runId` trailer — refuse the overwrite rather than fall through
|
|
7302
|
+
// to the CAS, so a transient read failure can never let a stale activation
|
|
7303
|
+
// clobber a newer (or unknown) run's recovery point (F10). The WIP ref +
|
|
7304
|
+
// TTL remain the fallback in every refused case.
|
|
7305
|
+
if (prevRecSha) {
|
|
7306
|
+
let ownerRunId = '';
|
|
7307
|
+
let ownerKnown = false;
|
|
7308
|
+
try { ownerRunId = (await fetchCheckpoint({ git, ref: recRef }))?.runId || ''; ownerKnown = true; } catch { ownerKnown = false; }
|
|
7309
|
+
const myRun = String(runId || '');
|
|
7310
|
+
const supersededRun = String(prior?.runId || '');
|
|
7311
|
+
const mine = ownerKnown && ownerRunId && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
|
|
7312
|
+
if (!mine) {
|
|
7313
|
+
const who = !ownerKnown ? 'could not be read (transient)' : (ownerRunId ? `owned by another run (${ownerRunId})` : 'carries no run identity');
|
|
7314
|
+
log.warn(`recovery checkpoint ${recRef} ${who} — refusing to overwrite it; the WIP ref ${ref} is still retained for recovery.`);
|
|
7315
|
+
return null;
|
|
7316
|
+
}
|
|
7317
|
+
}
|
|
7318
|
+
const r = await pushNamedRef({ git, ref: recRef, sha: ownedSha, expectSha: prevRecSha });
|
|
7319
|
+
if (r?.ok) {
|
|
7320
|
+
logger.info?.(`${prefix} (${corr}): preserved a durable recovery checkpoint ${recRef}@${ownedSha.slice(0, 12)} (${reason}) — it survives the per-checkpoint WIP cleanup and workspace reaping (the orphan GC spares it while its process stays active, up to the ${Math.round(cfg.ttlMs / 86400000)}d TTL backstop, which reclaims it regardless of process state) so the SLA-cancelled/force-stopped run can be recovered and resumed.`);
|
|
7321
|
+
return { ref: recRef, sha: ownedSha };
|
|
7322
|
+
}
|
|
7323
|
+
if (r?.staleLease) {
|
|
7324
|
+
// A newer activation already preserved a recovery point on this ref;
|
|
7325
|
+
// keep theirs. Our WIP ref is still retained as a fallback.
|
|
7326
|
+
log.warn(`recovery checkpoint ${recRef} was moved by a newer activation — keeping its recovery point; the WIP ref ${ref} is still retained for recovery.`);
|
|
7327
|
+
return null;
|
|
7328
|
+
}
|
|
7329
|
+
log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(r?.error || 'unknown', token)}; the WIP ref ${ref} is still retained for recovery.`);
|
|
7330
|
+
} catch (err) {
|
|
7331
|
+
log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(oneLineLog(err?.message || err), token)}; the WIP ref ${ref} is still retained for recovery.`);
|
|
7332
|
+
}
|
|
7333
|
+
return null;
|
|
7334
|
+
},
|
|
7335
|
+
// #296 (review, r4244827082): re-verify, IMMEDIATELY before the process-scoped
|
|
7336
|
+
// `resumeFromCheckpoint` recovery-pointer write, that THIS run still owns the
|
|
7337
|
+
// durable recovery ref it just preserved. `preserveRecovery` fences the GIT ref
|
|
7338
|
+
// (CAS + `Nano-Checkpoint-Run` ownership), but the subsequent process-scoped
|
|
7339
|
+
// variable write is unconditional last-writer-wins: a superseded activation that
|
|
7340
|
+
// won its ref earlier can still land its stale pointer AFTER a newer activation
|
|
7341
|
+
// (which re-activated the job on lease loss) published its own. Close that
|
|
7342
|
+
// residual race by re-observing the recovery ref now and confirming it still
|
|
7343
|
+
// resolves to the sha this run preserved AND is still owned by this run (or the
|
|
7344
|
+
// run it explicitly superseded) — the same ownership rule the GIT fence applies.
|
|
7345
|
+
// FAIL CLOSED: any read failure, a moved/absent ref, or an owner that is not
|
|
7346
|
+
// positively this run returns false, so the caller SKIPS the variable write and
|
|
7347
|
+
// leaves the newer activation's pointer intact (the pushed ref stays the durable
|
|
7348
|
+
// fallback). `expectedSha` is the sha this run's preserveRecovery returned.
|
|
7349
|
+
// #296 (review, r4244916101): accept a per-call `signal` and run the git reads
|
|
7350
|
+
// on a DEDICATED runner bound to it (AND the recovery deadline). The caller
|
|
7351
|
+
// races this against a 5s timeout; without a wired signal the losing branch
|
|
7352
|
+
// left the verify's git process live on the 60s runner — untracked by
|
|
7353
|
+
// checkpointer.stop()/gc — so recover() returned and the run dir was reaped
|
|
7354
|
+
// under a live git. Binding the runner to the per-call signal lets the caller
|
|
7355
|
+
// abort+await the verify on timeout so it is SETTLED before cleanup proceeds.
|
|
7356
|
+
async verifyRecoveryOwnership(expectedSha, { signal } = {}) {
|
|
7357
|
+
try {
|
|
7358
|
+
const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
|
|
7359
|
+
if (!recRef || !expectedSha) return false;
|
|
7360
|
+
// Combine the per-call abort with the recovery-deadline abort so EITHER
|
|
7361
|
+
// tears the git down (whichever fires first). AbortSignal.any is Node 20+.
|
|
7362
|
+
const signals = [recoveryController.signal, signal].filter(Boolean);
|
|
7363
|
+
const combined = signals.length > 1 ? AbortSignal.any(signals) : (signals[0] || undefined);
|
|
7364
|
+
const vgit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: combined });
|
|
7365
|
+
const curSha = await observeRemoteRefSha({ git: vgit, ref: recRef });
|
|
7366
|
+
if (!curSha || curSha !== expectedSha) return false;
|
|
7367
|
+
let ownerRunId = '';
|
|
7368
|
+
try { ownerRunId = (await fetchCheckpoint({ git: vgit, ref: recRef }))?.runId || ''; } catch { return false; }
|
|
7369
|
+
const myRun = String(runId || '');
|
|
7370
|
+
const supersededRun = String(prior?.runId || '');
|
|
7371
|
+
return Boolean(ownerRunId) && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
|
|
7372
|
+
} catch { return false; }
|
|
7373
|
+
},
|
|
5981
7374
|
// True when the workspace still holds checkpoint content NOT captured by a
|
|
5982
7375
|
// commit, so the pushed branch does not represent it (an uncommitted restored
|
|
5983
7376
|
// or working delta that lives only on the WIP ref). The discard-on-ack gate
|
|
@@ -5992,8 +7385,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
|
|
|
5992
7385
|
} catch { return true; }
|
|
5993
7386
|
},
|
|
5994
7387
|
// Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
|
|
5995
|
-
// run dir can be reaped safely.
|
|
5996
|
-
|
|
7388
|
+
// run dir can be reaped safely. `close()` is bounded by `closeTimeoutMs` so a
|
|
7389
|
+
// force-stopped worker is never held past the supervisor's SIGKILL backstop by
|
|
7390
|
+
// the (up to 120s) GC sweep: on the deadline we stop awaiting and let the
|
|
7391
|
+
// caller proceed (the in-flight checkpoint git AND the GC sweep are both
|
|
7392
|
+
// cancelled by the recovery DEADLINE — abortRecovery() — so the chain settles on
|
|
7393
|
+
// its own). The force-stop caller passes only the REMAINING of the one shared
|
|
7394
|
+
// force-stop budget, so recovery + this drain never exceed it together. Default
|
|
7395
|
+
// 0 = unbounded (normal/failed paths fully drain).
|
|
7396
|
+
async close({ timeoutMs = 0 } = {}) {
|
|
7397
|
+
const drain = (async () => { await checkpointer.stop(); await gc; })();
|
|
7398
|
+
if (!timeoutMs) return drain;
|
|
7399
|
+
let t;
|
|
7400
|
+
const cap = new Promise((resolve) => { t = setTimeout(resolve, timeoutMs); t?.unref?.(); });
|
|
7401
|
+
try { return await Promise.race([drain, cap]); } finally { clearTimeout(t); }
|
|
7402
|
+
},
|
|
5997
7403
|
async discard() {
|
|
5998
7404
|
await checkpointer.stop();
|
|
5999
7405
|
try {
|
|
@@ -6069,7 +7475,7 @@ function computeRelayCloseReason(result, runCompleted, gitResult) {
|
|
|
6069
7475
|
// branch.push), and reconcile the agent-opened PR (when task.allowPr). A push
|
|
6070
7476
|
// failure is reported (pushError) rather than thrown — the process model decides
|
|
6071
7477
|
// what to do next, and re-running the agent would be non-idempotent.
|
|
6072
|
-
function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, _runGit = null }) {
|
|
7478
|
+
function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch: effectiveBase, baseCloneSha = null, baseBaselineUnknown = false, hasPrBranch, envelope, token, logger = null, corr = '', budgetMs = 0, provisioned = false, keepRuns = false, credentialSource = null, _runGit = null, _pushGit = null, _reconcilePr = null, _postAttribution = null }) {
|
|
6073
7479
|
const log = logger || getLogger(); // route observability through the injected logger (tests) or the host logger
|
|
6074
7480
|
// Test-only seam: the late remote-reachability FILTER scans (movedStray /
|
|
6075
7481
|
// unpushedBranchCommits / final stranded) AND the remote-tip VERIFICATION calls
|
|
@@ -6080,8 +7486,25 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6080
7486
|
// probes cannot be made to fail deterministically. Defaults to the module `runGit`, so
|
|
6081
7487
|
// production behaviour is unchanged.
|
|
6082
7488
|
const lateScanGit = _runGit || runGit;
|
|
7489
|
+
// Test-only seam for the branch PUSH itself (the initial push AND the single
|
|
7490
|
+
// post-auth-failure retry). Routed separately from `lateScanGit` so a
|
|
7491
|
+
// hermetic test can drive the central push-recovery branches — changed-token
|
|
7492
|
+
// retry success, the unchanged-token guard, the credential-source gate, and
|
|
7493
|
+
// retry-still-fails — without a live remote, while the local enumeration and
|
|
7494
|
+
// late remote-reachability scans keep their own runner (Copilot review on
|
|
7495
|
+
// #293). Defaults to the module `runGit`, so production behaviour is unchanged.
|
|
7496
|
+
const pushGit = _pushGit || runGit;
|
|
7497
|
+
const reconcilePr = _reconcilePr || reconcileAgentPr;
|
|
7498
|
+
const postAttribution = _postAttribution || postAgentAttribution;
|
|
6083
7499
|
const cs = corr ? ` [${corr}]` : ''; // #229 cross-channel correlation suffix
|
|
6084
7500
|
const out = { branch: workingBranch, baseSha: startSha || null, headSha: null, commits: [], pushed: false, remote: null, pr: null };
|
|
7501
|
+
// The credential this finalize was HANDED. The push auth-failure recovery below
|
|
7502
|
+
// may rotate `token` to a freshly re-read gh credential mid-finalize; capturing
|
|
7503
|
+
// the original lets the return expose `rotatedToken` so the CALLER can propagate
|
|
7504
|
+
// the rotation to subsystems that captured the pre-rotation credential — notably
|
|
7505
|
+
// the WIP-checkpoint cleanup, whose `discardAfterAck()` would otherwise push a
|
|
7506
|
+
// ref deletion with the already-rejected token (Copilot review on #293).
|
|
7507
|
+
const originalToken = token;
|
|
6085
7508
|
|
|
6086
7509
|
// Bound EVERY finalize git op against ONE shared deadline derived from the recovery
|
|
6087
7510
|
// window (budgetMs) — the LOCAL graph scans below AND the sequential NETWORK ops
|
|
@@ -6465,7 +7888,100 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6465
7888
|
// qualified heads ref resolves unambiguously; --set-upstream still tracks the
|
|
6466
7889
|
// local branch.
|
|
6467
7890
|
const pushTimeout = netTimeoutMs();
|
|
6468
|
-
|
|
7891
|
+
let push = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: gitEnv, timeoutMs: pushTimeout });
|
|
7892
|
+
// #293: an AUTH rejection (401/403, or the OAuth `workflow`-scope refusal)
|
|
7893
|
+
// usually means the cached gh token went stale mid-run. Force a fresh
|
|
7894
|
+
// re-read and retry the push ONCE with it before stranding the commits — a
|
|
7895
|
+
// rotated credential self-heals instead of a `pushFailed`. Only when the
|
|
7896
|
+
// re-read yields a token DIFFERENT from the one that failed (a same-token
|
|
7897
|
+
// retry can't fix a genuine permission denial), and the retry runs with the
|
|
7898
|
+
// fresh token in GIT_TOKEN so the askpass hands it to git. The recovery is
|
|
7899
|
+
// NON-DESTRUCTIVE (see recoverGhTokenAfterAuthFailure): it never invalidates
|
|
7900
|
+
// the cache first, so a transiently-failing re-read can't erase a still-valid
|
|
7901
|
+
// — possibly newer — cached credential (Copilot review on #294).
|
|
7902
|
+
// GATED on the credential's SOURCE (Copilot review on #293): only a token
|
|
7903
|
+
// that CAME from the gh cache ('gh-cache') may be replaced by a fresh
|
|
7904
|
+
// `gh auth token` read — an explicit GITHUB_TOKEN ('env') or custom authRef
|
|
7905
|
+
// ('custom') is a deliberately configured identity that must fail as
|
|
7906
|
+
// configured, never be silently swapped for the operator's gh login.
|
|
7907
|
+
// The `token` guard is deliberately ABSENT (Copilot review on #293): a
|
|
7908
|
+
// default gh-cache MISS (token null, source 'gh-cache') is eligible too, so
|
|
7909
|
+
// a worker that cached "logged out" recovers null → fresh token after the
|
|
7910
|
+
// operator runs `gh auth login` instead of stranding the commits. That
|
|
7911
|
+
// recovery must REBUILD the credential plumbing the anonymous gitEnv lacked
|
|
7912
|
+
// (the askpass helper + GIT_ASKPASS, and global git config) — a bare
|
|
7913
|
+
// GIT_TOKEN swap leaves the push unauthenticated.
|
|
7914
|
+
// The credential that PRODUCED the ORIGINAL `push` — preserved separately so
|
|
7915
|
+
// the failure report below redacts the value that actually appeared in that
|
|
7916
|
+
// push's output, even once `token` is swapped to the fresh credential for
|
|
7917
|
+
// subsequent authenticated operations after a retry (Copilot review on #293).
|
|
7918
|
+
let pushFailureToken = token;
|
|
7919
|
+
// The timeout that bounded the reported `push` — promoted alongside
|
|
7920
|
+
// pushFailureToken when a failed retry is promoted to the reported result, so
|
|
7921
|
+
// describeGitFailure attributes the RETRY's timeout, not the original push's
|
|
7922
|
+
// (Copilot review on #293).
|
|
7923
|
+
let pushFailureTimeout = pushTimeout;
|
|
7924
|
+
if (push.status !== 0 && credentialSource === 'gh-cache' && isGitAuthFailure(describeGitFailure('git push', push, { token, timeoutMs: pushTimeout }))) {
|
|
7925
|
+
const fresh = recoverGhTokenAfterAuthFailure(token, { budgetMs: netTimeoutMs() });
|
|
7926
|
+
if (fresh) {
|
|
7927
|
+
log.info?.(`finalizeGit${cs}: git push auth failed — the cached gh token may be stale; re-read it (${describeGhAuthToken(ghAuthTokenInfo())}) and retrying the push once`);
|
|
7928
|
+
const retryEnv = { ...gitEnv, GIT_TOKEN: fresh };
|
|
7929
|
+
// Reuse the clone-time askpass helper when gitEnv already carries one
|
|
7930
|
+
// (the token-backed path): the helper emits GIT_TOKEN at runtime, so
|
|
7931
|
+
// pointing GIT_TOKEN at the fresh credential is enough — no rewrite.
|
|
7932
|
+
// On the null→fresh recovery the original gitEnv was the ANONYMOUS one
|
|
7933
|
+
// (no askpass helper, GIT_CONFIG_GLOBAL neutralized), so mint one and
|
|
7934
|
+
// restore global config. Mint it in the PARENT run directory
|
|
7935
|
+
// (dirname(workspaceDir) === runDir), NEVER inside workspaceDir: writing
|
|
7936
|
+
// askpass.js/.sh into the checked-out repo after the final commits leaves
|
|
7937
|
+
// untracked files that make hasUncommittedChanges() retain the WIP
|
|
7938
|
+
// checkpoint and pollute a preserved workspace (Copilot review on #294).
|
|
7939
|
+
if (!retryEnv.GIT_ASKPASS) {
|
|
7940
|
+
const retryAskpass = writeAskpass(dirname(workspaceDir), fresh);
|
|
7941
|
+
if (retryAskpass) {
|
|
7942
|
+
retryEnv.GIT_ASKPASS = retryAskpass;
|
|
7943
|
+
delete retryEnv.GIT_CONFIG_GLOBAL;
|
|
7944
|
+
}
|
|
7945
|
+
}
|
|
7946
|
+
const retryTimeout = netTimeoutMs();
|
|
7947
|
+
const retry = pushGit([...credArgs(), 'push', '--set-upstream', 'origin', `refs/heads/${workingBranch}:refs/heads/${workingBranch}`], { cwd: workspaceDir, env: retryEnv, timeoutMs: retryTimeout });
|
|
7948
|
+
// Rebind the git env to the FRESH credential whenever the retry ran —
|
|
7949
|
+
// regardless of its exit code — so the post-push remote-tip verification
|
|
7950
|
+
// below (`ls-remote`/`fetch`, all `env: gitEnv`) queries origin with the
|
|
7951
|
+
// credential the retry used, not the REJECTED old token. A retry that
|
|
7952
|
+
// exits nonzero because the server accepted the ref before a client-side
|
|
7953
|
+
// timeout would otherwise be verified with the stale token, ls-remote
|
|
7954
|
+
// would fail to authenticate, and a landed push would be misreported as
|
|
7955
|
+
// pushFailed/stranded (Copilot review on #293).
|
|
7956
|
+
gitEnv = retryEnv;
|
|
7957
|
+
if (retry.status === 0) {
|
|
7958
|
+
// Adopt the fresh token for the rest of finalization (the PR reconcile
|
|
7959
|
+
// + attribution below read `token`) and record the success.
|
|
7960
|
+
token = fresh;
|
|
7961
|
+
push = retry;
|
|
7962
|
+
} else {
|
|
7963
|
+
// The retry ALSO exited nonzero. It may still have LANDED the ref before
|
|
7964
|
+
// a client-side timeout (the transport-hiccup case verified below); if it
|
|
7965
|
+
// did, `landed` flips on and pushError is never reported. But if it did
|
|
7966
|
+
// NOT land, the RETRY — not the original stale-token rejection — is the
|
|
7967
|
+
// failure that actually stranded the commits: the original failure is
|
|
7968
|
+
// already RECOVERED (we re-read the token and retried), so reporting it
|
|
7969
|
+
// would point at a resolved cause instead of the real one. Promote `retry`
|
|
7970
|
+
// to the reported push result and carry ITS redaction credential (`fresh`,
|
|
7971
|
+
// which appeared in the retry's output) and ITS timeout into the final
|
|
7972
|
+
// error. The ORIGINAL rejected token must NOT be carried into the
|
|
7973
|
+
// subsequent authenticated operations (PR reconcile + attribution read
|
|
7974
|
+
// `token`): a landed retry would otherwise reconcile/attribute with the
|
|
7975
|
+
// rejected old token and fail despite the push. Adopt `fresh` for all
|
|
7976
|
+
// later ops; `gitEnv` is already rebound above (Copilot review on #293).
|
|
7977
|
+
push = retry;
|
|
7978
|
+
pushFailureToken = fresh;
|
|
7979
|
+
pushFailureTimeout = retryTimeout;
|
|
7980
|
+
token = fresh;
|
|
7981
|
+
log.warn?.(`finalizeGit${cs}: push still failed with the fresh gh token — ${oneLineLog(describeGitFailure('git push', retry, { token: fresh, timeoutMs: retryTimeout }))}`);
|
|
7982
|
+
}
|
|
7983
|
+
}
|
|
7984
|
+
}
|
|
6469
7985
|
if (push.status === 0) out.pushed = true;
|
|
6470
7986
|
else {
|
|
6471
7987
|
// A nonzero `git push` does NOT prove the ref was not updated: a timeout or
|
|
@@ -6547,7 +8063,7 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6547
8063
|
// new commit in this throwaway workspace. Surface the at-risk SHAs explicitly
|
|
6548
8064
|
// (issue #231, defense #2) so the failure is a hard, actionable signal with
|
|
6549
8065
|
// the commits to recover — not a soft pushError that reads as "completed".
|
|
6550
|
-
out.pushError = describeGitFailure('git push', push, { token, timeoutMs:
|
|
8066
|
+
out.pushError = describeGitFailure('git push', push, { token: pushFailureToken, timeoutMs: pushFailureTimeout });
|
|
6551
8067
|
out.pushFailed = true;
|
|
6552
8068
|
// Union the work-branch commit list into the stranded set: out.commits is now
|
|
6553
8069
|
// the promoted branchCommits, but if HEAD sat off the branch and promotion
|
|
@@ -6627,13 +8143,18 @@ function finalizeGit({ workspaceDir, gitEnv, startSha, workingBranch, baseBranch
|
|
|
6627
8143
|
// envelope + completion note carry only the `pushFailed`/`strandedCommits`
|
|
6628
8144
|
// recovery signal, not a misleading PR reference.
|
|
6629
8145
|
if (workingBranch && envelope.task?.allowPr && !out.pushFailed) {
|
|
6630
|
-
out.pr =
|
|
8146
|
+
out.pr = reconcilePr({ workspaceDir, token, branch: workingBranch, provider: envelope.repository?.provider || 'github' });
|
|
6631
8147
|
// Record the agent's authorship on the PR (commits carry the operator's
|
|
6632
8148
|
// identity now, so this preserves the machine-generated provenance).
|
|
6633
8149
|
if (out.pr?.found && out.pr.number && (envelope.repository?.provider || 'github') === 'github') {
|
|
6634
|
-
out.attribution =
|
|
8150
|
+
out.attribution = postAttribution({ workspaceDir, token, number: out.pr.number });
|
|
6635
8151
|
}
|
|
6636
8152
|
}
|
|
8153
|
+
// Surface a mid-finalize credential rotation (the push auth-failure recovery
|
|
8154
|
+
// re-read a fresh gh token) so the caller can propagate it to subsystems that
|
|
8155
|
+
// captured the pre-rotation credential — the WIP-checkpoint cleanup in
|
|
8156
|
+
// particular (Copilot review on #293). Absent when the token was never rotated.
|
|
8157
|
+
if (token && token !== originalToken) out.rotatedToken = token;
|
|
6637
8158
|
return out;
|
|
6638
8159
|
}
|
|
6639
8160
|
|
|
@@ -8657,6 +10178,11 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
|
|
|
8657
10178
|
if (git.branchMismatch) env.branchMismatch = git.branchMismatch;
|
|
8658
10179
|
if (git.pr) env.pr = git.pr;
|
|
8659
10180
|
if (git.error) env.gitError = git.error;
|
|
10181
|
+
// #295: a durable recovery ref (refs/nano/recovered/<eik>) an SLA-cancelled or
|
|
10182
|
+
// force-stopped run left behind — carried in the outcome so the escalation /
|
|
10183
|
+
// resume can recover the snapshot from it even after the workspace is reaped.
|
|
10184
|
+
if (git.recoveryRef) env.recoveryRef = git.recoveryRef;
|
|
10185
|
+
if (git.recoverySha) env.recoverySha = git.recoverySha;
|
|
8660
10186
|
}
|
|
8661
10187
|
return env;
|
|
8662
10188
|
}
|
|
@@ -9578,9 +11104,9 @@ function agenticStateForTarget(target, safeUrl = (u) => u) {
|
|
|
9578
11104
|
* `completeJob`/`failJob`, so the engine may still project it `CREATED`) is no
|
|
9579
11105
|
* longer running here, so it rides the marker as a flagged, observational entry
|
|
9580
11106
|
* without making an idle worker look busy or inflating the drain's in-flight count.
|
|
9581
|
-
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object }} fields
|
|
11107
|
+
* @param {{ pid:number, updatedAt:number, jobs:Array<{key:string,type:string,since:number,settlementPending?:boolean}>, engine:(string|null), agentic:object, readyAt:(number|null), ghAuth:(object|null) }} fields
|
|
9582
11108
|
*/
|
|
9583
|
-
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }) {
|
|
11109
|
+
function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt, ghAuth }) {
|
|
9584
11110
|
const jobList = Array.isArray(jobs) ? jobs : [];
|
|
9585
11111
|
return {
|
|
9586
11112
|
pid,
|
|
@@ -9593,6 +11119,11 @@ function buildActivityPayload({ pid, updatedAt, jobs, engine, agentic, readyAt }
|
|
|
9593
11119
|
// SDK client and begun leasing), the producer stamps this; null until then.
|
|
9594
11120
|
// The supervisor's rolling reload gates on it (a spawn/PID is not readiness).
|
|
9595
11121
|
readyAt: readyAt ?? null,
|
|
11122
|
+
// #293: the cached gh auth token's observable identity — { present,
|
|
11123
|
+
// fingerprint, scopes, since, ageMs, lastCheckAgoMs }, NEVER the value — so
|
|
11124
|
+
// `supervisor status` can show the credential's age/scopes per worker. Null
|
|
11125
|
+
// on a worker that has not primed the cache yet (or predates the field).
|
|
11126
|
+
ghAuth: ghAuth ?? null,
|
|
9596
11127
|
};
|
|
9597
11128
|
}
|
|
9598
11129
|
|
|
@@ -10193,10 +11724,27 @@ async function workAgent(req, flags, ctx) {
|
|
|
10193
11724
|
// for why per-job setup.env is deliberately excluded), so PATH resolves a stable
|
|
10194
11725
|
// representative harness binary.
|
|
10195
11726
|
const agentInstanceProbeOff = String(process.env.NANO_AGENT_INSTANCE || '').trim().toLowerCase() === 'off';
|
|
11727
|
+
// Harness transport protocol, resolved with env-override-then-profile precedence and
|
|
11728
|
+
// reduced to the EFFECTIVE protocol the worker actually runs (a container sandbox is
|
|
11729
|
+
// pipe-only regardless of `protocol`). Computed here — ahead of the durable-transcript
|
|
11730
|
+
// diagnostic below and the #275 mismatch refusal later — so the startup "ACP note" is
|
|
11731
|
+
// keyed on the transport the worker WILL use, not a brittle reparse of the command
|
|
11732
|
+
// line (issue #289 review).
|
|
11733
|
+
const envProtocol = (process.env.NANO_AGENTIC_PROTOCOL || '').trim().toLowerCase();
|
|
11734
|
+
const roleProtocol = resolveAgenticSetting(envProtocol, profile.protocol, PROTOCOLS, 'pipe');
|
|
11735
|
+
const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
|
|
10196
11736
|
const sdkSupportsAgentInstance =
|
|
10197
11737
|
!!camunda &&
|
|
10198
11738
|
typeof camunda.createAgentInstance === 'function' &&
|
|
10199
11739
|
typeof camunda.updateAgentInstance === 'function';
|
|
11740
|
+
// #289: best-effort host c8ctl / SDK versions, so the startup warning and the
|
|
11741
|
+
// per-job "producer unavailable" diagnostic can name the package that must be
|
|
11742
|
+
// upgraded (the macbook workers ran @camunda8/cli 4.1.0-alpha.2 bundling
|
|
11743
|
+
// @camunda8/orchestration-cluster-api 9.1.5, which has no AgentInstance verbs).
|
|
11744
|
+
// Resolved from the HOST c8ctl CLI's dependency graph (rooted at the real entry /
|
|
11745
|
+
// CLI package), not this plugin's sibling node_modules, so the nested host SDK
|
|
11746
|
+
// actually resolves (issue #289 review). Null when unresolvable.
|
|
11747
|
+
const hostSdkInfo = probeHostSdkInfo();
|
|
10200
11748
|
// The producer's `usable` gate also requires the ACP classifier (agent-instance.mjs);
|
|
10201
11749
|
// without it the producer is inert regardless of SDK support, so exclude the probe too.
|
|
10202
11750
|
const acpClassifierAvailable = typeof agenticSessionAcp?.classifyUpdate === 'function';
|
|
@@ -10267,13 +11815,113 @@ async function workAgent(req, flags, ctx) {
|
|
|
10267
11815
|
const extraNote = extraJobTypes.length > 0 ? ` (${extraJobTypes.length} via --job-type)` : '';
|
|
10268
11816
|
logger.info(` listening on ${jobTypes.length} job type(s)${extraNote}: ${jobTypes.join(' ')}`);
|
|
10269
11817
|
logger.info(` one job per worker (registry capacity 1 across all ${jobTypes.length} job type(s)); recovery window: ${recoveryWindowMs}ms; idle timeout: ${idleTimeoutMs}ms; hard cap: ${hardCapMs > 0 ? `${hardCapMs}ms` : 'off'}; poll timeout: ${pollTimeoutMs}ms`);
|
|
11818
|
+
// #289: fail LOUDLY at worker start — not silently per job — when this worker
|
|
11819
|
+
// cannot record a durable AgentInstance transcript on an agentic profile. Every
|
|
11820
|
+
// `nano work` profile runs an agent harness, so any worker that services external
|
|
11821
|
+
// agent jobs needs the host c8ctl SDK's createAgentInstance/updateAgentInstance
|
|
11822
|
+
// verbs. When the host SDK lacks them (e.g. @camunda8/cli 4.1.0-alpha.2 bundling
|
|
11823
|
+
// @camunda8/orchestration-cluster-api 9.1.5), EVERY job silently records no
|
|
11824
|
+
// transcript — the exact macbook-fleet defect. Name the host versions and the fix so
|
|
11825
|
+
// it is actionable from the startup log alone, rather than a per-job "producer
|
|
11826
|
+
// unavailable" line that merges two causes. NANO_AGENT_INSTANCE=off suppresses the
|
|
11827
|
+
// warning (transcripts were intentionally disabled).
|
|
11828
|
+
if (!agentInstanceProbeOff) {
|
|
11829
|
+
if (!sdkSupportsAgentInstance) {
|
|
11830
|
+
logger.warn(
|
|
11831
|
+
`[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the host c8ctl SDK lacks ` +
|
|
11832
|
+
`createAgentInstance/updateAgentInstance, so NO AgentInstance transcript will be ` +
|
|
11833
|
+
`recorded for any job this worker runs. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}. ` +
|
|
11834
|
+
`Upgrade the host c8ctl (the plugin version is irrelevant — the SDK ships with c8ctl): ` +
|
|
11835
|
+
`@camunda8/cli must bundle @camunda8/orchestration-cluster-api >= 10.0.0-alpha.45 ` +
|
|
11836
|
+
`(the first with the AgentInstance verbs). Set NANO_AGENT_INSTANCE=off to silence this ` +
|
|
11837
|
+
`if transcripts are intentionally disabled.`,
|
|
11838
|
+
);
|
|
11839
|
+
} else if (!acpClassifierAvailable) {
|
|
11840
|
+
logger.warn(
|
|
11841
|
+
`[${name}] DURABLE AGENT TRANSCRIPTS DISABLED — the ACP classifier ` +
|
|
11842
|
+
`(agenticSessionAcp.classifyUpdate) is unavailable, so no AgentInstance transcript ` +
|
|
11843
|
+
`will be recorded. This is unexpected (it is a static import) — the plugin install may ` +
|
|
11844
|
+
`be corrupt. Host SDK: ${formatHostSdkInfo(hostSdkInfo)}.`,
|
|
11845
|
+
);
|
|
11846
|
+
} else {
|
|
11847
|
+
// Transcripts CAN be recorded, but only for an ACP harness: a non-ACP harness
|
|
11848
|
+
// emits no session/update stream for the producer to classify, so it records
|
|
11849
|
+
// only the opening CONFIGURATION turn (no body). Surface this once at startup so
|
|
11850
|
+
// an operator running e.g. `copilot --allow-all` (no --acp) knows why a run has a
|
|
11851
|
+
// header-only transcript. Key the note on the EFFECTIVE transport the worker will
|
|
11852
|
+
// actually use (`effectiveProtocol === 'acp'`), NOT a reparse of the command line:
|
|
11853
|
+
// a `protocol: acp` profile is driven over ACP even when the command carries no
|
|
11854
|
+
// literal `--acp` (`ensureAcpFlag` appends it at spawn), and a hand-reparse also
|
|
11855
|
+
// misses supported selectors (`acp` subcommand, `*-acp` adapter, `--experimental-acp`)
|
|
11856
|
+
// while a container sandbox runs pipe-only regardless of `protocol` (issue #289 review).
|
|
11857
|
+
const protoNote = effectiveProtocol === 'acp'
|
|
11858
|
+
? ''
|
|
11859
|
+
: ` NOTE: the harness does not run in ACP mode (effective protocol "${effectiveProtocol}") — a non-ACP harness emits no ACP session/update stream, so only the opening CONFIGURATION turn is recorded (no body). Run the harness in ACP mode (--protocol acp on the host executor) for a full transcript.`;
|
|
11860
|
+
logger.info(`[${name}] Durable agent transcripts enabled (host SDK: ${formatHostSdkInfo(hostSdkInfo)}).${protoNote}`);
|
|
11861
|
+
}
|
|
11862
|
+
}
|
|
10270
11863
|
// Warm the gh-token cache now, off the job-handling path: githubCloneToken()
|
|
10271
11864
|
// may consult `gh auth token` (a synchronous spawn, up to 10s) as its default
|
|
10272
11865
|
// credential fallback, and doing that inside a job handler would block the
|
|
10273
11866
|
// event loop — stalling the lock heartbeat and delaying the job itself.
|
|
10274
11867
|
// Priming here pays that cost once at startup so every later lookup is a warm
|
|
10275
|
-
// cache hit.
|
|
10276
|
-
|
|
11868
|
+
// cache hit. Pass the logger so the token's fingerprint + scopes (never the
|
|
11869
|
+
// value) land in the worker log at startup — issue #293's "which credential is
|
|
11870
|
+
// this worker actually holding?" line.
|
|
11871
|
+
primeGhAuthToken({ logger });
|
|
11872
|
+
// …and keep it fresh for the worker's lifetime (issue #293): a process-lifetime
|
|
11873
|
+
// memo went silently stale the moment the operator refreshed the gh login, and
|
|
11874
|
+
// every job kept getting the OLD token until a restart. The TTL timer re-reads
|
|
11875
|
+
// `gh auth token` — ASYNCHRONOUSLY (spawn, never spawnSync) and DEFERRED while
|
|
11876
|
+
// a job is active (`isIdle`) — so the refresh can never block the event loop /
|
|
11877
|
+
// lock heartbeat the way a synchronous prime on the interval could (Copilot
|
|
11878
|
+
// review on #293). When the cached identity actually changes, `onChange`
|
|
11879
|
+
// republishes the activity marker so `supervisor status` shows the adopted
|
|
11880
|
+
// credential even while the worker sits idle. The supervisor's
|
|
11881
|
+
// `reload-credentials` broadcast (SIGUSR1) forces the same re-read on demand
|
|
11882
|
+
// (non-destructive — an atomic swap, never a drop).
|
|
11883
|
+
// `activeJobs`/`writeActivity` are declared below this point but are only
|
|
11884
|
+
// dereferenced when a tick fires (≥1s later) or a change lands — long after
|
|
11885
|
+
// the whole module body has run — so the closures capture the initialized
|
|
11886
|
+
// bindings (TDZ-safe). The handle is tracked so the stop-signal teardown can
|
|
11887
|
+
// clear it (a ref'd interval would otherwise keep the process alive past
|
|
11888
|
+
// SIGTERM/SIGINT — the --auto liveness test relies on the poll timer being
|
|
11889
|
+
// the sole keepalive).
|
|
11890
|
+
const ghRefreshTimer = startGhAuthTokenRefresh({
|
|
11891
|
+
logger,
|
|
11892
|
+
isIdle: () => activeJobs.size === 0,
|
|
11893
|
+
onChange: () => writeActivity(),
|
|
11894
|
+
});
|
|
11895
|
+
// `supervisor reload-credentials` asks every worker to re-read its gh token
|
|
11896
|
+
// without draining in-flight jobs. The daemon signals SIGUSR1; the handler
|
|
11897
|
+
// re-reads the CLI ASYNCHRONOUSLY (spawn, never spawnSync) and serialized
|
|
11898
|
+
// against the TTL timer, so even a worker mid-job keeps its event loop / lock
|
|
11899
|
+
// heartbeat responsive while the new credential is adopted. Deliberately NOT
|
|
11900
|
+
// the SIGUSR2 drain path — reloading a credential must not park the fleet.
|
|
11901
|
+
// Windows cannot deliver SIGUSR1 (Node terminates the process), so the daemon
|
|
11902
|
+
// refuses the op there. `writeActivityRef` is late-bound to `writeActivity`
|
|
11903
|
+
// once it is defined below so the handler can republish the token's
|
|
11904
|
+
// age/scopes once the async re-read settles.
|
|
11905
|
+
let writeActivityRef = null;
|
|
11906
|
+
if (process.platform !== 'win32') {
|
|
11907
|
+
process.on('SIGUSR1', () => {
|
|
11908
|
+
// Use the ASYNC refresh (spawn, never spawnSync) so re-reading/replacing
|
|
11909
|
+
// the cached credential can never pause the event loop / lock heartbeat of a
|
|
11910
|
+
// worker servicing a job — the whole promise of `reload-credentials` is
|
|
11911
|
+
// that it does NOT disrupt in-flight jobs (Copilot review on #293). It is
|
|
11912
|
+
// serialized against the periodic timer via `ghAuthRefreshInFlight`.
|
|
11913
|
+
// Deliberately NOT `invalidateGhAuthToken()` first: the async `force` read
|
|
11914
|
+
// re-reads `gh auth token` regardless of TTL and swaps the cache ATOMICALLY
|
|
11915
|
+
// when the new value lands, so the worker keeps serving the still-valid old
|
|
11916
|
+
// token for the ~10s read window instead of handing out null meanwhile.
|
|
11917
|
+
refreshGhAuthTokenAsync({ force: true })
|
|
11918
|
+
.then(({ changed }) => {
|
|
11919
|
+
logger.info(`Credentials reload requested — re-read the gh auth token: ${describeGhAuthToken(ghAuthTokenInfo())}${changed ? ' (token changed)' : ''}`);
|
|
11920
|
+
if (writeActivityRef) writeActivityRef();
|
|
11921
|
+
})
|
|
11922
|
+
.catch(() => { /* best effort — a wedged gh must not crash the worker */ });
|
|
11923
|
+
});
|
|
11924
|
+
}
|
|
10277
11925
|
logger.info('Polling for work — press Ctrl-C to stop.');
|
|
10278
11926
|
|
|
10279
11927
|
// When launched under the supervisor, report per-job activity — which job(s)
|
|
@@ -10458,7 +12106,10 @@ async function workAgent(req, flags, ctx) {
|
|
|
10458
12106
|
// Bound the last-activation guard map (TTL + settle-aware size cap). Shared
|
|
10459
12107
|
// with recordJobStart so a standalone worker still prunes it.
|
|
10460
12108
|
pruneLastActivation(nowMs);
|
|
10461
|
-
|
|
12109
|
+
// #293: publish the cached gh token's identity (fingerprint/scopes/age —
|
|
12110
|
+
// never the value) so `supervisor status` can answer "is this worker holding
|
|
12111
|
+
// a stale token?" in one line.
|
|
12112
|
+
const payload = buildActivityPayload({ pid: process.pid, updatedAt: Date.now(), jobs, engine: workerEngine, agentic: agenticState, readyAt, ghAuth: ghAuthTokenInfo() });
|
|
10462
12113
|
const tmp = `${activityFile}.${process.pid}.tmp`;
|
|
10463
12114
|
try {
|
|
10464
12115
|
mkdirSync(dirname(activityFile), { recursive: true });
|
|
@@ -10469,6 +12120,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
10469
12120
|
/* best effort — activity is advisory, never fail a job over it */
|
|
10470
12121
|
}
|
|
10471
12122
|
};
|
|
12123
|
+
// Late-bind the SIGUSR1 (reload-credentials) handler's republish hook now that
|
|
12124
|
+
// writeActivity exists (#293).
|
|
12125
|
+
writeActivityRef = writeActivity;
|
|
10472
12126
|
// The single-owner supervisor's agentic plane (issue #173): ONE multiplexed
|
|
10473
12127
|
// host connection carries presence/steer/ownership/transcript for EVERY
|
|
10474
12128
|
// supervised agent, retiring the per-`work`-process `createWorkChannel` fan-out
|
|
@@ -10745,14 +12399,14 @@ async function workAgent(req, flags, ctx) {
|
|
|
10745
12399
|
logger.info(` live terminal: ${roleTerminal === 'pty' ? 'PTY (streamed + steerable)' : 'pipe (streamed)'} on the supervised connection.`);
|
|
10746
12400
|
}
|
|
10747
12401
|
|
|
10748
|
-
// #110: the role's
|
|
10749
|
-
//
|
|
10750
|
-
//
|
|
10751
|
-
//
|
|
10752
|
-
//
|
|
10753
|
-
//
|
|
10754
|
-
|
|
10755
|
-
|
|
12402
|
+
// #110: the role's ACP permission policy (yolo|escalate|filter), resolved with
|
|
12403
|
+
// the same env-override-then-profile precedence as terminal. The harness protocol
|
|
12404
|
+
// (`envProtocol`/`roleProtocol`/`effectiveProtocol`) is resolved earlier, ahead of
|
|
12405
|
+
// the durable-transcript startup diagnostic (issue #289 review).
|
|
12406
|
+
// `NANO_AGENTIC_PERMISSION` overrides a one-off worker; otherwise the hire profile
|
|
12407
|
+
// decides; else the safe default (yolo). escalate/filter are carried through
|
|
12408
|
+
// verbatim — the acp-executor task enforces yolo and interim-handles the reserved
|
|
12409
|
+
// policies.
|
|
10756
12410
|
const envPermission = (process.env.NANO_AGENTIC_PERMISSION || '').trim().toLowerCase();
|
|
10757
12411
|
const rolePermission = resolveAgenticSetting(envPermission, profile.permission, PERMISSION_MODES, 'yolo');
|
|
10758
12412
|
|
|
@@ -10767,8 +12421,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
10767
12421
|
// container profile's command STILL pipes plain JSON to an ACP harness — the
|
|
10768
12422
|
// same husk. `effectiveHarnessProtocol` reduces a container's protocol to `pipe`
|
|
10769
12423
|
// so the refusal covers that pipe-only executor too, not just the host (thread
|
|
10770
|
-
// 4118327651).
|
|
10771
|
-
const effectiveProtocol = effectiveHarnessProtocol(roleProtocol, isContainer);
|
|
12424
|
+
// 4118327651). `effectiveProtocol` is computed earlier (issue #289 review).
|
|
10772
12425
|
const startupMismatch = detectProtocolMismatch({ command: profile.command, args: effectiveArgs, protocol: effectiveProtocol });
|
|
10773
12426
|
if (startupMismatch) {
|
|
10774
12427
|
// Non-restartable config failure: under the supervisor this must exit with
|
|
@@ -10894,7 +12547,14 @@ async function workAgent(req, flags, ctx) {
|
|
|
10894
12547
|
// Assemble + normalize the task envelope from headers (defaults) and
|
|
10895
12548
|
// variables (overrides), then resolve any secrets it references.
|
|
10896
12549
|
const envelope = normalizeTaskEnvelope(job.customHeaders ?? {}, job.variables ?? {}, { basePromptOverride });
|
|
10897
|
-
const { resolved, missing, names } = resolveJobSecrets(secretResolver, envelope);
|
|
12550
|
+
const { resolved, missing, names } = await resolveJobSecrets(secretResolver, envelope);
|
|
12551
|
+
// #222: recheck after the awaited secret resolution (stage: secrets) —
|
|
12552
|
+
// resolveJobSecrets AWAITS the gh-cache-miss forced refresh, so a
|
|
12553
|
+
// force-stop / lease loss can win DURING that await. Gate BEFORE the
|
|
12554
|
+
// missing-secret settlement below so an abort that landed mid-resolve
|
|
12555
|
+
// returns WITHOUT settling (which would race the force-stop yield),
|
|
12556
|
+
// mirroring the post-prompt-fetch gate above.
|
|
12557
|
+
if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'secrets', logger })) return;
|
|
10898
12558
|
if (missing.length > 0) {
|
|
10899
12559
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
10900
12560
|
const msg = `missing secret(s): ${missing.join(', ')} (resolver: ${secretResolver.kind})`;
|
|
@@ -10998,7 +12658,21 @@ async function workAgent(req, flags, ctx) {
|
|
|
10998
12658
|
} else if (agentInstanceProducer.retryPending) {
|
|
10999
12659
|
logger.info(`[${jobType}] AgentInstance producer not yet active for external agent job (${aiCorr}) — the initial createAgentInstance did not mint a key in time (transient rejection, no-key response, or bounded-out create), but the create stays armed to retry on the ACP hot path, so a run that recovers can still record a durable transcript (job completion unaffected).`);
|
|
11000
12660
|
} else {
|
|
11001
|
-
|
|
12661
|
+
// #289: report WHICH precondition failed (SDK create, SDK update, ACP
|
|
12662
|
+
// classifier, or not-external) plus the host SDK versions, instead of
|
|
12663
|
+
// merging the causes into one opaque line — so the no-transcript cause is
|
|
12664
|
+
// root-causable from this line alone. A PERMANENT create rejection (a 4xx
|
|
12665
|
+
// the engine will never mint on retry) is NOT a construction-time
|
|
12666
|
+
// precondition failure — activation WAS attempted and permanently
|
|
12667
|
+
// rejected — so report it distinctly rather than as "activation not
|
|
12668
|
+
// attempted: unknown precondition" (issue #289 review).
|
|
12669
|
+
const rejectedStatus = agentInstanceProducer.permanentlyRejected;
|
|
12670
|
+
if (rejectedStatus != null) {
|
|
12671
|
+
logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation attempted but permanently rejected (HTTP ${rejectedStatus}); host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
|
|
12672
|
+
} else {
|
|
12673
|
+
const reason = agentInstanceProducer.unavailableReason || 'unknown precondition';
|
|
12674
|
+
logger.info(`[${jobType}] AgentInstance producer unavailable for external agent job (${aiCorr}) — activation not attempted: ${reason}; host SDK: ${formatHostSdkInfo(hostSdkInfo)}; continuing without a durable transcript (job completion unaffected).`);
|
|
12675
|
+
}
|
|
11002
12676
|
}
|
|
11003
12677
|
} catch (err) {
|
|
11004
12678
|
logger.warn(`[${jobType}] AgentInstance producer activate() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; continuing without a durable transcript (job completion unaffected).`);
|
|
@@ -11137,15 +12811,38 @@ async function workAgent(req, flags, ctx) {
|
|
|
11137
12811
|
let cwd;
|
|
11138
12812
|
let extraEnv;
|
|
11139
12813
|
let repoToken = null;
|
|
12814
|
+
let repoCredentialSource = null;
|
|
11140
12815
|
if (hasRepo) {
|
|
11141
12816
|
const provider = envelope.repository.provider || 'github';
|
|
11142
12817
|
const authRef = envelope.repository.authRef;
|
|
11143
|
-
|
|
12818
|
+
// Resolve the credential AND its source once: the source gates whether
|
|
12819
|
+
// a later auth failure may fall back to the operator's `gh auth token`
|
|
12820
|
+
// (only a 'gh-cache' token may — never an explicit GITHUB_TOKEN or a
|
|
12821
|
+
// custom authRef; Copilot review on #293).
|
|
12822
|
+
const cred = githubCloneTokenWithSource({ provider, authRef, secretResolver });
|
|
12823
|
+
repoToken = cred.token; // absent → anonymous clone
|
|
12824
|
+
repoCredentialSource = cred.source;
|
|
11144
12825
|
try {
|
|
11145
12826
|
mkdirSync(workerNsDir, { recursive: true });
|
|
11146
12827
|
runDir = mkdtempSync(join(workerNsDir, 'run-'));
|
|
11147
12828
|
liveRunDirs.add(runDir);
|
|
11148
|
-
provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal });
|
|
12829
|
+
provisioned = provisionRepo({ envelope, token: repoToken, runDir, runId, timeoutMs: cloneTimeoutMs, logger, corr: aiCorr, abortSignal, credentialSource: repoCredentialSource });
|
|
12830
|
+
// #293: provisioning's clone auth-retry may have ROTATED the token
|
|
12831
|
+
// (adopting the fresh gh token into the workspace's askpass env).
|
|
12832
|
+
// Adopt that effective credential for the rest of the job —
|
|
12833
|
+
// finalizeGit's push/PR-reconcile and the checkpoint GC/ref-delete
|
|
12834
|
+
// askpass — so they don't keep failing with the stale pre-provision
|
|
12835
|
+
// token (Copilot review on #293).
|
|
12836
|
+
if (provisioned.effectiveToken) repoToken = provisioned.effectiveToken;
|
|
12837
|
+
if (provisioned.credentialSource) repoCredentialSource = provisioned.credentialSource;
|
|
12838
|
+
// #293: when the rotation above swapped a gh-cache credential, the
|
|
12839
|
+
// `resolved` secret map (built pre-provisioning, forwarded to the
|
|
12840
|
+
// harness as `secretEnv`) still holds the STALE token — sync it so an
|
|
12841
|
+
// allowPr harness's own gh/git runs use the credential provisioning
|
|
12842
|
+
// just adopted, not the one the clone proved dead (Copilot review on
|
|
12843
|
+
// #293). No-op for an env/custom identity, an unchanged token, or a
|
|
12844
|
+
// key the job never forwarded.
|
|
12845
|
+
applyRotatedGhCredential({ resolved, provider, authRef, source: repoCredentialSource, token: repoToken });
|
|
11149
12846
|
if (provisioned.baseFetchError) {
|
|
11150
12847
|
logger.warn(`[${jobType}] job ${job.jobKey} base fetch failed (${aiCorr}) — ${oneLineLog(provisioned.baseFetchError)}; base...head diffs may be unavailable`);
|
|
11151
12848
|
}
|
|
@@ -11244,6 +12941,28 @@ async function workAgent(req, flags, ctx) {
|
|
|
11244
12941
|
// performs a last-chance `failed` flush so edits since the last periodic
|
|
11245
12942
|
// checkpoint are not lost.
|
|
11246
12943
|
let checkpointFinalized = false;
|
|
12944
|
+
// #296 (F8/F9): set on the worker force-stop path so the `finally` below bounds
|
|
12945
|
+
// its `close()` drain to the same FORCE_STOP_RECOVERY_MS budget — a force-stop
|
|
12946
|
+
// must never be held past the supervisor's SIGKILL backstop by the (up to
|
|
12947
|
+
// 120s) GC sweep. The in-flight checkpoint git is cancelled by the recovery
|
|
12948
|
+
// DEADLINE (checkpointing.abortRecovery(), fired from the force-stop block),
|
|
12949
|
+
// not the job's abortSignal, so the chain settles on its own after we stop
|
|
12950
|
+
// awaiting while recovery can still run after the harness abort.
|
|
12951
|
+
let forceStopping = false;
|
|
12952
|
+
// #296: the SINGLE absolute force-stop deadline (set when the force-stop
|
|
12953
|
+
// recovery block begins). The recover race AND the `finally`'s close() drain
|
|
12954
|
+
// BOTH bound against it, so recovery + cleanup together fit inside ONE
|
|
12955
|
+
// FORCE_STOP_RECOVERY_MS window rather than consuming it twice (recovery race,
|
|
12956
|
+
// then an independent close() wait) and risking a SIGKILL during cleanup
|
|
12957
|
+
// (Copilot review on #296). 0 until a force-stop begins.
|
|
12958
|
+
let forceStopDeadline = 0;
|
|
12959
|
+
// #295: the durable recovery ref (and SHA) a preserveRecovery() pushed on an
|
|
12960
|
+
// SLA cancellation / failed run, surfaced in the job outcome (io.nanobpm.agentResult)
|
|
12961
|
+
// — but ONLY on the failed/SLA-cancel path, which settles normally and so reaches
|
|
12962
|
+
// buildResultEnvelope. A FORCE-STOPPED run is yielded (settle.fail, retries
|
|
12963
|
+
// preserved) and returns early below WITHOUT an outcome envelope, so its
|
|
12964
|
+
// recoveryRefInfo is intentionally NOT reported there (Copilot review on #295).
|
|
12965
|
+
let recoveryRefInfo = null;
|
|
11247
12966
|
// The per-job live-terminal relay session (issue #173): streams this job's
|
|
11248
12967
|
// harness terminal over the single-owner supervisor's ONE multiplexed host
|
|
11249
12968
|
// connection, keyed by this worker's instance + the jobKey, and accepts
|
|
@@ -11394,14 +13113,259 @@ async function workAgent(req, flags, ctx) {
|
|
|
11394
13113
|
// that yield. The finally below still runs (container/run-dir/relay
|
|
11395
13114
|
// cleanup); recordJobEnd fires in the outer finally.
|
|
11396
13115
|
if (result && result.aborted) {
|
|
11397
|
-
logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed;
|
|
13116
|
+
logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop or SLA/lease-loss cancellation) — harness killed; releasing the job (the broker re-activates it for retry when the run is still live).`);
|
|
11398
13117
|
// #264: last-chance WIP checkpoint so the retry can pick up the
|
|
11399
13118
|
// uncommitted work (bounded; the lease guards a newer owner).
|
|
11400
|
-
|
|
13119
|
+
// #295: then promote it to a durable recovery ref so a force-stopped or
|
|
13120
|
+
// SLA-cancelled run keeps a recovery point. The orphan GC spares that ref
|
|
13121
|
+
// while its process stays active, so it survives long enough to be
|
|
13122
|
+
// resumed. The ref is
|
|
13123
|
+
// pushed here, but it is NOT surfaced in a job outcome: this path yields
|
|
13124
|
+
// the job and returns early, never reaching buildResultEnvelope — a
|
|
13125
|
+
// force-stopped run produces no outcome envelope by design (Copilot
|
|
13126
|
+
// review on #295), so the recovery ref is recoverable from the pushed
|
|
13127
|
+
// refs/nano/recovered/<eik> ref itself, not from any job result.
|
|
13128
|
+
// #296: bound the WHOLE flush+promote to FORCE_STOP_RECOVERY_MS so it
|
|
13129
|
+
// reliably finishes (and this worker exits) inside the window the
|
|
13130
|
+
// supervisor's forceStopWorker now waits before its SIGKILL backstop —
|
|
13131
|
+
// otherwise the default flush (20s) + recovery git push (60s) could run
|
|
13132
|
+
// long and be SIGKILLed before refs/nano/recovered/<eik> is written.
|
|
13133
|
+
// Best-effort: if the budget elapses we give up on the recovery ref (the
|
|
13134
|
+
// WIP ref + TTL remain the fallback), never block the yield/exit.
|
|
13135
|
+
if (checkpointer) {
|
|
13136
|
+
forceStopping = true;
|
|
13137
|
+
const recoveryDeadline = FORCE_STOP_RECOVERY_MS;
|
|
13138
|
+
// #296: anchor the ONE shared force-stop deadline here. The recover race
|
|
13139
|
+
// below and the `finally`'s close() both bound against it, so the TOTAL
|
|
13140
|
+
// force-stop window is a single FORCE_STOP_RECOVERY_MS (not 2x), fitting
|
|
13141
|
+
// inside the supervisor's STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS backstop.
|
|
13142
|
+
forceStopDeadline = Date.now() + recoveryDeadline;
|
|
13143
|
+
// #296 (review): reserve a DRAIN window inside the deadline. abortRecovery()
|
|
13144
|
+
// only SIGTERMs the checkpoint git; createGitRunner then grants up to
|
|
13145
|
+
// gitKillGraceMs before SIGKILL. If abortRecovery() fires AT the deadline,
|
|
13146
|
+
// a stubborn git is still dying when close() hits its (expired) cap and the
|
|
13147
|
+
// finally reaps the workspace under a live process. Fire the abort EARLY —
|
|
13148
|
+
// gitKillGraceMs + a small margin before the deadline — so the SIGKILL lands
|
|
13149
|
+
// by `forceStopDeadline` and close() gets the remaining grace window to
|
|
13150
|
+
// actually drain the now-dead git, not the 1ms `Math.max(1, …)` would give.
|
|
13151
|
+
const killGrace = Math.max(0, Number(checkpointing?.gitKillGraceMs) || 0);
|
|
13152
|
+
const drainReserveMs = killGrace + 250;
|
|
13153
|
+
const abortAtMs = Math.max(1, recoveryDeadline - drainReserveMs);
|
|
13154
|
+
// #296 (F9): arm the recovery-deadline. When the budget expires, abort
|
|
13155
|
+
// the DEDICATED recovery git runner (not the job's abortSignal, which
|
|
13156
|
+
// has already fired and would fail every recovery command) so an
|
|
13157
|
+
// in-flight flush/preserveRecovery push is torn down and the chain
|
|
13158
|
+
// settles — letting the `finally`'s bounded `close()` drain inside the
|
|
13159
|
+
// supervisor's SIGKILL window. Fires `abortAtMs` (the deadline MINUS the
|
|
13160
|
+
// reserved drain window) so the git SIGKILL completes before the deadline;
|
|
13161
|
+
// unref'd so it never keeps the process alive.
|
|
13162
|
+
// #296 (review, r4244116341): the recover-race `Promise.race([recover,
|
|
13163
|
+
// budget])` below only bounds the AWAIT — when `budget` wins, the SDK
|
|
13164
|
+
// mutations inside `recover` (settleAbortMetaWrite's awaited WIP write and
|
|
13165
|
+
// the recovery-pointer write) keep running. The job then yields and a
|
|
13166
|
+
// reactivated worker can publish a NEWER `resumeFromCheckpoint`; a stale
|
|
13167
|
+
// `recover` continuation that completes afterward would clobber it with the
|
|
13168
|
+
// old recovery pointer. abortRecovery() cancels only git/GC, not these SDK
|
|
13169
|
+
// calls. So drive the SDK mutations off a DEDICATED recovery-deadline
|
|
13170
|
+
// signal, fired by the same deadline (and on budget expiry), and AWAIT
|
|
13171
|
+
// `recover` after the race so it is SETTLED/cancelled before we yield.
|
|
13172
|
+
const recoverySdkAbort = new AbortController();
|
|
13173
|
+
let deadlineTimer = setTimeout(() => {
|
|
13174
|
+
try { checkpointing.abortRecovery(); } catch { /* best effort */ }
|
|
13175
|
+
try { recoverySdkAbort.abort(); } catch { /* best effort */ }
|
|
13176
|
+
}, abortAtMs);
|
|
13177
|
+
deadlineTimer?.unref?.();
|
|
13178
|
+
const recover = (async () => {
|
|
13179
|
+
await checkpointer.flush('abort', { timeoutMs: recoveryDeadline }); checkpointFinalized = true;
|
|
13180
|
+
try { recoveryRefInfo = await checkpointing.preserveRecovery('force-stop') || recoveryRefInfo; } catch { /* best effort */ }
|
|
13181
|
+
// #296 (review, AC3): an SLA boundary cancel reaches THIS aborted branch
|
|
13182
|
+
// too — the supervisor heartbeat treats the resulting lease-loss 404 as
|
|
13183
|
+
// terminal and interrupts the runner (supervisor/src/dispatch.ts), so
|
|
13184
|
+
// runAgentJob returns aborted:true. The early return below then skips
|
|
13185
|
+
// buildResultEnvelope, so recoveryRef/recoverySha are never reported in a
|
|
13186
|
+
// job outcome. Persist the recovery ref through the PROCESS-scoped
|
|
13187
|
+
// `resumeFromCheckpoint` slot instead: it survives the element re-entry an
|
|
13188
|
+
// SLA escalation triggers, so the resumed run's predecessorCheckpointRef
|
|
13189
|
+
// can discover the durable refs/nano/recovered/<eik> point. Bounded and
|
|
13190
|
+
// best-effort — never holds the force-stop exit; the pushed ref itself
|
|
13191
|
+
// remains the authoritative recovery handle.
|
|
13192
|
+
// #296 (review): the abort flush fired its OWN WIP `resumeFromCheckpoint`
|
|
13193
|
+
// write fire-and-forget (see onCheckpoint). If that write is still in
|
|
13194
|
+
// flight it could land AFTER the recovery-pointer write below and
|
|
13195
|
+
// overwrite the recovery ref with a stale WIP pointer (which the orphan
|
|
13196
|
+
// GC then reclaims on TTL, losing the recovery discovery). Await the WIP
|
|
13197
|
+
// write FIRST — under the recovery-deadline signal so the await is
|
|
13198
|
+
// cancelled when the budget expires — so the recovery pointer is
|
|
13199
|
+
// guaranteed to land last. Bounded by the recover-race budget; best-effort.
|
|
13200
|
+
try { await checkpointing.settleAbortMetaWrite?.(recoverySdkAbort.signal); } catch { /* best effort */ }
|
|
13201
|
+
// #296 (review, r4244116341): if the recovery deadline has already fired,
|
|
13202
|
+
// do NOT write the recovery pointer — a write that lands after this branch
|
|
13203
|
+
// yields could overwrite a reactivated worker's newer resumeFromCheckpoint.
|
|
13204
|
+
if (recoverySdkAbort.signal.aborted) return;
|
|
13205
|
+
if (recoveryRefInfo && typeof camunda?.createElementInstanceVariables === 'function' && job?.processInstanceKey) {
|
|
13206
|
+
try {
|
|
13207
|
+
// #296 (review, r4244827082): gate the process-scoped pointer write on
|
|
13208
|
+
// PROVEN current ownership of the recovery ref. preserveRecovery only
|
|
13209
|
+
// fences the GIT ref; this variable write is last-writer-wins, so a
|
|
13210
|
+
// superseded activation could otherwise land its stale pointer after a
|
|
13211
|
+
// newer activation (which re-activated the job on lease loss) published
|
|
13212
|
+
// its own. Re-verify the ref still resolves to the sha THIS run
|
|
13213
|
+
// preserved and is still owned by this run; skip the write when it has
|
|
13214
|
+
// moved (a newer activation's pointer stays authoritative). Bounded by
|
|
13215
|
+
// the recovery deadline; a failed/aborted verify fails closed (skip).
|
|
13216
|
+
let stillOwns = false;
|
|
13217
|
+
try {
|
|
13218
|
+
let vt; let vOnAbort = null;
|
|
13219
|
+
// #296 (review, r4244916101): drive the verify off its OWN
|
|
13220
|
+
// AbortController so that when the 5s branch wins we can CANCEL
|
|
13221
|
+
// the in-flight git reads AND await the verify to full settlement
|
|
13222
|
+
// in the finally below — before recover() returns and the run dir
|
|
13223
|
+
// is reaped. Without this the losing verify kept running on the
|
|
13224
|
+
// 60s runner (untracked by stop()/gc) under a reaped workspace.
|
|
13225
|
+
const verifyAbort = new AbortController();
|
|
13226
|
+
const verifyOp = Promise.resolve(
|
|
13227
|
+
checkpointing.verifyRecoveryOwnership?.(recoveryRefInfo.sha, { signal: verifyAbort.signal }),
|
|
13228
|
+
).then((v) => v === true);
|
|
13229
|
+
stillOwns = await Promise.race([
|
|
13230
|
+
verifyOp,
|
|
13231
|
+
new Promise((res) => { vt = setTimeout(() => res(false), 5000); vt?.unref?.(); }),
|
|
13232
|
+
new Promise((res) => { vOnAbort = () => res(false); recoverySdkAbort.signal.addEventListener('abort', vOnAbort, { once: true }); }),
|
|
13233
|
+
]).finally(() => {
|
|
13234
|
+
clearTimeout(vt);
|
|
13235
|
+
if (vOnAbort) recoverySdkAbort.signal.removeEventListener('abort', vOnAbort);
|
|
13236
|
+
// The race settled (timeout/abort/complete). If the verify is
|
|
13237
|
+
// still mid-git (timeout/abort won), cancel its runner so the
|
|
13238
|
+
// git process is SIGTERM→SIGKILLed now, not at abortAtMs.
|
|
13239
|
+
try { verifyAbort.abort(); } catch { /* best effort */ }
|
|
13240
|
+
});
|
|
13241
|
+
// AWAIT the verify to full settlement (its git is now aborted, so
|
|
13242
|
+
// this resolves promptly) so no live git outlives recover()/close().
|
|
13243
|
+
try { await verifyOp; } catch { /* best effort */ }
|
|
13244
|
+
} catch { stillOwns = false; }
|
|
13245
|
+
if (!stillOwns) {
|
|
13246
|
+
logger.warn?.(`[${jobType}] job ${job.jobKey}: recovery ref ${recoveryRefInfo.ref} is no longer owned by this run (a newer activation superseded it) — skipping the process-scoped resumeFromCheckpoint write so the newer pointer stays authoritative.`);
|
|
13247
|
+
return;
|
|
13248
|
+
}
|
|
13249
|
+
// #296 (review, r4245472572): the Git-ref ownership check above cannot
|
|
13250
|
+
// fence THIS write across an SLA resume — a re-activated run is a NEW
|
|
13251
|
+
// element instance, so it owns a DIFFERENT refs/nano/recovered/<eik>
|
|
13252
|
+
// and this run still "owns" its old one (passing the check) even after
|
|
13253
|
+
// the new activation published a newer resumeFromCheckpoint. The write
|
|
13254
|
+
// is unconditional last-writer-wins, so it could then overwrite that
|
|
13255
|
+
// newer pointer. Two fences close the class (no atomic process-scoped
|
|
13256
|
+
// CAS exists — so the write is a best-effort DISCOVERY HINT, not an
|
|
13257
|
+
// authoritative fence, and these two narrowings reduce (not eliminate)
|
|
13258
|
+
// its staleness:
|
|
13259
|
+
// (1) OCCURRENCE-SCOPED READ-BACK — re-read the shared slot and skip
|
|
13260
|
+
// when it already names a DIFFERENT run's ref (a newer activation
|
|
13261
|
+
// beat us). Best-effort: a read failure/empty/absent result fails
|
|
13262
|
+
// OPEN (we still write) so a transient search outage never strands
|
|
13263
|
+
// recovery; eventual consistency (index lag) can hide a JUST-landed
|
|
13264
|
+
// newer write, which is exactly why fence (2) is also required.
|
|
13265
|
+
// (2) NATIVE ABORT SIGNAL on the write itself — the SDK options extend
|
|
13266
|
+
// RequestInit, so `signal` reaches fetch and aborts the in-flight
|
|
13267
|
+
// HTTP request the instant the recovery deadline fires, tearing the
|
|
13268
|
+
// mutation down at the transport (not just a best-effort .cancel()).
|
|
13269
|
+
// #296 (review, r4245620621): a read-then-write is NOT atomic, so a
|
|
13270
|
+
// replacement activation can still publish its ref between the read-back
|
|
13271
|
+
// and this write — the residual TOCTOU the reviewer notes is real. It is
|
|
13272
|
+
// BENIGN for recovery CORRECTNESS, which does not rest on this write:
|
|
13273
|
+
// the self-recorded OBJECT form of resumeFromCheckpoint is NEVER
|
|
13274
|
+
// auto-adopted on resume (`makeOccurrenceGuard` fails closed BY
|
|
13275
|
+
// CONSTRUCTION — only a process-delivered bare STRING is adopted), and
|
|
13276
|
+
// the authoritative durable recovery handle is the CAS-fenced git ref
|
|
13277
|
+
// refs/nano/recovered/<eik> (`preserveRecovery` → `pushNamedRef`
|
|
13278
|
+
// --force-with-lease). A stale object pointer therefore yields a
|
|
13279
|
+
// transcript-only resume (identical to an ABSENT pointer), never an
|
|
13280
|
+
// incorrect restore. These narrowings keep the HINT as fresh as is
|
|
13281
|
+
// cheaply possible; they are not relied on for a safety property.
|
|
13282
|
+
// #296 (review, r4245620587): the consistency options MUST be nested
|
|
13283
|
+
// under `consistency` (the real @camunda8 client throws "Missing
|
|
13284
|
+
// consistency options" otherwise — mirrors the 6994/6999 reads and
|
|
13285
|
+
// agent-resume's READ_CONSISTENCY) or the catch below would swallow the
|
|
13286
|
+
// throw and this read-back would silently fail OPEN on every call.
|
|
13287
|
+
if (typeof camunda?.searchVariables === 'function') {
|
|
13288
|
+
try {
|
|
13289
|
+
const found = await Promise.resolve(camunda.searchVariables(
|
|
13290
|
+
{ filter: { name: 'resumeFromCheckpoint', scopeKey: String(job.processInstanceKey) }, truncateValues: false },
|
|
13291
|
+
{ consistency: { waitUpToMs: 0 }, signal: recoverySdkAbort.signal },
|
|
13292
|
+
));
|
|
13293
|
+
const items = Array.isArray(found?.items) ? found.items : (Array.isArray(found?.data?.items) ? found.data.items : []);
|
|
13294
|
+
let slotRef = null;
|
|
13295
|
+
for (const it of items) {
|
|
13296
|
+
try { const v = JSON.parse(it?.value ?? 'null'); slotRef = (v && typeof v === 'object' ? v.ref : (typeof v === 'string' ? v : null)) ?? slotRef; } catch { /* unparseable entry — ignore */ }
|
|
13297
|
+
}
|
|
13298
|
+
if (slotRef && slotRef !== recoveryRefInfo.ref) {
|
|
13299
|
+
logger.warn?.(`[${jobType}] job ${job.jobKey}: the process-scoped resumeFromCheckpoint slot already names a newer activation's ref ${slotRef} (not this run's ${recoveryRefInfo.ref}) — skipping the stale overwrite so the newer pointer stays authoritative.`);
|
|
13300
|
+
return;
|
|
13301
|
+
}
|
|
13302
|
+
} catch { /* best effort — a read failure fails OPEN (write proceeds) */ }
|
|
13303
|
+
}
|
|
13304
|
+
const w = camunda.createElementInstanceVariables({
|
|
13305
|
+
elementInstanceKey: String(job.processInstanceKey),
|
|
13306
|
+
variables: { resumeFromCheckpoint: { ref: recoveryRefInfo.ref, sha: recoveryRefInfo.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, recovered: true, at: Date.now() } },
|
|
13307
|
+
local: true,
|
|
13308
|
+
signal: recoverySdkAbort.signal,
|
|
13309
|
+
});
|
|
13310
|
+
let wt; let onAbort = null;
|
|
13311
|
+
// Cancel the in-flight write when the recovery deadline fires (the SDK
|
|
13312
|
+
// returns a CancelablePromise), AND reject the race so the chain
|
|
13313
|
+
// settles — the mutation never lands late over a newer pointer.
|
|
13314
|
+
await Promise.race([
|
|
13315
|
+
Promise.resolve(w).finally(() => { clearTimeout(wt); if (onAbort) recoverySdkAbort.signal.removeEventListener('abort', onAbort); }),
|
|
13316
|
+
new Promise((_r, rej) => { wt = setTimeout(() => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write timed out')); }, 5000); wt?.unref?.(); }),
|
|
13317
|
+
new Promise((_r, rej) => { onAbort = () => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write aborted')); }; recoverySdkAbort.signal.addEventListener('abort', onAbort, { once: true }); }),
|
|
13318
|
+
]);
|
|
13319
|
+
} catch { /* best effort — the pushed ref is the fallback */ }
|
|
13320
|
+
}
|
|
13321
|
+
})();
|
|
13322
|
+
let budgetTimer = null;
|
|
13323
|
+
const budget = new Promise((res) => { budgetTimer = setTimeout(res, recoveryDeadline); if (budgetTimer.unref) budgetTimer.unref(); });
|
|
13324
|
+
try { await Promise.race([recover, budget]); } catch { /* best effort */ }
|
|
13325
|
+
// Clear the recover-race budget timer (it has fired or the race settled).
|
|
13326
|
+
// deadlineTimer is deliberately NOT cleared: even when the recover race
|
|
13327
|
+
// settles early, abortRecovery() must still fire at abortAtMs to abort the
|
|
13328
|
+
// background GC sweep (gcAbort) — otherwise close() below would await a
|
|
13329
|
+
// sweep that can run up to GC_OVERALL_TIMEOUT_MS (120s), holding the
|
|
13330
|
+
// force-stop open past the supervisor's SIGKILL window (the F8 failure).
|
|
13331
|
+
if (budgetTimer) clearTimeout(budgetTimer);
|
|
13332
|
+
// #296 (review, r4244116341): the race above only bounds the await — when
|
|
13333
|
+
// `budget` wins, `recover` may still be mid-SDK-write. Abort the recovery
|
|
13334
|
+
// signal (cancels the in-flight SDK mutation and short-circuits recover's
|
|
13335
|
+
// remaining steps) and AWAIT `recover` so it is fully settled BEFORE we
|
|
13336
|
+
// yield the job. This guarantees no stale continuation writes the old
|
|
13337
|
+
// recovery pointer after a reactivated worker publishes a newer one. The
|
|
13338
|
+
// signal + each step's internal deadline keep this await bounded.
|
|
13339
|
+
try { recoverySdkAbort.abort(); } catch { /* best effort */ }
|
|
13340
|
+
try { await recover; } catch { /* best effort */ }
|
|
13341
|
+
checkpointFinalized = true;
|
|
13342
|
+
}
|
|
11401
13343
|
return;
|
|
11402
13344
|
}
|
|
11403
13345
|
// #264: a failed run is retried — checkpoint what it left behind.
|
|
11404
|
-
|
|
13346
|
+
// #295: an SLA-cancelled run surfaces here as a non-ok (timed-out) result;
|
|
13347
|
+
// preserve a durable recovery ref and report it in the job outcome below.
|
|
13348
|
+
if (checkpointer && !result.ok) {
|
|
13349
|
+
await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true;
|
|
13350
|
+
try { recoveryRefInfo = await checkpointing.preserveRecovery('failed') || recoveryRefInfo; } catch { /* best effort */ }
|
|
13351
|
+
}
|
|
13352
|
+
// #296 (review, r4244991586): the failed-result path can outlive its lease.
|
|
13353
|
+
// `result.aborted` is only true when the harness observed the abort WHILE the
|
|
13354
|
+
// process ran; an abortSignal that fires AFTER runAgentJob() returned (with a
|
|
13355
|
+
// genuine non-ok result) but DURING the flush/promote above leaves
|
|
13356
|
+
// `result.aborted === false`. Effect has already interrupted the raw runner and
|
|
13357
|
+
// released the slot, so if we now fall through to settleJob.fail (and the
|
|
13358
|
+
// checkpoint-variable/envelope writes) we overlap a replacement activation with
|
|
13359
|
+
// STALE writes + a STALE settle. Recheck the job signal here — after the
|
|
13360
|
+
// post-run flush/promote, before any settlement — and switch to the
|
|
13361
|
+
// interrupted/yield path (return WITHOUT settling, exactly like the
|
|
13362
|
+
// result.aborted branch and settleRunError's abort idiom) when it has fired.
|
|
13363
|
+
// The recovery ref preserved above still stands (the durable handle); we simply
|
|
13364
|
+
// do not settle a job whose lease is already gone. The broker re-activates it.
|
|
13365
|
+
if (!result.ok && abortSignal?.aborted === true) {
|
|
13366
|
+
logger.warn(`[${jobType}] job ${job.jobKey}: abortSignal fired after the failed run returned (lease loss/force-stop won the race) — yielding WITHOUT settleJob.fail so a replacement activation does not overlap stale checkpoint/settle writes (the preserved recovery ref remains the durable handle).`);
|
|
13367
|
+
return;
|
|
13368
|
+
}
|
|
11405
13369
|
|
|
11406
13370
|
// Gap 2 (#678): a clean run that emitted no machine-readable result gets
|
|
11407
13371
|
// ONE bounded re-emit nudge in the same workspace, feeding back its own
|
|
@@ -11498,6 +13462,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
11498
13462
|
hasPrBranch: provisioned.hasPrBranch,
|
|
11499
13463
|
envelope,
|
|
11500
13464
|
token: repoToken,
|
|
13465
|
+
credentialSource: repoCredentialSource,
|
|
11501
13466
|
logger,
|
|
11502
13467
|
corr: aiCorr,
|
|
11503
13468
|
keepRuns,
|
|
@@ -11542,6 +13507,21 @@ async function workAgent(req, flags, ctx) {
|
|
|
11542
13507
|
} else if (provisioned) {
|
|
11543
13508
|
gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
|
|
11544
13509
|
}
|
|
13510
|
+
// #295: surface the durable recovery ref (pushed on an SLA cancellation /
|
|
13511
|
+
// failed run above) in the job outcome so the escalation can point a resume
|
|
13512
|
+
// at it. Attach to whatever gitResult was built for this (provisioned) job.
|
|
13513
|
+
if (recoveryRefInfo && gitResult) { gitResult.recoveryRef = recoveryRefInfo.ref; gitResult.recoverySha = recoveryRefInfo.sha; }
|
|
13514
|
+
// Propagate a mid-finalize credential rotation (finalizeGit's push
|
|
13515
|
+
// auth-failure recovery re-read a fresh gh token) to every subsystem that
|
|
13516
|
+
// captured the pre-rotation token: the outer `repoToken` (later redaction /
|
|
13517
|
+
// consumers) and the WIP-checkpoint cleanup, whose post-ack ref deletion
|
|
13518
|
+
// would otherwise authenticate with the credential origin already rejected
|
|
13519
|
+
// and leave the remote WIP ref behind until a GC sweep (Copilot review on
|
|
13520
|
+
// #293). No-op when the token was not rotated.
|
|
13521
|
+
if (gitResult?.rotatedToken) {
|
|
13522
|
+
repoToken = gitResult.rotatedToken;
|
|
13523
|
+
checkpointing?.updateCredential?.(gitResult.rotatedToken);
|
|
13524
|
+
}
|
|
11545
13525
|
// #264: the work is durable on the pushed branch, so the WIP ref goes —
|
|
11546
13526
|
// but only once the engine acks job.complete (below): until then a lost
|
|
11547
13527
|
// lease or failed settle redelivers the job, and the next activation must
|
|
@@ -11577,7 +13557,24 @@ async function workAgent(req, flags, ctx) {
|
|
|
11577
13557
|
// last periodic checkpoint. `flush` forces past the post-stop guard and is
|
|
11578
13558
|
// bounded; the lease still protects a newer owner. Then close (drain).
|
|
11579
13559
|
if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
|
|
11580
|
-
|
|
13560
|
+
// #296 (F8): on a force-stop, bound the drain to the REMAINING of the ONE
|
|
13561
|
+
// shared force-stop deadline (set when the recovery block began) — NOT a
|
|
13562
|
+
// fresh full FORCE_STOP_RECOVERY_MS. The recovery race may already have
|
|
13563
|
+
// consumed most of the budget, so a fresh window here could run the total
|
|
13564
|
+
// (recovery + cleanup) to ~2x the budget and be SIGKILLed mid-cleanup
|
|
13565
|
+
// (Copilot review on #296). The recovery DEADLINE (abortRecovery) tears down
|
|
13566
|
+
// both the checkpoint git AND the GC sweep — and it now fires gitKillGraceMs
|
|
13567
|
+
// BEFORE the deadline, so by the time we get here the git SIGKILL has landed
|
|
13568
|
+
// (or is in its grace window) and the drain settles on its own. The cap is the
|
|
13569
|
+
// REMAINING budget; the floor is the git kill grace (not 1ms) so that even when
|
|
13570
|
+
// the recover race consumed the whole window the drain still awaits the actual
|
|
13571
|
+
// SIGKILL of the already-SIGTERMed git instead of abandoning a live process to
|
|
13572
|
+
// have its workspace reaped under it (Copilot review on #296, 1ms finding).
|
|
13573
|
+
const closeFloorMs = Math.max(1, Number(checkpointing?.gitKillGraceMs) || 0);
|
|
13574
|
+
const closeOpts = forceStopping
|
|
13575
|
+
? { timeoutMs: Math.max(closeFloorMs, forceStopDeadline - Date.now()) }
|
|
13576
|
+
: undefined;
|
|
13577
|
+
try { await checkpointing.close(closeOpts); } catch { /* best effort */ }
|
|
11581
13578
|
}
|
|
11582
13579
|
if (isContainer) liveRunIds.delete(runId);
|
|
11583
13580
|
// #205: clear the in-flight job marker now the harness has stopped — the
|
|
@@ -12038,6 +14035,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12038
14035
|
if (finished) return;
|
|
12039
14036
|
finished = true;
|
|
12040
14037
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
14038
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12041
14039
|
if (reaperTimer) clearInterval(reaperTimer);
|
|
12042
14040
|
if (runDirTimer) clearInterval(runDirTimer);
|
|
12043
14041
|
if (agenticPlane) {
|
|
@@ -12068,6 +14066,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
12068
14066
|
aborting = true;
|
|
12069
14067
|
draining = true;
|
|
12070
14068
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
14069
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12071
14070
|
logger.info(`Received ${signal} — aborting in-flight work and stopping worker...`);
|
|
12072
14071
|
// Snapshot in-flight jobs (with their retry budget) BEFORE the interrupt
|
|
12073
14072
|
// clears the ownership registry, so we can yield each one afterwards.
|
|
@@ -12113,6 +14112,11 @@ async function workAgent(req, flags, ctx) {
|
|
|
12113
14112
|
if (!autoMode) { try { unwatchFile(configFile); } catch { /* best effort */ } }
|
|
12114
14113
|
const n0 = inFlightCount();
|
|
12115
14114
|
logger.info(`Received ${signal} — draining: polling stopped; waiting on ${n0} in-flight job(s) to finish (send SIGTERM to abort).`);
|
|
14115
|
+
// The refresh timer is ref'd (so it fires on cadence even under a busy
|
|
14116
|
+
// loop) and there may be NO in-flight job during a drain — clear it now
|
|
14117
|
+
// so it can't keep the process alive once the drain completes and the
|
|
14118
|
+
// runtime is interrupted.
|
|
14119
|
+
if (ghRefreshTimer) clearInterval(ghRefreshTimer);
|
|
12116
14120
|
// Poll until every in-flight job has settled and released its slot, then
|
|
12117
14121
|
// interrupt the now-idle runtime and exit. Wait INDEFINITELY — a force
|
|
12118
14122
|
// abort (SIGTERM) is the only escape hatch.
|
|
@@ -12508,8 +14512,9 @@ function formatDuration(ms) {
|
|
|
12508
14512
|
return `${d}d${h % 24}h`;
|
|
12509
14513
|
}
|
|
12510
14514
|
|
|
12511
|
-
/** Project a live/stored worker record to a status row (pure w.r.t. `now`).
|
|
12512
|
-
|
|
14515
|
+
/** Project a live/stored worker record to a status row (pure w.r.t. `now`).
|
|
14516
|
+
* `readActivity` is an injectable seam over readWorkerActivity (testing). */
|
|
14517
|
+
function summarizeSupervisorWorker(w, now = Date.now(), { readActivity = readWorkerActivity } = {}) {
|
|
12513
14518
|
const alive = isPidAlive(w.pid);
|
|
12514
14519
|
// Absolute base epoch (ms) the worker started, so an attached console can
|
|
12515
14520
|
// re-age `uptimeMs` locally on each tick (see reageSupervisorStatus) without
|
|
@@ -12523,8 +14528,9 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12523
14528
|
let activity = null; // { state: 'busy'|'idle', jobs: [{ key, type, sinceMs, sinceEpochMs }] }
|
|
12524
14529
|
let engine = null; // job-polling engine base URL this worker reported (#99)
|
|
12525
14530
|
let agentic = null; // { status, mode, url, discovered } agentic-channel state (#99)
|
|
14531
|
+
let ghAuth = null; // #293: { present, fingerprint, scopes, since, ageMs, lastCheckAgoMs } — never the token
|
|
12526
14532
|
if (alive) {
|
|
12527
|
-
const act =
|
|
14533
|
+
const act = readActivity(w.id);
|
|
12528
14534
|
if (act && act.pid === w.pid) {
|
|
12529
14535
|
const jobs = Array.isArray(act.jobs)
|
|
12530
14536
|
? act.jobs
|
|
@@ -12556,6 +14562,24 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12556
14562
|
// stale incarnation can't show a dead worker as connected to a hub.
|
|
12557
14563
|
engine = typeof act.engine === 'string' && act.engine ? act.engine : null;
|
|
12558
14564
|
agentic = act.agentic && typeof act.agentic === 'object' ? act.agentic : null;
|
|
14565
|
+
// #293: the gh token identity rides the same marker. Re-age the ticking
|
|
14566
|
+
// durations from the stamped `since`/marker `updatedAt` so a console that
|
|
14567
|
+
// re-ages locally (and a status snapshot read seconds later) shows a
|
|
14568
|
+
// current age, not the write-time one. Never carries the token value.
|
|
14569
|
+
if (act.ghAuth && typeof act.ghAuth === 'object') {
|
|
14570
|
+
const g = act.ghAuth;
|
|
14571
|
+
const ageMs = Number.isFinite(g.since) ? Math.max(0, now - g.since) : (Number.isFinite(g.ageMs) ? g.ageMs : null);
|
|
14572
|
+
const lastCheckAgoMs = Number.isFinite(act.updatedAt) && Number.isFinite(g.lastCheckAgoMs)
|
|
14573
|
+
? Math.max(0, now - act.updatedAt + g.lastCheckAgoMs)
|
|
14574
|
+
: (Number.isFinite(g.lastCheckAgoMs) ? g.lastCheckAgoMs : null);
|
|
14575
|
+
// Coerce the identity strings at the (untrusted) marker boundary: a
|
|
14576
|
+
// malformed/tampered/older marker could carry a non-string `scopes` or
|
|
14577
|
+
// `fingerprint` (a number, array, object), and a downstream string op
|
|
14578
|
+
// (e.g. `scopes.split(',')` in supervisorGhAuthCell) would THROW and
|
|
14579
|
+
// crash `supervisor status` (Copilot review on #294). Keep only real
|
|
14580
|
+
// strings; anything else degrades to null (rendered as `?`/unknown).
|
|
14581
|
+
ghAuth = { present: g.present === true, fingerprint: typeof g.fingerprint === 'string' ? g.fingerprint : null, scopes: typeof g.scopes === 'string' ? g.scopes : null, since: Number.isFinite(g.since) ? g.since : null, ageMs, lastCheckAgoMs };
|
|
14582
|
+
}
|
|
12559
14583
|
}
|
|
12560
14584
|
// No marker (or a stale-pid one): leave activity null → rendered as unknown.
|
|
12561
14585
|
}
|
|
@@ -12580,6 +14604,7 @@ function summarizeSupervisorWorker(w, now = Date.now()) {
|
|
|
12580
14604
|
activity,
|
|
12581
14605
|
engine,
|
|
12582
14606
|
agentic,
|
|
14607
|
+
ghAuth,
|
|
12583
14608
|
};
|
|
12584
14609
|
}
|
|
12585
14610
|
|
|
@@ -12614,6 +14639,16 @@ function supervisorStatusSignature(workers) {
|
|
|
12614
14639
|
// change is a real transition that must repaint attached consoles (#99).
|
|
12615
14640
|
w.engine ?? '',
|
|
12616
14641
|
w.agentic ? (w.agentic.status ?? '') : null,
|
|
14642
|
+
// #293: the gh credential IDENTITY — presence, fingerprint, scopes, and the
|
|
14643
|
+
// token's `since` epoch. A token rotation / scope change / logout on an
|
|
14644
|
+
// otherwise-idle worker is a real transition the attached console must
|
|
14645
|
+
// repaint for (the GH AUTH cell changes), so fold it into the signature.
|
|
14646
|
+
// Deliberately EXCLUDES the ticking durations (ageMs, lastCheckAgoMs) —
|
|
14647
|
+
// those merely elapse and are re-aged locally (see reageSupervisorStatus),
|
|
14648
|
+
// so including them would make every clock tick a spurious "change".
|
|
14649
|
+
w.ghAuth
|
|
14650
|
+
? `${w.ghAuth.present ? 1 : 0}\u0000${w.ghAuth.fingerprint ?? ''}\u0000${w.ghAuth.scopes ?? ''}\u0000${w.ghAuth.since ?? ''}`
|
|
14651
|
+
: null,
|
|
12617
14652
|
]),
|
|
12618
14653
|
);
|
|
12619
14654
|
}
|
|
@@ -12676,6 +14711,32 @@ function supervisorAgenticCell(w) {
|
|
|
12676
14711
|
return String(w.agentic.status);
|
|
12677
14712
|
}
|
|
12678
14713
|
|
|
14714
|
+
/**
|
|
14715
|
+
* GH AUTH cell (#293): the cached gh token's age + scopes, so "this worker is
|
|
14716
|
+
* holding a stale token" is a one-line check instead of a manual
|
|
14717
|
+
* /proc/<pid>/environ vs `gh auth token` comparison. `-` for a down/stopping
|
|
14718
|
+
* worker, `?` for a live worker not (yet) reporting or on an older build whose
|
|
14719
|
+
* marker predates the field, `none` when gh is missing/not logged in. The
|
|
14720
|
+
* fingerprint is available in the status JSON (and the worker log) when two
|
|
14721
|
+
* workers need to be told apart; the table keeps the cell narrow (age + scopes).
|
|
14722
|
+
*/
|
|
14723
|
+
function supervisorGhAuthCell(w) {
|
|
14724
|
+
if (w.state !== 'running') return '-';
|
|
14725
|
+
if (!w.activity) return '?'; // alive but not reporting
|
|
14726
|
+
const g = w.ghAuth;
|
|
14727
|
+
if (!g) return '?'; // marker predates the ghAuth field
|
|
14728
|
+
if (!g.present) return 'none';
|
|
14729
|
+
const age = g.ageMs != null ? formatDuration(g.ageMs) : '?';
|
|
14730
|
+
// Defensive against a malformed activity marker (untrusted JSON): only call
|
|
14731
|
+
// string ops on a real string, so a non-string `scopes` (a corrupt/older/
|
|
14732
|
+
// tampered marker) renders `?` instead of throwing and crashing the whole
|
|
14733
|
+
// status table (Copilot review on #294). statusFromActivity already coerces at
|
|
14734
|
+
// the marker boundary; this is the belt-and-braces at the render edge, which
|
|
14735
|
+
// also runs on frames built elsewhere (e.g. reageSupervisorStatus passthrough).
|
|
14736
|
+
const scopes = typeof g.scopes === 'string' && g.scopes ? g.scopes.split(',').map((s) => s.trim()).filter(Boolean).join(',') : '?';
|
|
14737
|
+
return `${age} [${scopes}]`;
|
|
14738
|
+
}
|
|
14739
|
+
|
|
12679
14740
|
/**
|
|
12680
14741
|
* Re-age a supervisor status snapshot to `now`, recomputing the ticking
|
|
12681
14742
|
* durations (`uptimeMs`, per-job `sinceMs`) from the absolute base epochs the
|
|
@@ -12718,7 +14779,17 @@ function reageSupervisorStatus(status, now = Date.now()) {
|
|
|
12718
14779
|
const state = jobs.some((j) => j && typeof j === 'object' && !j.settlementPending) ? 'busy' : 'idle';
|
|
12719
14780
|
activity = { ...activity, jobs, state };
|
|
12720
14781
|
}
|
|
12721
|
-
|
|
14782
|
+
// #293: re-age the gh credential's AGE from its absolute `since` epoch, the
|
|
14783
|
+
// same way uptime/job-age tick locally — otherwise an attached console that
|
|
14784
|
+
// re-ages the cached snapshot on every tick (never rebuilding it via
|
|
14785
|
+
// summarizeSupervisorWorker) would freeze the GH AUTH cell's age at its
|
|
14786
|
+
// snapshot-time value. `lastCheckAgoMs` is RELATIVE (no absolute base rides
|
|
14787
|
+
// the frame), so it cannot be re-aged here and is left as-is.
|
|
14788
|
+
let ghAuth = w.ghAuth;
|
|
14789
|
+
if (ghAuth && typeof ghAuth === 'object' && Number.isFinite(ghAuth.since)) {
|
|
14790
|
+
ghAuth = { ...ghAuth, ageMs: Math.max(0, now - ghAuth.since) };
|
|
14791
|
+
}
|
|
14792
|
+
return { ...w, uptimeMs, activity, ghAuth };
|
|
12722
14793
|
}),
|
|
12723
14794
|
};
|
|
12724
14795
|
}
|
|
@@ -12878,6 +14949,7 @@ function formatSupervisorStatus(status) {
|
|
|
12878
14949
|
state: String(w.state) === 'down' && w.configFailed === true ? 'down (config)' : String(w.state),
|
|
12879
14950
|
engine: supervisorEngineCell(w),
|
|
12880
14951
|
agentic: supervisorAgenticCell(w),
|
|
14952
|
+
ghAuth: supervisorGhAuthCell(w),
|
|
12881
14953
|
job: supervisorJobCell(w),
|
|
12882
14954
|
pid: w.pid ? String(w.pid) : '-',
|
|
12883
14955
|
restarts: String(w.restarts),
|
|
@@ -12887,8 +14959,8 @@ function formatSupervisorStatus(status) {
|
|
|
12887
14959
|
// ENGINE + AGENTIC sit early (just after STATE) so the pinned live view's
|
|
12888
14960
|
// width clamp (which trims from the right) drops the least-critical columns
|
|
12889
14961
|
// (LAST EXIT, UPTIME) first and keeps the visibility diagnostics visible.
|
|
12890
|
-
const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
|
|
12891
|
-
const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'job', 'pid', 'restarts', 'uptime', 'last'];
|
|
14962
|
+
const head = { id: 'ID', profile: 'PROFILE', state: 'STATE', engine: 'ENGINE', agentic: 'AGENTIC', ghAuth: 'GH AUTH', job: 'JOB', pid: 'PID', restarts: 'RESTARTS', uptime: 'UPTIME', last: 'LAST EXIT' };
|
|
14963
|
+
const cols = ['id', 'profile', 'state', 'engine', 'agentic', 'ghAuth', 'job', 'pid', 'restarts', 'uptime', 'last'];
|
|
12892
14964
|
const width = {};
|
|
12893
14965
|
for (const c of cols) width[c] = Math.max(head[c].length, ...rows.map((r) => r[c].length));
|
|
12894
14966
|
const fmt = (r) => ' ' + cols.map((c) => r[c].padEnd(width[c])).join(' ');
|
|
@@ -13353,7 +15425,12 @@ async function runSupervisorDaemon() {
|
|
|
13353
15425
|
const pid = w.pid;
|
|
13354
15426
|
if (w.child && pid) {
|
|
13355
15427
|
try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
|
|
13356
|
-
|
|
15428
|
+
// #296: wait STOP_GRACE_MS for a routine exit PLUS the bounded recovery budget,
|
|
15429
|
+
// so a worker that is flushing + promoting a durable recovery ref (#295) for an
|
|
15430
|
+
// interrupted agent job can finish before the SIGKILL backstop. A worker that
|
|
15431
|
+
// exits sooner is reaped at once — this only lengthens the ceiling for one that
|
|
15432
|
+
// is actively preserving recovery.
|
|
15433
|
+
await waitForChildExit(w.child, STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS);
|
|
13357
15434
|
if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
|
|
13358
15435
|
}
|
|
13359
15436
|
};
|
|
@@ -13800,6 +15877,40 @@ async function runSupervisorDaemon() {
|
|
|
13800
15877
|
attachClients.add(sock);
|
|
13801
15878
|
sock.write(encodeFrame(statusFrame(false)));
|
|
13802
15879
|
break;
|
|
15880
|
+
case 'reload-credentials': {
|
|
15881
|
+
// #293: tell every worker to re-read its gh auth token (SIGUSR1,
|
|
15882
|
+
// async — the still-valid old token is served until the new one
|
|
15883
|
+
// atomically lands) WITHOUT draining in-flight jobs — the operator
|
|
15884
|
+
// re-logged in and wants
|
|
15885
|
+
// the fleet to pick up the fresh credential now, not after a restart.
|
|
15886
|
+
// Windows cannot deliver SIGUSR1 (Node terminates the process), so
|
|
15887
|
+
// refuse there; the TTL refresh still adopts a rotated token.
|
|
15888
|
+
if (osPlatform() === 'win32') {
|
|
15889
|
+
sock.write(encodeFrame({ ok: false, error: 'reload-credentials is not supported on Windows (it relies on SIGUSR1, which Windows cannot deliver) — the workers\' TTL refresh still picks up a new token, or use `supervisor restart`', final: true }));
|
|
15890
|
+
break;
|
|
15891
|
+
}
|
|
15892
|
+
const rawCreds = Array.isArray(req.targets)
|
|
15893
|
+
? req.targets.flatMap((t) => resolveTargets(t))
|
|
15894
|
+
: resolveTargets(req.target);
|
|
15895
|
+
const credIds = [...new Set(rawCreds)];
|
|
15896
|
+
const signaled = [];
|
|
15897
|
+
const failed = [];
|
|
15898
|
+
for (const id of credIds) {
|
|
15899
|
+
const w = workers.get(id);
|
|
15900
|
+
const alive = !!(w && isPidAlive(w.pid));
|
|
15901
|
+
// #293: only signal a worker whose SIGUSR1 (reload) listener is
|
|
15902
|
+
// PROVEN installed (reloadCredentialsReady) — otherwise the signal
|
|
15903
|
+
// hits Node's default action (start the inspector) and the daemon
|
|
15904
|
+
// would wrongly record "reloaded". Skipped workers land in `failed`
|
|
15905
|
+
// so `ok` stays honest; they pick up the fresh token at their own
|
|
15906
|
+
// startup prime.
|
|
15907
|
+
if (!reloadCredentialsReady(w, alive ? readWorkerActivity(id) : null, alive)) { failed.push(id); continue; }
|
|
15908
|
+
try { process.kill(w.pid, 'SIGUSR1'); signaled.push(id); }
|
|
15909
|
+
catch { failed.push(id); }
|
|
15910
|
+
}
|
|
15911
|
+
sock.write(encodeFrame({ ok: failed.length === 0, type: 'credentials-reloaded', signaled, failed, final: true }));
|
|
15912
|
+
break;
|
|
15913
|
+
}
|
|
13803
15914
|
case 'stop': {
|
|
13804
15915
|
// #202: default is a GRACEFUL DRAIN — quiesce workers, let in-flight
|
|
13805
15916
|
// jobs finish, exit when idle. `req.force` hard-aborts (kill harness,
|
|
@@ -14305,6 +16416,45 @@ async function supervisorReloadCmd(req) {
|
|
|
14305
16416
|
if (outcome === 'error') process.exit(1);
|
|
14306
16417
|
}
|
|
14307
16418
|
|
|
16419
|
+
/**
|
|
16420
|
+
* Re-read every supervised worker's cached gh auth token WITHOUT draining
|
|
16421
|
+
* in-flight jobs (issue #293). The daemon signals each worker (SIGUSR1); the
|
|
16422
|
+
* worker re-reads `gh auth token` NON-DESTRUCTIVELY (a forced async re-read that
|
|
16423
|
+
* swaps the cache atomically when the new value lands — the still-valid old
|
|
16424
|
+
* token is served meanwhile, never dropped), and logs the new fingerprint/
|
|
16425
|
+
* scopes. Use after a `gh auth login`/`refresh` so the fleet picks up the fresh
|
|
16426
|
+
* credential immediately instead of waiting for the TTL refresh or a restart.
|
|
16427
|
+
*/
|
|
16428
|
+
async function supervisorReloadCredentialsCmd(req) {
|
|
16429
|
+
const logger = getLogger();
|
|
16430
|
+
const target = req.positional[1] || 'all';
|
|
16431
|
+
const running = await liveSupervisor();
|
|
16432
|
+
if (!running) { logger.error('Supervisor is not running.'); process.exit(1); }
|
|
16433
|
+
const socketPath = running.socket || getSupervisorSocketPath();
|
|
16434
|
+
let res;
|
|
16435
|
+
try {
|
|
16436
|
+
res = await supervisorRequest({ op: 'reload-credentials', target }, { socketPath });
|
|
16437
|
+
} catch (err) {
|
|
16438
|
+
logger.error(`Could not reach the supervisor control socket to reload credentials: ${err?.message || err}`);
|
|
16439
|
+
process.exit(1);
|
|
16440
|
+
}
|
|
16441
|
+
// A structured credentials-reloaded result carries `signaled`/`failed` arrays
|
|
16442
|
+
// even on a PARTIAL failure (daemon returns `ok:false` with no `error` when
|
|
16443
|
+
// some workers were down). Report those lists BEFORE the generic failure guard
|
|
16444
|
+
// — exactly as the rolling-reload client does — so an operator can see which
|
|
16445
|
+
// workers picked up the new token and which are down, instead of a bare
|
|
16446
|
+
// "reload-credentials failed". The generic guard then only catches a truly
|
|
16447
|
+
// unstructured failure (unreachable socket, the Windows refusal with `error`).
|
|
16448
|
+
if (!res) { logger.error('reload-credentials failed'); process.exit(1); }
|
|
16449
|
+
const hasStructured = Array.isArray(res.signaled) || Array.isArray(res.failed);
|
|
16450
|
+
if (res.ok === false && !hasStructured) { logger.error(res.error || 'reload-credentials failed'); process.exit(1); }
|
|
16451
|
+
const signaled = Array.isArray(res.signaled) ? res.signaled : [];
|
|
16452
|
+
const failed = Array.isArray(res.failed) ? res.failed : [];
|
|
16453
|
+
if (signaled.length > 0) logger.info(`Told ${signaled.length} worker(s) to re-read their gh auth token: ${signaled.join(', ')}. Each re-reads \`gh auth token\` (non-destructively — the still-valid old token is served until the new one lands) and logs the new fingerprint/scopes.`);
|
|
16454
|
+
else logger.warn('No running workers matched — nothing to reload.');
|
|
16455
|
+
if (failed.length > 0) { logger.error(`Could not signal: ${failed.join(', ')} (down or gone).`); process.exit(1); }
|
|
16456
|
+
}
|
|
16457
|
+
|
|
14308
16458
|
/**
|
|
14309
16459
|
* Count the in-flight jobs across a fleet snapshot (array of
|
|
14310
16460
|
* `summarizeSupervisorWorker` results) — the number an operator is waiting on
|
|
@@ -14411,9 +16561,11 @@ async function supervisorStopCmd(force = false) {
|
|
|
14411
16561
|
|
|
14412
16562
|
// Wait for the daemon process itself to exit. A drain waits INDEFINITELY (the
|
|
14413
16563
|
// operator opted to wait); a force stop applies the grace window + a SIGKILL
|
|
14414
|
-
// backstop so a wedged daemon/worker group can't linger.
|
|
16564
|
+
// backstop so a wedged daemon/worker group can't linger. The window must cover
|
|
16565
|
+
// the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
|
|
16566
|
+
// group while a worker is still promoting its durable recovery ref (#295).
|
|
14415
16567
|
if (force) {
|
|
14416
|
-
const deadline = Date.now() +
|
|
16568
|
+
const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
|
|
14417
16569
|
while (Date.now() < deadline) {
|
|
14418
16570
|
if (!isPidAlive(daemonPid)) break;
|
|
14419
16571
|
await new Promise((r) => setTimeout(r, 150));
|
|
@@ -14515,6 +16667,10 @@ async function attachSupervisorConsole(state) {
|
|
|
14515
16667
|
out(`• removed: ${frame.removed.join(', ') || '(none matched)'}.`);
|
|
14516
16668
|
} else if (frame.type === 'restarted') {
|
|
14517
16669
|
out(`• restarted: ${frame.restarted.join(', ') || '(none matched)'}.`);
|
|
16670
|
+
} else if (frame.type === 'credentials-reloaded') {
|
|
16671
|
+
const sig = Array.isArray(frame.signaled) ? frame.signaled : [];
|
|
16672
|
+
const fail = Array.isArray(frame.failed) ? frame.failed : [];
|
|
16673
|
+
out(`• credentials reload signaled: ${sig.join(', ') || '(none matched)'}${fail.length ? ` — could not signal: ${fail.join(', ')}` : ''}.`);
|
|
14518
16674
|
} else if (frame.ok === false) {
|
|
14519
16675
|
out(`! ${frame.error}`);
|
|
14520
16676
|
}
|
|
@@ -14560,10 +16716,15 @@ async function attachSupervisorConsole(state) {
|
|
|
14560
16716
|
case '': break;
|
|
14561
16717
|
case 'help':
|
|
14562
16718
|
out('Commands: status | add <profile> [--name <worker>] [--instances <n>] [work flags] |');
|
|
14563
|
-
out(' remove <id|profile|all> | restart <id|profile|all> |');
|
|
16719
|
+
out(' remove <id|profile|all> | restart <id|profile|all> | reload-credentials [id|profile|all] |');
|
|
14564
16720
|
out(' logs [id] | detach | stop | help');
|
|
14565
16721
|
break;
|
|
14566
16722
|
case 'status': sock.write(encodeFrame({ op: 'status' })); break;
|
|
16723
|
+
case 'reload-credentials': {
|
|
16724
|
+
const target = parts.shift() || 'all';
|
|
16725
|
+
sock.write(encodeFrame({ op: 'reload-credentials', target }));
|
|
16726
|
+
break;
|
|
16727
|
+
}
|
|
14567
16728
|
case 'add': {
|
|
14568
16729
|
const profile = parts.shift();
|
|
14569
16730
|
if (!profile) { out('usage: add <profile> [--name <worker>] [--instances <n>] [work flags]'); break; }
|
|
@@ -14882,7 +17043,9 @@ function systemdUserAvailable() {
|
|
|
14882
17043
|
async function stopSupervisorProcess(running) {
|
|
14883
17044
|
try { await supervisorRequest({ op: 'stop' }); }
|
|
14884
17045
|
catch { try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ } }
|
|
14885
|
-
|
|
17046
|
+
// #296: cover the daemon's own worker force-stop grace so the group SIGKILL below
|
|
17047
|
+
// does not reap a worker mid-recovery (promoting its durable recovery ref, #295).
|
|
17048
|
+
const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
|
|
14886
17049
|
while (Date.now() < deadline) {
|
|
14887
17050
|
if (!isPidAlive(running.pid)) break;
|
|
14888
17051
|
await new Promise((r) => setTimeout(r, 150));
|
|
@@ -15109,6 +17272,9 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
15109
17272
|
case 'reload':
|
|
15110
17273
|
await supervisorReloadCmd(req);
|
|
15111
17274
|
return;
|
|
17275
|
+
case 'reload-credentials':
|
|
17276
|
+
await supervisorReloadCredentialsCmd(req);
|
|
17277
|
+
return;
|
|
15112
17278
|
case 'stop':
|
|
15113
17279
|
await supervisorStopCmd(coerceBool(flags?.force, false));
|
|
15114
17280
|
return;
|
|
@@ -15117,7 +17283,7 @@ async function supervisorCommand(req, flags, ctx) {
|
|
|
15117
17283
|
supervisorLogsCmd(req);
|
|
15118
17284
|
return;
|
|
15119
17285
|
default:
|
|
15120
|
-
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach`);
|
|
17286
|
+
getLogger().error(`Unknown supervisor action "${action}". Use: start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach`);
|
|
15121
17287
|
process.exit(1);
|
|
15122
17288
|
}
|
|
15123
17289
|
}
|
|
@@ -17922,8 +20088,30 @@ export {
|
|
|
17922
20088
|
namespaceHasLiveHarness,
|
|
17923
20089
|
authUrl,
|
|
17924
20090
|
githubCloneToken,
|
|
20091
|
+
githubCloneTokenWithSource,
|
|
17925
20092
|
ghAuthTokenFromCli,
|
|
17926
20093
|
primeGhAuthToken,
|
|
20094
|
+
refreshGhAuthToken,
|
|
20095
|
+
refreshGhAuthTokenAsync,
|
|
20096
|
+
invalidateGhAuthToken,
|
|
20097
|
+
recoverGhTokenAfterAuthFailure,
|
|
20098
|
+
setGhAuthTokenReader,
|
|
20099
|
+
resetGhAuthTokenReader,
|
|
20100
|
+
classifyGhAuthRead,
|
|
20101
|
+
GH_AUTH_READ_FAILED,
|
|
20102
|
+
adoptGhAuthToken,
|
|
20103
|
+
adoptGhAuthTokenAsync,
|
|
20104
|
+
readGhAuthTokenAsync,
|
|
20105
|
+
ghCredentialEnvName,
|
|
20106
|
+
applyRotatedGhCredential,
|
|
20107
|
+
ghAuthTokenInfo,
|
|
20108
|
+
describeGhAuthToken,
|
|
20109
|
+
ghTokenFingerprint,
|
|
20110
|
+
ghTokenScopes,
|
|
20111
|
+
ghTokenScopesAsync,
|
|
20112
|
+
isGitAuthFailure,
|
|
20113
|
+
startGhAuthTokenRefresh,
|
|
20114
|
+
GH_AUTH_TOKEN_TTL_MS,
|
|
17927
20115
|
ghAuthEnv,
|
|
17928
20116
|
redactToken,
|
|
17929
20117
|
agentRunsRoot,
|
|
@@ -18004,10 +20192,12 @@ export {
|
|
|
18004
20192
|
countSupervisorInFlight,
|
|
18005
20193
|
supervisorEngineCell,
|
|
18006
20194
|
supervisorAgenticCell,
|
|
20195
|
+
supervisorGhAuthCell,
|
|
18007
20196
|
agenticStateForTarget,
|
|
18008
20197
|
normalizeAgenticMessage,
|
|
18009
20198
|
buildActivityPayload,
|
|
18010
20199
|
activityMarkerReadyFor,
|
|
20200
|
+
reloadCredentialsReady,
|
|
18011
20201
|
waitForChildExit,
|
|
18012
20202
|
supervisorWorkerActivityFile,
|
|
18013
20203
|
SETTLEMENT_PENDING_GHOST_TTL_MS,
|
|
@@ -18019,6 +20209,7 @@ export {
|
|
|
18019
20209
|
supervisorStartCmd,
|
|
18020
20210
|
supervisorAddCmd,
|
|
18021
20211
|
supervisorReloadCmd,
|
|
20212
|
+
supervisorReloadCredentialsCmd,
|
|
18022
20213
|
workforceReloadCmd,
|
|
18023
20214
|
runningSupervisor,
|
|
18024
20215
|
readSupervisorState,
|
|
@@ -18127,6 +20318,7 @@ export const metadata = {
|
|
|
18127
20318
|
{ command: 'c8ctl nano supervisor add reviewer --instances 3', description: 'Add 3 distinct auto-named instances of a profile in one call' },
|
|
18128
20319
|
{ command: 'c8ctl nano supervisor restart reviewer', description: 'Restart a supervised worker by id or profile' },
|
|
18129
20320
|
{ command: 'c8ctl nano supervisor reload', description: 'Adopt updated harness code with zero downtime: after `nano update`, roll through the fleet draining in-flight jobs and respawning each worker so it re-reads the new plugin (workers only; restart the daemon for new supervisor code)' },
|
|
20321
|
+
{ command: 'c8ctl nano supervisor reload-credentials', description: 'After a `gh auth login`/`refresh`, tell every worker to re-read its gh token from the CLI now (async; the still-valid old token is served until the new one lands; no drain, in-flight jobs keep running) — status shows each worker\'s token age/scopes' },
|
|
18130
20322
|
{ command: 'c8ctl nano supervisor stop', description: 'Stop the supervisor daemon and all its workers' },
|
|
18131
20323
|
{ command: 'c8ctl nano workforce add copilot --instances 5 --auto', description: 'Compose a reusable fleet: 5 copilot workers serving every deployed agent job type (--auto)' },
|
|
18132
20324
|
{ command: 'c8ctl nano workforce add qwen --instances 2 --roles pr-review,feature', description: "Add an entry mapped to explicit job types (<rank>:pr-review, <rank>:feature, where <rank> is the qwen hire's rank at start) — does not mutate the hired profile" },
|
|
@@ -18367,7 +20559,7 @@ function printUsage() {
|
|
|
18367
20559
|
console.log(' c8ctl nano hire [--name <n>] [--rank <r>] [--command <c>] [--arg <switch> ...] [--model <m>] [--capabilities <a,b>] [--sandbox none|docker|podman] [--image <ref>] [--terminal pty|pipe] [--protocol pipe|acp] [--permission yolo|escalate|filter] [--env NAME=VALUE ...] [--list]');
|
|
18368
20560
|
console.log(' c8ctl nano assign <profileName> <cap[,cap...]> [--name <n>] [--capabilities <a,b>]');
|
|
18369
20561
|
console.log(' c8ctl nano work <profileName> [--auto [--auto-scope <p>]] [--arg <switch> ...] [--recovery-window <ms>] [--idle-timeout <ms>] [--job-timeout <ms>] [--poll-timeout <ms>] [--job-type <token> ...] [--sandbox none|docker|podman] [--image <ref>] [--env NAME=VALUE ...] [--secret-resolver host] [--min-free-mb <n>] [--clone-timeout <ms>] [--keep-runs] [--stream]');
|
|
18370
|
-
console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|stop|logs|attach] ... (manage many workers from one terminal)');
|
|
20562
|
+
console.log(' c8ctl nano supervisor [start|install|uninstall|status|add|remove|restart|reload|reload-credentials|stop|logs|attach] ... (manage many workers from one terminal)');
|
|
18371
20563
|
console.log(' c8ctl nano workforce [add|remove|list|start|status|stop|reload] ... [--manifest <manifest>] (declarative, reusable fleet manifests)');
|
|
18372
20564
|
console.log('');
|
|
18373
20565
|
console.log('Subcommands:');
|