c8ctl-plugin-nano 1.70.3 → 1.70.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -90,7 +90,7 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
90
90
  // so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
91
91
  // a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
92
92
  import { resolveEffectiveEnvelope } from './agent-resume.mjs';
93
- import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
93
+ import { checkpointConfig, checkpointEligibility, checkpointRef, recoveredCheckpointRef, predecessorCheckpointRef, resumeRefIsExplicit, pushNamedRef, observeRemoteRefSha, makeOrphanProbe, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
94
94
 
95
95
  const requireFromHere = createRequire(import.meta.url);
96
96
  const pluginDir = dirname(fileURLToPath(import.meta.url));
@@ -260,6 +260,24 @@ const READINESS_TIMEOUT_MS = 60_000;
260
260
  const READINESS_POLL_MS = 500;
261
261
  const HEALTH_TIMEOUT_MS = 1_500;
262
262
  const STOP_GRACE_MS = 8_000;
263
+ // #296: a `stop --force` SIGTERMs a `nano work` child, which (on an interrupted
264
+ // agent job) flushes a last WIP checkpoint and promotes it to a durable recovery
265
+ // ref (#295) before exiting. That flush+promote must fit inside the window the
266
+ // supervisor allows before its SIGKILL backstop, or the recovery ref is never
267
+ // written. This is the bounded budget the worker gives that force-stop recovery,
268
+ // and the EXTRA grace `forceStopWorker` waits beyond STOP_GRACE_MS before SIGKILL
269
+ // (a child that exits sooner is reaped immediately — this only lengthens the
270
+ // SIGKILL backstop for a worker that is actively preserving recovery).
271
+ const FORCE_STOP_RECOVERY_MS = 20_000;
272
+ // #296: the backstop deadline an OUTER daemon-stop path (the socket-unreachable
273
+ // `stop --force` fallback and `stopSupervisorProcess`) waits for the DAEMON — and
274
+ // its whole process group, which includes the workers — to exit before SIGKILLing
275
+ // the group. It must exceed what the daemon itself allows a worker before ITS
276
+ // SIGKILL backstop (`forceStopWorker` waits STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS),
277
+ // plus a small margin for the daemon to observe the worker's exit and shut down.
278
+ // Otherwise the outer path SIGKILLs the group ~18s early and kills a worker that is
279
+ // still flushing + promoting its durable recovery ref (#295).
280
+ const FORCE_STOP_DAEMON_GRACE_MS = STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS + 2_000;
263
281
  // #202: the signal a graceful `supervisor stop`/`workforce stop` sends each
264
282
  // `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
265
283
  // then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
@@ -2877,6 +2895,19 @@ const RESERVED_RESULT_KEYS = new Set([
2877
2895
  // push-checkpoint / points world-restore at an arbitrary SHA, so reserve it
2878
2896
  // alongside the other host-owned git-result keys (thread 4117276817).
2879
2897
  'worldMarker',
2898
+ // `resumeFromCheckpoint` is the host/process-owned resume pointer (issue #295):
2899
+ // the plugin writes it at the PROCESS scope so a re-entered (new element
2900
+ // instance) activation can find the predecessor's WIP checkpoint. When it is a
2901
+ // bare STRING it is trusted as an EXPLICIT, occurrence-authoritative ref that
2902
+ // BYPASSES both the elementId correlation AND the ambiguity guard
2903
+ // (resumeRefIsExplicit). Because a successful agent result is merged into the
2904
+ // process variables, an unreserved `resumeFromCheckpoint` lets an untrusted
2905
+ // harness emit a bare-string ref that a later activation then trusts as
2906
+ // occurrence-authoritative — restoring an ARBITRARY WIP ref from the repository
2907
+ // with no correlation or guard. Reserve it so the agent result can never inject
2908
+ // or shadow the host/process-owned pointer (Copilot review on #296, thread
2909
+ // 4243776734).
2910
+ 'resumeFromCheckpoint',
2880
2911
  ]);
2881
2912
 
2882
2913
  // Parse `text` as a JSON object, returning it only when it is a plain object.
@@ -3138,18 +3169,49 @@ const NUDGE_CONTEXT_CAP_CHARS = 24_000;
3138
3169
 
3139
3170
  // The bespoke prompt for the re-emit turn: derive the status from the work the
3140
3171
  // agent already did, write ONLY the result, change nothing else.
3141
- function buildResultNudgePrompt(priorStdout, { hasResultFile = true } = {}) {
3172
+ // ACP `StopReason` values (#301). Only `end_turn` means the agent finished its
3173
+ // turn on its own; anything else (`max_turn_requests`, `max_tokens`, `refusal`,
3174
+ // `cancelled`, …) means the turn was CUT OFF, so the work may be incomplete.
3175
+ export function acpStopReasonFrom(promptResult) {
3176
+ const r = promptResult?.stopReason;
3177
+ return typeof r === 'string' && r.trim() ? r.trim().slice(0, 64) : null;
3178
+ }
3179
+ export function isTruncatedStopReason(stopReason) {
3180
+ return typeof stopReason === 'string' && stopReason !== '' && stopReason !== 'end_turn';
3181
+ }
3182
+
3183
+ function buildResultNudgePrompt(priorStdout, { hasResultFile = true, stopReason = null } = {}) {
3142
3184
  const ctx = typeof priorStdout === 'string' ? priorStdout.slice(-NUDGE_CONTEXT_CAP_CHARS) : '';
3143
- const intro = [
3144
- 'You already completed the task in your previous turn, but you did NOT emit a',
3145
- 'machine-readable result, so the orchestrator cannot read your status and the',
3146
- 'run cannot advance.',
3147
- '',
3148
- 'Do NOT redo the work, re-run tools, edit files, push, or open/modify a PR. Just',
3149
- 'emit the result for the work you already did: a single flat JSON object of your',
3150
- 'result variables (at minimum {"status":"..."}).',
3151
- '',
3152
- ];
3185
+ // #301: never ASSERT completion. A budget-cut turn gets a bounded continuation
3186
+ // that asks the agent to preserve its work and report honestly; a clean
3187
+ // end_turn gets a softened re-emit that still allows `blocked`.
3188
+ const intro = isTruncatedStopReason(stopReason)
3189
+ ? [
3190
+ `Your previous turn was CUT OFF before you finished (stop reason: ${stopReason}).`,
3191
+ 'You did NOT emit a machine-readable result, and the task may be INCOMPLETE.',
3192
+ '',
3193
+ 'Do not start new work. If you have uncommitted changes that belong to the task,',
3194
+ 'commit them LOCALLY (`git commit -s`) so they are not lost — do NOT push:',
3195
+ 'this turn runs without the task\'s branch/push policy, and the harness pushes',
3196
+ 'for you afterwards when the task allows it (a push here could bypass that',
3197
+ 'policy and publish to the wrong branch).',
3198
+ 'Then report your TRUE status as a single flat JSON object of result variables.',
3199
+ 'Use "blocked" (with a "summary" and a "question" explaining what remains) unless',
3200
+ 'you actually delivered the full task (e.g. the PR is open). Never claim',
3201
+ '"completed" for work that was not committed and delivered.',
3202
+ '',
3203
+ ]
3204
+ : [
3205
+ 'Your previous turn ended, but you did NOT emit a machine-readable result, so',
3206
+ 'the orchestrator cannot read your status and the run cannot advance.',
3207
+ '',
3208
+ 'Do NOT redo the work, re-run tools, edit files, push, or open/modify a PR. Just',
3209
+ 'report your TRUE status for the work so far: a single flat JSON object of your',
3210
+ 'result variables (at minimum {"status":"..."}). If the task is not actually',
3211
+ 'finished, report "blocked" with a "summary" and a "question" — do not claim',
3212
+ 'completion for work you did not deliver.',
3213
+ '',
3214
+ ];
3153
3215
  // When the harness could not create the result file, AGENT_RESULT_FILE is NOT
3154
3216
  // exported (runAgentJob only sets it for a truthy resultFile), so the usual
3155
3217
  // "write to $AGENT_RESULT_FILE" instruction is impossible. Lead with the stdout
@@ -3198,6 +3260,49 @@ function capStdoutTail(s) {
3198
3260
  return { text: buf.subarray(start).toString('utf8'), truncated: true };
3199
3261
  }
3200
3262
 
3263
+ // #301: statuses that already report an honest non-success; left untouched.
3264
+ // `needs_input` belongs here too: it is a supported non-success result that
3265
+ // carries the actual `question` a human must answer. Rewriting it to `blocked`
3266
+ // would replace that question with the generic budget-retry prompt and lose the
3267
+ // information needed to unblock the task (Copilot review on #302, "previously
3268
+ // missed" — see agent-workers.test.mjs for needs_input as a supported status).
3269
+ const HONEST_NON_SUCCESS_STATUSES = new Set(['blocked', 'escalated', 'failed', 'incomplete', 'needs_input']);
3270
+ export function isWorkDelivered(git) {
3271
+ if (!git) return false;
3272
+ // A PR only proves delivery while it is OPEN or MERGED. `reconcileAgentPr`
3273
+ // searches `--state all`, so `found: true` also matches a CLOSED, unmerged PR
3274
+ // — e.g. an old, abandoned PR on the same branch. Treating that as delivered
3275
+ // would leave a claimed `completed` result unchanged for work that never
3276
+ // shipped, so a closed-unmerged PR is NOT delivery proof (Copilot review on
3277
+ // #302, discussion_r4464312576). `state` is the gh `pr list` field
3278
+ // (OPEN/CLOSED/MERGED); an absent/unknown state fails closed (not delivered).
3279
+ if (git.pr?.found) {
3280
+ const state = typeof git.pr.state === 'string' ? git.pr.state.trim().toUpperCase() : '';
3281
+ if (state === 'OPEN' || state === 'MERGED') return true;
3282
+ // CLOSED (unmerged) or unknown: fall through to the commit proof below.
3283
+ }
3284
+ return git.pushed === true && Array.isArray(git.commits) && git.commits.length > 0;
3285
+ }
3286
+ // Given the picked raw result of a run whose ACP turn ended with `stopReason`,
3287
+ // return an honest `blocked` result when the turn was cut off (non-`end_turn`)
3288
+ // and the work was NOT verifiably delivered. Otherwise return `raw` unchanged.
3289
+ export function downgradeBudgetStopResult(raw, { stopReason, git } = {}) {
3290
+ if (!isTruncatedStopReason(stopReason)) return raw;
3291
+ if (isWorkDelivered(git)) return raw;
3292
+ const status = isPlainObject(raw) && typeof raw.status === 'string' ? raw.status.trim().toLowerCase() : null;
3293
+ if (status && HONEST_NON_SUCCESS_STATUSES.has(status)) return raw;
3294
+ const prior = isPlainObject(raw) && typeof raw.summary === 'string' && raw.summary.trim() ? ` Agent's own summary: ${raw.summary.trim().slice(0, 2000)}` : '';
3295
+ const msg = `The agent's turn was cut off (stopReason=${stopReason}) before it delivered the task (no PR found, no commits pushed).${prior}`;
3296
+ return {
3297
+ ...(isPlainObject(raw) ? raw : {}),
3298
+ status: 'blocked',
3299
+ summary: msg,
3300
+ question: `Agent run stopped early (${stopReason}) without delivering. Retry with a larger budget, or recover the WIP checkpoint and continue?`,
3301
+ stopReason,
3302
+ claimedStatus: status,
3303
+ };
3304
+ }
3305
+
3201
3306
  async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger, logPrefix = '' }) {
3202
3307
  const stdout0 = result && typeof result.stdout === 'string' ? result.stdout : '';
3203
3308
  // A parsed result only counts as usable if it still carries at least one
@@ -3228,7 +3333,8 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
3228
3333
  }
3229
3334
  let nudge = null;
3230
3335
  let nudgeError = null;
3231
- try { nudge = await rerun(buildResultNudgePrompt(stdout0, { hasResultFile: resultFile != null })); } catch (err) { nudge = null; nudgeError = err; }
3336
+ const stopReason = result?.stopReason ?? null;
3337
+ try { nudge = await rerun(buildResultNudgePrompt(stdout0, { hasResultFile: resultFile != null, stopReason })); } catch (err) { nudge = null; nudgeError = err; }
3232
3338
  const nudgeOut = nudge && typeof nudge.stdout === 'string' ? nudge.stdout : '';
3233
3339
  // Re-apply the per-stream capture cap after concatenation: `result.stdout` is
3234
3340
  // forwarded verbatim into the job vars (`output`) and the audit envelope, so
@@ -3245,13 +3351,22 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
3245
3351
  // no-nudge early return above).
3246
3352
  const truncated = result?.truncated === true || capped.truncated;
3247
3353
  const recovered = hasUsableResult(readAgentResultFile(resultFile) ?? parseResultFromStdout(stdout));
3354
+ // The nudge turn has its OWN ACP `stopReason` (#301, Copilot review on #302,
3355
+ // discussion_r4464312610). It must NOT be discarded: when the first turn ended
3356
+ // cleanly (`end_turn`) but the NUDGE was itself cut off (`max_turn_requests`,
3357
+ // `max_tokens`, …) after writing `status:"completed"`, the delivery check must
3358
+ // still see a truncated reason or it would permit completion without delivery
3359
+ // and the logs/audit would show only the first turn's reason. Prefer the
3360
+ // nudge's reason when it is truncated; otherwise keep the first turn's.
3361
+ const nudgeStopReason = nudge && typeof nudge.stopReason === 'string' && nudge.stopReason.trim() ? nudge.stopReason.trim() : null;
3362
+ const effectiveStopReason = (isTruncatedStopReason(nudgeStopReason) ? nudgeStopReason : null) ?? stopReason;
3248
3363
  if (nudgeError && logger?.warn) {
3249
3364
  logger.warn(`${logPrefix} re-emit nudge rerun threw — ${nudgeError?.message ?? nudgeError}`);
3250
3365
  }
3251
3366
  if (logger?.info) {
3252
3367
  logger.info(recovered
3253
- ? `${logPrefix} no result on the first turn — recovered it via one re-emit nudge`
3254
- : `${logPrefix} no result on the first turn — re-emit nudge did not recover one`);
3368
+ ? `${logPrefix} no result on the first turn${stopReason ? ` (stopReason=${stopReason})` : ''} — recovered it via one ${isTruncatedStopReason(stopReason) ? 'budget-continuation' : 're-emit'} nudge${nudgeStopReason ? ` (nudge stopReason=${nudgeStopReason})` : ''}`
3369
+ : `${logPrefix} no result on the first turn${stopReason ? ` (stopReason=${stopReason})` : ''} — ${isTruncatedStopReason(stopReason) ? 'budget-continuation' : 're-emit'} nudge did not recover one${nudgeStopReason ? ` (nudge stopReason=${nudgeStopReason})` : ''}`);
3255
3370
  }
3256
3371
  // Propagate the rerun turn's OWN validated ACP outcome back to the caller (#263):
3257
3372
  // the nudge is a continuation, so a fresh `_meta.outcome` it reports (e.g. a
@@ -3265,7 +3380,7 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
3265
3380
  // agent work (thread 4118913047).
3266
3381
  const transcriptFloorOnly = result?.acpTranscriptFloorOnly === true
3267
3382
  && (!nudgeOut.trim() || nudge?.acpTranscriptFloorOnly === true);
3268
- return { stdout, nudged: true, truncated, acpOutcome: nudge?.acpOutcome ?? result?.acpOutcome ?? null, transcriptFloorOnly };
3383
+ return { stdout, nudged: true, truncated, acpOutcome: nudge?.acpOutcome ?? result?.acpOutcome ?? null, transcriptFloorOnly, stopReason: effectiveStopReason, nudgeStopReason };
3269
3384
  }
3270
3385
 
3271
3386
  function coerceBool(v, dflt = false) {
@@ -6750,7 +6865,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6750
6865
  ...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
6751
6866
  ...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
6752
6867
  };
6753
- const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
6868
+ // #296 (F9): the checkpoint-path git runner must stay USABLE after the harness
6869
+ // abort. The force-stop RECOVERY path (flush + preserveRecovery, promoting the
6870
+ // durable recovery ref #295) runs only once `result.aborted` is observed — i.e.
6871
+ // AFTER the job's `abortSignal` has already fired. Binding this runner to
6872
+ // `abortSignal` (the round-4 F8 attempt) made `createGitRunner` fail every
6873
+ // recovery command (`[aborted]`), so the abort flush could not push its snapshot
6874
+ // and preserveRecovery could not even observe/create the recovery ref — recovery
6875
+ // was self-defeating. Instead drive it from a DEDICATED recovery-deadline
6876
+ // controller that is NEVER aborted by the harness stop, only when the recovery
6877
+ // budget (FORCE_STOP_RECOVERY_MS) expires. That still delivers F8's goal — an
6878
+ // in-flight push is torn down at the deadline so the chain settles and `close()`
6879
+ // drains inside the supervisor's SIGKILL window — without killing the recovery
6880
+ // before it can write the ref.
6881
+ const recoveryController = new AbortController();
6882
+ const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: recoveryController.signal });
6754
6883
  // The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
6755
6884
  // if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
6756
6885
  // git child is killed and later ops fail fast, so setup unwinds instead of holding
@@ -6761,6 +6890,40 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6761
6890
  // which time the workspace (and its askpass helper) has been reaped.
6762
6891
  const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
6763
6892
 
6893
+ // Bound every checkpoint-path SDK call: a timed-out call rejects; a late settle is
6894
+ // swallowed so it can never surface as an unhandled rejection. Hoisted here (ahead
6895
+ // of the GC block, which reuses them) so the #295 ambiguity guard below is bounded.
6896
+ const CP_SDK_TIMEOUT_MS = cfg.sdkTimeoutMs;
6897
+ const withDeadline = (p, ms, label, signal = null) => {
6898
+ let timer;
6899
+ let onAbort = null;
6900
+ // #296 (review): a bare Promise.race REJECTS on timeout but leaves the SDK
6901
+ // mutation in flight — a timed-out `resumeFromCheckpoint` write could then
6902
+ // complete AFTER a retry already wrote a newer pointer and overwrite it with
6903
+ // the stale ref. The SDK returns a CancelablePromise (toCancelable): cancel it
6904
+ // on timeout so the underlying fetch is aborted and the late settle can never
6905
+ // land. `p.cancel?.()` is a no-op for a plain (non-cancelable) promise, and a
6906
+ // cancelled SDK call rejects with CancelSdkError, which the `.catch(() => {})`
6907
+ // below swallows.
6908
+ // #296 (review r4245146247): the SAME stale-write hazard applies to the job's
6909
+ // ABORT (lease loss), not just the per-call timeout. An optional `signal` lets a
6910
+ // setup-path side-effect write be cancelled the instant the job loses its lease,
6911
+ // so a write left pending after a replacement activation published a newer
6912
+ // pointer is torn down and its late settle can never overwrite that newer value.
6913
+ const race = new Promise((_r, reject) => {
6914
+ timer = setTimeout(() => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} timed out after ${ms}ms`)); }, ms);
6915
+ timer?.unref?.();
6916
+ if (signal) {
6917
+ if (signal.aborted === true) { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); return; }
6918
+ onAbort = () => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); };
6919
+ signal.addEventListener('abort', onAbort, { once: true });
6920
+ }
6921
+ });
6922
+ Promise.resolve(p).catch(() => {});
6923
+ const cleanup = () => { clearTimeout(timer); if (signal && onAbort) { try { signal.removeEventListener('abort', onAbort); } catch { /* best effort */ } } };
6924
+ return Promise.race([Promise.resolve(p).finally(cleanup), race]);
6925
+ };
6926
+
6764
6927
  let prior = null;
6765
6928
  let restored = null;
6766
6929
  let startupFetchFailed = false;
@@ -6770,6 +6933,93 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6770
6933
  startupFetchFailed = true;
6771
6934
  log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
6772
6935
  }
6936
+ // #295: nothing at this run's deterministic ref, but a human may have answered an
6937
+ // SLA escalation and re-entered the task as a NEW element instance. The predecessor
6938
+ // recorded its ref in a process-scoped `resumeFromCheckpoint` variable (survives
6939
+ // re-entry); adopt it under THIS run's key so the restore/ownership/discard paths
6940
+ // below operate unchanged and no GC can reclaim it between activations.
6941
+ if (!prior && !startupFetchFailed) {
6942
+ // Correlate the shared process-scoped slot to THIS task. Copilot review on #296
6943
+ // (r4244212483): a SELF-RECORDED (object) predecessor is correlated only by the
6944
+ // STATIC elementId, which is NOT unique under parallel multi-instance of one
6945
+ // agent-task element (N occurrences share the elementId AND the one process-scoped
6946
+ // resume slot). Counting ACTIVE occurrences cannot establish occurrence identity
6947
+ // (index lag can hide a live sibling; a terminal sibling drops out of the ACTIVE
6948
+ // result yet may have been the slot's last writer), so NO engine query makes a
6949
+ // self-record adoption provably safe. We therefore adopt ONLY an EXPLICIT,
6950
+ // occurrence-authoritative ref — a bare STRING the process delivers naming THIS
6951
+ // occurrence's exact predecessor (nano-workforce#891, option 2). A self-recorded
6952
+ // object is NEVER auto-adopted (fail closed to a clean start); until #891 delivers
6953
+ // the explicit ref, a resume is transcript-only rather than risking a sibling's
6954
+ // state. `predecessorCheckpointRef` still parses/locates the candidate ref; the
6955
+ // explicit check below is what authorizes adopting it.
6956
+ const predRef = predecessorCheckpointRef(job?.variables, ref, job?.elementId ?? null);
6957
+ if (predRef) {
6958
+ const adoptAllowed = resumeRefIsExplicit(job?.variables);
6959
+ if (!adoptAllowed) {
6960
+ log.warn(`not adopting predecessor WIP checkpoint ${predRef} — it is an elementId-correlated self-record, which cannot be proven to name THIS occurrence (parallel multi-instance of one agent-task element shares the elementId and the process-scoped resume slot); starting clean. Resume is transcript-only until the process delivers an explicit occurrence-authoritative ref (nano-workforce#891).`);
6961
+ }
6962
+ let predCp = null;
6963
+ if (adoptAllowed) {
6964
+ try { predCp = await fetchCp({ git: startupGit, ref: predRef }); } catch (err) { log.debug(`could not fetch predecessor WIP checkpoint ${predRef} — ${oneLineLog(err?.message || err)}.`); }
6965
+ }
6966
+ if (predCp?.sha) {
6967
+ // #296 (review): after ANY adoption the process-scoped slot still names the
6968
+ // stale PREDECESSOR ref. If this resumed activation then succeeds WITHOUT
6969
+ // taking another checkpoint (a short run before the safety timer), its adopted
6970
+ // WIP ref is deleted on ack while the slot keeps pointing at the predecessor —
6971
+ // a later activation would re-restore that old snapshot. Advance the slot to
6972
+ // THIS run's key (with the sha we actually adopted) so the pointer tracks the
6973
+ // current owner. Factored into one helper so EVERY adoption branch advances it
6974
+ // (the create-only win AND the lost-race re-read, whose race winner's own
6975
+ // best-effort advance may have failed — this makes the advance idempotent
6976
+ // across the race rather than reliant on the winner). Best-effort; a failure
6977
+ // never blocks setup.
6978
+ const advanceResumePointer = async (adoptedSha) => {
6979
+ if (!adoptedSha || typeof camunda?.createElementInstanceVariables !== 'function' || !job?.processInstanceKey) return;
6980
+ // #296 (review r4245146247): recheck the job signal before this process-scoped
6981
+ // side effect. The startup git work above (fetch/push) may have straddled a
6982
+ // lease loss; once aborted, a replacement activation owns this pointer slot and
6983
+ // may already have published a newer value, so writing our (now stale) ref here
6984
+ // would overwrite it. Skip when aborted, and bind the SDK call to abortSignal so
6985
+ // a lease loss DURING the write cancels it before its late settle can land.
6986
+ if (abortSignal?.aborted === true) return;
6987
+ try {
6988
+ await withDeadline(camunda.createElementInstanceVariables({
6989
+ elementInstanceKey: String(job.processInstanceKey),
6990
+ variables: { resumeFromCheckpoint: { ref, sha: adoptedSha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch: provisioned.workingBranch || null, at: Date.now(), adoptedFrom: predRef } },
6991
+ local: true,
6992
+ }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', abortSignal || undefined);
6993
+ } catch (err) { log.debug(`could not advance the resumeFromCheckpoint pointer to ${ref} — ${oneLineLog(err?.message || err)}.`); }
6994
+ };
6995
+ // Create-only adoption (default expectSha=''): only write our key while it
6996
+ // is ABSENT, so a concurrent same-key retry that already adopted + advanced
6997
+ // the ref is never clobbered (Copilot review on #296).
6998
+ const copied = await pushNamedRef({ git: startupGit, ref, sha: predCp.sha });
6999
+ if (copied?.ok) {
7000
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = { ...predCp, ref }; }
7001
+ if (!prior) prior = { ...predCp, ref };
7002
+ logger.info?.(`${prefix} (${corr}): adopted predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} under this run's key ${ref} (SLA-escalation resume); restoring it below.`);
7003
+ await advanceResumePointer(predCp.sha);
7004
+ } else if (copied?.staleLease) {
7005
+ // A concurrent activation populated this key first — adopt what IS there
7006
+ // rather than force over it; the ref now holds the race winner's state.
7007
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = null; }
7008
+ if (prior) {
7009
+ logger.info?.(`${prefix} (${corr}): a concurrent activation already populated ${ref}; adopting its checkpoint ${prior.sha ? '@' + String(prior.sha).slice(0, 12) : ''} (SLA-escalation resume); restoring it below.`);
7010
+ // Advance the pointer to the re-read sha too: the race winner's own
7011
+ // best-effort advance may have failed, which would leave the slot naming
7012
+ // the bare predecessor and risk a later re-adoption of stale work.
7013
+ await advanceResumePointer(prior.sha);
7014
+ } else {
7015
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but ${ref} was concurrently moved and could not be re-read — the predecessor ref is retained for explicit recovery.`);
7016
+ }
7017
+ } else {
7018
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but could not copy it to ${ref} — ${copied?.error || 'unknown'}; the predecessor ref is retained for explicit recovery.`);
7019
+ }
7020
+ }
7021
+ }
7022
+ }
6773
7023
  if (prior) {
6774
7024
  const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
6775
7025
  if (r?.restored) {
@@ -6797,12 +7047,12 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6797
7047
 
6798
7048
  // Orphan GC for this remote, throttled per worker process, in the background.
6799
7049
  let gc = Promise.resolve();
7050
+ // #296: the GC sweep's abort controller, hoisted so the force-stop recovery
7051
+ // DEADLINE (abortRecovery) can tear the (up to 120s) sweep down too — otherwise
7052
+ // the single shared force-stop budget would bound only the recovery git, and a
7053
+ // still-running GC could hold close()'s drain past the supervisor's SIGKILL window.
7054
+ let gcAbort = null;
6800
7055
  const gcKey = provisioned.remote || provisioned.workspaceDir;
6801
- // Bound every checkpoint-path SDK call: the GC sweep and the variable write are
6802
- // awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
6803
- // call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
6804
- // late settle is swallowed so it can never surface as an unhandled rejection.
6805
- const CP_SDK_TIMEOUT_MS = 10000;
6806
7056
  // A single overall bound on the whole GC sweep. Individual ops are already
6807
7057
  // bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
6808
7058
  // but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
@@ -6813,23 +7063,27 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6813
7063
  // reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
6814
7064
  // 4099558206 / 4099762442).
6815
7065
  const GC_OVERALL_TIMEOUT_MS = 120000;
6816
- const withDeadline = (p, ms, label) => {
6817
- let timer;
6818
- const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
6819
- Promise.resolve(p).catch(() => {});
6820
- return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
6821
- };
6822
7066
  if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
6823
- const isTerminal = typeof camunda?.getElementInstance === 'function'
6824
- ? async (key) => {
6825
- try {
6826
- const ei = await withDeadline(
7067
+ // #295: a nano-managed ref (WIP or durable recovery) is an orphan only when its
7068
+ // element is terminal AND its PROCESS instance is finished too. An element
7069
+ // cancelled by an SLA boundary timer, inside a process that is still active (human
7070
+ // escalation pending), must NOT be reclaimed — the TTL is the final backstop.
7071
+ // `makeOrphanProbe` encodes that; it degrades to the legacy element-only check when
7072
+ // the host SDK exposes no `getProcessInstance`. The sweep applies this to BOTH the
7073
+ // `refs/nano/wip/*` and `refs/nano/recovered/*` namespaces so recovery refs cannot
7074
+ // grow without bound once their escalation is resolved.
7075
+ const isTerminal = makeOrphanProbe({
7076
+ getElementInstance: typeof camunda?.getElementInstance === 'function'
7077
+ ? (key) => withDeadline(
6827
7078
  camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
6828
- CP_SDK_TIMEOUT_MS, 'getElementInstance');
6829
- return ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED';
6830
- } catch { return false; }
6831
- }
6832
- : null;
7079
+ CP_SDK_TIMEOUT_MS, 'getElementInstance')
7080
+ : undefined,
7081
+ getProcessInstance: typeof camunda?.getProcessInstance === 'function'
7082
+ ? (pik) => withDeadline(
7083
+ camunda.getProcessInstance({ processInstanceKey: String(pik) }, { consistency: { waitUpToMs: 0 } }),
7084
+ CP_SDK_TIMEOUT_MS, 'getProcessInstance')
7085
+ : null,
7086
+ });
6833
7087
  // Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
6834
7088
  // job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
6835
7089
  // push-deletes stale ones, while the workspace is concurrently driven by the harness
@@ -6839,6 +7093,7 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6839
7093
  // decouples them; its dir is reaped when the sweep settles. On the overall deadline
6840
7094
  // we abort the sweep's cancellable git so the promise unwinds instead of hanging.
6841
7095
  const sweepAbort = new AbortController();
7096
+ gcAbort = sweepAbort;
6842
7097
  let sweepDir = null;
6843
7098
  try {
6844
7099
  if (!originUrl) throw new Error('workspace had no origin URL');
@@ -6862,10 +7117,10 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6862
7117
  // the sweep's git is still live. The deadline only bounds it by aborting the git.
6863
7118
  gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
6864
7119
  .then((r) => {
6865
- if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned WIP checkpoint ref(s): ${r.deleted.map((d) => `${d.ref} [${d.why}]`).join(', ')}.`);
6866
- else if (sweepAbort.signal.aborted) log.debug(`WIP checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
6867
- else if (r?.error) log.debug(`WIP checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
6868
- }, (err) => log.debug(`WIP checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
7120
+ if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned checkpoint ref(s): ${r.deleted.map((d) => `${d.ref}@${String(d.sha || '') || '?'} [${d.why}]`).join(', ')}.`);
7121
+ else if (sweepAbort.signal.aborted) log.debug(`checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
7122
+ else if (r?.error) log.debug(`checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
7123
+ }, (err) => log.debug(`checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
6869
7124
  .finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
6870
7125
  } catch (err) {
6871
7126
  if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
@@ -6879,17 +7134,80 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6879
7134
  // delete against it so a stale activation cannot delete a newer owner's WIP.
6880
7135
  let ownedSha = prior?.sha || '';
6881
7136
  let varWarned = false;
7137
+ let resumeVarWarned = false;
7138
+ // #296 (review): the in-flight ABORT-path metadata write (`setVar` fired
7139
+ // fire-and-forget on a force-stop flush, below). The force-stop recovery path
7140
+ // later writes the durable recovery ref to the SAME process-scoped
7141
+ // `resumeFromCheckpoint` slot. If this earlier WIP write is still in flight and
7142
+ // lands AFTER the recovery-pointer write, it overwrites the recovery pointer with
7143
+ // a stale WIP ref — and once the WIP TTL expires, auto-resume can no longer
7144
+ // discover the still-existing recovery ref. Track the promise so the recovery
7145
+ // path can AWAIT it (bounded) before writing the recovery pointer, guaranteeing
7146
+ // the recovery write lands last. The WIP write still lands on its own when
7147
+ // promotion fails (the fallback is retained); we only serialize the ordering.
7148
+ let abortMetaWrite = null;
6882
7149
  const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
6883
- ? async (res) => {
6884
- try {
6885
- await withDeadline(camunda.createElementInstanceVariables({
7150
+ ? (res, signal = null) => {
7151
+ // #296 (review r4245278256): bind these shared process-scoped writes to the
7152
+ // job abort signal for NON-`abort` checkpoints (periodic/`failed`). A periodic
7153
+ // or failed checkpoint can be AWAITING this mutation (`await setVar(res).promise`
7154
+ // below) when heartbeat lease loss interrupts the raw runner and releases the
7155
+ // slot; a replacement activation may then publish a newer `resumeFromCheckpoint`
7156
+ // pointer before this stale call completes and overwrites it — and the later
7157
+ // post-flush signal recheck cannot undo a write that already landed. Passing
7158
+ // `abortSignal` makes `withDeadline` cancel the in-flight SDK call the instant
7159
+ // the lease is lost, so the stale write is torn down and can never land late.
7160
+ // The `abort`-reason path deliberately passes NO signal: that write is owned by
7161
+ // the dedicated `settleAbortMetaWrite` flow (recovery-deadline governed), which
7162
+ // must still fire AFTER the job abort during the force-stop recovery window.
7163
+ // #296: run the element-scope and process-scope writes CONCURRENTLY, not
7164
+ // sequentially. Each carries its own CP_SDK_TIMEOUT_MS deadline, so a
7165
+ // sequential pair could stall for 2× the timeout (20s) — enough to exhaust
7166
+ // the whole FORCE_STOP_RECOVERY_MS budget during a force-stop flush and
7167
+ // starve the recovery git promotion. Racing them caps the worst case at a
7168
+ // single timeout, and each has an independent try/catch so one failing (or
7169
+ // being absent, e.g. no processInstanceKey) never blocks the other.
7170
+ // #296 (review, r4244281967): RETAIN the underlying CancelablePromise(s) so
7171
+ // the abort path can CANCEL them when the recovery deadline fires — not just
7172
+ // stop awaiting. Otherwise a fire-and-forget abort write whose own SDK timeout
7173
+ // outlasts the recovery window stays in flight after the worker yields and can
7174
+ // land LATE over a newer activation's `resumeFromCheckpoint`. The SDK returns a
7175
+ // CancelablePromise (toCancelable); `.cancel?.()` is a no-op for a plain promise
7176
+ // (the test stubs), so capturing the raw return is enough.
7177
+ const rawWrites = [
7178
+ camunda.createElementInstanceVariables({
6886
7179
  elementInstanceKey: String(job.elementInstanceKey),
6887
7180
  variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
6888
7181
  local: true,
6889
- }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables');
6890
- } catch (err) {
6891
- if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7182
+ }),
7183
+ ];
7184
+ const writes = [
7185
+ withDeadline(rawWrites[0], CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7186
+ if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7187
+ }),
7188
+ ];
7189
+ // #295: also record the ref at the PROCESS instance scope (local on the root
7190
+ // scope key = the process instance key). Unlike the local `agentCheckpoint`
7191
+ // above, this survives the element re-entry an SLA escalation triggers, so a
7192
+ // resumed run (a NEW element instance) can find and adopt its predecessor's
7193
+ // checkpoint via `resumeFromCheckpoint`. Best-effort and independent of the
7194
+ // local write; a failure never blocks the push.
7195
+ if (job?.processInstanceKey) {
7196
+ const rawProcess = camunda.createElementInstanceVariables({
7197
+ elementInstanceKey: String(job.processInstanceKey),
7198
+ variables: { resumeFromCheckpoint: { ref, sha: res.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch, at: res.at } },
7199
+ local: true,
7200
+ });
7201
+ rawWrites.push(rawProcess);
7202
+ writes.push(withDeadline(rawProcess, CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7203
+ if (!resumeVarWarned) { resumeVarWarned = true; log.warn(`could not set the resumeFromCheckpoint variable — ${oneLineLog(err?.message || err)}; a resumed run may not auto-find ${ref}.`); }
7204
+ }));
6892
7205
  }
7206
+ const promise = Promise.all(writes);
7207
+ // Cancel every in-flight underlying write (best-effort; no-op for plain
7208
+ // promises). Used by the abort path when the recovery deadline fires.
7209
+ const cancel = () => { for (const w of rawWrites) { try { w?.cancel?.(); } catch { /* best effort */ } } };
7210
+ return { promise, cancel };
6893
7211
  }
6894
7212
  : null;
6895
7213
 
@@ -6929,7 +7247,35 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6929
7247
  onCheckpoint: async (res) => {
6930
7248
  ownedSha = res.sha;
6931
7249
  log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
6932
- if (setVar) await setVar(res);
7250
+ if (!setVar) return;
7251
+ // #296: `ownedSha` (above) is the only state preserveRecovery() needs from
7252
+ // this callback, and it is already set. Only a FORCE-STOP (`abort`) flush
7253
+ // fires the metadata writes off the serialized flush/promote chain: that path
7254
+ // is bounded by FORCE_STOP_RECOVERY_MS and the worker returns early (no
7255
+ // settle), so awaiting a hung engine there could burn the whole recovery
7256
+ // budget before the recovery git promotion runs — and the durable recovery
7257
+ // ref, not these vars, is the durable artifact of a force-stopped run.
7258
+ // A `failed` flush is DIFFERENT: the job settles (settleJob.fail) right after
7259
+ // the flush returns, and a retry/reactivation can read variables before a
7260
+ // fire-and-forget `resumeFromCheckpoint` write is durable — worse, a delayed
7261
+ // process-scoped write could overwrite a NEWER activation's pointer. So the
7262
+ // failed path (and every normal checkpoint) AWAITS the write before
7263
+ // settlement, keeping the predecessor pointer durable before the job settles.
7264
+ // #296 (review r4245278256): and it passes `abortSignal`, so if heartbeat lease
7265
+ // loss fires DURING this awaited write the in-flight SDK call is cancelled — a
7266
+ // replacement activation then owns the slot and this run's now-stale pointer can
7267
+ // never land late over its newer value.
7268
+ // #296 (review): on the abort path, CAPTURE the fire-and-forget write (promise +
7269
+ // cancel) so the force-stop recovery path can await it before writing the
7270
+ // recovery pointer to the same slot — otherwise a slow WIP write lands after and
7271
+ // clobbers the recovery ref. `.promise.catch(() => {})` keeps it fire-and-forget
7272
+ // for the flush; the settled/rejected result is irrelevant to the caller. The
7273
+ // retained `cancel` lets settleAbortMetaWrite tear down the in-flight SDK call
7274
+ // when the recovery deadline fires (r4244281967), so it can never land late. This
7275
+ // path deliberately passes NO abortSignal to setVar: the abort write is owned by
7276
+ // settleAbortMetaWrite's recovery-deadline flow and must outlive the job abort.
7277
+ if (res.reason === 'abort') { const w = setVar(res); abortMetaWrite = { promise: w.promise.catch(() => {}), cancel: w.cancel }; return; }
7278
+ await setVar(res, abortSignal || undefined).promise;
6933
7279
  },
6934
7280
  });
6935
7281
  return {
@@ -6939,6 +7285,23 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6939
7285
  restored,
6940
7286
  priorNotRestored,
6941
7287
  flushTimeoutMs: cfg.flushTimeoutMs,
7288
+ // #296 (review): the git SIGKILL grace window, surfaced so the force-stop path
7289
+ // can reserve a real drain window for close() instead of the 1ms an expired
7290
+ // shared deadline would otherwise hand it (see the force-stop block).
7291
+ gitKillGraceMs: cfg.gitKillGraceMs,
7292
+ // #296 (F9): abort the recovery-deadline controller — tears down an in-flight
7293
+ // checkpoint/recovery git push so `stop()`/`close()` settle promptly. Driven by
7294
+ // the force-stop path once the recovery budget expires (NOT by the job's
7295
+ // abortSignal, which fires BEFORE recovery runs and would fail every recovery
7296
+ // command). Also aborts the background GC sweep so the ONE shared force-stop
7297
+ // deadline bounds recovery AND cleanup together — otherwise a still-running
7298
+ // (up to 120s) sweep could hold close()'s drain past the supervisor's SIGKILL
7299
+ // window even after recovery settled. Idempotent / best-effort; a no-op once
7300
+ // already aborted.
7301
+ abortRecovery() {
7302
+ try { recoveryController.abort(); } catch { /* best effort */ }
7303
+ try { gcAbort?.abort(); } catch { /* best effort */ }
7304
+ },
6942
7305
  // Adopt a credential the job ROTATED after this checkpointer was created —
6943
7306
  // e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
6944
7307
  // (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
@@ -6950,6 +7313,148 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6950
7313
  updateCredential(nextToken) {
6951
7314
  if (nextToken && nextToken !== token) token = nextToken;
6952
7315
  },
7316
+ // #296 (review): await the in-flight ABORT-path metadata write (the WIP
7317
+ // `resumeFromCheckpoint` write fired fire-and-forget on a force-stop flush),
7318
+ // bounded, so the force-stop recovery path can guarantee ITS recovery-pointer
7319
+ // write to the same slot lands LAST (a slow WIP write must not overwrite the
7320
+ // durable recovery ref). Best-effort, never throws; resolves immediately when
7321
+ // no abort write is in flight. The caller bounds the wait inside the shared
7322
+ // force-stop budget, so a hung engine cannot stall the force-stop exit.
7323
+ // #296 (review, r4244116341): accept the recovery-deadline `signal` so this SDK
7324
+ // await is cancelled by the SAME deadline that tears down the recovery git — the
7325
+ // recover chain must SETTLE (not dangle) when the budget expires, so a stale
7326
+ // continuation can never write the old recovery pointer over a reactivated
7327
+ // worker's newer one.
7328
+ // #296 (review, r4244281967): stopping the AWAIT is not enough — the underlying
7329
+ // `createElementInstanceVariables` mutation would otherwise stay in flight past
7330
+ // the deadline and could land LATE over a newer activation's `resumeFromCheckpoint`.
7331
+ // So when `signal` aborts we also CANCEL the retained CancelablePromise(s)
7332
+ // (`abortMetaWrite.cancel()`), aborting the fetch so the late settle can never land.
7333
+ async settleAbortMetaWrite(signal = null) {
7334
+ const w = abortMetaWrite;
7335
+ if (!w) return;
7336
+ if (signal?.aborted) { try { w.cancel?.(); } catch { /* best effort */ } return; }
7337
+ if (!signal) { try { await w.promise; } catch { /* best effort */ } return; }
7338
+ let onAbort = null;
7339
+ try {
7340
+ await Promise.race([
7341
+ w.promise.catch(() => {}),
7342
+ new Promise((res) => { onAbort = () => { try { w.cancel?.(); } catch { /* best effort */ } res(); }; signal.addEventListener('abort', onAbort, { once: true }); }),
7343
+ ]);
7344
+ } catch { /* best effort */ } finally { if (onAbort) signal.removeEventListener('abort', onAbort); }
7345
+ },
7346
+ // #295: promote the latest snapshot to a durable recovery ref on an SLA
7347
+ // cancellation or a force-stop. `refs/nano/recovered/<elementInstanceKey>`
7348
+ // lives OUTSIDE the `refs/nano/wip/*` namespace, so it survives the
7349
+ // per-checkpoint WIP cleanup (`discardAfterAck`, which only deletes WIP refs)
7350
+ // and workspace reaping — long enough for the human escalation to be answered
7351
+ // and the work recovered. It is NOT exempt from the orphan GC, though: the
7352
+ // sweep reclaims it under the same process-terminal / 7d-TTL policy as a WIP
7353
+ // ref (#296), sparing it only while its process is still ACTIVE. Returns
7354
+ // `{ ref, sha }` (surfaced in the job outcome) or
7355
+ // null when there is nothing durable to preserve / the push failed. Best-effort;
7356
+ // never throws. The live WIP ref is kept too (this is an additional durable copy).
7357
+ async preserveRecovery(reason = 'recovery') {
7358
+ // Copilot review on #295: the caller invokes this right after a BOUNDED
7359
+ // `flush()`, which can RETURN while its forced checkpoint is still in flight
7360
+ // — and `ownedSha` is only updated later, by `onCheckpoint`, once that
7361
+ // checkpoint's push resolves. Reading `ownedSha` immediately could therefore
7362
+ // see `null`/the pre-flush SHA and promote an older snapshot (or nothing).
7363
+ // DRAIN the checkpoint chain first (`stop()` resolves once the in-flight
7364
+ // chain settles, letting `onCheckpoint` record the latest SHA) so the SHA
7365
+ // selected below is the newest snapshot this run actually pushed. This
7366
+ // mirrors the success path, which already `stop()`s before finalizeGit.
7367
+ try { await checkpointer.stop(); } catch { /* best effort — fall through to the last recorded ownedSha */ }
7368
+ if (!ownedSha) return null;
7369
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7370
+ if (!recRef) return null;
7371
+ try {
7372
+ // Fence the recovery promotion against the sha we OBSERVE on the ref now,
7373
+ // so a slower/stale activation that observed an older/absent value loses
7374
+ // the lease once a newer run moved it, instead of blind-forcing over
7375
+ // whatever is current (Copilot review on #296).
7376
+ const prevRecSha = await observeRemoteRefSha({ git, ref: recRef });
7377
+ // #296 (F7/F10): a bare CAS against the just-observed value is NOT ownership
7378
+ // — a superseded activation can observe the newer run's `shaB` here and
7379
+ // still overwrite it with its older `ownedShaA`. Mirror the WIP path's
7380
+ // `Nano-Checkpoint-Run` ownership check: only overwrite an EXISTING recovery
7381
+ // ref when it is POSITIVELY identified as either ours (a re-preservation by
7382
+ // this same run) or the run we explicitly superseded (`prior?.runId`). An
7383
+ // absent ref is a create-only write. FAIL CLOSED: if the owner cannot be
7384
+ // positively identified — the ownership fetch THREW/failed, or the ref
7385
+ // carries no `runId` trailer — refuse the overwrite rather than fall through
7386
+ // to the CAS, so a transient read failure can never let a stale activation
7387
+ // clobber a newer (or unknown) run's recovery point (F10). The WIP ref +
7388
+ // TTL remain the fallback in every refused case.
7389
+ if (prevRecSha) {
7390
+ let ownerRunId = '';
7391
+ let ownerKnown = false;
7392
+ try { ownerRunId = (await fetchCheckpoint({ git, ref: recRef }))?.runId || ''; ownerKnown = true; } catch { ownerKnown = false; }
7393
+ const myRun = String(runId || '');
7394
+ const supersededRun = String(prior?.runId || '');
7395
+ const mine = ownerKnown && ownerRunId && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7396
+ if (!mine) {
7397
+ const who = !ownerKnown ? 'could not be read (transient)' : (ownerRunId ? `owned by another run (${ownerRunId})` : 'carries no run identity');
7398
+ log.warn(`recovery checkpoint ${recRef} ${who} — refusing to overwrite it; the WIP ref ${ref} is still retained for recovery.`);
7399
+ return null;
7400
+ }
7401
+ }
7402
+ const r = await pushNamedRef({ git, ref: recRef, sha: ownedSha, expectSha: prevRecSha });
7403
+ if (r?.ok) {
7404
+ logger.info?.(`${prefix} (${corr}): preserved a durable recovery checkpoint ${recRef}@${ownedSha.slice(0, 12)} (${reason}) — it survives the per-checkpoint WIP cleanup and workspace reaping (the orphan GC spares it while its process stays active, up to the ${Math.round(cfg.ttlMs / 86400000)}d TTL backstop, which reclaims it regardless of process state) so the SLA-cancelled/force-stopped run can be recovered and resumed.`);
7405
+ return { ref: recRef, sha: ownedSha };
7406
+ }
7407
+ if (r?.staleLease) {
7408
+ // A newer activation already preserved a recovery point on this ref;
7409
+ // keep theirs. Our WIP ref is still retained as a fallback.
7410
+ log.warn(`recovery checkpoint ${recRef} was moved by a newer activation — keeping its recovery point; the WIP ref ${ref} is still retained for recovery.`);
7411
+ return null;
7412
+ }
7413
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(r?.error || 'unknown', token)}; the WIP ref ${ref} is still retained for recovery.`);
7414
+ } catch (err) {
7415
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(oneLineLog(err?.message || err), token)}; the WIP ref ${ref} is still retained for recovery.`);
7416
+ }
7417
+ return null;
7418
+ },
7419
+ // #296 (review, r4244827082): re-verify, IMMEDIATELY before the process-scoped
7420
+ // `resumeFromCheckpoint` recovery-pointer write, that THIS run still owns the
7421
+ // durable recovery ref it just preserved. `preserveRecovery` fences the GIT ref
7422
+ // (CAS + `Nano-Checkpoint-Run` ownership), but the subsequent process-scoped
7423
+ // variable write is unconditional last-writer-wins: a superseded activation that
7424
+ // won its ref earlier can still land its stale pointer AFTER a newer activation
7425
+ // (which re-activated the job on lease loss) published its own. Close that
7426
+ // residual race by re-observing the recovery ref now and confirming it still
7427
+ // resolves to the sha this run preserved AND is still owned by this run (or the
7428
+ // run it explicitly superseded) — the same ownership rule the GIT fence applies.
7429
+ // FAIL CLOSED: any read failure, a moved/absent ref, or an owner that is not
7430
+ // positively this run returns false, so the caller SKIPS the variable write and
7431
+ // leaves the newer activation's pointer intact (the pushed ref stays the durable
7432
+ // fallback). `expectedSha` is the sha this run's preserveRecovery returned.
7433
+ // #296 (review, r4244916101): accept a per-call `signal` and run the git reads
7434
+ // on a DEDICATED runner bound to it (AND the recovery deadline). The caller
7435
+ // races this against a 5s timeout; without a wired signal the losing branch
7436
+ // left the verify's git process live on the 60s runner — untracked by
7437
+ // checkpointer.stop()/gc — so recover() returned and the run dir was reaped
7438
+ // under a live git. Binding the runner to the per-call signal lets the caller
7439
+ // abort+await the verify on timeout so it is SETTLED before cleanup proceeds.
7440
+ async verifyRecoveryOwnership(expectedSha, { signal } = {}) {
7441
+ try {
7442
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7443
+ if (!recRef || !expectedSha) return false;
7444
+ // Combine the per-call abort with the recovery-deadline abort so EITHER
7445
+ // tears the git down (whichever fires first). AbortSignal.any is Node 20+.
7446
+ const signals = [recoveryController.signal, signal].filter(Boolean);
7447
+ const combined = signals.length > 1 ? AbortSignal.any(signals) : (signals[0] || undefined);
7448
+ const vgit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: combined });
7449
+ const curSha = await observeRemoteRefSha({ git: vgit, ref: recRef });
7450
+ if (!curSha || curSha !== expectedSha) return false;
7451
+ let ownerRunId = '';
7452
+ try { ownerRunId = (await fetchCheckpoint({ git: vgit, ref: recRef }))?.runId || ''; } catch { return false; }
7453
+ const myRun = String(runId || '');
7454
+ const supersededRun = String(prior?.runId || '');
7455
+ return Boolean(ownerRunId) && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7456
+ } catch { return false; }
7457
+ },
6953
7458
  // True when the workspace still holds checkpoint content NOT captured by a
6954
7459
  // commit, so the pushed branch does not represent it (an uncommitted restored
6955
7460
  // or working delta that lives only on the WIP ref). The discard-on-ack gate
@@ -6964,8 +7469,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6964
7469
  } catch { return true; }
6965
7470
  },
6966
7471
  // Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
6967
- // run dir can be reaped safely.
6968
- async close() { await checkpointer.stop(); await gc; },
7472
+ // run dir can be reaped safely. `close()` is bounded by `closeTimeoutMs` so a
7473
+ // force-stopped worker is never held past the supervisor's SIGKILL backstop by
7474
+ // the (up to 120s) GC sweep: on the deadline we stop awaiting and let the
7475
+ // caller proceed (the in-flight checkpoint git AND the GC sweep are both
7476
+ // cancelled by the recovery DEADLINE — abortRecovery() — so the chain settles on
7477
+ // its own). The force-stop caller passes only the REMAINING of the one shared
7478
+ // force-stop budget, so recovery + this drain never exceed it together. Default
7479
+ // 0 = unbounded (normal/failed paths fully drain).
7480
+ async close({ timeoutMs = 0 } = {}) {
7481
+ const drain = (async () => { await checkpointer.stop(); await gc; })();
7482
+ if (!timeoutMs) return drain;
7483
+ let t;
7484
+ const cap = new Promise((resolve) => { t = setTimeout(resolve, timeoutMs); t?.unref?.(); });
7485
+ try { return await Promise.race([drain, cap]); } finally { clearTimeout(t); }
7486
+ },
6969
7487
  async discard() {
6970
7488
  await checkpointer.stop();
6971
7489
  try {
@@ -8705,6 +9223,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
8705
9223
  let childClosed = null; // { code, signal } once the child exits
8706
9224
  let promptResolved = false; // the main session/prompt turn sent + resolved
8707
9225
  let acpOutcome = null; // the turn's `_meta.outcome`, if the agent reported one
9226
+ let acpStopReason = null; // the turn's ACP `stopReason` (#301)
8708
9227
  let settleTimer = null; // post-turn grace before force-reaping a lingering agent
8709
9228
  // One-time warning latch for the reserved escalate/filter policies so the
8710
9229
  // deferral is observable (not silent) but never spams a warning per request.
@@ -8811,6 +9330,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
8811
9330
  // floor (thread 4118913047).
8812
9331
  const floorOnly = floorEmitted && !sawRealTranscript;
8813
9332
  let resolved = acpOutcome ? { ...result, acpOutcome } : result;
9333
+ if (acpStopReason) resolved = { ...resolved, stopReason: acpStopReason };
8814
9334
  if (floorOnly) resolved = { ...resolved, acpTranscriptFloorOnly: true };
8815
9335
  resolve(resolved);
8816
9336
  };
@@ -9275,6 +9795,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
9275
9795
  prompt: [{ type: 'text', text: String(stdinData ?? '') }],
9276
9796
  });
9277
9797
  acpOutcome = acpOutcomeFrom(prompted);
9798
+ acpStopReason = acpStopReasonFrom(prompted);
9278
9799
  // End-of-turn: the main session/prompt request resolved. Mark it so the
9279
9800
  // `close` handler can tell a completed turn from an early/handshake exit.
9280
9801
  promptResolved = true;
@@ -9706,6 +10227,12 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
9706
10227
  // The agent's explicit ACP end-of-task report (`_meta.outcome`), host-recorded
9707
10228
  // alongside whatever result vars it returned.
9708
10229
  if (result.acpOutcome) env.outcome = result.acpOutcome;
10230
+ // #301: why the (first) ACP turn ended — `end_turn` or a budget cut-off. After a
10231
+ // nudge this is the EFFECTIVE reason (the nudge's when it was truncated), and the
10232
+ // nudge turn's own reason is recorded separately so the audit shows both turns.
10233
+ if (result.stopReason) env.stopReason = result.stopReason;
10234
+ if (result.nudgeStopReason) env.nudgeStopReason = result.nudgeStopReason;
10235
+ if (result.budgetDowngraded) env.budgetDowngraded = true;
9709
10236
  // Audit (issue #63): record which linked-resource key supplied the base prompt.
9710
10237
  // The engine only keeps `latest` per resourceId (no pinning), so recording the
9711
10238
  // resolved key is the only reproducibility handle for which prompt version ran.
@@ -9744,6 +10271,11 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
9744
10271
  if (git.branchMismatch) env.branchMismatch = git.branchMismatch;
9745
10272
  if (git.pr) env.pr = git.pr;
9746
10273
  if (git.error) env.gitError = git.error;
10274
+ // #295: a durable recovery ref (refs/nano/recovered/<eik>) an SLA-cancelled or
10275
+ // force-stopped run left behind — carried in the outcome so the escalation /
10276
+ // resume can recover the snapshot from it even after the workspace is reaped.
10277
+ if (git.recoveryRef) env.recoveryRef = git.recoveryRef;
10278
+ if (git.recoverySha) env.recoverySha = git.recoverySha;
9747
10279
  }
9748
10280
  return env;
9749
10281
  }
@@ -12502,6 +13034,28 @@ async function workAgent(req, flags, ctx) {
12502
13034
  // performs a last-chance `failed` flush so edits since the last periodic
12503
13035
  // checkpoint are not lost.
12504
13036
  let checkpointFinalized = false;
13037
+ // #296 (F8/F9): set on the worker force-stop path so the `finally` below bounds
13038
+ // its `close()` drain to the same FORCE_STOP_RECOVERY_MS budget — a force-stop
13039
+ // must never be held past the supervisor's SIGKILL backstop by the (up to
13040
+ // 120s) GC sweep. The in-flight checkpoint git is cancelled by the recovery
13041
+ // DEADLINE (checkpointing.abortRecovery(), fired from the force-stop block),
13042
+ // not the job's abortSignal, so the chain settles on its own after we stop
13043
+ // awaiting while recovery can still run after the harness abort.
13044
+ let forceStopping = false;
13045
+ // #296: the SINGLE absolute force-stop deadline (set when the force-stop
13046
+ // recovery block begins). The recover race AND the `finally`'s close() drain
13047
+ // BOTH bound against it, so recovery + cleanup together fit inside ONE
13048
+ // FORCE_STOP_RECOVERY_MS window rather than consuming it twice (recovery race,
13049
+ // then an independent close() wait) and risking a SIGKILL during cleanup
13050
+ // (Copilot review on #296). 0 until a force-stop begins.
13051
+ let forceStopDeadline = 0;
13052
+ // #295: the durable recovery ref (and SHA) a preserveRecovery() pushed on an
13053
+ // SLA cancellation / failed run, surfaced in the job outcome (io.nanobpm.agentResult)
13054
+ // — but ONLY on the failed/SLA-cancel path, which settles normally and so reaches
13055
+ // buildResultEnvelope. A FORCE-STOPPED run is yielded (settle.fail, retries
13056
+ // preserved) and returns early below WITHOUT an outcome envelope, so its
13057
+ // recoveryRefInfo is intentionally NOT reported there (Copilot review on #295).
13058
+ let recoveryRefInfo = null;
12505
13059
  // The per-job live-terminal relay session (issue #173): streams this job's
12506
13060
  // harness terminal over the single-owner supervisor's ONE multiplexed host
12507
13061
  // connection, keyed by this worker's instance + the jobKey, and accepts
@@ -12652,14 +13206,259 @@ async function workAgent(req, flags, ctx) {
12652
13206
  // that yield. The finally below still runs (container/run-dir/relay
12653
13207
  // cleanup); recordJobEnd fires in the outer finally.
12654
13208
  if (result && result.aborted) {
12655
- logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
13209
+ logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop or SLA/lease-loss cancellation) — harness killed; releasing the job (the broker re-activates it for retry when the run is still live).`);
12656
13210
  // #264: last-chance WIP checkpoint so the retry can pick up the
12657
13211
  // uncommitted work (bounded; the lease guards a newer owner).
12658
- if (checkpointer) { await checkpointer.flush('abort', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13212
+ // #295: then promote it to a durable recovery ref so a force-stopped or
13213
+ // SLA-cancelled run keeps a recovery point. The orphan GC spares that ref
13214
+ // while its process stays active, so it survives long enough to be
13215
+ // resumed. The ref is
13216
+ // pushed here, but it is NOT surfaced in a job outcome: this path yields
13217
+ // the job and returns early, never reaching buildResultEnvelope — a
13218
+ // force-stopped run produces no outcome envelope by design (Copilot
13219
+ // review on #295), so the recovery ref is recoverable from the pushed
13220
+ // refs/nano/recovered/<eik> ref itself, not from any job result.
13221
+ // #296: bound the WHOLE flush+promote to FORCE_STOP_RECOVERY_MS so it
13222
+ // reliably finishes (and this worker exits) inside the window the
13223
+ // supervisor's forceStopWorker now waits before its SIGKILL backstop —
13224
+ // otherwise the default flush (20s) + recovery git push (60s) could run
13225
+ // long and be SIGKILLed before refs/nano/recovered/<eik> is written.
13226
+ // Best-effort: if the budget elapses we give up on the recovery ref (the
13227
+ // WIP ref + TTL remain the fallback), never block the yield/exit.
13228
+ if (checkpointer) {
13229
+ forceStopping = true;
13230
+ const recoveryDeadline = FORCE_STOP_RECOVERY_MS;
13231
+ // #296: anchor the ONE shared force-stop deadline here. The recover race
13232
+ // below and the `finally`'s close() both bound against it, so the TOTAL
13233
+ // force-stop window is a single FORCE_STOP_RECOVERY_MS (not 2x), fitting
13234
+ // inside the supervisor's STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS backstop.
13235
+ forceStopDeadline = Date.now() + recoveryDeadline;
13236
+ // #296 (review): reserve a DRAIN window inside the deadline. abortRecovery()
13237
+ // only SIGTERMs the checkpoint git; createGitRunner then grants up to
13238
+ // gitKillGraceMs before SIGKILL. If abortRecovery() fires AT the deadline,
13239
+ // a stubborn git is still dying when close() hits its (expired) cap and the
13240
+ // finally reaps the workspace under a live process. Fire the abort EARLY —
13241
+ // gitKillGraceMs + a small margin before the deadline — so the SIGKILL lands
13242
+ // by `forceStopDeadline` and close() gets the remaining grace window to
13243
+ // actually drain the now-dead git, not the 1ms `Math.max(1, …)` would give.
13244
+ const killGrace = Math.max(0, Number(checkpointing?.gitKillGraceMs) || 0);
13245
+ const drainReserveMs = killGrace + 250;
13246
+ const abortAtMs = Math.max(1, recoveryDeadline - drainReserveMs);
13247
+ // #296 (F9): arm the recovery-deadline. When the budget expires, abort
13248
+ // the DEDICATED recovery git runner (not the job's abortSignal, which
13249
+ // has already fired and would fail every recovery command) so an
13250
+ // in-flight flush/preserveRecovery push is torn down and the chain
13251
+ // settles — letting the `finally`'s bounded `close()` drain inside the
13252
+ // supervisor's SIGKILL window. Fires `abortAtMs` (the deadline MINUS the
13253
+ // reserved drain window) so the git SIGKILL completes before the deadline;
13254
+ // unref'd so it never keeps the process alive.
13255
+ // #296 (review, r4244116341): the recover-race `Promise.race([recover,
13256
+ // budget])` below only bounds the AWAIT — when `budget` wins, the SDK
13257
+ // mutations inside `recover` (settleAbortMetaWrite's awaited WIP write and
13258
+ // the recovery-pointer write) keep running. The job then yields and a
13259
+ // reactivated worker can publish a NEWER `resumeFromCheckpoint`; a stale
13260
+ // `recover` continuation that completes afterward would clobber it with the
13261
+ // old recovery pointer. abortRecovery() cancels only git/GC, not these SDK
13262
+ // calls. So drive the SDK mutations off a DEDICATED recovery-deadline
13263
+ // signal, fired by the same deadline (and on budget expiry), and AWAIT
13264
+ // `recover` after the race so it is SETTLED/cancelled before we yield.
13265
+ const recoverySdkAbort = new AbortController();
13266
+ let deadlineTimer = setTimeout(() => {
13267
+ try { checkpointing.abortRecovery(); } catch { /* best effort */ }
13268
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13269
+ }, abortAtMs);
13270
+ deadlineTimer?.unref?.();
13271
+ const recover = (async () => {
13272
+ await checkpointer.flush('abort', { timeoutMs: recoveryDeadline }); checkpointFinalized = true;
13273
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('force-stop') || recoveryRefInfo; } catch { /* best effort */ }
13274
+ // #296 (review, AC3): an SLA boundary cancel reaches THIS aborted branch
13275
+ // too — the supervisor heartbeat treats the resulting lease-loss 404 as
13276
+ // terminal and interrupts the runner (supervisor/src/dispatch.ts), so
13277
+ // runAgentJob returns aborted:true. The early return below then skips
13278
+ // buildResultEnvelope, so recoveryRef/recoverySha are never reported in a
13279
+ // job outcome. Persist the recovery ref through the PROCESS-scoped
13280
+ // `resumeFromCheckpoint` slot instead: it survives the element re-entry an
13281
+ // SLA escalation triggers, so the resumed run's predecessorCheckpointRef
13282
+ // can discover the durable refs/nano/recovered/<eik> point. Bounded and
13283
+ // best-effort — never holds the force-stop exit; the pushed ref itself
13284
+ // remains the authoritative recovery handle.
13285
+ // #296 (review): the abort flush fired its OWN WIP `resumeFromCheckpoint`
13286
+ // write fire-and-forget (see onCheckpoint). If that write is still in
13287
+ // flight it could land AFTER the recovery-pointer write below and
13288
+ // overwrite the recovery ref with a stale WIP pointer (which the orphan
13289
+ // GC then reclaims on TTL, losing the recovery discovery). Await the WIP
13290
+ // write FIRST — under the recovery-deadline signal so the await is
13291
+ // cancelled when the budget expires — so the recovery pointer is
13292
+ // guaranteed to land last. Bounded by the recover-race budget; best-effort.
13293
+ try { await checkpointing.settleAbortMetaWrite?.(recoverySdkAbort.signal); } catch { /* best effort */ }
13294
+ // #296 (review, r4244116341): if the recovery deadline has already fired,
13295
+ // do NOT write the recovery pointer — a write that lands after this branch
13296
+ // yields could overwrite a reactivated worker's newer resumeFromCheckpoint.
13297
+ if (recoverySdkAbort.signal.aborted) return;
13298
+ if (recoveryRefInfo && typeof camunda?.createElementInstanceVariables === 'function' && job?.processInstanceKey) {
13299
+ try {
13300
+ // #296 (review, r4244827082): gate the process-scoped pointer write on
13301
+ // PROVEN current ownership of the recovery ref. preserveRecovery only
13302
+ // fences the GIT ref; this variable write is last-writer-wins, so a
13303
+ // superseded activation could otherwise land its stale pointer after a
13304
+ // newer activation (which re-activated the job on lease loss) published
13305
+ // its own. Re-verify the ref still resolves to the sha THIS run
13306
+ // preserved and is still owned by this run; skip the write when it has
13307
+ // moved (a newer activation's pointer stays authoritative). Bounded by
13308
+ // the recovery deadline; a failed/aborted verify fails closed (skip).
13309
+ let stillOwns = false;
13310
+ try {
13311
+ let vt; let vOnAbort = null;
13312
+ // #296 (review, r4244916101): drive the verify off its OWN
13313
+ // AbortController so that when the 5s branch wins we can CANCEL
13314
+ // the in-flight git reads AND await the verify to full settlement
13315
+ // in the finally below — before recover() returns and the run dir
13316
+ // is reaped. Without this the losing verify kept running on the
13317
+ // 60s runner (untracked by stop()/gc) under a reaped workspace.
13318
+ const verifyAbort = new AbortController();
13319
+ const verifyOp = Promise.resolve(
13320
+ checkpointing.verifyRecoveryOwnership?.(recoveryRefInfo.sha, { signal: verifyAbort.signal }),
13321
+ ).then((v) => v === true);
13322
+ stillOwns = await Promise.race([
13323
+ verifyOp,
13324
+ new Promise((res) => { vt = setTimeout(() => res(false), 5000); vt?.unref?.(); }),
13325
+ new Promise((res) => { vOnAbort = () => res(false); recoverySdkAbort.signal.addEventListener('abort', vOnAbort, { once: true }); }),
13326
+ ]).finally(() => {
13327
+ clearTimeout(vt);
13328
+ if (vOnAbort) recoverySdkAbort.signal.removeEventListener('abort', vOnAbort);
13329
+ // The race settled (timeout/abort/complete). If the verify is
13330
+ // still mid-git (timeout/abort won), cancel its runner so the
13331
+ // git process is SIGTERM→SIGKILLed now, not at abortAtMs.
13332
+ try { verifyAbort.abort(); } catch { /* best effort */ }
13333
+ });
13334
+ // AWAIT the verify to full settlement (its git is now aborted, so
13335
+ // this resolves promptly) so no live git outlives recover()/close().
13336
+ try { await verifyOp; } catch { /* best effort */ }
13337
+ } catch { stillOwns = false; }
13338
+ if (!stillOwns) {
13339
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: recovery ref ${recoveryRefInfo.ref} is no longer owned by this run (a newer activation superseded it) — skipping the process-scoped resumeFromCheckpoint write so the newer pointer stays authoritative.`);
13340
+ return;
13341
+ }
13342
+ // #296 (review, r4245472572): the Git-ref ownership check above cannot
13343
+ // fence THIS write across an SLA resume — a re-activated run is a NEW
13344
+ // element instance, so it owns a DIFFERENT refs/nano/recovered/<eik>
13345
+ // and this run still "owns" its old one (passing the check) even after
13346
+ // the new activation published a newer resumeFromCheckpoint. The write
13347
+ // is unconditional last-writer-wins, so it could then overwrite that
13348
+ // newer pointer. Two fences close the class (no atomic process-scoped
13349
+ // CAS exists — so the write is a best-effort DISCOVERY HINT, not an
13350
+ // authoritative fence, and these two narrowings reduce (not eliminate)
13351
+ // its staleness:
13352
+ // (1) OCCURRENCE-SCOPED READ-BACK — re-read the shared slot and skip
13353
+ // when it already names a DIFFERENT run's ref (a newer activation
13354
+ // beat us). Best-effort: a read failure/empty/absent result fails
13355
+ // OPEN (we still write) so a transient search outage never strands
13356
+ // recovery; eventual consistency (index lag) can hide a JUST-landed
13357
+ // newer write, which is exactly why fence (2) is also required.
13358
+ // (2) NATIVE ABORT SIGNAL on the write itself — the SDK options extend
13359
+ // RequestInit, so `signal` reaches fetch and aborts the in-flight
13360
+ // HTTP request the instant the recovery deadline fires, tearing the
13361
+ // mutation down at the transport (not just a best-effort .cancel()).
13362
+ // #296 (review, r4245620621): a read-then-write is NOT atomic, so a
13363
+ // replacement activation can still publish its ref between the read-back
13364
+ // and this write — the residual TOCTOU the reviewer notes is real. It is
13365
+ // BENIGN for recovery CORRECTNESS, which does not rest on this write:
13366
+ // the self-recorded OBJECT form of resumeFromCheckpoint is NEVER
13367
+ // auto-adopted on resume (`makeOccurrenceGuard` fails closed BY
13368
+ // CONSTRUCTION — only a process-delivered bare STRING is adopted), and
13369
+ // the authoritative durable recovery handle is the CAS-fenced git ref
13370
+ // refs/nano/recovered/<eik> (`preserveRecovery` → `pushNamedRef`
13371
+ // --force-with-lease). A stale object pointer therefore yields a
13372
+ // transcript-only resume (identical to an ABSENT pointer), never an
13373
+ // incorrect restore. These narrowings keep the HINT as fresh as is
13374
+ // cheaply possible; they are not relied on for a safety property.
13375
+ // #296 (review, r4245620587): the consistency options MUST be nested
13376
+ // under `consistency` (the real @camunda8 client throws "Missing
13377
+ // consistency options" otherwise — mirrors the 6994/6999 reads and
13378
+ // agent-resume's READ_CONSISTENCY) or the catch below would swallow the
13379
+ // throw and this read-back would silently fail OPEN on every call.
13380
+ if (typeof camunda?.searchVariables === 'function') {
13381
+ try {
13382
+ const found = await Promise.resolve(camunda.searchVariables(
13383
+ { filter: { name: 'resumeFromCheckpoint', scopeKey: String(job.processInstanceKey) }, truncateValues: false },
13384
+ { consistency: { waitUpToMs: 0 }, signal: recoverySdkAbort.signal },
13385
+ ));
13386
+ const items = Array.isArray(found?.items) ? found.items : (Array.isArray(found?.data?.items) ? found.data.items : []);
13387
+ let slotRef = null;
13388
+ for (const it of items) {
13389
+ try { const v = JSON.parse(it?.value ?? 'null'); slotRef = (v && typeof v === 'object' ? v.ref : (typeof v === 'string' ? v : null)) ?? slotRef; } catch { /* unparseable entry — ignore */ }
13390
+ }
13391
+ if (slotRef && slotRef !== recoveryRefInfo.ref) {
13392
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: the process-scoped resumeFromCheckpoint slot already names a newer activation's ref ${slotRef} (not this run's ${recoveryRefInfo.ref}) — skipping the stale overwrite so the newer pointer stays authoritative.`);
13393
+ return;
13394
+ }
13395
+ } catch { /* best effort — a read failure fails OPEN (write proceeds) */ }
13396
+ }
13397
+ const w = camunda.createElementInstanceVariables({
13398
+ elementInstanceKey: String(job.processInstanceKey),
13399
+ variables: { resumeFromCheckpoint: { ref: recoveryRefInfo.ref, sha: recoveryRefInfo.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, recovered: true, at: Date.now() } },
13400
+ local: true,
13401
+ signal: recoverySdkAbort.signal,
13402
+ });
13403
+ let wt; let onAbort = null;
13404
+ // Cancel the in-flight write when the recovery deadline fires (the SDK
13405
+ // returns a CancelablePromise), AND reject the race so the chain
13406
+ // settles — the mutation never lands late over a newer pointer.
13407
+ await Promise.race([
13408
+ Promise.resolve(w).finally(() => { clearTimeout(wt); if (onAbort) recoverySdkAbort.signal.removeEventListener('abort', onAbort); }),
13409
+ new Promise((_r, rej) => { wt = setTimeout(() => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write timed out')); }, 5000); wt?.unref?.(); }),
13410
+ new Promise((_r, rej) => { onAbort = () => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write aborted')); }; recoverySdkAbort.signal.addEventListener('abort', onAbort, { once: true }); }),
13411
+ ]);
13412
+ } catch { /* best effort — the pushed ref is the fallback */ }
13413
+ }
13414
+ })();
13415
+ let budgetTimer = null;
13416
+ const budget = new Promise((res) => { budgetTimer = setTimeout(res, recoveryDeadline); if (budgetTimer.unref) budgetTimer.unref(); });
13417
+ try { await Promise.race([recover, budget]); } catch { /* best effort */ }
13418
+ // Clear the recover-race budget timer (it has fired or the race settled).
13419
+ // deadlineTimer is deliberately NOT cleared: even when the recover race
13420
+ // settles early, abortRecovery() must still fire at abortAtMs to abort the
13421
+ // background GC sweep (gcAbort) — otherwise close() below would await a
13422
+ // sweep that can run up to GC_OVERALL_TIMEOUT_MS (120s), holding the
13423
+ // force-stop open past the supervisor's SIGKILL window (the F8 failure).
13424
+ if (budgetTimer) clearTimeout(budgetTimer);
13425
+ // #296 (review, r4244116341): the race above only bounds the await — when
13426
+ // `budget` wins, `recover` may still be mid-SDK-write. Abort the recovery
13427
+ // signal (cancels the in-flight SDK mutation and short-circuits recover's
13428
+ // remaining steps) and AWAIT `recover` so it is fully settled BEFORE we
13429
+ // yield the job. This guarantees no stale continuation writes the old
13430
+ // recovery pointer after a reactivated worker publishes a newer one. The
13431
+ // signal + each step's internal deadline keep this await bounded.
13432
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13433
+ try { await recover; } catch { /* best effort */ }
13434
+ checkpointFinalized = true;
13435
+ }
12659
13436
  return;
12660
13437
  }
12661
13438
  // #264: a failed run is retried — checkpoint what it left behind.
12662
- if (checkpointer && !result.ok) { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13439
+ // #295: an SLA-cancelled run surfaces here as a non-ok (timed-out) result;
13440
+ // preserve a durable recovery ref and report it in the job outcome below.
13441
+ if (checkpointer && !result.ok) {
13442
+ await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true;
13443
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('failed') || recoveryRefInfo; } catch { /* best effort */ }
13444
+ }
13445
+ // #296 (review, r4244991586): the failed-result path can outlive its lease.
13446
+ // `result.aborted` is only true when the harness observed the abort WHILE the
13447
+ // process ran; an abortSignal that fires AFTER runAgentJob() returned (with a
13448
+ // genuine non-ok result) but DURING the flush/promote above leaves
13449
+ // `result.aborted === false`. Effect has already interrupted the raw runner and
13450
+ // released the slot, so if we now fall through to settleJob.fail (and the
13451
+ // checkpoint-variable/envelope writes) we overlap a replacement activation with
13452
+ // STALE writes + a STALE settle. Recheck the job signal here — after the
13453
+ // post-run flush/promote, before any settlement — and switch to the
13454
+ // interrupted/yield path (return WITHOUT settling, exactly like the
13455
+ // result.aborted branch and settleRunError's abort idiom) when it has fired.
13456
+ // The recovery ref preserved above still stands (the durable handle); we simply
13457
+ // do not settle a job whose lease is already gone. The broker re-activates it.
13458
+ if (!result.ok && abortSignal?.aborted === true) {
13459
+ logger.warn(`[${jobType}] job ${job.jobKey}: abortSignal fired after the failed run returned (lease loss/force-stop won the race) — yielding WITHOUT settleJob.fail so a replacement activation does not overlap stale checkpoint/settle writes (the preserved recovery ref remains the durable handle).`);
13460
+ return;
13461
+ }
12663
13462
 
12664
13463
  // Gap 2 (#678): a clean run that emitted no machine-readable result gets
12665
13464
  // ONE bounded re-emit nudge in the same workspace, feeding back its own
@@ -12669,7 +13468,7 @@ async function workAgent(req, flags, ctx) {
12669
13468
  // the result is recoverable only via the stdout `::nano:result::`
12670
13469
  // sentinel, which `resolveAgentResultWithNudge` handles directly.
12671
13470
  if (result.ok) {
12672
- const { stdout, nudged, truncated, acpOutcome, transcriptFloorOnly } = await resolveAgentResultWithNudge({
13471
+ const { stdout, nudged, truncated, acpOutcome, transcriptFloorOnly, stopReason: effectiveStopReason, nudgeStopReason } = await resolveAgentResultWithNudge({
12673
13472
  result,
12674
13473
  resultFile,
12675
13474
  logger,
@@ -12701,6 +13500,15 @@ async function workAgent(req, flags, ctx) {
12701
13500
  // Adopt the nudge turn's outcome so the escalation / envelope reflect
12702
13501
  // the latest continuation, not only the first turn (#263).
12703
13502
  result.acpOutcome = acpOutcome;
13503
+ // Adopt the EFFECTIVE stopReason (#301, Copilot review on #302,
13504
+ // discussion_r4464312610): when the nudge turn was itself cut off
13505
+ // (e.g. first turn `end_turn`, nudge `max_turn_requests`), the
13506
+ // delivery check + logs/audit must reflect the truncated nudge turn,
13507
+ // not only the first turn. `effectiveStopReason` is the nudge's reason
13508
+ // when truncated, else the first turn's. Record the nudge's own reason
13509
+ // separately for audit.
13510
+ if (effectiveStopReason) result.stopReason = effectiveStopReason;
13511
+ if (nudgeStopReason) result.nudgeStopReason = nudgeStopReason;
12704
13512
  }
12705
13513
  }
12706
13514
 
@@ -12801,6 +13609,10 @@ async function workAgent(req, flags, ctx) {
12801
13609
  } else if (provisioned) {
12802
13610
  gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
12803
13611
  }
13612
+ // #295: surface the durable recovery ref (pushed on an SLA cancellation /
13613
+ // failed run above) in the job outcome so the escalation can point a resume
13614
+ // at it. Attach to whatever gitResult was built for this (provisioned) job.
13615
+ if (recoveryRefInfo && gitResult) { gitResult.recoveryRef = recoveryRefInfo.ref; gitResult.recoverySha = recoveryRefInfo.sha; }
12804
13616
  // Propagate a mid-finalize credential rotation (finalizeGit's push
12805
13617
  // auth-failure recovery re-read a fresh gh token) to every subsystem that
12806
13618
  // captured the pre-rotation token: the outer `repoToken` (later redaction /
@@ -12847,7 +13659,24 @@ async function workAgent(req, flags, ctx) {
12847
13659
  // last periodic checkpoint. `flush` forces past the post-stop guard and is
12848
13660
  // bounded; the lease still protects a newer owner. Then close (drain).
12849
13661
  if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
12850
- try { await checkpointing.close(); } catch { /* best effort */ }
13662
+ // #296 (F8): on a force-stop, bound the drain to the REMAINING of the ONE
13663
+ // shared force-stop deadline (set when the recovery block began) — NOT a
13664
+ // fresh full FORCE_STOP_RECOVERY_MS. The recovery race may already have
13665
+ // consumed most of the budget, so a fresh window here could run the total
13666
+ // (recovery + cleanup) to ~2x the budget and be SIGKILLed mid-cleanup
13667
+ // (Copilot review on #296). The recovery DEADLINE (abortRecovery) tears down
13668
+ // both the checkpoint git AND the GC sweep — and it now fires gitKillGraceMs
13669
+ // BEFORE the deadline, so by the time we get here the git SIGKILL has landed
13670
+ // (or is in its grace window) and the drain settles on its own. The cap is the
13671
+ // REMAINING budget; the floor is the git kill grace (not 1ms) so that even when
13672
+ // the recover race consumed the whole window the drain still awaits the actual
13673
+ // SIGKILL of the already-SIGTERMed git instead of abandoning a live process to
13674
+ // have its workspace reaped under it (Copilot review on #296, 1ms finding).
13675
+ const closeFloorMs = Math.max(1, Number(checkpointing?.gitKillGraceMs) || 0);
13676
+ const closeOpts = forceStopping
13677
+ ? { timeoutMs: Math.max(closeFloorMs, forceStopDeadline - Date.now()) }
13678
+ : undefined;
13679
+ try { await checkpointing.close(closeOpts); } catch { /* best effort */ }
12851
13680
  }
12852
13681
  if (isContainer) liveRunIds.delete(runId);
12853
13682
  // #205: clear the in-flight job marker now the harness has stopped — the
@@ -12916,11 +13745,21 @@ async function workAgent(req, flags, ctx) {
12916
13745
  // outputs. Read before deleting the temp dir.
12917
13746
  // Last resort: vars implied by the agent's ACP `_meta.outcome` (a `blocked`
12918
13747
  // report → an escalation) when it wrote no result of its own.
12919
- const rawResult = pickAgentResult(
13748
+ let rawResult = pickAgentResult(
12920
13749
  () => readAgentResultFile(resultFile),
12921
13750
  () => parseResultFromStdout(result.stdout),
12922
13751
  () => resultVarsFromAcpOutcome(result.acpOutcome),
12923
13752
  );
13753
+ // #301: a turn cut off by its budget is never recorded as a success
13754
+ // unless the work was verifiably delivered (PR found / commits pushed).
13755
+ {
13756
+ const honest = downgradeBudgetStopResult(rawResult, { stopReason: result.stopReason, git: gitResult });
13757
+ if (honest !== rawResult) {
13758
+ logger.warn(`[${jobType}] job ${job.jobKey}: turn ended with stopReason=${result.stopReason} and the work was not delivered — recording status "blocked" instead of ${JSON.stringify(rawResult?.status ?? null)}`);
13759
+ rawResult = honest;
13760
+ result.budgetDowngraded = true;
13761
+ }
13762
+ }
12924
13763
  if (resultDir) { try { rmSync(resultDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(resultDir); }
12925
13764
  const resultVars = sanitizeResultVars(rawResult);
12926
13765
 
@@ -12929,7 +13768,7 @@ async function workAgent(req, flags, ctx) {
12929
13768
  const gitNote = gitResult
12930
13769
  ? ` [${gitResult.branch ? `branch ${gitResult.branch}` : 'detached HEAD'}: ${gitResult.commits.length} commit(s), ${gitResult.branch ? (gitResult.pushed ? 'pushed' : (gitResult.pushFailed ? 'push FAILED' : 'not pushed')) : 'no branch to push'}${gitResult.pr?.found ? `, PR #${gitResult.pr.number}` : ''}]`
12931
13770
  : '';
12932
- logger.info(`[${jobType}] job ${job.jobKey} complete (exit 0)${result.truncated ? ' [output truncated]' : ''}${gitNote}`);
13771
+ logger.info(`[${jobType}] job ${job.jobKey} complete (exit 0)${result.stopReason ? ` [stopReason=${result.stopReason}]` : ''}${result.truncated ? ' [output truncated]' : ''}${gitNote}`);
12933
13772
  if (gitResult?.pushFailed) {
12934
13773
  // Elevate a push failure from a terse info tail to a loud, actionable
12935
13774
  // error naming the stranded commit SHAs (issue #231): these commits are
@@ -14698,7 +15537,12 @@ async function runSupervisorDaemon() {
14698
15537
  const pid = w.pid;
14699
15538
  if (w.child && pid) {
14700
15539
  try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
14701
- await waitForChildExit(w.child, STOP_GRACE_MS);
15540
+ // #296: wait STOP_GRACE_MS for a routine exit PLUS the bounded recovery budget,
15541
+ // so a worker that is flushing + promoting a durable recovery ref (#295) for an
15542
+ // interrupted agent job can finish before the SIGKILL backstop. A worker that
15543
+ // exits sooner is reaped at once — this only lengthens the ceiling for one that
15544
+ // is actively preserving recovery.
15545
+ await waitForChildExit(w.child, STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS);
14702
15546
  if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
14703
15547
  }
14704
15548
  };
@@ -15829,9 +16673,11 @@ async function supervisorStopCmd(force = false) {
15829
16673
 
15830
16674
  // Wait for the daemon process itself to exit. A drain waits INDEFINITELY (the
15831
16675
  // operator opted to wait); a force stop applies the grace window + a SIGKILL
15832
- // backstop so a wedged daemon/worker group can't linger.
16676
+ // backstop so a wedged daemon/worker group can't linger. The window must cover
16677
+ // the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
16678
+ // group while a worker is still promoting its durable recovery ref (#295).
15833
16679
  if (force) {
15834
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
16680
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
15835
16681
  while (Date.now() < deadline) {
15836
16682
  if (!isPidAlive(daemonPid)) break;
15837
16683
  await new Promise((r) => setTimeout(r, 150));
@@ -16309,7 +17155,9 @@ function systemdUserAvailable() {
16309
17155
  async function stopSupervisorProcess(running) {
16310
17156
  try { await supervisorRequest({ op: 'stop' }); }
16311
17157
  catch { try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ } }
16312
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
17158
+ // #296: cover the daemon's own worker force-stop grace so the group SIGKILL below
17159
+ // does not reap a worker mid-recovery (promoting its durable recovery ref, #295).
17160
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
16313
17161
  while (Date.now() < deadline) {
16314
17162
  if (!isPidAlive(running.pid)) break;
16315
17163
  await new Promise((r) => setTimeout(r, 150));