c8ctl-plugin-nano 1.70.3 → 1.70.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/c8ctl-plugin.js CHANGED
@@ -90,7 +90,7 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
90
90
  // so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
91
91
  // a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
92
92
  import { resolveEffectiveEnvelope } from './agent-resume.mjs';
93
- import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
93
+ import { checkpointConfig, checkpointEligibility, checkpointRef, recoveredCheckpointRef, predecessorCheckpointRef, resumeRefIsExplicit, pushNamedRef, observeRemoteRefSha, makeOrphanProbe, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
94
94
 
95
95
  const requireFromHere = createRequire(import.meta.url);
96
96
  const pluginDir = dirname(fileURLToPath(import.meta.url));
@@ -260,6 +260,24 @@ const READINESS_TIMEOUT_MS = 60_000;
260
260
  const READINESS_POLL_MS = 500;
261
261
  const HEALTH_TIMEOUT_MS = 1_500;
262
262
  const STOP_GRACE_MS = 8_000;
263
+ // #296: a `stop --force` SIGTERMs a `nano work` child, which (on an interrupted
264
+ // agent job) flushes a last WIP checkpoint and promotes it to a durable recovery
265
+ // ref (#295) before exiting. That flush+promote must fit inside the window the
266
+ // supervisor allows before its SIGKILL backstop, or the recovery ref is never
267
+ // written. This is the bounded budget the worker gives that force-stop recovery,
268
+ // and the EXTRA grace `forceStopWorker` waits beyond STOP_GRACE_MS before SIGKILL
269
+ // (a child that exits sooner is reaped immediately — this only lengthens the
270
+ // SIGKILL backstop for a worker that is actively preserving recovery).
271
+ const FORCE_STOP_RECOVERY_MS = 20_000;
272
+ // #296: the backstop deadline an OUTER daemon-stop path (the socket-unreachable
273
+ // `stop --force` fallback and `stopSupervisorProcess`) waits for the DAEMON — and
274
+ // its whole process group, which includes the workers — to exit before SIGKILLing
275
+ // the group. It must exceed what the daemon itself allows a worker before ITS
276
+ // SIGKILL backstop (`forceStopWorker` waits STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS),
277
+ // plus a small margin for the daemon to observe the worker's exit and shut down.
278
+ // Otherwise the outer path SIGKILLs the group ~18s early and kills a worker that is
279
+ // still flushing + promoting its durable recovery ref (#295).
280
+ const FORCE_STOP_DAEMON_GRACE_MS = STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS + 2_000;
263
281
  // #202: the signal a graceful `supervisor stop`/`workforce stop` sends each
264
282
  // `nano work` child to quiesce it — stop leasing new jobs, finish in-flight work,
265
283
  // then exit. SIGTERM/SIGINT remain the FORCE abort (kill harness, yield jobs).
@@ -2877,6 +2895,19 @@ const RESERVED_RESULT_KEYS = new Set([
2877
2895
  // push-checkpoint / points world-restore at an arbitrary SHA, so reserve it
2878
2896
  // alongside the other host-owned git-result keys (thread 4117276817).
2879
2897
  'worldMarker',
2898
+ // `resumeFromCheckpoint` is the host/process-owned resume pointer (issue #295):
2899
+ // the plugin writes it at the PROCESS scope so a re-entered (new element
2900
+ // instance) activation can find the predecessor's WIP checkpoint. When it is a
2901
+ // bare STRING it is trusted as an EXPLICIT, occurrence-authoritative ref that
2902
+ // BYPASSES both the elementId correlation AND the ambiguity guard
2903
+ // (resumeRefIsExplicit). Because a successful agent result is merged into the
2904
+ // process variables, an unreserved `resumeFromCheckpoint` lets an untrusted
2905
+ // harness emit a bare-string ref that a later activation then trusts as
2906
+ // occurrence-authoritative — restoring an ARBITRARY WIP ref from the repository
2907
+ // with no correlation or guard. Reserve it so the agent result can never inject
2908
+ // or shadow the host/process-owned pointer (Copilot review on #296, thread
2909
+ // 4243776734).
2910
+ 'resumeFromCheckpoint',
2880
2911
  ]);
2881
2912
 
2882
2913
  // Parse `text` as a JSON object, returning it only when it is a plain object.
@@ -6750,7 +6781,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6750
6781
  ...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
6751
6782
  ...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
6752
6783
  };
6753
- const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
6784
+ // #296 (F9): the checkpoint-path git runner must stay USABLE after the harness
6785
+ // abort. The force-stop RECOVERY path (flush + preserveRecovery, promoting the
6786
+ // durable recovery ref #295) runs only once `result.aborted` is observed — i.e.
6787
+ // AFTER the job's `abortSignal` has already fired. Binding this runner to
6788
+ // `abortSignal` (the round-4 F8 attempt) made `createGitRunner` fail every
6789
+ // recovery command (`[aborted]`), so the abort flush could not push its snapshot
6790
+ // and preserveRecovery could not even observe/create the recovery ref — recovery
6791
+ // was self-defeating. Instead drive it from a DEDICATED recovery-deadline
6792
+ // controller that is NEVER aborted by the harness stop, only when the recovery
6793
+ // budget (FORCE_STOP_RECOVERY_MS) expires. That still delivers F8's goal — an
6794
+ // in-flight push is torn down at the deadline so the chain settles and `close()`
6795
+ // drains inside the supervisor's SIGKILL window — without killing the recovery
6796
+ // before it can write the ref.
6797
+ const recoveryController = new AbortController();
6798
+ const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: recoveryController.signal });
6754
6799
  // The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
6755
6800
  // if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
6756
6801
  // git child is killed and later ops fail fast, so setup unwinds instead of holding
@@ -6761,6 +6806,40 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6761
6806
  // which time the workspace (and its askpass helper) has been reaped.
6762
6807
  const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
6763
6808
 
6809
+ // Bound every checkpoint-path SDK call: a timed-out call rejects; a late settle is
6810
+ // swallowed so it can never surface as an unhandled rejection. Hoisted here (ahead
6811
+ // of the GC block, which reuses them) so the #295 ambiguity guard below is bounded.
6812
+ const CP_SDK_TIMEOUT_MS = cfg.sdkTimeoutMs;
6813
+ const withDeadline = (p, ms, label, signal = null) => {
6814
+ let timer;
6815
+ let onAbort = null;
6816
+ // #296 (review): a bare Promise.race REJECTS on timeout but leaves the SDK
6817
+ // mutation in flight — a timed-out `resumeFromCheckpoint` write could then
6818
+ // complete AFTER a retry already wrote a newer pointer and overwrite it with
6819
+ // the stale ref. The SDK returns a CancelablePromise (toCancelable): cancel it
6820
+ // on timeout so the underlying fetch is aborted and the late settle can never
6821
+ // land. `p.cancel?.()` is a no-op for a plain (non-cancelable) promise, and a
6822
+ // cancelled SDK call rejects with CancelSdkError, which the `.catch(() => {})`
6823
+ // below swallows.
6824
+ // #296 (review r4245146247): the SAME stale-write hazard applies to the job's
6825
+ // ABORT (lease loss), not just the per-call timeout. An optional `signal` lets a
6826
+ // setup-path side-effect write be cancelled the instant the job loses its lease,
6827
+ // so a write left pending after a replacement activation published a newer
6828
+ // pointer is torn down and its late settle can never overwrite that newer value.
6829
+ const race = new Promise((_r, reject) => {
6830
+ timer = setTimeout(() => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} timed out after ${ms}ms`)); }, ms);
6831
+ timer?.unref?.();
6832
+ if (signal) {
6833
+ if (signal.aborted === true) { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); return; }
6834
+ onAbort = () => { try { p?.cancel?.(); } catch { /* best effort */ } reject(new Error(`${label} aborted`)); };
6835
+ signal.addEventListener('abort', onAbort, { once: true });
6836
+ }
6837
+ });
6838
+ Promise.resolve(p).catch(() => {});
6839
+ const cleanup = () => { clearTimeout(timer); if (signal && onAbort) { try { signal.removeEventListener('abort', onAbort); } catch { /* best effort */ } } };
6840
+ return Promise.race([Promise.resolve(p).finally(cleanup), race]);
6841
+ };
6842
+
6764
6843
  let prior = null;
6765
6844
  let restored = null;
6766
6845
  let startupFetchFailed = false;
@@ -6770,6 +6849,93 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6770
6849
  startupFetchFailed = true;
6771
6850
  log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
6772
6851
  }
6852
+ // #295: nothing at this run's deterministic ref, but a human may have answered an
6853
+ // SLA escalation and re-entered the task as a NEW element instance. The predecessor
6854
+ // recorded its ref in a process-scoped `resumeFromCheckpoint` variable (survives
6855
+ // re-entry); adopt it under THIS run's key so the restore/ownership/discard paths
6856
+ // below operate unchanged and no GC can reclaim it between activations.
6857
+ if (!prior && !startupFetchFailed) {
6858
+ // Correlate the shared process-scoped slot to THIS task. Copilot review on #296
6859
+ // (r4244212483): a SELF-RECORDED (object) predecessor is correlated only by the
6860
+ // STATIC elementId, which is NOT unique under parallel multi-instance of one
6861
+ // agent-task element (N occurrences share the elementId AND the one process-scoped
6862
+ // resume slot). Counting ACTIVE occurrences cannot establish occurrence identity
6863
+ // (index lag can hide a live sibling; a terminal sibling drops out of the ACTIVE
6864
+ // result yet may have been the slot's last writer), so NO engine query makes a
6865
+ // self-record adoption provably safe. We therefore adopt ONLY an EXPLICIT,
6866
+ // occurrence-authoritative ref — a bare STRING the process delivers naming THIS
6867
+ // occurrence's exact predecessor (nano-workforce#891, option 2). A self-recorded
6868
+ // object is NEVER auto-adopted (fail closed to a clean start); until #891 delivers
6869
+ // the explicit ref, a resume is transcript-only rather than risking a sibling's
6870
+ // state. `predecessorCheckpointRef` still parses/locates the candidate ref; the
6871
+ // explicit check below is what authorizes adopting it.
6872
+ const predRef = predecessorCheckpointRef(job?.variables, ref, job?.elementId ?? null);
6873
+ if (predRef) {
6874
+ const adoptAllowed = resumeRefIsExplicit(job?.variables);
6875
+ if (!adoptAllowed) {
6876
+ log.warn(`not adopting predecessor WIP checkpoint ${predRef} — it is an elementId-correlated self-record, which cannot be proven to name THIS occurrence (parallel multi-instance of one agent-task element shares the elementId and the process-scoped resume slot); starting clean. Resume is transcript-only until the process delivers an explicit occurrence-authoritative ref (nano-workforce#891).`);
6877
+ }
6878
+ let predCp = null;
6879
+ if (adoptAllowed) {
6880
+ try { predCp = await fetchCp({ git: startupGit, ref: predRef }); } catch (err) { log.debug(`could not fetch predecessor WIP checkpoint ${predRef} — ${oneLineLog(err?.message || err)}.`); }
6881
+ }
6882
+ if (predCp?.sha) {
6883
+ // #296 (review): after ANY adoption the process-scoped slot still names the
6884
+ // stale PREDECESSOR ref. If this resumed activation then succeeds WITHOUT
6885
+ // taking another checkpoint (a short run before the safety timer), its adopted
6886
+ // WIP ref is deleted on ack while the slot keeps pointing at the predecessor —
6887
+ // a later activation would re-restore that old snapshot. Advance the slot to
6888
+ // THIS run's key (with the sha we actually adopted) so the pointer tracks the
6889
+ // current owner. Factored into one helper so EVERY adoption branch advances it
6890
+ // (the create-only win AND the lost-race re-read, whose race winner's own
6891
+ // best-effort advance may have failed — this makes the advance idempotent
6892
+ // across the race rather than reliant on the winner). Best-effort; a failure
6893
+ // never blocks setup.
6894
+ const advanceResumePointer = async (adoptedSha) => {
6895
+ if (!adoptedSha || typeof camunda?.createElementInstanceVariables !== 'function' || !job?.processInstanceKey) return;
6896
+ // #296 (review r4245146247): recheck the job signal before this process-scoped
6897
+ // side effect. The startup git work above (fetch/push) may have straddled a
6898
+ // lease loss; once aborted, a replacement activation owns this pointer slot and
6899
+ // may already have published a newer value, so writing our (now stale) ref here
6900
+ // would overwrite it. Skip when aborted, and bind the SDK call to abortSignal so
6901
+ // a lease loss DURING the write cancels it before its late settle can land.
6902
+ if (abortSignal?.aborted === true) return;
6903
+ try {
6904
+ await withDeadline(camunda.createElementInstanceVariables({
6905
+ elementInstanceKey: String(job.processInstanceKey),
6906
+ variables: { resumeFromCheckpoint: { ref, sha: adoptedSha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch: provisioned.workingBranch || null, at: Date.now(), adoptedFrom: predRef } },
6907
+ local: true,
6908
+ }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', abortSignal || undefined);
6909
+ } catch (err) { log.debug(`could not advance the resumeFromCheckpoint pointer to ${ref} — ${oneLineLog(err?.message || err)}.`); }
6910
+ };
6911
+ // Create-only adoption (default expectSha=''): only write our key while it
6912
+ // is ABSENT, so a concurrent same-key retry that already adopted + advanced
6913
+ // the ref is never clobbered (Copilot review on #296).
6914
+ const copied = await pushNamedRef({ git: startupGit, ref, sha: predCp.sha });
6915
+ if (copied?.ok) {
6916
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = { ...predCp, ref }; }
6917
+ if (!prior) prior = { ...predCp, ref };
6918
+ logger.info?.(`${prefix} (${corr}): adopted predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} under this run's key ${ref} (SLA-escalation resume); restoring it below.`);
6919
+ await advanceResumePointer(predCp.sha);
6920
+ } else if (copied?.staleLease) {
6921
+ // A concurrent activation populated this key first — adopt what IS there
6922
+ // rather than force over it; the ref now holds the race winner's state.
6923
+ try { prior = await fetchCp({ git: startupGit, ref }); } catch { prior = null; }
6924
+ if (prior) {
6925
+ logger.info?.(`${prefix} (${corr}): a concurrent activation already populated ${ref}; adopting its checkpoint ${prior.sha ? '@' + String(prior.sha).slice(0, 12) : ''} (SLA-escalation resume); restoring it below.`);
6926
+ // Advance the pointer to the re-read sha too: the race winner's own
6927
+ // best-effort advance may have failed, which would leave the slot naming
6928
+ // the bare predecessor and risk a later re-adoption of stale work.
6929
+ await advanceResumePointer(prior.sha);
6930
+ } else {
6931
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but ${ref} was concurrently moved and could not be re-read — the predecessor ref is retained for explicit recovery.`);
6932
+ }
6933
+ } else {
6934
+ log.warn(`found predecessor WIP checkpoint ${predRef}@${predCp.sha.slice(0, 12)} but could not copy it to ${ref} — ${copied?.error || 'unknown'}; the predecessor ref is retained for explicit recovery.`);
6935
+ }
6936
+ }
6937
+ }
6938
+ }
6773
6939
  if (prior) {
6774
6940
  const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
6775
6941
  if (r?.restored) {
@@ -6797,12 +6963,12 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6797
6963
 
6798
6964
  // Orphan GC for this remote, throttled per worker process, in the background.
6799
6965
  let gc = Promise.resolve();
6966
+ // #296: the GC sweep's abort controller, hoisted so the force-stop recovery
6967
+ // DEADLINE (abortRecovery) can tear the (up to 120s) sweep down too — otherwise
6968
+ // the single shared force-stop budget would bound only the recovery git, and a
6969
+ // still-running GC could hold close()'s drain past the supervisor's SIGKILL window.
6970
+ let gcAbort = null;
6800
6971
  const gcKey = provisioned.remote || provisioned.workspaceDir;
6801
- // Bound every checkpoint-path SDK call: the GC sweep and the variable write are
6802
- // awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
6803
- // call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
6804
- // late settle is swallowed so it can never surface as an unhandled rejection.
6805
- const CP_SDK_TIMEOUT_MS = 10000;
6806
6972
  // A single overall bound on the whole GC sweep. Individual ops are already
6807
6973
  // bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
6808
6974
  // but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
@@ -6813,23 +6979,27 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6813
6979
  // reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
6814
6980
  // 4099558206 / 4099762442).
6815
6981
  const GC_OVERALL_TIMEOUT_MS = 120000;
6816
- const withDeadline = (p, ms, label) => {
6817
- let timer;
6818
- const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
6819
- Promise.resolve(p).catch(() => {});
6820
- return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
6821
- };
6822
6982
  if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
6823
- const isTerminal = typeof camunda?.getElementInstance === 'function'
6824
- ? async (key) => {
6825
- try {
6826
- const ei = await withDeadline(
6983
+ // #295: a nano-managed ref (WIP or durable recovery) is an orphan only when its
6984
+ // element is terminal AND its PROCESS instance is finished too. An element
6985
+ // cancelled by an SLA boundary timer, inside a process that is still active (human
6986
+ // escalation pending), must NOT be reclaimed — the TTL is the final backstop.
6987
+ // `makeOrphanProbe` encodes that; it degrades to the legacy element-only check when
6988
+ // the host SDK exposes no `getProcessInstance`. The sweep applies this to BOTH the
6989
+ // `refs/nano/wip/*` and `refs/nano/recovered/*` namespaces so recovery refs cannot
6990
+ // grow without bound once their escalation is resolved.
6991
+ const isTerminal = makeOrphanProbe({
6992
+ getElementInstance: typeof camunda?.getElementInstance === 'function'
6993
+ ? (key) => withDeadline(
6827
6994
  camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
6828
- CP_SDK_TIMEOUT_MS, 'getElementInstance');
6829
- return ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED';
6830
- } catch { return false; }
6831
- }
6832
- : null;
6995
+ CP_SDK_TIMEOUT_MS, 'getElementInstance')
6996
+ : undefined,
6997
+ getProcessInstance: typeof camunda?.getProcessInstance === 'function'
6998
+ ? (pik) => withDeadline(
6999
+ camunda.getProcessInstance({ processInstanceKey: String(pik) }, { consistency: { waitUpToMs: 0 } }),
7000
+ CP_SDK_TIMEOUT_MS, 'getProcessInstance')
7001
+ : null,
7002
+ });
6833
7003
  // Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
6834
7004
  // job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
6835
7005
  // push-deletes stale ones, while the workspace is concurrently driven by the harness
@@ -6839,6 +7009,7 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6839
7009
  // decouples them; its dir is reaped when the sweep settles. On the overall deadline
6840
7010
  // we abort the sweep's cancellable git so the promise unwinds instead of hanging.
6841
7011
  const sweepAbort = new AbortController();
7012
+ gcAbort = sweepAbort;
6842
7013
  let sweepDir = null;
6843
7014
  try {
6844
7015
  if (!originUrl) throw new Error('workspace had no origin URL');
@@ -6862,10 +7033,10 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6862
7033
  // the sweep's git is still live. The deadline only bounds it by aborting the git.
6863
7034
  gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
6864
7035
  .then((r) => {
6865
- if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned WIP checkpoint ref(s): ${r.deleted.map((d) => `${d.ref} [${d.why}]`).join(', ')}.`);
6866
- else if (sweepAbort.signal.aborted) log.debug(`WIP checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
6867
- else if (r?.error) log.debug(`WIP checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
6868
- }, (err) => log.debug(`WIP checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
7036
+ if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned checkpoint ref(s): ${r.deleted.map((d) => `${d.ref}@${String(d.sha || '') || '?'} [${d.why}]`).join(', ')}.`);
7037
+ else if (sweepAbort.signal.aborted) log.debug(`checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
7038
+ else if (r?.error) log.debug(`checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
7039
+ }, (err) => log.debug(`checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
6869
7040
  .finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
6870
7041
  } catch (err) {
6871
7042
  if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
@@ -6879,17 +7050,80 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6879
7050
  // delete against it so a stale activation cannot delete a newer owner's WIP.
6880
7051
  let ownedSha = prior?.sha || '';
6881
7052
  let varWarned = false;
7053
+ let resumeVarWarned = false;
7054
+ // #296 (review): the in-flight ABORT-path metadata write (`setVar` fired
7055
+ // fire-and-forget on a force-stop flush, below). The force-stop recovery path
7056
+ // later writes the durable recovery ref to the SAME process-scoped
7057
+ // `resumeFromCheckpoint` slot. If this earlier WIP write is still in flight and
7058
+ // lands AFTER the recovery-pointer write, it overwrites the recovery pointer with
7059
+ // a stale WIP ref — and once the WIP TTL expires, auto-resume can no longer
7060
+ // discover the still-existing recovery ref. Track the promise so the recovery
7061
+ // path can AWAIT it (bounded) before writing the recovery pointer, guaranteeing
7062
+ // the recovery write lands last. The WIP write still lands on its own when
7063
+ // promotion fails (the fallback is retained); we only serialize the ordering.
7064
+ let abortMetaWrite = null;
6882
7065
  const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
6883
- ? async (res) => {
6884
- try {
6885
- await withDeadline(camunda.createElementInstanceVariables({
7066
+ ? (res, signal = null) => {
7067
+ // #296 (review r4245278256): bind these shared process-scoped writes to the
7068
+ // job abort signal for NON-`abort` checkpoints (periodic/`failed`). A periodic
7069
+ // or failed checkpoint can be AWAITING this mutation (`await setVar(res).promise`
7070
+ // below) when heartbeat lease loss interrupts the raw runner and releases the
7071
+ // slot; a replacement activation may then publish a newer `resumeFromCheckpoint`
7072
+ // pointer before this stale call completes and overwrites it — and the later
7073
+ // post-flush signal recheck cannot undo a write that already landed. Passing
7074
+ // `abortSignal` makes `withDeadline` cancel the in-flight SDK call the instant
7075
+ // the lease is lost, so the stale write is torn down and can never land late.
7076
+ // The `abort`-reason path deliberately passes NO signal: that write is owned by
7077
+ // the dedicated `settleAbortMetaWrite` flow (recovery-deadline governed), which
7078
+ // must still fire AFTER the job abort during the force-stop recovery window.
7079
+ // #296: run the element-scope and process-scope writes CONCURRENTLY, not
7080
+ // sequentially. Each carries its own CP_SDK_TIMEOUT_MS deadline, so a
7081
+ // sequential pair could stall for 2× the timeout (20s) — enough to exhaust
7082
+ // the whole FORCE_STOP_RECOVERY_MS budget during a force-stop flush and
7083
+ // starve the recovery git promotion. Racing them caps the worst case at a
7084
+ // single timeout, and each has an independent try/catch so one failing (or
7085
+ // being absent, e.g. no processInstanceKey) never blocks the other.
7086
+ // #296 (review, r4244281967): RETAIN the underlying CancelablePromise(s) so
7087
+ // the abort path can CANCEL them when the recovery deadline fires — not just
7088
+ // stop awaiting. Otherwise a fire-and-forget abort write whose own SDK timeout
7089
+ // outlasts the recovery window stays in flight after the worker yields and can
7090
+ // land LATE over a newer activation's `resumeFromCheckpoint`. The SDK returns a
7091
+ // CancelablePromise (toCancelable); `.cancel?.()` is a no-op for a plain promise
7092
+ // (the test stubs), so capturing the raw return is enough.
7093
+ const rawWrites = [
7094
+ camunda.createElementInstanceVariables({
6886
7095
  elementInstanceKey: String(job.elementInstanceKey),
6887
7096
  variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
6888
7097
  local: true,
6889
- }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables');
6890
- } catch (err) {
6891
- if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7098
+ }),
7099
+ ];
7100
+ const writes = [
7101
+ withDeadline(rawWrites[0], CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7102
+ if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
7103
+ }),
7104
+ ];
7105
+ // #295: also record the ref at the PROCESS instance scope (local on the root
7106
+ // scope key = the process instance key). Unlike the local `agentCheckpoint`
7107
+ // above, this survives the element re-entry an SLA escalation triggers, so a
7108
+ // resumed run (a NEW element instance) can find and adopt its predecessor's
7109
+ // checkpoint via `resumeFromCheckpoint`. Best-effort and independent of the
7110
+ // local write; a failure never blocks the push.
7111
+ if (job?.processInstanceKey) {
7112
+ const rawProcess = camunda.createElementInstanceVariables({
7113
+ elementInstanceKey: String(job.processInstanceKey),
7114
+ variables: { resumeFromCheckpoint: { ref, sha: res.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, branch, at: res.at } },
7115
+ local: true,
7116
+ });
7117
+ rawWrites.push(rawProcess);
7118
+ writes.push(withDeadline(rawProcess, CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables', signal || undefined).catch((err) => {
7119
+ if (!resumeVarWarned) { resumeVarWarned = true; log.warn(`could not set the resumeFromCheckpoint variable — ${oneLineLog(err?.message || err)}; a resumed run may not auto-find ${ref}.`); }
7120
+ }));
6892
7121
  }
7122
+ const promise = Promise.all(writes);
7123
+ // Cancel every in-flight underlying write (best-effort; no-op for plain
7124
+ // promises). Used by the abort path when the recovery deadline fires.
7125
+ const cancel = () => { for (const w of rawWrites) { try { w?.cancel?.(); } catch { /* best effort */ } } };
7126
+ return { promise, cancel };
6893
7127
  }
6894
7128
  : null;
6895
7129
 
@@ -6929,7 +7163,35 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6929
7163
  onCheckpoint: async (res) => {
6930
7164
  ownedSha = res.sha;
6931
7165
  log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
6932
- if (setVar) await setVar(res);
7166
+ if (!setVar) return;
7167
+ // #296: `ownedSha` (above) is the only state preserveRecovery() needs from
7168
+ // this callback, and it is already set. Only a FORCE-STOP (`abort`) flush
7169
+ // fires the metadata writes off the serialized flush/promote chain: that path
7170
+ // is bounded by FORCE_STOP_RECOVERY_MS and the worker returns early (no
7171
+ // settle), so awaiting a hung engine there could burn the whole recovery
7172
+ // budget before the recovery git promotion runs — and the durable recovery
7173
+ // ref, not these vars, is the durable artifact of a force-stopped run.
7174
+ // A `failed` flush is DIFFERENT: the job settles (settleJob.fail) right after
7175
+ // the flush returns, and a retry/reactivation can read variables before a
7176
+ // fire-and-forget `resumeFromCheckpoint` write is durable — worse, a delayed
7177
+ // process-scoped write could overwrite a NEWER activation's pointer. So the
7178
+ // failed path (and every normal checkpoint) AWAITS the write before
7179
+ // settlement, keeping the predecessor pointer durable before the job settles.
7180
+ // #296 (review r4245278256): and it passes `abortSignal`, so if heartbeat lease
7181
+ // loss fires DURING this awaited write the in-flight SDK call is cancelled — a
7182
+ // replacement activation then owns the slot and this run's now-stale pointer can
7183
+ // never land late over its newer value.
7184
+ // #296 (review): on the abort path, CAPTURE the fire-and-forget write (promise +
7185
+ // cancel) so the force-stop recovery path can await it before writing the
7186
+ // recovery pointer to the same slot — otherwise a slow WIP write lands after and
7187
+ // clobbers the recovery ref. `.promise.catch(() => {})` keeps it fire-and-forget
7188
+ // for the flush; the settled/rejected result is irrelevant to the caller. The
7189
+ // retained `cancel` lets settleAbortMetaWrite tear down the in-flight SDK call
7190
+ // when the recovery deadline fires (r4244281967), so it can never land late. This
7191
+ // path deliberately passes NO abortSignal to setVar: the abort write is owned by
7192
+ // settleAbortMetaWrite's recovery-deadline flow and must outlive the job abort.
7193
+ if (res.reason === 'abort') { const w = setVar(res); abortMetaWrite = { promise: w.promise.catch(() => {}), cancel: w.cancel }; return; }
7194
+ await setVar(res, abortSignal || undefined).promise;
6933
7195
  },
6934
7196
  });
6935
7197
  return {
@@ -6939,6 +7201,23 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6939
7201
  restored,
6940
7202
  priorNotRestored,
6941
7203
  flushTimeoutMs: cfg.flushTimeoutMs,
7204
+ // #296 (review): the git SIGKILL grace window, surfaced so the force-stop path
7205
+ // can reserve a real drain window for close() instead of the 1ms an expired
7206
+ // shared deadline would otherwise hand it (see the force-stop block).
7207
+ gitKillGraceMs: cfg.gitKillGraceMs,
7208
+ // #296 (F9): abort the recovery-deadline controller — tears down an in-flight
7209
+ // checkpoint/recovery git push so `stop()`/`close()` settle promptly. Driven by
7210
+ // the force-stop path once the recovery budget expires (NOT by the job's
7211
+ // abortSignal, which fires BEFORE recovery runs and would fail every recovery
7212
+ // command). Also aborts the background GC sweep so the ONE shared force-stop
7213
+ // deadline bounds recovery AND cleanup together — otherwise a still-running
7214
+ // (up to 120s) sweep could hold close()'s drain past the supervisor's SIGKILL
7215
+ // window even after recovery settled. Idempotent / best-effort; a no-op once
7216
+ // already aborted.
7217
+ abortRecovery() {
7218
+ try { recoveryController.abort(); } catch { /* best effort */ }
7219
+ try { gcAbort?.abort(); } catch { /* best effort */ }
7220
+ },
6942
7221
  // Adopt a credential the job ROTATED after this checkpointer was created —
6943
7222
  // e.g. finalizeGit's push auth-failure recovery re-read a fresh gh token
6944
7223
  // (its `rotatedToken`). The post-ack ref deletion (`discardAfterAck`) rebuilds
@@ -6950,6 +7229,148 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6950
7229
  updateCredential(nextToken) {
6951
7230
  if (nextToken && nextToken !== token) token = nextToken;
6952
7231
  },
7232
+ // #296 (review): await the in-flight ABORT-path metadata write (the WIP
7233
+ // `resumeFromCheckpoint` write fired fire-and-forget on a force-stop flush),
7234
+ // bounded, so the force-stop recovery path can guarantee ITS recovery-pointer
7235
+ // write to the same slot lands LAST (a slow WIP write must not overwrite the
7236
+ // durable recovery ref). Best-effort, never throws; resolves immediately when
7237
+ // no abort write is in flight. The caller bounds the wait inside the shared
7238
+ // force-stop budget, so a hung engine cannot stall the force-stop exit.
7239
+ // #296 (review, r4244116341): accept the recovery-deadline `signal` so this SDK
7240
+ // await is cancelled by the SAME deadline that tears down the recovery git — the
7241
+ // recover chain must SETTLE (not dangle) when the budget expires, so a stale
7242
+ // continuation can never write the old recovery pointer over a reactivated
7243
+ // worker's newer one.
7244
+ // #296 (review, r4244281967): stopping the AWAIT is not enough — the underlying
7245
+ // `createElementInstanceVariables` mutation would otherwise stay in flight past
7246
+ // the deadline and could land LATE over a newer activation's `resumeFromCheckpoint`.
7247
+ // So when `signal` aborts we also CANCEL the retained CancelablePromise(s)
7248
+ // (`abortMetaWrite.cancel()`), aborting the fetch so the late settle can never land.
7249
+ async settleAbortMetaWrite(signal = null) {
7250
+ const w = abortMetaWrite;
7251
+ if (!w) return;
7252
+ if (signal?.aborted) { try { w.cancel?.(); } catch { /* best effort */ } return; }
7253
+ if (!signal) { try { await w.promise; } catch { /* best effort */ } return; }
7254
+ let onAbort = null;
7255
+ try {
7256
+ await Promise.race([
7257
+ w.promise.catch(() => {}),
7258
+ new Promise((res) => { onAbort = () => { try { w.cancel?.(); } catch { /* best effort */ } res(); }; signal.addEventListener('abort', onAbort, { once: true }); }),
7259
+ ]);
7260
+ } catch { /* best effort */ } finally { if (onAbort) signal.removeEventListener('abort', onAbort); }
7261
+ },
7262
+ // #295: promote the latest snapshot to a durable recovery ref on an SLA
7263
+ // cancellation or a force-stop. `refs/nano/recovered/<elementInstanceKey>`
7264
+ // lives OUTSIDE the `refs/nano/wip/*` namespace, so it survives the
7265
+ // per-checkpoint WIP cleanup (`discardAfterAck`, which only deletes WIP refs)
7266
+ // and workspace reaping — long enough for the human escalation to be answered
7267
+ // and the work recovered. It is NOT exempt from the orphan GC, though: the
7268
+ // sweep reclaims it under the same process-terminal / 7d-TTL policy as a WIP
7269
+ // ref (#296), sparing it only while its process is still ACTIVE. Returns
7270
+ // `{ ref, sha }` (surfaced in the job outcome) or
7271
+ // null when there is nothing durable to preserve / the push failed. Best-effort;
7272
+ // never throws. The live WIP ref is kept too (this is an additional durable copy).
7273
+ async preserveRecovery(reason = 'recovery') {
7274
+ // Copilot review on #295: the caller invokes this right after a BOUNDED
7275
+ // `flush()`, which can RETURN while its forced checkpoint is still in flight
7276
+ // — and `ownedSha` is only updated later, by `onCheckpoint`, once that
7277
+ // checkpoint's push resolves. Reading `ownedSha` immediately could therefore
7278
+ // see `null`/the pre-flush SHA and promote an older snapshot (or nothing).
7279
+ // DRAIN the checkpoint chain first (`stop()` resolves once the in-flight
7280
+ // chain settles, letting `onCheckpoint` record the latest SHA) so the SHA
7281
+ // selected below is the newest snapshot this run actually pushed. This
7282
+ // mirrors the success path, which already `stop()`s before finalizeGit.
7283
+ try { await checkpointer.stop(); } catch { /* best effort — fall through to the last recorded ownedSha */ }
7284
+ if (!ownedSha) return null;
7285
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7286
+ if (!recRef) return null;
7287
+ try {
7288
+ // Fence the recovery promotion against the sha we OBSERVE on the ref now,
7289
+ // so a slower/stale activation that observed an older/absent value loses
7290
+ // the lease once a newer run moved it, instead of blind-forcing over
7291
+ // whatever is current (Copilot review on #296).
7292
+ const prevRecSha = await observeRemoteRefSha({ git, ref: recRef });
7293
+ // #296 (F7/F10): a bare CAS against the just-observed value is NOT ownership
7294
+ // — a superseded activation can observe the newer run's `shaB` here and
7295
+ // still overwrite it with its older `ownedShaA`. Mirror the WIP path's
7296
+ // `Nano-Checkpoint-Run` ownership check: only overwrite an EXISTING recovery
7297
+ // ref when it is POSITIVELY identified as either ours (a re-preservation by
7298
+ // this same run) or the run we explicitly superseded (`prior?.runId`). An
7299
+ // absent ref is a create-only write. FAIL CLOSED: if the owner cannot be
7300
+ // positively identified — the ownership fetch THREW/failed, or the ref
7301
+ // carries no `runId` trailer — refuse the overwrite rather than fall through
7302
+ // to the CAS, so a transient read failure can never let a stale activation
7303
+ // clobber a newer (or unknown) run's recovery point (F10). The WIP ref +
7304
+ // TTL remain the fallback in every refused case.
7305
+ if (prevRecSha) {
7306
+ let ownerRunId = '';
7307
+ let ownerKnown = false;
7308
+ try { ownerRunId = (await fetchCheckpoint({ git, ref: recRef }))?.runId || ''; ownerKnown = true; } catch { ownerKnown = false; }
7309
+ const myRun = String(runId || '');
7310
+ const supersededRun = String(prior?.runId || '');
7311
+ const mine = ownerKnown && ownerRunId && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7312
+ if (!mine) {
7313
+ const who = !ownerKnown ? 'could not be read (transient)' : (ownerRunId ? `owned by another run (${ownerRunId})` : 'carries no run identity');
7314
+ log.warn(`recovery checkpoint ${recRef} ${who} — refusing to overwrite it; the WIP ref ${ref} is still retained for recovery.`);
7315
+ return null;
7316
+ }
7317
+ }
7318
+ const r = await pushNamedRef({ git, ref: recRef, sha: ownedSha, expectSha: prevRecSha });
7319
+ if (r?.ok) {
7320
+ logger.info?.(`${prefix} (${corr}): preserved a durable recovery checkpoint ${recRef}@${ownedSha.slice(0, 12)} (${reason}) — it survives the per-checkpoint WIP cleanup and workspace reaping (the orphan GC spares it while its process stays active, up to the ${Math.round(cfg.ttlMs / 86400000)}d TTL backstop, which reclaims it regardless of process state) so the SLA-cancelled/force-stopped run can be recovered and resumed.`);
7321
+ return { ref: recRef, sha: ownedSha };
7322
+ }
7323
+ if (r?.staleLease) {
7324
+ // A newer activation already preserved a recovery point on this ref;
7325
+ // keep theirs. Our WIP ref is still retained as a fallback.
7326
+ log.warn(`recovery checkpoint ${recRef} was moved by a newer activation — keeping its recovery point; the WIP ref ${ref} is still retained for recovery.`);
7327
+ return null;
7328
+ }
7329
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(r?.error || 'unknown', token)}; the WIP ref ${ref} is still retained for recovery.`);
7330
+ } catch (err) {
7331
+ log.warn(`could not preserve recovery checkpoint ${recRef} — ${redactToken(oneLineLog(err?.message || err), token)}; the WIP ref ${ref} is still retained for recovery.`);
7332
+ }
7333
+ return null;
7334
+ },
7335
+ // #296 (review, r4244827082): re-verify, IMMEDIATELY before the process-scoped
7336
+ // `resumeFromCheckpoint` recovery-pointer write, that THIS run still owns the
7337
+ // durable recovery ref it just preserved. `preserveRecovery` fences the GIT ref
7338
+ // (CAS + `Nano-Checkpoint-Run` ownership), but the subsequent process-scoped
7339
+ // variable write is unconditional last-writer-wins: a superseded activation that
7340
+ // won its ref earlier can still land its stale pointer AFTER a newer activation
7341
+ // (which re-activated the job on lease loss) published its own. Close that
7342
+ // residual race by re-observing the recovery ref now and confirming it still
7343
+ // resolves to the sha this run preserved AND is still owned by this run (or the
7344
+ // run it explicitly superseded) — the same ownership rule the GIT fence applies.
7345
+ // FAIL CLOSED: any read failure, a moved/absent ref, or an owner that is not
7346
+ // positively this run returns false, so the caller SKIPS the variable write and
7347
+ // leaves the newer activation's pointer intact (the pushed ref stays the durable
7348
+ // fallback). `expectedSha` is the sha this run's preserveRecovery returned.
7349
+ // #296 (review, r4244916101): accept a per-call `signal` and run the git reads
7350
+ // on a DEDICATED runner bound to it (AND the recovery deadline). The caller
7351
+ // races this against a 5s timeout; without a wired signal the losing branch
7352
+ // left the verify's git process live on the 60s runner — untracked by
7353
+ // checkpointer.stop()/gc — so recover() returned and the run dir was reaped
7354
+ // under a live git. Binding the runner to the per-call signal lets the caller
7355
+ // abort+await the verify on timeout so it is SETTLED before cleanup proceeds.
7356
+ async verifyRecoveryOwnership(expectedSha, { signal } = {}) {
7357
+ try {
7358
+ const recRef = recoveredCheckpointRef(job?.elementInstanceKey);
7359
+ if (!recRef || !expectedSha) return false;
7360
+ // Combine the per-call abort with the recovery-deadline abort so EITHER
7361
+ // tears the git down (whichever fires first). AbortSignal.any is Node 20+.
7362
+ const signals = [recoveryController.signal, signal].filter(Boolean);
7363
+ const combined = signals.length > 1 ? AbortSignal.any(signals) : (signals[0] || undefined);
7364
+ const vgit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: combined });
7365
+ const curSha = await observeRemoteRefSha({ git: vgit, ref: recRef });
7366
+ if (!curSha || curSha !== expectedSha) return false;
7367
+ let ownerRunId = '';
7368
+ try { ownerRunId = (await fetchCheckpoint({ git: vgit, ref: recRef }))?.runId || ''; } catch { return false; }
7369
+ const myRun = String(runId || '');
7370
+ const supersededRun = String(prior?.runId || '');
7371
+ return Boolean(ownerRunId) && (ownerRunId === myRun || (supersededRun && ownerRunId === supersededRun));
7372
+ } catch { return false; }
7373
+ },
6953
7374
  // True when the workspace still holds checkpoint content NOT captured by a
6954
7375
  // commit, so the pushed branch does not represent it (an uncommitted restored
6955
7376
  // or working delta that lives only on the WIP ref). The discard-on-ack gate
@@ -6964,8 +7385,21 @@ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token =
6964
7385
  } catch { return true; }
6965
7386
  },
6966
7387
  // Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
6967
- // run dir can be reaped safely.
6968
- async close() { await checkpointer.stop(); await gc; },
7388
+ // run dir can be reaped safely. `close()` is bounded by `closeTimeoutMs` so a
7389
+ // force-stopped worker is never held past the supervisor's SIGKILL backstop by
7390
+ // the (up to 120s) GC sweep: on the deadline we stop awaiting and let the
7391
+ // caller proceed (the in-flight checkpoint git AND the GC sweep are both
7392
+ // cancelled by the recovery DEADLINE — abortRecovery() — so the chain settles on
7393
+ // its own). The force-stop caller passes only the REMAINING of the one shared
7394
+ // force-stop budget, so recovery + this drain never exceed it together. Default
7395
+ // 0 = unbounded (normal/failed paths fully drain).
7396
+ async close({ timeoutMs = 0 } = {}) {
7397
+ const drain = (async () => { await checkpointer.stop(); await gc; })();
7398
+ if (!timeoutMs) return drain;
7399
+ let t;
7400
+ const cap = new Promise((resolve) => { t = setTimeout(resolve, timeoutMs); t?.unref?.(); });
7401
+ try { return await Promise.race([drain, cap]); } finally { clearTimeout(t); }
7402
+ },
6969
7403
  async discard() {
6970
7404
  await checkpointer.stop();
6971
7405
  try {
@@ -9744,6 +10178,11 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
9744
10178
  if (git.branchMismatch) env.branchMismatch = git.branchMismatch;
9745
10179
  if (git.pr) env.pr = git.pr;
9746
10180
  if (git.error) env.gitError = git.error;
10181
+ // #295: a durable recovery ref (refs/nano/recovered/<eik>) an SLA-cancelled or
10182
+ // force-stopped run left behind — carried in the outcome so the escalation /
10183
+ // resume can recover the snapshot from it even after the workspace is reaped.
10184
+ if (git.recoveryRef) env.recoveryRef = git.recoveryRef;
10185
+ if (git.recoverySha) env.recoverySha = git.recoverySha;
9747
10186
  }
9748
10187
  return env;
9749
10188
  }
@@ -12502,6 +12941,28 @@ async function workAgent(req, flags, ctx) {
12502
12941
  // performs a last-chance `failed` flush so edits since the last periodic
12503
12942
  // checkpoint are not lost.
12504
12943
  let checkpointFinalized = false;
12944
+ // #296 (F8/F9): set on the worker force-stop path so the `finally` below bounds
12945
+ // its `close()` drain to the same FORCE_STOP_RECOVERY_MS budget — a force-stop
12946
+ // must never be held past the supervisor's SIGKILL backstop by the (up to
12947
+ // 120s) GC sweep. The in-flight checkpoint git is cancelled by the recovery
12948
+ // DEADLINE (checkpointing.abortRecovery(), fired from the force-stop block),
12949
+ // not the job's abortSignal, so the chain settles on its own after we stop
12950
+ // awaiting while recovery can still run after the harness abort.
12951
+ let forceStopping = false;
12952
+ // #296: the SINGLE absolute force-stop deadline (set when the force-stop
12953
+ // recovery block begins). The recover race AND the `finally`'s close() drain
12954
+ // BOTH bound against it, so recovery + cleanup together fit inside ONE
12955
+ // FORCE_STOP_RECOVERY_MS window rather than consuming it twice (recovery race,
12956
+ // then an independent close() wait) and risking a SIGKILL during cleanup
12957
+ // (Copilot review on #296). 0 until a force-stop begins.
12958
+ let forceStopDeadline = 0;
12959
+ // #295: the durable recovery ref (and SHA) a preserveRecovery() pushed on an
12960
+ // SLA cancellation / failed run, surfaced in the job outcome (io.nanobpm.agentResult)
12961
+ // — but ONLY on the failed/SLA-cancel path, which settles normally and so reaches
12962
+ // buildResultEnvelope. A FORCE-STOPPED run is yielded (settle.fail, retries
12963
+ // preserved) and returns early below WITHOUT an outcome envelope, so its
12964
+ // recoveryRefInfo is intentionally NOT reported there (Copilot review on #295).
12965
+ let recoveryRefInfo = null;
12505
12966
  // The per-job live-terminal relay session (issue #173): streams this job's
12506
12967
  // harness terminal over the single-owner supervisor's ONE multiplexed host
12507
12968
  // connection, keyed by this worker's instance + the jobKey, and accepts
@@ -12652,14 +13113,259 @@ async function workAgent(req, flags, ctx) {
12652
13113
  // that yield. The finally below still runs (container/run-dir/relay
12653
13114
  // cleanup); recordJobEnd fires in the outer finally.
12654
13115
  if (result && result.aborted) {
12655
- logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
13116
+ logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop or SLA/lease-loss cancellation) — harness killed; releasing the job (the broker re-activates it for retry when the run is still live).`);
12656
13117
  // #264: last-chance WIP checkpoint so the retry can pick up the
12657
13118
  // uncommitted work (bounded; the lease guards a newer owner).
12658
- if (checkpointer) { await checkpointer.flush('abort', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13119
+ // #295: then promote it to a durable recovery ref so a force-stopped or
13120
+ // SLA-cancelled run keeps a recovery point. The orphan GC spares that ref
13121
+ // while its process stays active, so it survives long enough to be
13122
+ // resumed. The ref is
13123
+ // pushed here, but it is NOT surfaced in a job outcome: this path yields
13124
+ // the job and returns early, never reaching buildResultEnvelope — a
13125
+ // force-stopped run produces no outcome envelope by design (Copilot
13126
+ // review on #295), so the recovery ref is recoverable from the pushed
13127
+ // refs/nano/recovered/<eik> ref itself, not from any job result.
13128
+ // #296: bound the WHOLE flush+promote to FORCE_STOP_RECOVERY_MS so it
13129
+ // reliably finishes (and this worker exits) inside the window the
13130
+ // supervisor's forceStopWorker now waits before its SIGKILL backstop —
13131
+ // otherwise the default flush (20s) + recovery git push (60s) could run
13132
+ // long and be SIGKILLed before refs/nano/recovered/<eik> is written.
13133
+ // Best-effort: if the budget elapses we give up on the recovery ref (the
13134
+ // WIP ref + TTL remain the fallback), never block the yield/exit.
13135
+ if (checkpointer) {
13136
+ forceStopping = true;
13137
+ const recoveryDeadline = FORCE_STOP_RECOVERY_MS;
13138
+ // #296: anchor the ONE shared force-stop deadline here. The recover race
13139
+ // below and the `finally`'s close() both bound against it, so the TOTAL
13140
+ // force-stop window is a single FORCE_STOP_RECOVERY_MS (not 2x), fitting
13141
+ // inside the supervisor's STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS backstop.
13142
+ forceStopDeadline = Date.now() + recoveryDeadline;
13143
+ // #296 (review): reserve a DRAIN window inside the deadline. abortRecovery()
13144
+ // only SIGTERMs the checkpoint git; createGitRunner then grants up to
13145
+ // gitKillGraceMs before SIGKILL. If abortRecovery() fires AT the deadline,
13146
+ // a stubborn git is still dying when close() hits its (expired) cap and the
13147
+ // finally reaps the workspace under a live process. Fire the abort EARLY —
13148
+ // gitKillGraceMs + a small margin before the deadline — so the SIGKILL lands
13149
+ // by `forceStopDeadline` and close() gets the remaining grace window to
13150
+ // actually drain the now-dead git, not the 1ms `Math.max(1, …)` would give.
13151
+ const killGrace = Math.max(0, Number(checkpointing?.gitKillGraceMs) || 0);
13152
+ const drainReserveMs = killGrace + 250;
13153
+ const abortAtMs = Math.max(1, recoveryDeadline - drainReserveMs);
13154
+ // #296 (F9): arm the recovery-deadline. When the budget expires, abort
13155
+ // the DEDICATED recovery git runner (not the job's abortSignal, which
13156
+ // has already fired and would fail every recovery command) so an
13157
+ // in-flight flush/preserveRecovery push is torn down and the chain
13158
+ // settles — letting the `finally`'s bounded `close()` drain inside the
13159
+ // supervisor's SIGKILL window. Fires `abortAtMs` (the deadline MINUS the
13160
+ // reserved drain window) so the git SIGKILL completes before the deadline;
13161
+ // unref'd so it never keeps the process alive.
13162
+ // #296 (review, r4244116341): the recover-race `Promise.race([recover,
13163
+ // budget])` below only bounds the AWAIT — when `budget` wins, the SDK
13164
+ // mutations inside `recover` (settleAbortMetaWrite's awaited WIP write and
13165
+ // the recovery-pointer write) keep running. The job then yields and a
13166
+ // reactivated worker can publish a NEWER `resumeFromCheckpoint`; a stale
13167
+ // `recover` continuation that completes afterward would clobber it with the
13168
+ // old recovery pointer. abortRecovery() cancels only git/GC, not these SDK
13169
+ // calls. So drive the SDK mutations off a DEDICATED recovery-deadline
13170
+ // signal, fired by the same deadline (and on budget expiry), and AWAIT
13171
+ // `recover` after the race so it is SETTLED/cancelled before we yield.
13172
+ const recoverySdkAbort = new AbortController();
13173
+ let deadlineTimer = setTimeout(() => {
13174
+ try { checkpointing.abortRecovery(); } catch { /* best effort */ }
13175
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13176
+ }, abortAtMs);
13177
+ deadlineTimer?.unref?.();
13178
+ const recover = (async () => {
13179
+ await checkpointer.flush('abort', { timeoutMs: recoveryDeadline }); checkpointFinalized = true;
13180
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('force-stop') || recoveryRefInfo; } catch { /* best effort */ }
13181
+ // #296 (review, AC3): an SLA boundary cancel reaches THIS aborted branch
13182
+ // too — the supervisor heartbeat treats the resulting lease-loss 404 as
13183
+ // terminal and interrupts the runner (supervisor/src/dispatch.ts), so
13184
+ // runAgentJob returns aborted:true. The early return below then skips
13185
+ // buildResultEnvelope, so recoveryRef/recoverySha are never reported in a
13186
+ // job outcome. Persist the recovery ref through the PROCESS-scoped
13187
+ // `resumeFromCheckpoint` slot instead: it survives the element re-entry an
13188
+ // SLA escalation triggers, so the resumed run's predecessorCheckpointRef
13189
+ // can discover the durable refs/nano/recovered/<eik> point. Bounded and
13190
+ // best-effort — never holds the force-stop exit; the pushed ref itself
13191
+ // remains the authoritative recovery handle.
13192
+ // #296 (review): the abort flush fired its OWN WIP `resumeFromCheckpoint`
13193
+ // write fire-and-forget (see onCheckpoint). If that write is still in
13194
+ // flight it could land AFTER the recovery-pointer write below and
13195
+ // overwrite the recovery ref with a stale WIP pointer (which the orphan
13196
+ // GC then reclaims on TTL, losing the recovery discovery). Await the WIP
13197
+ // write FIRST — under the recovery-deadline signal so the await is
13198
+ // cancelled when the budget expires — so the recovery pointer is
13199
+ // guaranteed to land last. Bounded by the recover-race budget; best-effort.
13200
+ try { await checkpointing.settleAbortMetaWrite?.(recoverySdkAbort.signal); } catch { /* best effort */ }
13201
+ // #296 (review, r4244116341): if the recovery deadline has already fired,
13202
+ // do NOT write the recovery pointer — a write that lands after this branch
13203
+ // yields could overwrite a reactivated worker's newer resumeFromCheckpoint.
13204
+ if (recoverySdkAbort.signal.aborted) return;
13205
+ if (recoveryRefInfo && typeof camunda?.createElementInstanceVariables === 'function' && job?.processInstanceKey) {
13206
+ try {
13207
+ // #296 (review, r4244827082): gate the process-scoped pointer write on
13208
+ // PROVEN current ownership of the recovery ref. preserveRecovery only
13209
+ // fences the GIT ref; this variable write is last-writer-wins, so a
13210
+ // superseded activation could otherwise land its stale pointer after a
13211
+ // newer activation (which re-activated the job on lease loss) published
13212
+ // its own. Re-verify the ref still resolves to the sha THIS run
13213
+ // preserved and is still owned by this run; skip the write when it has
13214
+ // moved (a newer activation's pointer stays authoritative). Bounded by
13215
+ // the recovery deadline; a failed/aborted verify fails closed (skip).
13216
+ let stillOwns = false;
13217
+ try {
13218
+ let vt; let vOnAbort = null;
13219
+ // #296 (review, r4244916101): drive the verify off its OWN
13220
+ // AbortController so that when the 5s branch wins we can CANCEL
13221
+ // the in-flight git reads AND await the verify to full settlement
13222
+ // in the finally below — before recover() returns and the run dir
13223
+ // is reaped. Without this the losing verify kept running on the
13224
+ // 60s runner (untracked by stop()/gc) under a reaped workspace.
13225
+ const verifyAbort = new AbortController();
13226
+ const verifyOp = Promise.resolve(
13227
+ checkpointing.verifyRecoveryOwnership?.(recoveryRefInfo.sha, { signal: verifyAbort.signal }),
13228
+ ).then((v) => v === true);
13229
+ stillOwns = await Promise.race([
13230
+ verifyOp,
13231
+ new Promise((res) => { vt = setTimeout(() => res(false), 5000); vt?.unref?.(); }),
13232
+ new Promise((res) => { vOnAbort = () => res(false); recoverySdkAbort.signal.addEventListener('abort', vOnAbort, { once: true }); }),
13233
+ ]).finally(() => {
13234
+ clearTimeout(vt);
13235
+ if (vOnAbort) recoverySdkAbort.signal.removeEventListener('abort', vOnAbort);
13236
+ // The race settled (timeout/abort/complete). If the verify is
13237
+ // still mid-git (timeout/abort won), cancel its runner so the
13238
+ // git process is SIGTERM→SIGKILLed now, not at abortAtMs.
13239
+ try { verifyAbort.abort(); } catch { /* best effort */ }
13240
+ });
13241
+ // AWAIT the verify to full settlement (its git is now aborted, so
13242
+ // this resolves promptly) so no live git outlives recover()/close().
13243
+ try { await verifyOp; } catch { /* best effort */ }
13244
+ } catch { stillOwns = false; }
13245
+ if (!stillOwns) {
13246
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: recovery ref ${recoveryRefInfo.ref} is no longer owned by this run (a newer activation superseded it) — skipping the process-scoped resumeFromCheckpoint write so the newer pointer stays authoritative.`);
13247
+ return;
13248
+ }
13249
+ // #296 (review, r4245472572): the Git-ref ownership check above cannot
13250
+ // fence THIS write across an SLA resume — a re-activated run is a NEW
13251
+ // element instance, so it owns a DIFFERENT refs/nano/recovered/<eik>
13252
+ // and this run still "owns" its old one (passing the check) even after
13253
+ // the new activation published a newer resumeFromCheckpoint. The write
13254
+ // is unconditional last-writer-wins, so it could then overwrite that
13255
+ // newer pointer. Two fences close the class (no atomic process-scoped
13256
+ // CAS exists — so the write is a best-effort DISCOVERY HINT, not an
13257
+ // authoritative fence, and these two narrowings reduce (not eliminate)
13258
+ // its staleness:
13259
+ // (1) OCCURRENCE-SCOPED READ-BACK — re-read the shared slot and skip
13260
+ // when it already names a DIFFERENT run's ref (a newer activation
13261
+ // beat us). Best-effort: a read failure/empty/absent result fails
13262
+ // OPEN (we still write) so a transient search outage never strands
13263
+ // recovery; eventual consistency (index lag) can hide a JUST-landed
13264
+ // newer write, which is exactly why fence (2) is also required.
13265
+ // (2) NATIVE ABORT SIGNAL on the write itself — the SDK options extend
13266
+ // RequestInit, so `signal` reaches fetch and aborts the in-flight
13267
+ // HTTP request the instant the recovery deadline fires, tearing the
13268
+ // mutation down at the transport (not just a best-effort .cancel()).
13269
+ // #296 (review, r4245620621): a read-then-write is NOT atomic, so a
13270
+ // replacement activation can still publish its ref between the read-back
13271
+ // and this write — the residual TOCTOU the reviewer notes is real. It is
13272
+ // BENIGN for recovery CORRECTNESS, which does not rest on this write:
13273
+ // the self-recorded OBJECT form of resumeFromCheckpoint is NEVER
13274
+ // auto-adopted on resume (`makeOccurrenceGuard` fails closed BY
13275
+ // CONSTRUCTION — only a process-delivered bare STRING is adopted), and
13276
+ // the authoritative durable recovery handle is the CAS-fenced git ref
13277
+ // refs/nano/recovered/<eik> (`preserveRecovery` → `pushNamedRef`
13278
+ // --force-with-lease). A stale object pointer therefore yields a
13279
+ // transcript-only resume (identical to an ABSENT pointer), never an
13280
+ // incorrect restore. These narrowings keep the HINT as fresh as is
13281
+ // cheaply possible; they are not relied on for a safety property.
13282
+ // #296 (review, r4245620587): the consistency options MUST be nested
13283
+ // under `consistency` (the real @camunda8 client throws "Missing
13284
+ // consistency options" otherwise — mirrors the 6994/6999 reads and
13285
+ // agent-resume's READ_CONSISTENCY) or the catch below would swallow the
13286
+ // throw and this read-back would silently fail OPEN on every call.
13287
+ if (typeof camunda?.searchVariables === 'function') {
13288
+ try {
13289
+ const found = await Promise.resolve(camunda.searchVariables(
13290
+ { filter: { name: 'resumeFromCheckpoint', scopeKey: String(job.processInstanceKey) }, truncateValues: false },
13291
+ { consistency: { waitUpToMs: 0 }, signal: recoverySdkAbort.signal },
13292
+ ));
13293
+ const items = Array.isArray(found?.items) ? found.items : (Array.isArray(found?.data?.items) ? found.data.items : []);
13294
+ let slotRef = null;
13295
+ for (const it of items) {
13296
+ try { const v = JSON.parse(it?.value ?? 'null'); slotRef = (v && typeof v === 'object' ? v.ref : (typeof v === 'string' ? v : null)) ?? slotRef; } catch { /* unparseable entry — ignore */ }
13297
+ }
13298
+ if (slotRef && slotRef !== recoveryRefInfo.ref) {
13299
+ logger.warn?.(`[${jobType}] job ${job.jobKey}: the process-scoped resumeFromCheckpoint slot already names a newer activation's ref ${slotRef} (not this run's ${recoveryRefInfo.ref}) — skipping the stale overwrite so the newer pointer stays authoritative.`);
13300
+ return;
13301
+ }
13302
+ } catch { /* best effort — a read failure fails OPEN (write proceeds) */ }
13303
+ }
13304
+ const w = camunda.createElementInstanceVariables({
13305
+ elementInstanceKey: String(job.processInstanceKey),
13306
+ variables: { resumeFromCheckpoint: { ref: recoveryRefInfo.ref, sha: recoveryRefInfo.sha, elementInstanceKey: String(job.elementInstanceKey ?? ''), elementId: job.elementId ?? null, recovered: true, at: Date.now() } },
13307
+ local: true,
13308
+ signal: recoverySdkAbort.signal,
13309
+ });
13310
+ let wt; let onAbort = null;
13311
+ // Cancel the in-flight write when the recovery deadline fires (the SDK
13312
+ // returns a CancelablePromise), AND reject the race so the chain
13313
+ // settles — the mutation never lands late over a newer pointer.
13314
+ await Promise.race([
13315
+ Promise.resolve(w).finally(() => { clearTimeout(wt); if (onAbort) recoverySdkAbort.signal.removeEventListener('abort', onAbort); }),
13316
+ new Promise((_r, rej) => { wt = setTimeout(() => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write timed out')); }, 5000); wt?.unref?.(); }),
13317
+ new Promise((_r, rej) => { onAbort = () => { try { w?.cancel?.(); } catch { /* best effort */ } rej(new Error('resumeFromCheckpoint recovery write aborted')); }; recoverySdkAbort.signal.addEventListener('abort', onAbort, { once: true }); }),
13318
+ ]);
13319
+ } catch { /* best effort — the pushed ref is the fallback */ }
13320
+ }
13321
+ })();
13322
+ let budgetTimer = null;
13323
+ const budget = new Promise((res) => { budgetTimer = setTimeout(res, recoveryDeadline); if (budgetTimer.unref) budgetTimer.unref(); });
13324
+ try { await Promise.race([recover, budget]); } catch { /* best effort */ }
13325
+ // Clear the recover-race budget timer (it has fired or the race settled).
13326
+ // deadlineTimer is deliberately NOT cleared: even when the recover race
13327
+ // settles early, abortRecovery() must still fire at abortAtMs to abort the
13328
+ // background GC sweep (gcAbort) — otherwise close() below would await a
13329
+ // sweep that can run up to GC_OVERALL_TIMEOUT_MS (120s), holding the
13330
+ // force-stop open past the supervisor's SIGKILL window (the F8 failure).
13331
+ if (budgetTimer) clearTimeout(budgetTimer);
13332
+ // #296 (review, r4244116341): the race above only bounds the await — when
13333
+ // `budget` wins, `recover` may still be mid-SDK-write. Abort the recovery
13334
+ // signal (cancels the in-flight SDK mutation and short-circuits recover's
13335
+ // remaining steps) and AWAIT `recover` so it is fully settled BEFORE we
13336
+ // yield the job. This guarantees no stale continuation writes the old
13337
+ // recovery pointer after a reactivated worker publishes a newer one. The
13338
+ // signal + each step's internal deadline keep this await bounded.
13339
+ try { recoverySdkAbort.abort(); } catch { /* best effort */ }
13340
+ try { await recover; } catch { /* best effort */ }
13341
+ checkpointFinalized = true;
13342
+ }
12659
13343
  return;
12660
13344
  }
12661
13345
  // #264: a failed run is retried — checkpoint what it left behind.
12662
- if (checkpointer && !result.ok) { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
13346
+ // #295: an SLA-cancelled run surfaces here as a non-ok (timed-out) result;
13347
+ // preserve a durable recovery ref and report it in the job outcome below.
13348
+ if (checkpointer && !result.ok) {
13349
+ await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true;
13350
+ try { recoveryRefInfo = await checkpointing.preserveRecovery('failed') || recoveryRefInfo; } catch { /* best effort */ }
13351
+ }
13352
+ // #296 (review, r4244991586): the failed-result path can outlive its lease.
13353
+ // `result.aborted` is only true when the harness observed the abort WHILE the
13354
+ // process ran; an abortSignal that fires AFTER runAgentJob() returned (with a
13355
+ // genuine non-ok result) but DURING the flush/promote above leaves
13356
+ // `result.aborted === false`. Effect has already interrupted the raw runner and
13357
+ // released the slot, so if we now fall through to settleJob.fail (and the
13358
+ // checkpoint-variable/envelope writes) we overlap a replacement activation with
13359
+ // STALE writes + a STALE settle. Recheck the job signal here — after the
13360
+ // post-run flush/promote, before any settlement — and switch to the
13361
+ // interrupted/yield path (return WITHOUT settling, exactly like the
13362
+ // result.aborted branch and settleRunError's abort idiom) when it has fired.
13363
+ // The recovery ref preserved above still stands (the durable handle); we simply
13364
+ // do not settle a job whose lease is already gone. The broker re-activates it.
13365
+ if (!result.ok && abortSignal?.aborted === true) {
13366
+ logger.warn(`[${jobType}] job ${job.jobKey}: abortSignal fired after the failed run returned (lease loss/force-stop won the race) — yielding WITHOUT settleJob.fail so a replacement activation does not overlap stale checkpoint/settle writes (the preserved recovery ref remains the durable handle).`);
13367
+ return;
13368
+ }
12663
13369
 
12664
13370
  // Gap 2 (#678): a clean run that emitted no machine-readable result gets
12665
13371
  // ONE bounded re-emit nudge in the same workspace, feeding back its own
@@ -12801,6 +13507,10 @@ async function workAgent(req, flags, ctx) {
12801
13507
  } else if (provisioned) {
12802
13508
  gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
12803
13509
  }
13510
+ // #295: surface the durable recovery ref (pushed on an SLA cancellation /
13511
+ // failed run above) in the job outcome so the escalation can point a resume
13512
+ // at it. Attach to whatever gitResult was built for this (provisioned) job.
13513
+ if (recoveryRefInfo && gitResult) { gitResult.recoveryRef = recoveryRefInfo.ref; gitResult.recoverySha = recoveryRefInfo.sha; }
12804
13514
  // Propagate a mid-finalize credential rotation (finalizeGit's push
12805
13515
  // auth-failure recovery re-read a fresh gh token) to every subsystem that
12806
13516
  // captured the pre-rotation token: the outer `repoToken` (later redaction /
@@ -12847,7 +13557,24 @@ async function workAgent(req, flags, ctx) {
12847
13557
  // last periodic checkpoint. `flush` forces past the post-stop guard and is
12848
13558
  // bounded; the lease still protects a newer owner. Then close (drain).
12849
13559
  if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
12850
- try { await checkpointing.close(); } catch { /* best effort */ }
13560
+ // #296 (F8): on a force-stop, bound the drain to the REMAINING of the ONE
13561
+ // shared force-stop deadline (set when the recovery block began) — NOT a
13562
+ // fresh full FORCE_STOP_RECOVERY_MS. The recovery race may already have
13563
+ // consumed most of the budget, so a fresh window here could run the total
13564
+ // (recovery + cleanup) to ~2x the budget and be SIGKILLed mid-cleanup
13565
+ // (Copilot review on #296). The recovery DEADLINE (abortRecovery) tears down
13566
+ // both the checkpoint git AND the GC sweep — and it now fires gitKillGraceMs
13567
+ // BEFORE the deadline, so by the time we get here the git SIGKILL has landed
13568
+ // (or is in its grace window) and the drain settles on its own. The cap is the
13569
+ // REMAINING budget; the floor is the git kill grace (not 1ms) so that even when
13570
+ // the recover race consumed the whole window the drain still awaits the actual
13571
+ // SIGKILL of the already-SIGTERMed git instead of abandoning a live process to
13572
+ // have its workspace reaped under it (Copilot review on #296, 1ms finding).
13573
+ const closeFloorMs = Math.max(1, Number(checkpointing?.gitKillGraceMs) || 0);
13574
+ const closeOpts = forceStopping
13575
+ ? { timeoutMs: Math.max(closeFloorMs, forceStopDeadline - Date.now()) }
13576
+ : undefined;
13577
+ try { await checkpointing.close(closeOpts); } catch { /* best effort */ }
12851
13578
  }
12852
13579
  if (isContainer) liveRunIds.delete(runId);
12853
13580
  // #205: clear the in-flight job marker now the harness has stopped — the
@@ -14698,7 +15425,12 @@ async function runSupervisorDaemon() {
14698
15425
  const pid = w.pid;
14699
15426
  if (w.child && pid) {
14700
15427
  try { process.kill(pid, 'SIGTERM'); } catch { /* already gone */ }
14701
- await waitForChildExit(w.child, STOP_GRACE_MS);
15428
+ // #296: wait STOP_GRACE_MS for a routine exit PLUS the bounded recovery budget,
15429
+ // so a worker that is flushing + promoting a durable recovery ref (#295) for an
15430
+ // interrupted agent job can finish before the SIGKILL backstop. A worker that
15431
+ // exits sooner is reaped at once — this only lengthens the ceiling for one that
15432
+ // is actively preserving recovery.
15433
+ await waitForChildExit(w.child, STOP_GRACE_MS + FORCE_STOP_RECOVERY_MS);
14702
15434
  if (isPidAlive(pid)) { try { process.kill(pid, 'SIGKILL'); } catch { /* ignore */ } }
14703
15435
  }
14704
15436
  };
@@ -15829,9 +16561,11 @@ async function supervisorStopCmd(force = false) {
15829
16561
 
15830
16562
  // Wait for the daemon process itself to exit. A drain waits INDEFINITELY (the
15831
16563
  // operator opted to wait); a force stop applies the grace window + a SIGKILL
15832
- // backstop so a wedged daemon/worker group can't linger.
16564
+ // backstop so a wedged daemon/worker group can't linger. The window must cover
16565
+ // the daemon's own worker force-stop grace (#296), or the SIGKILL below reaps the
16566
+ // group while a worker is still promoting its durable recovery ref (#295).
15833
16567
  if (force) {
15834
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
16568
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
15835
16569
  while (Date.now() < deadline) {
15836
16570
  if (!isPidAlive(daemonPid)) break;
15837
16571
  await new Promise((r) => setTimeout(r, 150));
@@ -16309,7 +17043,9 @@ function systemdUserAvailable() {
16309
17043
  async function stopSupervisorProcess(running) {
16310
17044
  try { await supervisorRequest({ op: 'stop' }); }
16311
17045
  catch { try { process.kill(running.pid, 'SIGTERM'); } catch { /* already gone */ } }
16312
- const deadline = Date.now() + STOP_GRACE_MS + 2_000;
17046
+ // #296: cover the daemon's own worker force-stop grace so the group SIGKILL below
17047
+ // does not reap a worker mid-recovery (promoting its durable recovery ref, #295).
17048
+ const deadline = Date.now() + FORCE_STOP_DAEMON_GRACE_MS;
16313
17049
  while (Date.now() < deadline) {
16314
17050
  if (!isPidAlive(running.pid)) break;
16315
17051
  await new Promise((r) => setTimeout(r, 150));