c8ctl-plugin-nano 1.68.0 → 1.69.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/agent-resume.mjs CHANGED
@@ -31,11 +31,12 @@
31
31
  // (which gets an ephemeral `nano/agent-work/<base>-<runId>` fallback that differs
32
32
  // per activation) — all degrade to a transcript-only continuation (a full
33
33
  // prior-branch resolve+checkout is the later increment).
34
- // - UNCOMMITTED working-tree state is NOT recoverable unless the workspace/microVM
35
- // persists across activations (the isolated-context increment). This increment
34
+ // - UNCOMMITTED working-tree state is NOT recoverable by this module. This module
36
35
  // resumes from last-pushed commit + transcript and treats uncommitted deltas as
37
36
  // lost. The resume preamble tells the agent this explicitly so it re-derives any
38
- // uncommitted work rather than assuming it survived.
37
+ // uncommitted work rather than assuming it survived. WIP checkpoints
38
+ // (`agent-checkpoint.mjs`, NANO_AGENT_CHECKPOINT, default auto, #264) restore that state
39
+ // separately and append a note that supersedes this statement.
39
40
  //
40
41
  // Everything at the process edge (the SDK read) is injected, so the orchestration is
41
42
  // driven deterministically under `node --test` with an in-memory fake client. The
package/c8ctl-plugin.js CHANGED
@@ -90,6 +90,7 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
90
90
  // so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
91
91
  // a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
92
92
  import { resolveEffectiveEnvelope } from './agent-resume.mjs';
93
+ import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
93
94
 
94
95
  const requireFromHere = createRequire(import.meta.url);
95
96
  const pluginDir = dirname(fileURLToPath(import.meta.url));
@@ -4498,21 +4499,10 @@ function authUrl(url, provider, hasToken) {
4498
4499
 
4499
4500
  class ProvisionError extends Error {}
4500
4501
 
4501
- // Reduce an arbitrary string to a single git-ref-safe path segment: git refname
4502
- // rules forbid spaces and ~^:?*[\, leading/trailing/doubled dots, a trailing
4503
- // ".lock", etc. Used to build the deterministic fallback work branch (issue #231)
4504
- // off the base branch name + run id, so a run with no branch.create still commits
4505
- // on a fresh, always-fast-forwardable branch instead of on the base.
4506
- function sanitizeBranchSegment(s) {
4507
- const cleaned = String(s == null ? '' : s)
4508
- .replace(/[^0-9A-Za-z._-]+/g, '-') // collapse anything unusual to a dash
4509
- .replace(/\.{2,}/g, '.') // no doubled dots (git forbids "..")
4510
- .replace(/^[-.]+/, '') // no leading dot or dash
4511
- .slice(0, 60) // bound the segment BEFORE the trailing
4512
- .replace(/[-.]+$/g, '') // checks, so truncating at char 60 can't
4513
- .replace(/\.lock$/i, 'lock'); // re-introduce a trailing dot/dash or ".lock"
4514
- return cleaned || 'base';
4515
- }
4502
+ // `sanitizeBranchSegment` (the deterministic fallback-work-branch segment builder,
4503
+ // issue #231) now lives in ./agent-checkpoint.mjs as the single source of truth: the
4504
+ // restore path reconstructs the generated `nano/agent-work/<base>-<runId>` identity to
4505
+ // validate a benign per-run rename, so provisioning and restore must share one impl.
4516
4506
 
4517
4507
  // Never let git invoke the host's configured credential helper for our clone/
4518
4508
  // push. Reset the helper list ("") so no helper runs — even when we DO have a
@@ -5315,7 +5305,12 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
5315
5305
  // exactly when a real work branch was resolved that is not the base itself
5316
5306
  // (false for a read-only base checkout with push disabled, or a detached HEAD).
5317
5307
  const hasPrBranch = !!workingBranch && workingBranch !== effectiveBase;
5318
- return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
5308
+ // Authentication is not synonymous with a token: an author-embedded HTTPS
5309
+ // userinfo URL or an SSH remote authenticates the push with no `token`. Stamp
5310
+ // the authoritative result so eligibility (auto checkpointing) doesn't
5311
+ // under-detect an authenticated clone.
5312
+ const authenticated = !!token || isAuthenticatedRemote(repo.url);
5313
+ return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
5319
5314
  }
5320
5315
 
5321
5316
  // Look up a PR for this branch (2a does NOT open it — the harness does, driven
@@ -5439,6 +5434,306 @@ function shouldPreserveRunDir(gitResult) {
5439
5434
  return !!gitResult?.pushFailed;
5440
5435
  }
5441
5436
 
5437
+ // #264: WIP checkpoint wiring for one host-provisioned job (see agent-checkpoint.mjs).
5438
+ // Returns null when disabled/ineligible/unavailable, else `{ checkpointer,
5439
+ // restored, flushTimeoutMs, discard(), close() }`. Never throws — best-effort.
5440
+ async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token = null, secretValues = [], job, jobType, runId, camunda, logger, corr = '', env = process.env, deps = {}, abortSignal = null }) {
5441
+ const cfg = checkpointConfig(env);
5442
+ const prefix = `[${jobType}] job ${job?.jobKey}`;
5443
+ // Cancellation contract: if the runner was already interrupted (lock loss /
5444
+ // stop --force) before we begin, do no git work — the Effect slot may already be
5445
+ // released and the throwaway workspace is about to be reaped.
5446
+ if (abortSignal?.aborted === true) return null;
5447
+ const eligible = checkpointEligibility({ mode: cfg.mode, provisioned, envelope, token });
5448
+ if (!eligible.enabled) {
5449
+ if (provisioned?.workspaceDir) logger.debug?.(`${prefix}: WIP checkpoints off for this job — ${eligible.reason}.`);
5450
+ return null;
5451
+ }
5452
+ const ref = checkpointRef(job?.elementInstanceKey);
5453
+ if (!ref) {
5454
+ if (cfg.mode === 'on') logger.warn?.(`${prefix}: WIP checkpoints enabled but the job has no usable elementInstanceKey — checkpoints off for this job.`);
5455
+ return null;
5456
+ }
5457
+ const {
5458
+ createGitRunner: mkGit = createGitRunner,
5459
+ fetchCheckpoint: fetchCp = fetchCheckpoint,
5460
+ restoreCheckpoint: restoreCp = restoreCheckpoint,
5461
+ deleteCheckpointRef: deleteCp = deleteCheckpointRef,
5462
+ sweep = sweepStaleCheckpoints,
5463
+ now = () => Date.now(),
5464
+ } = deps;
5465
+ const log = {
5466
+ warn: (m) => logger.warn?.(`${prefix} (${corr}): ${m}`),
5467
+ debug: (m) => logger.debug?.(`${prefix} (${corr}): ${m}`),
5468
+ };
5469
+ try {
5470
+ const committer = provisioned.committer || {};
5471
+ const gitEnv = {
5472
+ ...(provisioned.gitEnv || process.env),
5473
+ ...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
5474
+ ...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
5475
+ };
5476
+ const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
5477
+ // The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
5478
+ // if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
5479
+ // git child is killed and later ops fail fast, so setup unwinds instead of holding
5480
+ // a released Effect slot until a git timeout. The checkpointer keeps the UNBOUND
5481
+ // `git` runner above so its abort-flush can still push a final snapshot.
5482
+ const startupGit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: abortSignal || undefined });
5483
+ // Captured now: the ref is deleted only after the engine acks completion, by
5484
+ // which time the workspace (and its askpass helper) has been reaped.
5485
+ const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
5486
+
5487
+ let prior = null;
5488
+ let restored = null;
5489
+ let startupFetchFailed = false;
5490
+ try {
5491
+ prior = await fetchCp({ git: startupGit, ref });
5492
+ } catch (err) {
5493
+ startupFetchFailed = true;
5494
+ log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
5495
+ }
5496
+ if (prior) {
5497
+ const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
5498
+ if (r?.restored) {
5499
+ restored = r;
5500
+ logger.info?.(`${prefix} (${corr}): restored WIP checkpoint ${ref}@${prior.sha.slice(0, 12)} (${r.mode}${r.commitsRecovered ? ', local commits recovered' : ''}) as uncommitted changes.`);
5501
+ } else {
5502
+ const rb = r?.rollbackFailed
5503
+ ? ' — WARNING: the workspace could NOT be rolled back to its pristine state and may hold partially-recovered content'
5504
+ : '';
5505
+ log.warn(`found WIP checkpoint ${ref}@${prior.sha.slice(0, 12)} but did not restore it — ${r?.reason || 'unknown'}${rb}; the ref is RETAINED (not overwritten) for this activation, so its content can still be recovered explicitly.`);
5506
+ }
5507
+ }
5508
+ // Cancellation recheck: after the startup git work, bail before starting the
5509
+ // background GC or the checkpointer if the runner was interrupted — a released
5510
+ // slot must not leave a GC sweep or a live checkpointer overlapping the next
5511
+ // activation on a workspace that is about to be reaped.
5512
+ if (abortSignal?.aborted === true) {
5513
+ log.debug('setup interrupted after restore — not starting GC/checkpointer for this (yielded) job.');
5514
+ return null;
5515
+ }
5516
+ // The ref held a prior checkpoint we could NOT lay into the workspace (restore
5517
+ // skipped): its content lives ONLY on the ref, so discarding it on ack would be
5518
+ // the same data loss as discarding uncommitted content. Keep the ref in that case.
5519
+ const priorNotRestored = Boolean(prior) && !restored;
5520
+
5521
+ // Orphan GC for this remote, throttled per worker process, in the background.
5522
+ let gc = Promise.resolve();
5523
+ const gcKey = provisioned.remote || provisioned.workspaceDir;
5524
+ // Bound every checkpoint-path SDK call: the GC sweep and the variable write are
5525
+ // awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
5526
+ // call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
5527
+ // late settle is swallowed so it can never surface as an unhandled rejection.
5528
+ const CP_SDK_TIMEOUT_MS = 10000;
5529
+ // A single overall bound on the whole GC sweep. Individual ops are already
5530
+ // bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
5531
+ // but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
5532
+ // remote could otherwise hold `close()`/`discard()` (awaited in job cleanup /
5533
+ // force-stop) for many minutes and starve worker capacity. Cap the total; on the
5534
+ // deadline we ABORT the sweep's cancellable git (below) and still AWAIT it to fully
5535
+ // settle — never abandon it to run git against a workspace the job is about to
5536
+ // reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
5537
+ // 4099558206 / 4099762442).
5538
+ const GC_OVERALL_TIMEOUT_MS = 120000;
5539
+ const withDeadline = (p, ms, label) => {
5540
+ let timer;
5541
+ const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
5542
+ Promise.resolve(p).catch(() => {});
5543
+ return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
5544
+ };
5545
+ if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
5546
+ const isTerminal = typeof camunda?.getElementInstance === 'function'
5547
+ ? async (key) => {
5548
+ try {
5549
+ const ei = await withDeadline(
5550
+ camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
5551
+ CP_SDK_TIMEOUT_MS, 'getElementInstance');
5552
+ return ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED';
5553
+ } catch { return false; }
5554
+ }
5555
+ : null;
5556
+ // Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
5557
+ // job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
5558
+ // push-deletes stale ones, while the workspace is concurrently driven by the harness
5559
+ // and later by `finalizeGit` (which fetches the base and reads FETCH_HEAD). Sharing
5560
+ // one `.git` risks ref-lock contention and FETCH_HEAD being clobbered between the
5561
+ // finalizer's fetch and read. A scratch repo with only `origin` configured fully
5562
+ // decouples them; its dir is reaped when the sweep settles. On the overall deadline
5563
+ // we abort the sweep's cancellable git so the promise unwinds instead of hanging.
5564
+ const sweepAbort = new AbortController();
5565
+ let sweepDir = null;
5566
+ try {
5567
+ if (!originUrl) throw new Error('workspace had no origin URL');
5568
+ sweepDir = mkdtempSync(join(tmpdir(), 'nano-wip-gc-'));
5569
+ const sweepEnv = { ...(provisioned.gitEnv || process.env), GIT_TERMINAL_PROMPT: '0' };
5570
+ delete sweepEnv.GIT_ASKPASS;
5571
+ delete sweepEnv.GIT_TOKEN;
5572
+ const askpass = token ? writeAskpass(sweepDir, token) : null;
5573
+ if (askpass) { sweepEnv.GIT_ASKPASS = askpass; sweepEnv.GIT_TOKEN = token; }
5574
+ const sweepGit = mkGit({ cwd: sweepDir, env: sweepEnv, timeoutMs: cfg.gitTimeoutMs, signal: sweepAbort.signal });
5575
+ const init = await sweepGit(['init', '--quiet', '.']);
5576
+ if (init.status !== 0) throw new Error(`git init failed: ${oneLineLog(init.stderr || init.stdout || '')}`);
5577
+ // Configure origin through the protected `.git/config` (0700 tmpdir) rather than
5578
+ // argv, so any userinfo embedded in `originUrl` never lands on `git`'s argv/`ps`.
5579
+ const escUrl = originUrl.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
5580
+ appendFileSync(join(sweepDir, '.git', 'config'), `[remote "origin"]\n\turl = "${escUrl}"\n`);
5581
+ const deadlineTimer = setTimeout(() => sweepAbort.abort(), GC_OVERALL_TIMEOUT_MS);
5582
+ deadlineTimer.unref?.();
5583
+ // `gc` is tied to the sweep's ACTUAL completion (not a race that abandons it),
5584
+ // so `close()`/`discard()` — which await `gc` — never reap the scratch repo while
5585
+ // the sweep's git is still live. The deadline only bounds it by aborting the git.
5586
+ gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
5587
+ .then((r) => {
5588
+ if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned WIP checkpoint ref(s): ${r.deleted.map((d) => `${d.ref} [${d.why}]`).join(', ')}.`);
5589
+ else if (sweepAbort.signal.aborted) log.debug(`WIP checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
5590
+ else if (r?.error) log.debug(`WIP checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
5591
+ }, (err) => log.debug(`WIP checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
5592
+ .finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
5593
+ } catch (err) {
5594
+ if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
5595
+ log.debug(`WIP checkpoint GC not started — ${redactToken(oneLineLog(err?.message || err), token)}.`);
5596
+ }
5597
+ }
5598
+
5599
+ const branch = provisioned.workingBranch || null;
5600
+ // The SHA this run currently owns on the ref: the checkpoint we restored to
5601
+ // begin with, then each checkpoint we successfully push. `discard()` leases its
5602
+ // delete against it so a stale activation cannot delete a newer owner's WIP.
5603
+ let ownedSha = prior?.sha || '';
5604
+ let varWarned = false;
5605
+ const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
5606
+ ? async (res) => {
5607
+ try {
5608
+ await withDeadline(camunda.createElementInstanceVariables({
5609
+ elementInstanceKey: String(job.elementInstanceKey),
5610
+ variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
5611
+ local: true,
5612
+ }), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables');
5613
+ } catch (err) {
5614
+ if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
5615
+ }
5616
+ }
5617
+ : null;
5618
+
5619
+ const checkpoint = createWorkspaceCheckpoint({
5620
+ git,
5621
+ ref,
5622
+ baseSha: provisioned.startSha || '',
5623
+ // The RAW base ref this run was provisioned against — recorded UNSANITIZED so a
5624
+ // later restore can prove two per-run fallbacks share a base without relying on
5625
+ // the ambiguous sanitized branch segment (see CHECKPOINT_BASEREF_TRAILER).
5626
+ baseRefName: provisioned.baseBranch || '',
5627
+ runId: String(runId || ''),
5628
+ // When a prior snapshot EXISTED but could NOT be restored (branch-mismatch /
5629
+ // detached refusal, or a diverged patch that did not apply), its content lives
5630
+ // ONLY on the ref. Do NOT hand this run ownership of it: with `expectSha`/
5631
+ // `priorRunId` set, the first checkpoint would `--force-with-lease` over the
5632
+ // ref and destroy the only copy of that un-restored snapshot. Withholding
5633
+ // ownership makes the first push a create-only that the existing ref rejects,
5634
+ // so checkpointing disables for this activation and the old ref is left intact
5635
+ // for explicit recovery (matching `priorNotRestored` retaining it on discard).
5636
+ expectSha: priorNotRestored ? '' : (prior?.sha || ''),
5637
+ priorRunId: priorNotRestored ? '' : (prior?.runId || ''),
5638
+ startupFetchFailed,
5639
+ maxFileBytes: cfg.maxFileBytes,
5640
+ // Include any credential embedded in the origin URL's userinfo: provisioning
5641
+ // preserves author-supplied userinfo (authUrl), and for such credential-bearing
5642
+ // URLs `token` is null, so the URL password would otherwise be absent from the
5643
+ // scan set and a changed file echoing it could enter the readable WIP ref.
5644
+ secretValues: normalizeSecretValues([...(secretValues || []), token, ...credentialsFromUrl(originUrl)]),
5645
+ message: `nano: WIP checkpoint for job ${job.jobKey}`,
5646
+ });
5647
+ const checkpointer = createCheckpointer({
5648
+ checkpoint,
5649
+ minIntervalMs: cfg.minIntervalMs,
5650
+ intervalMs: cfg.intervalMs,
5651
+ logger: log,
5652
+ onCheckpoint: async (res) => {
5653
+ ownedSha = res.sha;
5654
+ log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
5655
+ if (setVar) await setVar(res);
5656
+ },
5657
+ });
5658
+ return {
5659
+ ref,
5660
+ mode: cfg.mode,
5661
+ checkpointer,
5662
+ restored,
5663
+ priorNotRestored,
5664
+ flushTimeoutMs: cfg.flushTimeoutMs,
5665
+ // True when the workspace still holds checkpoint content NOT captured by a
5666
+ // commit, so the pushed branch does not represent it (an uncommitted restored
5667
+ // or working delta that lives only on the WIP ref). The discard-on-ack gate
5668
+ // uses this: `gitResult.pushed` alone is not proof the WIP is durable — the
5669
+ // agent can push an unrelated commit while restored changes stay uncommitted.
5670
+ // Best-effort; on any git error we report "dirty" so the caller KEEPS the ref.
5671
+ async hasUncommittedChanges() {
5672
+ try {
5673
+ const r = await git(['status', '--porcelain', '--untracked-files=all']);
5674
+ if (r.status !== 0) return true;
5675
+ return (r.stdout || '').trim().length > 0;
5676
+ } catch { return true; }
5677
+ },
5678
+ // Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
5679
+ // run dir can be reaped safely.
5680
+ async close() { await checkpointer.stop(); await gc; },
5681
+ async discard() {
5682
+ await checkpointer.stop();
5683
+ try {
5684
+ const r = await deleteCp({ git, ref, expectSha: ownedSha });
5685
+ if (r?.staleLease) log.debug(`WIP checkpoint ${ref} left intact — the ref moved to a newer activation; not deleting another owner's WIP.`);
5686
+ else if (!r?.ok) log.warn(`could not delete WIP checkpoint ${ref} — ${r?.error || 'unknown'}.`);
5687
+ } catch (err) {
5688
+ log.warn(`could not delete WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}.`);
5689
+ }
5690
+ },
5691
+ // Delete the ref once the engine has acknowledged the job's completion.
5692
+ // Independent of the (by then reaped) workspace: pushes the delete from a
5693
+ // throwaway scratch repo with its own askpass helper. Never throws; a ref left
5694
+ // behind is reclaimed by the orphan GC once the element instance is terminal.
5695
+ async discardAfterAck() {
5696
+ await checkpointer.stop();
5697
+ await gc;
5698
+ if (!originUrl) { log.warn(`could not delete WIP checkpoint ${ref} — workspace had no origin URL; the orphan GC will reclaim it.`); return; }
5699
+ let dir = null;
5700
+ try {
5701
+ dir = mkdtempSync(join(tmpdir(), 'nano-wip-'));
5702
+ const gitEnv = { ...(provisioned.gitEnv || process.env), GIT_TERMINAL_PROMPT: '0' };
5703
+ delete gitEnv.GIT_ASKPASS;
5704
+ delete gitEnv.GIT_TOKEN;
5705
+ const askpass = token ? writeAskpass(dir, token) : null;
5706
+ if (askpass) { gitEnv.GIT_ASKPASS = askpass; gitEnv.GIT_TOKEN = token; }
5707
+ // A plain (non-bare) repo: `safe.bareRepository=explicit` refuses implicit bare dirs.
5708
+ const scratch = mkGit({ cwd: dir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
5709
+ const init = await scratch(['init', '--quiet', '.']);
5710
+ if (init.status !== 0) throw new Error(`git init failed: ${oneLineLog(init.stderr || init.stdout || '')}`);
5711
+ // Configure origin through the throwaway repo's protected `.git/config`
5712
+ // (a 0700 tmpdir) rather than passing `originUrl` positionally to
5713
+ // `git push`: `authUrl` preserves any userinfo an author embedded in
5714
+ // `repository.url`, and an argv URL would leak those credentials into the
5715
+ // push process's argv / `ps` listing. Writing the config file directly
5716
+ // keeps the URL off argv entirely (a `git remote add` would still expose
5717
+ // it); auth still flows through GIT_ASKPASS (thread 4099558181).
5718
+ const escUrl = originUrl.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
5719
+ appendFileSync(join(dir, '.git', 'config'), `[remote "origin"]\n\turl = "${escUrl}"\n`);
5720
+ const r = await deleteCp({ git: scratch, ref, expectSha: ownedSha });
5721
+ if (r?.ok) log.debug(`deleted WIP checkpoint ${ref} after the engine acknowledged completion.`);
5722
+ else if (r?.staleLease) log.debug(`WIP checkpoint ${ref} left intact — the ref moved to a newer activation; the orphan GC will reclaim it if needed.`);
5723
+ else log.warn(`could not delete WIP checkpoint ${ref} — ${redactToken(r?.error || 'unknown', token)}; the orphan GC will reclaim it.`);
5724
+ } catch (err) {
5725
+ log.warn(`could not delete WIP checkpoint ${ref} — ${redactToken(oneLineLog(err?.message || err), token)}; the orphan GC will reclaim it.`);
5726
+ } finally {
5727
+ if (dir) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } }
5728
+ }
5729
+ },
5730
+ };
5731
+ } catch (err) {
5732
+ log.warn(`WIP checkpoint setup failed — ${oneLineLog(err?.message || err)}; continuing without checkpoints.`);
5733
+ return null;
5734
+ }
5735
+ }
5736
+
5442
5737
  // Decide the agentic/transcript relay's close reason (normal / job-killed / error)
5443
5738
  // for a finished job. Pure + exported so the close-reason contract is unit-tested
5444
5739
  // without spinning a full worker job. `error` must cover ANY hard finalization
@@ -10409,6 +10704,13 @@ async function workAgent(req, flags, ctx) {
10409
10704
  // as a `normal` close. This flag flips true only once we reach the end of the
10410
10705
  // try body, so an early throw is correctly reported as `error`.
10411
10706
  let runCompleted = false;
10707
+ let discardCheckpointOnAck = false;
10708
+ // #264: whether a TERMINAL WIP checkpoint decision (abort/failed flush, or the
10709
+ // success-path stop) was already taken on the normal control flow. If the run
10710
+ // throws before any of those (e.g. runAgentJob rejects), the finally below
10711
+ // performs a last-chance `failed` flush so edits since the last periodic
10712
+ // checkpoint are not lost.
10713
+ let checkpointFinalized = false;
10412
10714
  // The per-job live-terminal relay session (issue #173): streams this job's
10413
10715
  // harness terminal over the single-owner supervisor's ONE multiplexed host
10414
10716
  // connection, keyed by this worker's instance + the jobKey, and accepts
@@ -10433,6 +10735,30 @@ async function workAgent(req, flags, ctx) {
10433
10735
  return;
10434
10736
  }
10435
10737
 
10738
+ // #264: WIP checkpoints (NANO_AGENT_CHECKPOINT=auto|on|off, default auto). Restore a prior
10739
+ // attempt's snapshot into the fresh clone, then snapshot the workspace to
10740
+ // `refs/nano/wip/<elementInstanceKey>` as the run progresses. Best-effort:
10741
+ // any failure logs and degrades to the transcript-only resume.
10742
+ const checkpointing = await setupWorkspaceCheckpoints({ provisioned, envelope, token: repoToken, secretValues: Object.values(resolved || {}), job, jobType, runId, camunda, logger, corr: aiCorr, abortSignal });
10743
+ // #264 cancellation contract: setup does cancellable git work (fetch/restore)
10744
+ // and may start a background GC + live checkpointer. If a lock-loss/force-stop
10745
+ // won during that await, stop here before opening the relay/harness so a
10746
+ // released slot never proceeds into the runner path or overlaps the next
10747
+ // activation. (setupWorkspaceCheckpoints already declined to start GC/the
10748
+ // checkpointer on an observed abort, so there is nothing to tear down.)
10749
+ if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'relay-open', logger })) {
10750
+ // A checkpointer may have been created just before the abort landed; stop
10751
+ // it (and drain its GC) so no snapshot timer or sweep overlaps the next
10752
+ // activation on the about-to-be-reaped workspace.
10753
+ if (checkpointing?.close) { try { await checkpointing.close(); } catch { /* best effort */ } }
10754
+ await discardAgentInstanceProducer();
10755
+ if (runDir) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(runDir); }
10756
+ if (isContainer) liveRunIds.delete(runId);
10757
+ return;
10758
+ }
10759
+ if (checkpointing?.restored) effectiveEnvelope = withCheckpointNote(effectiveEnvelope, checkpointing.restored);
10760
+ const checkpointer = checkpointing?.checkpointer ?? null;
10761
+
10436
10762
  let relaySession = null;
10437
10763
  if (agenticPlane) {
10438
10764
  // #229: thread the correlation join keys + a lazy AgentInstance-key getter
@@ -10514,7 +10840,11 @@ async function workAgent(req, flags, ctx) {
10514
10840
  // producer so its turn is appended to the engine's AgentHistory. Inert
10515
10841
  // when no producer was minted (non-external job / off / mint failed) and
10516
10842
  // for non-ACP harnesses (no session/updates are emitted). Best-effort.
10517
- onAcpUpdate: agentInstanceProducer ? (u) => agentInstanceProducer.ingest(u) : undefined,
10843
+ // #264: the same updates pace WIP checkpoints (a completed tool call or a
10844
+ // plan change is a natural snapshot boundary; rate-limited inside).
10845
+ onAcpUpdate: (agentInstanceProducer || checkpointer)
10846
+ ? (u) => { agentInstanceProducer?.ingest(u); checkpointer?.notify(u); }
10847
+ : undefined,
10518
10848
  };
10519
10849
  result = await runAgentJob(profile, job, runOpts);
10520
10850
 
@@ -10526,8 +10856,13 @@ async function workAgent(req, flags, ctx) {
10526
10856
  // cleanup); recordJobEnd fires in the outer finally.
10527
10857
  if (result && result.aborted) {
10528
10858
  logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
10859
+ // #264: last-chance WIP checkpoint so the retry can pick up the
10860
+ // uncommitted work (bounded; the lease guards a newer owner).
10861
+ if (checkpointer) { await checkpointer.flush('abort', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
10529
10862
  return;
10530
10863
  }
10864
+ // #264: a failed run is retried — checkpoint what it left behind.
10865
+ if (checkpointer && !result.ok) { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
10531
10866
 
10532
10867
  // Gap 2 (#678): a clean run that emitted no machine-readable result gets
10533
10868
  // ONE bounded re-emit nudge in the same workspace, feeding back its own
@@ -10576,6 +10911,20 @@ async function workAgent(req, flags, ctx) {
10576
10911
  try { await agentInstanceProducer.complete(result.ok); } catch (err) { logger.warn(`[${jobType}] AgentInstance producer complete() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; job settlement unaffected.`); }
10577
10912
  }
10578
10913
 
10914
+ // #264: the run is done — take a FINAL snapshot (so edits still pending in
10915
+ // the rate-limiter's trailing timer are not lost) and stop snapshotting
10916
+ // before finalizeGit touches the workspace. `flush('final')` forces one
10917
+ // last checkpoint past the post-stop guard, then latches the scheduler
10918
+ // stopped; on a successful run with no branch commit (or a crash before
10919
+ // job.complete is acked) the retained WIP ref then still holds the latest
10920
+ // work rather than only the last periodic checkpoint (advisory 10713).
10921
+ // `flush` is bounded by a timeout, so it can RETURN while its snapshot is
10922
+ // still queued/in-flight; that snapshot reads the same workspace/HEAD that
10923
+ // finalizeGit is about to commit and push. DRAIN the scheduler (stop()
10924
+ // resolves once the in-flight checkpoint chain settles) before finalizeGit
10925
+ // so finalization never races a late snapshot (thread 4099675063).
10926
+ if (checkpointer && result.ok) { await checkpointer.flush('final'); await checkpointer.stop(); checkpointFinalized = true; }
10927
+
10579
10928
  // Finalize git only when the harness succeeded — never push a
10580
10929
  // half-finished workspace.
10581
10930
  if (provisioned && result.ok) {
@@ -10635,10 +10984,43 @@ async function workAgent(req, flags, ctx) {
10635
10984
  } else if (provisioned) {
10636
10985
  gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
10637
10986
  }
10987
+ // #264: the work is durable on the pushed branch, so the WIP ref goes —
10988
+ // but only once the engine acks job.complete (below): until then a lost
10989
+ // lease or failed settle redelivers the job, and the next activation must
10990
+ // still be able to restore from it. Kept too when finalization failed.
10991
+ // REQUIRE `gitResult.pushed`: the README/AGENTS contract deletes the ref
10992
+ // only "for a run whose branch push SUCCEEDED". A clean run can restore a
10993
+ // checkpoint as uncommitted changes and make no commit (so finalizeGit
10994
+ // pushes nothing and `pushed` is false while none of the error flags trip)
10995
+ // — discarding then would drop the only durable copy of the recovered WIP.
10996
+ // When not pushed, keep the ref; the orphan GC reclaims it once the element
10997
+ // instance is terminal.
10998
+ //
10999
+ // `pushed` alone is NOT proof the WIP is durable, though: the agent can push
11000
+ // an UNRELATED commit while restored/working checkpoint content stays
11001
+ // uncommitted (or the prior ref was never restored at all). Only discard when
11002
+ // the workspace is clean (everything the checkpointer would snapshot is
11003
+ // committed, hence on the pushed branch) AND no prior checkpoint was left
11004
+ // unrestored — otherwise the WIP ref is the only durable copy, so keep it
11005
+ // (thread 4099856563).
11006
+ let checkpointContentUncommitted = false;
11007
+ if (checkpointing) {
11008
+ try { checkpointContentUncommitted = await checkpointing.hasUncommittedChanges(); } catch { checkpointContentUncommitted = true; }
11009
+ }
11010
+ discardCheckpointOnAck = Boolean(checkpointing && result.ok && gitResult?.pushed && !gitFinalizeFailed && !gitResult?.error && !gitResult?.pushFailed && !gitResult?.pushError && !checkpointContentUncommitted && !checkpointing.priorNotRestored);
10638
11011
  // Reached the end of the run + finalization without throwing. A git
10639
11012
  // finalization error keeps this false so the relay close is 'error'.
10640
11013
  runCompleted = !gitFinalizeFailed;
10641
11014
  } finally {
11015
+ if (checkpointing) {
11016
+ // #264: exception path — if the run threw before any terminal checkpoint
11017
+ // decision was taken (abort/failed flush or the success-path stop), take a
11018
+ // last-chance `failed` flush so the retry can recover edits made since the
11019
+ // last periodic checkpoint. `flush` forces past the post-stop guard and is
11020
+ // bounded; the lease still protects a newer owner. Then close (drain).
11021
+ if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
11022
+ try { await checkpointing.close(); } catch { /* best effort */ }
11023
+ }
10642
11024
  if (isContainer) liveRunIds.delete(runId);
10643
11025
  // #205: clear the in-flight job marker now the harness has stopped — the
10644
11026
  // owning lifecycle's finally is the authoritative "job finished" signal,
@@ -10780,7 +11162,7 @@ async function workAgent(req, flags, ctx) {
10780
11162
  const resultKeys = Object.keys(resultVars);
10781
11163
  if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
10782
11164
  else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
10783
- return await settleJob.complete({
11165
+ const settled = await settleJob.complete({
10784
11166
  ...resultVars,
10785
11167
  [AGENT_RESULT_KEY]: resultEnvelope,
10786
11168
  output: result.stdout,
@@ -10791,6 +11173,9 @@ async function workAgent(req, flags, ctx) {
10791
11173
  ? { branch: gitResult.branch, commits: gitResult.commits, pushed: gitResult.pushed, pullRequest: gitResult.pr || null }
10792
11174
  : {}),
10793
11175
  });
11176
+ // The engine acknowledged completion: the WIP checkpoint is no longer needed.
11177
+ if (discardCheckpointOnAck) await checkpointing.discardAfterAck();
11178
+ return settled;
10794
11179
  }
10795
11180
  const retries = Math.max(0, (Number(job.retries) || 1) - 1);
10796
11181
  const detail = result.error
@@ -16833,6 +17218,7 @@ export {
16833
17218
  runGit,
16834
17219
  sanitizeBranchSegment,
16835
17220
  shouldPreserveRunDir,
17221
+ setupWorkspaceCheckpoints,
16836
17222
  computeRelayCloseReason,
16837
17223
  describeGitFailure,
16838
17224
  boundGitOutput,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "c8ctl-plugin-nano",
3
- "version": "1.68.0",
3
+ "version": "1.69.0",
4
4
  "type": "module",
5
5
  "description": "c8ctl plugin to start, inspect, and stop a local Nano BPM (nanobpmn) cluster",
6
6
  "main": "c8ctl-plugin.js",
@@ -23,6 +23,7 @@
23
23
  "c8ctl-plugin.js",
24
24
  "agent-instance.mjs",
25
25
  "agent-resume.mjs",
26
+ "agent-checkpoint.mjs",
26
27
  "acp-transcript-producer.mjs",
27
28
  "platforms.mjs",
28
29
  "agentic.mjs",
@@ -75,12 +76,12 @@
75
76
  },
76
77
  "optionalDependencies": {
77
78
  "node-pty": "^1.0.0",
78
- "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.68.0",
79
- "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.68.0",
80
- "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.68.0",
81
- "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.68.0",
82
- "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.68.0",
83
- "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.68.0",
84
- "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.68.0"
79
+ "@nanobpm/c8ctl-plugin-nano-darwin-arm64": "1.69.0",
80
+ "@nanobpm/c8ctl-plugin-nano-darwin-x64": "1.69.0",
81
+ "@nanobpm/c8ctl-plugin-nano-linux-x64": "1.69.0",
82
+ "@nanobpm/c8ctl-plugin-nano-linux-arm64": "1.69.0",
83
+ "@nanobpm/c8ctl-plugin-nano-linux-armv7": "1.69.0",
84
+ "@nanobpm/c8ctl-plugin-nano-linux-armv6": "1.69.0",
85
+ "@nanobpm/c8ctl-plugin-nano-win32-x64": "1.69.0"
85
86
  }
86
87
  }