c8ctl-plugin-nano 1.67.0 → 1.69.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -5
- package/agent-checkpoint.mjs +944 -0
- package/agent-resume.mjs +4 -3
- package/c8ctl-plugin.js +504 -27
- package/package.json +9 -8
package/c8ctl-plugin.js
CHANGED
|
@@ -90,6 +90,7 @@ import { createAgentInstanceProducer, isExternalAgentJob } from './agent-instanc
|
|
|
90
90
|
// so the new agent CONTINUES rather than cold-reruns — at-least-once delivery becomes
|
|
91
91
|
// a continuation, not a duplicate. Best-effort; degrades to the legacy cold rerun.
|
|
92
92
|
import { resolveEffectiveEnvelope } from './agent-resume.mjs';
|
|
93
|
+
import { checkpointConfig, checkpointEligibility, checkpointRef, normalizeSecretValues, credentialsFromUrl, isAuthenticatedRemote, shouldSweep, sweepStaleCheckpoints, createGitRunner, fetchCheckpoint, restoreCheckpoint, createWorkspaceCheckpoint, createCheckpointer, deleteCheckpointRef, withCheckpointNote, sanitizeBranchSegment } from './agent-checkpoint.mjs';
|
|
93
94
|
|
|
94
95
|
const requireFromHere = createRequire(import.meta.url);
|
|
95
96
|
const pluginDir = dirname(fileURLToPath(import.meta.url));
|
|
@@ -2739,6 +2740,45 @@ function sanitizeResultVars(obj) {
|
|
|
2739
2740
|
return out;
|
|
2740
2741
|
}
|
|
2741
2742
|
|
|
2743
|
+
// True when `obj` carries at least one EFFECTIVE result var: a key that survives
|
|
2744
|
+
// sanitizeResultVars (reserved / io.nanobpm.* / proto keys stripped) AND whose
|
|
2745
|
+
// value is not null/undefined. A key present with a null/undefined value (e.g.
|
|
2746
|
+
// `{"status":null}`) is NOT effective — it leaves downstream gateways with no
|
|
2747
|
+
// decision value — so it must neither shadow a usable ACP outcome (pickAgentResult)
|
|
2748
|
+
// nor suppress the dropped-result nudge (resolveAgentResultWithNudge). Both
|
|
2749
|
+
// predicates route through here so they stay consistent.
|
|
2750
|
+
function hasEffectiveResultVars(obj) {
|
|
2751
|
+
for (const v of Object.values(sanitizeResultVars(obj))) {
|
|
2752
|
+
if (v !== null && v !== undefined) return true;
|
|
2753
|
+
}
|
|
2754
|
+
return false;
|
|
2755
|
+
}
|
|
2756
|
+
|
|
2757
|
+
// Choose the agent's structured result from its candidate sources (result file,
|
|
2758
|
+
// stdout sentinel, ACP outcome) in priority order. Prefer the first candidate
|
|
2759
|
+
// that carries at least one EFFECTIVE var (i.e. survives sanitizeResultVars —
|
|
2760
|
+
// the reserved / io.nanobpm.* / proto keys stripped, with a non-null value) so an
|
|
2761
|
+
// empty `{}`, a reserved-keys-only, or a null-valued-only result file/sentinel
|
|
2762
|
+
// does NOT shadow a usable `blocked`
|
|
2763
|
+
// ACP outcome via raw `??` nullish precedence: `??` only falls through on
|
|
2764
|
+
// null/undefined, so a present-but-empty object would otherwise win and force
|
|
2765
|
+
// the dropped-result nudge even though the ACP outcome already escalates. Falls
|
|
2766
|
+
// back to the FIRST present-but-empty candidate when none carries effective vars
|
|
2767
|
+
// so the raw object is still surfaced to the audit envelope and the
|
|
2768
|
+
// dropped-result nudge still fires. Candidates are thunks so a later source is
|
|
2769
|
+
// only computed when an earlier one is absent/empty (preserving the previous
|
|
2770
|
+
// short-circuit evaluation and never double-reading the result file needlessly).
|
|
2771
|
+
function pickAgentResult(...thunks) {
|
|
2772
|
+
let firstPresent;
|
|
2773
|
+
for (const thunk of thunks) {
|
|
2774
|
+
const candidate = thunk();
|
|
2775
|
+
if (candidate == null) continue;
|
|
2776
|
+
if (firstPresent === undefined) firstPresent = candidate;
|
|
2777
|
+
if (hasEffectiveResultVars(candidate)) return candidate;
|
|
2778
|
+
}
|
|
2779
|
+
return firstPresent ?? null;
|
|
2780
|
+
}
|
|
2781
|
+
|
|
2742
2782
|
const SANDBOXES = ['none', 'docker', 'podman'];
|
|
2743
2783
|
// Only container-based sandboxes need an image / disk hygiene / a runtime bin.
|
|
2744
2784
|
const CONTAINER_SANDBOXES = new Set(['docker', 'podman']);
|
|
@@ -2824,11 +2864,17 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
|
|
|
2824
2864
|
const stdout0 = result && typeof result.stdout === 'string' ? result.stdout : '';
|
|
2825
2865
|
// A parsed result only counts as usable if it still carries at least one
|
|
2826
2866
|
// *effective* var after sanitizeResultVars strips the reserved / io.nanobpm.* /
|
|
2827
|
-
// proto keys. An empty `{}
|
|
2867
|
+
// proto keys AND ignoring null/undefined values. An empty `{}`, a reserved-keys-
|
|
2868
|
+
// only, or a null-valued-only (e.g. `{"status":null}`) object leaves downstream
|
|
2828
2869
|
// gateways with no status/decision vars — exactly the dropped-result case the
|
|
2829
2870
|
// nudge exists to recover — so gate on effective vars, not raw object presence.
|
|
2830
|
-
const hasUsableResult = (parsed) =>
|
|
2831
|
-
|
|
2871
|
+
const hasUsableResult = (parsed) => hasEffectiveResultVars(parsed);
|
|
2872
|
+
// A `blocked` ACP outcome is a usable result (it escalates), so it needs no nudge.
|
|
2873
|
+
const already = pickAgentResult(
|
|
2874
|
+
() => readAgentResultFile(resultFile),
|
|
2875
|
+
() => parseResultFromStdout(stdout0),
|
|
2876
|
+
() => resultVarsFromAcpOutcome(result?.acpOutcome),
|
|
2877
|
+
);
|
|
2832
2878
|
// Only nudge a clean run that produced NO usable result but DID produce output
|
|
2833
2879
|
// (silence means a crash/hang the idle path already handles, not a dropped result).
|
|
2834
2880
|
// A null `resultFile` (temp-dir creation failed) is NOT a reason to skip: the
|
|
@@ -2840,7 +2886,7 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
|
|
|
2840
2886
|
// `truncated` flag consistent with it — echo the incoming `result.truncated`
|
|
2841
2887
|
// rather than hardcoding false, so callers that trust the return value don't
|
|
2842
2888
|
// see an already-truncated stdout reported as untruncated.
|
|
2843
|
-
return { stdout: stdout0, nudged: false, truncated: result?.truncated === true };
|
|
2889
|
+
return { stdout: stdout0, nudged: false, truncated: result?.truncated === true, acpOutcome: result?.acpOutcome ?? null };
|
|
2844
2890
|
}
|
|
2845
2891
|
let nudge = null;
|
|
2846
2892
|
let nudgeError = null;
|
|
@@ -2869,7 +2915,13 @@ async function resolveAgentResultWithNudge({ result, resultFile, rerun, logger,
|
|
|
2869
2915
|
? `${logPrefix} no result on the first turn — recovered it via one re-emit nudge`
|
|
2870
2916
|
: `${logPrefix} no result on the first turn — re-emit nudge did not recover one`);
|
|
2871
2917
|
}
|
|
2872
|
-
|
|
2918
|
+
// Propagate the rerun turn's OWN validated ACP outcome back to the caller (#263):
|
|
2919
|
+
// the nudge is a continuation, so a fresh `_meta.outcome` it reports (e.g. a
|
|
2920
|
+
// `blocked` that wrote no file/sentinel) is the latest state and must reach
|
|
2921
|
+
// `workAgent` — otherwise the escalation and the recorded envelope outcome would
|
|
2922
|
+
// silently reflect only the first turn's outcome. Fall back to the first result's
|
|
2923
|
+
// outcome when the nudge reported none.
|
|
2924
|
+
return { stdout, nudged: true, truncated, acpOutcome: nudge?.acpOutcome ?? result?.acpOutcome ?? null };
|
|
2873
2925
|
}
|
|
2874
2926
|
|
|
2875
2927
|
function coerceBool(v, dflt = false) {
|
|
@@ -4447,21 +4499,10 @@ function authUrl(url, provider, hasToken) {
|
|
|
4447
4499
|
|
|
4448
4500
|
class ProvisionError extends Error {}
|
|
4449
4501
|
|
|
4450
|
-
//
|
|
4451
|
-
//
|
|
4452
|
-
//
|
|
4453
|
-
//
|
|
4454
|
-
// on a fresh, always-fast-forwardable branch instead of on the base.
|
|
4455
|
-
function sanitizeBranchSegment(s) {
|
|
4456
|
-
const cleaned = String(s == null ? '' : s)
|
|
4457
|
-
.replace(/[^0-9A-Za-z._-]+/g, '-') // collapse anything unusual to a dash
|
|
4458
|
-
.replace(/\.{2,}/g, '.') // no doubled dots (git forbids "..")
|
|
4459
|
-
.replace(/^[-.]+/, '') // no leading dot or dash
|
|
4460
|
-
.slice(0, 60) // bound the segment BEFORE the trailing
|
|
4461
|
-
.replace(/[-.]+$/g, '') // checks, so truncating at char 60 can't
|
|
4462
|
-
.replace(/\.lock$/i, 'lock'); // re-introduce a trailing dot/dash or ".lock"
|
|
4463
|
-
return cleaned || 'base';
|
|
4464
|
-
}
|
|
4502
|
+
// `sanitizeBranchSegment` (the deterministic fallback-work-branch segment builder,
|
|
4503
|
+
// issue #231) now lives in ./agent-checkpoint.mjs as the single source of truth: the
|
|
4504
|
+
// restore path reconstructs the generated `nano/agent-work/<base>-<runId>` identity to
|
|
4505
|
+
// validate a benign per-run rename, so provisioning and restore must share one impl.
|
|
4465
4506
|
|
|
4466
4507
|
// Never let git invoke the host's configured credential helper for our clone/
|
|
4467
4508
|
// push. Reset the helper list ("") so no helper runs — even when we DO have a
|
|
@@ -5264,7 +5305,12 @@ function provisionRepo({ envelope, token, runDir, runId, timeoutMs = 120_000, lo
|
|
|
5264
5305
|
// exactly when a real work branch was resolved that is not the base itself
|
|
5265
5306
|
// (false for a read-only base checkout with push disabled, or a detached HEAD).
|
|
5266
5307
|
const hasPrBranch = !!workingBranch && workingBranch !== effectiveBase;
|
|
5267
|
-
|
|
5308
|
+
// Authentication is not synonymous with a token: an author-embedded HTTPS
|
|
5309
|
+
// userinfo URL or an SSH remote authenticates the push with no `token`. Stamp
|
|
5310
|
+
// the authoritative result so eligibility (auto checkpointing) doesn't
|
|
5311
|
+
// under-detect an authenticated clone.
|
|
5312
|
+
const authenticated = !!token || isAuthenticatedRemote(repo.url);
|
|
5313
|
+
return { workspaceDir, gitEnv, committer, startSha: sha.status === 0 ? (sha.stdout || '').trim() : '', workingBranch, fallbackBranch, hasPrBranch, authenticated, baseBranch: effectiveBase || null, baseCloneSha, baseBaselineUnknown, detached: !workingBranch, ref: commitSha || branchName || '', base, baseFetchError, remote: redactToken(repo.url, token) };
|
|
5268
5314
|
}
|
|
5269
5315
|
|
|
5270
5316
|
// Look up a PR for this branch (2a does NOT open it — the harness does, driven
|
|
@@ -5388,6 +5434,306 @@ function shouldPreserveRunDir(gitResult) {
|
|
|
5388
5434
|
return !!gitResult?.pushFailed;
|
|
5389
5435
|
}
|
|
5390
5436
|
|
|
5437
|
+
// #264: WIP checkpoint wiring for one host-provisioned job (see agent-checkpoint.mjs).
|
|
5438
|
+
// Returns null when disabled/ineligible/unavailable, else `{ checkpointer,
|
|
5439
|
+
// restored, flushTimeoutMs, discard(), close() }`. Never throws — best-effort.
|
|
5440
|
+
async function setupWorkspaceCheckpoints({ provisioned, envelope = null, token = null, secretValues = [], job, jobType, runId, camunda, logger, corr = '', env = process.env, deps = {}, abortSignal = null }) {
|
|
5441
|
+
const cfg = checkpointConfig(env);
|
|
5442
|
+
const prefix = `[${jobType}] job ${job?.jobKey}`;
|
|
5443
|
+
// Cancellation contract: if the runner was already interrupted (lock loss /
|
|
5444
|
+
// stop --force) before we begin, do no git work — the Effect slot may already be
|
|
5445
|
+
// released and the throwaway workspace is about to be reaped.
|
|
5446
|
+
if (abortSignal?.aborted === true) return null;
|
|
5447
|
+
const eligible = checkpointEligibility({ mode: cfg.mode, provisioned, envelope, token });
|
|
5448
|
+
if (!eligible.enabled) {
|
|
5449
|
+
if (provisioned?.workspaceDir) logger.debug?.(`${prefix}: WIP checkpoints off for this job — ${eligible.reason}.`);
|
|
5450
|
+
return null;
|
|
5451
|
+
}
|
|
5452
|
+
const ref = checkpointRef(job?.elementInstanceKey);
|
|
5453
|
+
if (!ref) {
|
|
5454
|
+
if (cfg.mode === 'on') logger.warn?.(`${prefix}: WIP checkpoints enabled but the job has no usable elementInstanceKey — checkpoints off for this job.`);
|
|
5455
|
+
return null;
|
|
5456
|
+
}
|
|
5457
|
+
const {
|
|
5458
|
+
createGitRunner: mkGit = createGitRunner,
|
|
5459
|
+
fetchCheckpoint: fetchCp = fetchCheckpoint,
|
|
5460
|
+
restoreCheckpoint: restoreCp = restoreCheckpoint,
|
|
5461
|
+
deleteCheckpointRef: deleteCp = deleteCheckpointRef,
|
|
5462
|
+
sweep = sweepStaleCheckpoints,
|
|
5463
|
+
now = () => Date.now(),
|
|
5464
|
+
} = deps;
|
|
5465
|
+
const log = {
|
|
5466
|
+
warn: (m) => logger.warn?.(`${prefix} (${corr}): ${m}`),
|
|
5467
|
+
debug: (m) => logger.debug?.(`${prefix} (${corr}): ${m}`),
|
|
5468
|
+
};
|
|
5469
|
+
try {
|
|
5470
|
+
const committer = provisioned.committer || {};
|
|
5471
|
+
const gitEnv = {
|
|
5472
|
+
...(provisioned.gitEnv || process.env),
|
|
5473
|
+
...(committer.name ? { GIT_AUTHOR_NAME: committer.name, GIT_COMMITTER_NAME: committer.name } : {}),
|
|
5474
|
+
...(committer.email ? { GIT_AUTHOR_EMAIL: committer.email, GIT_COMMITTER_EMAIL: committer.email } : {}),
|
|
5475
|
+
};
|
|
5476
|
+
const git = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
|
|
5477
|
+
// The STARTUP git work (origin lookup, checkpoint fetch, restore) is cancellable:
|
|
5478
|
+
// if the runner is interrupted (lock loss / stop --force) mid-setup, the in-flight
|
|
5479
|
+
// git child is killed and later ops fail fast, so setup unwinds instead of holding
|
|
5480
|
+
// a released Effect slot until a git timeout. The checkpointer keeps the UNBOUND
|
|
5481
|
+
// `git` runner above so its abort-flush can still push a final snapshot.
|
|
5482
|
+
const startupGit = mkGit({ cwd: provisioned.workspaceDir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs, signal: abortSignal || undefined });
|
|
5483
|
+
// Captured now: the ref is deleted only after the engine acks completion, by
|
|
5484
|
+
// which time the workspace (and its askpass helper) has been reaped.
|
|
5485
|
+
const originUrl = ((await startupGit(['remote', 'get-url', 'origin'])).stdout || '').trim();
|
|
5486
|
+
|
|
5487
|
+
let prior = null;
|
|
5488
|
+
let restored = null;
|
|
5489
|
+
let startupFetchFailed = false;
|
|
5490
|
+
try {
|
|
5491
|
+
prior = await fetchCp({ git: startupGit, ref });
|
|
5492
|
+
} catch (err) {
|
|
5493
|
+
startupFetchFailed = true;
|
|
5494
|
+
log.warn(`could not fetch WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}; starting without it.`);
|
|
5495
|
+
}
|
|
5496
|
+
if (prior) {
|
|
5497
|
+
const r = await restoreCp({ git: startupGit, checkpoint: prior, expectedBranch: provisioned.workingBranch || null, expectedBranchEphemeral: provisioned.fallbackBranch === true, expectedBaseRef: provisioned.baseBranch || '' });
|
|
5498
|
+
if (r?.restored) {
|
|
5499
|
+
restored = r;
|
|
5500
|
+
logger.info?.(`${prefix} (${corr}): restored WIP checkpoint ${ref}@${prior.sha.slice(0, 12)} (${r.mode}${r.commitsRecovered ? ', local commits recovered' : ''}) as uncommitted changes.`);
|
|
5501
|
+
} else {
|
|
5502
|
+
const rb = r?.rollbackFailed
|
|
5503
|
+
? ' — WARNING: the workspace could NOT be rolled back to its pristine state and may hold partially-recovered content'
|
|
5504
|
+
: '';
|
|
5505
|
+
log.warn(`found WIP checkpoint ${ref}@${prior.sha.slice(0, 12)} but did not restore it — ${r?.reason || 'unknown'}${rb}; the ref is RETAINED (not overwritten) for this activation, so its content can still be recovered explicitly.`);
|
|
5506
|
+
}
|
|
5507
|
+
}
|
|
5508
|
+
// Cancellation recheck: after the startup git work, bail before starting the
|
|
5509
|
+
// background GC or the checkpointer if the runner was interrupted — a released
|
|
5510
|
+
// slot must not leave a GC sweep or a live checkpointer overlapping the next
|
|
5511
|
+
// activation on a workspace that is about to be reaped.
|
|
5512
|
+
if (abortSignal?.aborted === true) {
|
|
5513
|
+
log.debug('setup interrupted after restore — not starting GC/checkpointer for this (yielded) job.');
|
|
5514
|
+
return null;
|
|
5515
|
+
}
|
|
5516
|
+
// The ref held a prior checkpoint we could NOT lay into the workspace (restore
|
|
5517
|
+
// skipped): its content lives ONLY on the ref, so discarding it on ack would be
|
|
5518
|
+
// the same data loss as discarding uncommitted content. Keep the ref in that case.
|
|
5519
|
+
const priorNotRestored = Boolean(prior) && !restored;
|
|
5520
|
+
|
|
5521
|
+
// Orphan GC for this remote, throttled per worker process, in the background.
|
|
5522
|
+
let gc = Promise.resolve();
|
|
5523
|
+
const gcKey = provisioned.remote || provisioned.workspaceDir;
|
|
5524
|
+
// Bound every checkpoint-path SDK call: the GC sweep and the variable write are
|
|
5525
|
+
// awaited by `stop()`/`close()` (and `onCheckpoint`), so an unbounded engine
|
|
5526
|
+
// call could wedge worker shutdown / the ACP loop. A timed-out call rejects; a
|
|
5527
|
+
// late settle is swallowed so it can never surface as an unhandled rejection.
|
|
5528
|
+
const CP_SDK_TIMEOUT_MS = 10000;
|
|
5529
|
+
// A single overall bound on the whole GC sweep. Individual ops are already
|
|
5530
|
+
// bounded (each terminal lookup CP_SDK_TIMEOUT_MS, each git op cfg.gitTimeoutMs),
|
|
5531
|
+
// but a sweep chains up to maxLookups + maxDeletes of them, so a slow/unreachable
|
|
5532
|
+
// remote could otherwise hold `close()`/`discard()` (awaited in job cleanup /
|
|
5533
|
+
// force-stop) for many minutes and starve worker capacity. Cap the total; on the
|
|
5534
|
+
// deadline we ABORT the sweep's cancellable git (below) and still AWAIT it to fully
|
|
5535
|
+
// settle — never abandon it to run git against a workspace the job is about to
|
|
5536
|
+
// reap. Any refs it did not reach are reclaimed on a future hourly sweep (thread
|
|
5537
|
+
// 4099558206 / 4099762442).
|
|
5538
|
+
const GC_OVERALL_TIMEOUT_MS = 120000;
|
|
5539
|
+
const withDeadline = (p, ms, label) => {
|
|
5540
|
+
let timer;
|
|
5541
|
+
const timeout = new Promise((_r, reject) => { timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms); });
|
|
5542
|
+
Promise.resolve(p).catch(() => {});
|
|
5543
|
+
return Promise.race([Promise.resolve(p).finally(() => clearTimeout(timer)), timeout]);
|
|
5544
|
+
};
|
|
5545
|
+
if (shouldSweep(gcKey, { everyMs: cfg.gcEveryMs, now: now(), registry: deps.sweepRegistry })) {
|
|
5546
|
+
const isTerminal = typeof camunda?.getElementInstance === 'function'
|
|
5547
|
+
? async (key) => {
|
|
5548
|
+
try {
|
|
5549
|
+
const ei = await withDeadline(
|
|
5550
|
+
camunda.getElementInstance({ elementInstanceKey: key }, { consistency: { waitUpToMs: 0 } }),
|
|
5551
|
+
CP_SDK_TIMEOUT_MS, 'getElementInstance');
|
|
5552
|
+
return ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED';
|
|
5553
|
+
} catch { return false; }
|
|
5554
|
+
}
|
|
5555
|
+
: null;
|
|
5556
|
+
// Run the sweep in an ISOLATED throwaway scratch repo (its own `.git`), NEVER the
|
|
5557
|
+
// job workspace. The sweep fetches remote WIP refs, writes local throwaway refs and
|
|
5558
|
+
// push-deletes stale ones, while the workspace is concurrently driven by the harness
|
|
5559
|
+
// and later by `finalizeGit` (which fetches the base and reads FETCH_HEAD). Sharing
|
|
5560
|
+
// one `.git` risks ref-lock contention and FETCH_HEAD being clobbered between the
|
|
5561
|
+
// finalizer's fetch and read. A scratch repo with only `origin` configured fully
|
|
5562
|
+
// decouples them; its dir is reaped when the sweep settles. On the overall deadline
|
|
5563
|
+
// we abort the sweep's cancellable git so the promise unwinds instead of hanging.
|
|
5564
|
+
const sweepAbort = new AbortController();
|
|
5565
|
+
let sweepDir = null;
|
|
5566
|
+
try {
|
|
5567
|
+
if (!originUrl) throw new Error('workspace had no origin URL');
|
|
5568
|
+
sweepDir = mkdtempSync(join(tmpdir(), 'nano-wip-gc-'));
|
|
5569
|
+
const sweepEnv = { ...(provisioned.gitEnv || process.env), GIT_TERMINAL_PROMPT: '0' };
|
|
5570
|
+
delete sweepEnv.GIT_ASKPASS;
|
|
5571
|
+
delete sweepEnv.GIT_TOKEN;
|
|
5572
|
+
const askpass = token ? writeAskpass(sweepDir, token) : null;
|
|
5573
|
+
if (askpass) { sweepEnv.GIT_ASKPASS = askpass; sweepEnv.GIT_TOKEN = token; }
|
|
5574
|
+
const sweepGit = mkGit({ cwd: sweepDir, env: sweepEnv, timeoutMs: cfg.gitTimeoutMs, signal: sweepAbort.signal });
|
|
5575
|
+
const init = await sweepGit(['init', '--quiet', '.']);
|
|
5576
|
+
if (init.status !== 0) throw new Error(`git init failed: ${oneLineLog(init.stderr || init.stdout || '')}`);
|
|
5577
|
+
// Configure origin through the protected `.git/config` (0700 tmpdir) rather than
|
|
5578
|
+
// argv, so any userinfo embedded in `originUrl` never lands on `git`'s argv/`ps`.
|
|
5579
|
+
const escUrl = originUrl.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
5580
|
+
appendFileSync(join(sweepDir, '.git', 'config'), `[remote "origin"]\n\turl = "${escUrl}"\n`);
|
|
5581
|
+
const deadlineTimer = setTimeout(() => sweepAbort.abort(), GC_OVERALL_TIMEOUT_MS);
|
|
5582
|
+
deadlineTimer.unref?.();
|
|
5583
|
+
// `gc` is tied to the sweep's ACTUAL completion (not a race that abandons it),
|
|
5584
|
+
// so `close()`/`discard()` — which await `gc` — never reap the scratch repo while
|
|
5585
|
+
// the sweep's git is still live. The deadline only bounds it by aborting the git.
|
|
5586
|
+
gc = Promise.resolve(sweep({ git: sweepGit, ownRef: ref, ttlMs: cfg.ttlMs, graceMs: cfg.gcGraceMs, isTerminal, now, signal: sweepAbort.signal }))
|
|
5587
|
+
.then((r) => {
|
|
5588
|
+
if (r?.deleted?.length) logger.info?.(`${prefix} (${corr}): reclaimed ${r.deleted.length} orphaned WIP checkpoint ref(s): ${r.deleted.map((d) => `${d.ref} [${d.why}]`).join(', ')}.`);
|
|
5589
|
+
else if (sweepAbort.signal.aborted) log.debug(`WIP checkpoint GC bounded at ${GC_OVERALL_TIMEOUT_MS}ms — aborted its in-flight git and drained; leftover refs reclaimed on a later sweep.`);
|
|
5590
|
+
else if (r?.error) log.debug(`WIP checkpoint GC skipped — ${redactToken(oneLineLog(r.error), token)}.`);
|
|
5591
|
+
}, (err) => log.debug(`WIP checkpoint GC threw — ${redactToken(oneLineLog(err?.message || err), token)}.`))
|
|
5592
|
+
.finally(() => { clearTimeout(deadlineTimer); if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } } });
|
|
5593
|
+
} catch (err) {
|
|
5594
|
+
if (sweepDir) { try { rmSync(sweepDir, { recursive: true, force: true }); } catch { /* best effort */ } }
|
|
5595
|
+
log.debug(`WIP checkpoint GC not started — ${redactToken(oneLineLog(err?.message || err), token)}.`);
|
|
5596
|
+
}
|
|
5597
|
+
}
|
|
5598
|
+
|
|
5599
|
+
const branch = provisioned.workingBranch || null;
|
|
5600
|
+
// The SHA this run currently owns on the ref: the checkpoint we restored to
|
|
5601
|
+
// begin with, then each checkpoint we successfully push. `discard()` leases its
|
|
5602
|
+
// delete against it so a stale activation cannot delete a newer owner's WIP.
|
|
5603
|
+
let ownedSha = prior?.sha || '';
|
|
5604
|
+
let varWarned = false;
|
|
5605
|
+
const setVar = typeof camunda?.createElementInstanceVariables === 'function' && job?.elementInstanceKey
|
|
5606
|
+
? async (res) => {
|
|
5607
|
+
try {
|
|
5608
|
+
await withDeadline(camunda.createElementInstanceVariables({
|
|
5609
|
+
elementInstanceKey: String(job.elementInstanceKey),
|
|
5610
|
+
variables: { agentCheckpoint: { ref, sha: res.sha, head: res.head, at: res.at, reason: res.reason, branch } },
|
|
5611
|
+
local: true,
|
|
5612
|
+
}), CP_SDK_TIMEOUT_MS, 'createElementInstanceVariables');
|
|
5613
|
+
} catch (err) {
|
|
5614
|
+
if (!varWarned) { varWarned = true; log.warn(`could not set the agentCheckpoint variable — ${oneLineLog(err?.message || err)}; checkpoints still pushed to ${ref}.`); }
|
|
5615
|
+
}
|
|
5616
|
+
}
|
|
5617
|
+
: null;
|
|
5618
|
+
|
|
5619
|
+
const checkpoint = createWorkspaceCheckpoint({
|
|
5620
|
+
git,
|
|
5621
|
+
ref,
|
|
5622
|
+
baseSha: provisioned.startSha || '',
|
|
5623
|
+
// The RAW base ref this run was provisioned against — recorded UNSANITIZED so a
|
|
5624
|
+
// later restore can prove two per-run fallbacks share a base without relying on
|
|
5625
|
+
// the ambiguous sanitized branch segment (see CHECKPOINT_BASEREF_TRAILER).
|
|
5626
|
+
baseRefName: provisioned.baseBranch || '',
|
|
5627
|
+
runId: String(runId || ''),
|
|
5628
|
+
// When a prior snapshot EXISTED but could NOT be restored (branch-mismatch /
|
|
5629
|
+
// detached refusal, or a diverged patch that did not apply), its content lives
|
|
5630
|
+
// ONLY on the ref. Do NOT hand this run ownership of it: with `expectSha`/
|
|
5631
|
+
// `priorRunId` set, the first checkpoint would `--force-with-lease` over the
|
|
5632
|
+
// ref and destroy the only copy of that un-restored snapshot. Withholding
|
|
5633
|
+
// ownership makes the first push a create-only that the existing ref rejects,
|
|
5634
|
+
// so checkpointing disables for this activation and the old ref is left intact
|
|
5635
|
+
// for explicit recovery (matching `priorNotRestored` retaining it on discard).
|
|
5636
|
+
expectSha: priorNotRestored ? '' : (prior?.sha || ''),
|
|
5637
|
+
priorRunId: priorNotRestored ? '' : (prior?.runId || ''),
|
|
5638
|
+
startupFetchFailed,
|
|
5639
|
+
maxFileBytes: cfg.maxFileBytes,
|
|
5640
|
+
// Include any credential embedded in the origin URL's userinfo: provisioning
|
|
5641
|
+
// preserves author-supplied userinfo (authUrl), and for such credential-bearing
|
|
5642
|
+
// URLs `token` is null, so the URL password would otherwise be absent from the
|
|
5643
|
+
// scan set and a changed file echoing it could enter the readable WIP ref.
|
|
5644
|
+
secretValues: normalizeSecretValues([...(secretValues || []), token, ...credentialsFromUrl(originUrl)]),
|
|
5645
|
+
message: `nano: WIP checkpoint for job ${job.jobKey}`,
|
|
5646
|
+
});
|
|
5647
|
+
const checkpointer = createCheckpointer({
|
|
5648
|
+
checkpoint,
|
|
5649
|
+
minIntervalMs: cfg.minIntervalMs,
|
|
5650
|
+
intervalMs: cfg.intervalMs,
|
|
5651
|
+
logger: log,
|
|
5652
|
+
onCheckpoint: async (res) => {
|
|
5653
|
+
ownedSha = res.sha;
|
|
5654
|
+
log.debug(`WIP checkpoint ${ref}@${res.sha.slice(0, 12)} pushed (${res.reason})${res.excluded?.length ? `; excluded ${res.excluded.map((e) => `${e.path} [${e.why}]`).join(', ')}` : ''}.`);
|
|
5655
|
+
if (setVar) await setVar(res);
|
|
5656
|
+
},
|
|
5657
|
+
});
|
|
5658
|
+
return {
|
|
5659
|
+
ref,
|
|
5660
|
+
mode: cfg.mode,
|
|
5661
|
+
checkpointer,
|
|
5662
|
+
restored,
|
|
5663
|
+
priorNotRestored,
|
|
5664
|
+
flushTimeoutMs: cfg.flushTimeoutMs,
|
|
5665
|
+
// True when the workspace still holds checkpoint content NOT captured by a
|
|
5666
|
+
// commit, so the pushed branch does not represent it (an uncommitted restored
|
|
5667
|
+
// or working delta that lives only on the WIP ref). The discard-on-ack gate
|
|
5668
|
+
// uses this: `gitResult.pushed` alone is not proof the WIP is durable — the
|
|
5669
|
+
// agent can push an unrelated commit while restored changes stay uncommitted.
|
|
5670
|
+
// Best-effort; on any git error we report "dirty" so the caller KEEPS the ref.
|
|
5671
|
+
async hasUncommittedChanges() {
|
|
5672
|
+
try {
|
|
5673
|
+
const r = await git(['status', '--porcelain', '--untracked-files=all']);
|
|
5674
|
+
if (r.status !== 0) return true;
|
|
5675
|
+
return (r.stdout || '').trim().length > 0;
|
|
5676
|
+
} catch { return true; }
|
|
5677
|
+
},
|
|
5678
|
+
// Stop snapshotting and wait for in-flight git work (checkpoint + GC) so the
|
|
5679
|
+
// run dir can be reaped safely.
|
|
5680
|
+
async close() { await checkpointer.stop(); await gc; },
|
|
5681
|
+
async discard() {
|
|
5682
|
+
await checkpointer.stop();
|
|
5683
|
+
try {
|
|
5684
|
+
const r = await deleteCp({ git, ref, expectSha: ownedSha });
|
|
5685
|
+
if (r?.staleLease) log.debug(`WIP checkpoint ${ref} left intact — the ref moved to a newer activation; not deleting another owner's WIP.`);
|
|
5686
|
+
else if (!r?.ok) log.warn(`could not delete WIP checkpoint ${ref} — ${r?.error || 'unknown'}.`);
|
|
5687
|
+
} catch (err) {
|
|
5688
|
+
log.warn(`could not delete WIP checkpoint ${ref} — ${oneLineLog(err?.message || err)}.`);
|
|
5689
|
+
}
|
|
5690
|
+
},
|
|
5691
|
+
// Delete the ref once the engine has acknowledged the job's completion.
|
|
5692
|
+
// Independent of the (by then reaped) workspace: pushes the delete from a
|
|
5693
|
+
// throwaway scratch repo with its own askpass helper. Never throws; a ref left
|
|
5694
|
+
// behind is reclaimed by the orphan GC once the element instance is terminal.
|
|
5695
|
+
async discardAfterAck() {
|
|
5696
|
+
await checkpointer.stop();
|
|
5697
|
+
await gc;
|
|
5698
|
+
if (!originUrl) { log.warn(`could not delete WIP checkpoint ${ref} — workspace had no origin URL; the orphan GC will reclaim it.`); return; }
|
|
5699
|
+
let dir = null;
|
|
5700
|
+
try {
|
|
5701
|
+
dir = mkdtempSync(join(tmpdir(), 'nano-wip-'));
|
|
5702
|
+
const gitEnv = { ...(provisioned.gitEnv || process.env), GIT_TERMINAL_PROMPT: '0' };
|
|
5703
|
+
delete gitEnv.GIT_ASKPASS;
|
|
5704
|
+
delete gitEnv.GIT_TOKEN;
|
|
5705
|
+
const askpass = token ? writeAskpass(dir, token) : null;
|
|
5706
|
+
if (askpass) { gitEnv.GIT_ASKPASS = askpass; gitEnv.GIT_TOKEN = token; }
|
|
5707
|
+
// A plain (non-bare) repo: `safe.bareRepository=explicit` refuses implicit bare dirs.
|
|
5708
|
+
const scratch = mkGit({ cwd: dir, env: gitEnv, timeoutMs: cfg.gitTimeoutMs });
|
|
5709
|
+
const init = await scratch(['init', '--quiet', '.']);
|
|
5710
|
+
if (init.status !== 0) throw new Error(`git init failed: ${oneLineLog(init.stderr || init.stdout || '')}`);
|
|
5711
|
+
// Configure origin through the throwaway repo's protected `.git/config`
|
|
5712
|
+
// (a 0700 tmpdir) rather than passing `originUrl` positionally to
|
|
5713
|
+
// `git push`: `authUrl` preserves any userinfo an author embedded in
|
|
5714
|
+
// `repository.url`, and an argv URL would leak those credentials into the
|
|
5715
|
+
// push process's argv / `ps` listing. Writing the config file directly
|
|
5716
|
+
// keeps the URL off argv entirely (a `git remote add` would still expose
|
|
5717
|
+
// it); auth still flows through GIT_ASKPASS (thread 4099558181).
|
|
5718
|
+
const escUrl = originUrl.replace(/\\/g, '\\\\').replace(/"/g, '\\"');
|
|
5719
|
+
appendFileSync(join(dir, '.git', 'config'), `[remote "origin"]\n\turl = "${escUrl}"\n`);
|
|
5720
|
+
const r = await deleteCp({ git: scratch, ref, expectSha: ownedSha });
|
|
5721
|
+
if (r?.ok) log.debug(`deleted WIP checkpoint ${ref} after the engine acknowledged completion.`);
|
|
5722
|
+
else if (r?.staleLease) log.debug(`WIP checkpoint ${ref} left intact — the ref moved to a newer activation; the orphan GC will reclaim it if needed.`);
|
|
5723
|
+
else log.warn(`could not delete WIP checkpoint ${ref} — ${redactToken(r?.error || 'unknown', token)}; the orphan GC will reclaim it.`);
|
|
5724
|
+
} catch (err) {
|
|
5725
|
+
log.warn(`could not delete WIP checkpoint ${ref} — ${redactToken(oneLineLog(err?.message || err), token)}; the orphan GC will reclaim it.`);
|
|
5726
|
+
} finally {
|
|
5727
|
+
if (dir) { try { rmSync(dir, { recursive: true, force: true }); } catch { /* best effort */ } }
|
|
5728
|
+
}
|
|
5729
|
+
},
|
|
5730
|
+
};
|
|
5731
|
+
} catch (err) {
|
|
5732
|
+
log.warn(`WIP checkpoint setup failed — ${oneLineLog(err?.message || err)}; continuing without checkpoints.`);
|
|
5733
|
+
return null;
|
|
5734
|
+
}
|
|
5735
|
+
}
|
|
5736
|
+
|
|
5391
5737
|
// Decide the agentic/transcript relay's close reason (normal / job-killed / error)
|
|
5392
5738
|
// for a finished job. Pure + exported so the close-reason contract is unit-tested
|
|
5393
5739
|
// without spinning a full worker job. `error` must cover ANY hard finalization
|
|
@@ -6799,6 +7145,30 @@ export function acpSessionNewParams({ cwd, init, resumePlan }) {
|
|
|
6799
7145
|
return params;
|
|
6800
7146
|
}
|
|
6801
7147
|
|
|
7148
|
+
// The agent's explicit end-of-task report, when its `session/prompt` result carries
|
|
7149
|
+
// `_meta.outcome` (rusty-harness's `report_outcome` tool): `{ status: 'completed' |
|
|
7150
|
+
// 'blocked', summary }`. Anything else — no `_meta`, another agent, a malformed
|
|
7151
|
+
// value — is `null`, so agents that don't send it are handled exactly as before.
|
|
7152
|
+
const ACP_OUTCOME_STATUSES = new Set(['completed', 'blocked']);
|
|
7153
|
+
const ACP_OUTCOME_SUMMARY_MAX_CHARS = 8_000;
|
|
7154
|
+
export function acpOutcomeFrom(promptResult) {
|
|
7155
|
+
const o = promptResult?._meta?.outcome;
|
|
7156
|
+
if (!isPlainObject(o) || !ACP_OUTCOME_STATUSES.has(o.status)) return null;
|
|
7157
|
+
const summary = typeof o.summary === 'string' ? o.summary.trim().slice(0, ACP_OUTCOME_SUMMARY_MAX_CHARS) : '';
|
|
7158
|
+
if (!summary) return null;
|
|
7159
|
+
return { status: o.status, summary };
|
|
7160
|
+
}
|
|
7161
|
+
|
|
7162
|
+
// Result vars implied by an ACP outcome, used ONLY when the agent wrote no result
|
|
7163
|
+
// file / sentinel of its own. `blocked` maps to the workforce escalation contract
|
|
7164
|
+
// (`status: "blocked"` with a non-blank `question`), so the job escalates instead of
|
|
7165
|
+
// spending a re-emit nudge turn. `completed` implies no vars: job-specific status
|
|
7166
|
+
// values can't be guessed, so the normal result / nudge path still applies.
|
|
7167
|
+
export function resultVarsFromAcpOutcome(outcome) {
|
|
7168
|
+
if (outcome?.status !== 'blocked') return null;
|
|
7169
|
+
return { status: 'blocked', summary: outcome.summary, question: outcome.summary };
|
|
7170
|
+
}
|
|
7171
|
+
|
|
6802
7172
|
function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, idleTimeoutMs, recoveryWindowMs, relayTap = null, stream = false, streamPrefix = '', onStreamOut, onStreamErr, permission = 'yolo', shell = false, onAcpUpdate = null, abortSignal = null, onSpawn = null, resumePlan = null }) {
|
|
6803
7173
|
return new Promise((resolve) => {
|
|
6804
7174
|
const logger = getLogger();
|
|
@@ -6819,6 +7189,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
6819
7189
|
const pending = new Map();
|
|
6820
7190
|
let childClosed = null; // { code, signal } once the child exits
|
|
6821
7191
|
let promptResolved = false; // the main session/prompt turn sent + resolved
|
|
7192
|
+
let acpOutcome = null; // the turn's `_meta.outcome`, if the agent reported one
|
|
6822
7193
|
let settleTimer = null; // post-turn grace before force-reaping a lingering agent
|
|
6823
7194
|
// One-time warning latch for the reserved escalate/filter policies so the
|
|
6824
7195
|
// deferral is observable (not silent) but never spams a warning per request.
|
|
@@ -6913,7 +7284,7 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
6913
7284
|
if (teeSink) { tee('', true); teeErr('', true); }
|
|
6914
7285
|
// Reap the child if it is still alive (turn resolved but agent lingering).
|
|
6915
7286
|
try { if (child && childClosed === null) killTree(child); } catch { /* best effort */ }
|
|
6916
|
-
resolve(result);
|
|
7287
|
+
resolve(acpOutcome ? { ...result, acpOutcome } : result);
|
|
6917
7288
|
};
|
|
6918
7289
|
|
|
6919
7290
|
// --- JSON-RPC 2.0 plumbing (newline-delimited framing) -------------------
|
|
@@ -7368,10 +7739,11 @@ function spawnCaptureAcp({ command, args = [], cwd, env, stdinData, timeoutMs, i
|
|
|
7368
7739
|
attachSteerIfAny();
|
|
7369
7740
|
// Deliver the task envelope as the prompt (from stdinData, matching the
|
|
7370
7741
|
// pipe/pty paths which write the same payload to stdin).
|
|
7371
|
-
await request('session/prompt', {
|
|
7742
|
+
const prompted = await request('session/prompt', {
|
|
7372
7743
|
sessionId,
|
|
7373
7744
|
prompt: [{ type: 'text', text: String(stdinData ?? '') }],
|
|
7374
7745
|
});
|
|
7746
|
+
acpOutcome = acpOutcomeFrom(prompted);
|
|
7375
7747
|
// End-of-turn: the main session/prompt request resolved. Mark it so the
|
|
7376
7748
|
// `close` handler can tell a completed turn from an early/handshake exit.
|
|
7377
7749
|
promptResolved = true;
|
|
@@ -7732,6 +8104,9 @@ function buildResultEnvelope(result, { sandbox, image, git, result: agentResult,
|
|
|
7732
8104
|
// shutdown (didn't exit on its own within the post-turn grace) — surfaced so
|
|
7733
8105
|
// consistently-hanging agents are visible rather than hidden behind a success.
|
|
7734
8106
|
if (result.forcedReap) env.forcedReap = true;
|
|
8107
|
+
// The agent's explicit ACP end-of-task report (`_meta.outcome`), host-recorded
|
|
8108
|
+
// alongside whatever result vars it returned.
|
|
8109
|
+
if (result.acpOutcome) env.outcome = result.acpOutcome;
|
|
7735
8110
|
// Audit (issue #63): record which linked-resource key supplied the base prompt.
|
|
7736
8111
|
// The engine only keeps `latest` per resourceId (no pinning), so recording the
|
|
7737
8112
|
// resolved key is the only reproducibility handle for which prompt version ran.
|
|
@@ -10329,6 +10704,13 @@ async function workAgent(req, flags, ctx) {
|
|
|
10329
10704
|
// as a `normal` close. This flag flips true only once we reach the end of the
|
|
10330
10705
|
// try body, so an early throw is correctly reported as `error`.
|
|
10331
10706
|
let runCompleted = false;
|
|
10707
|
+
let discardCheckpointOnAck = false;
|
|
10708
|
+
// #264: whether a TERMINAL WIP checkpoint decision (abort/failed flush, or the
|
|
10709
|
+
// success-path stop) was already taken on the normal control flow. If the run
|
|
10710
|
+
// throws before any of those (e.g. runAgentJob rejects), the finally below
|
|
10711
|
+
// performs a last-chance `failed` flush so edits since the last periodic
|
|
10712
|
+
// checkpoint are not lost.
|
|
10713
|
+
let checkpointFinalized = false;
|
|
10332
10714
|
// The per-job live-terminal relay session (issue #173): streams this job's
|
|
10333
10715
|
// harness terminal over the single-owner supervisor's ONE multiplexed host
|
|
10334
10716
|
// connection, keyed by this worker's instance + the jobKey, and accepts
|
|
@@ -10353,6 +10735,30 @@ async function workAgent(req, flags, ctx) {
|
|
|
10353
10735
|
return;
|
|
10354
10736
|
}
|
|
10355
10737
|
|
|
10738
|
+
// #264: WIP checkpoints (NANO_AGENT_CHECKPOINT=auto|on|off, default auto). Restore a prior
|
|
10739
|
+
// attempt's snapshot into the fresh clone, then snapshot the workspace to
|
|
10740
|
+
// `refs/nano/wip/<elementInstanceKey>` as the run progresses. Best-effort:
|
|
10741
|
+
// any failure logs and degrades to the transcript-only resume.
|
|
10742
|
+
const checkpointing = await setupWorkspaceCheckpoints({ provisioned, envelope, token: repoToken, secretValues: Object.values(resolved || {}), job, jobType, runId, camunda, logger, corr: aiCorr, abortSignal });
|
|
10743
|
+
// #264 cancellation contract: setup does cancellable git work (fetch/restore)
|
|
10744
|
+
// and may start a background GC + live checkpointer. If a lock-loss/force-stop
|
|
10745
|
+
// won during that await, stop here before opening the relay/harness so a
|
|
10746
|
+
// released slot never proceeds into the runner path or overlaps the next
|
|
10747
|
+
// activation. (setupWorkspaceCheckpoints already declined to start GC/the
|
|
10748
|
+
// checkpointer on an observed abort, so there is nothing to tear down.)
|
|
10749
|
+
if (checkSetupAbort(abortSignal, { jobType, jobKey: job.jobKey, stage: 'relay-open', logger })) {
|
|
10750
|
+
// A checkpointer may have been created just before the abort landed; stop
|
|
10751
|
+
// it (and drain its GC) so no snapshot timer or sweep overlaps the next
|
|
10752
|
+
// activation on the about-to-be-reaped workspace.
|
|
10753
|
+
if (checkpointing?.close) { try { await checkpointing.close(); } catch { /* best effort */ } }
|
|
10754
|
+
await discardAgentInstanceProducer();
|
|
10755
|
+
if (runDir) { try { rmSync(runDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(runDir); }
|
|
10756
|
+
if (isContainer) liveRunIds.delete(runId);
|
|
10757
|
+
return;
|
|
10758
|
+
}
|
|
10759
|
+
if (checkpointing?.restored) effectiveEnvelope = withCheckpointNote(effectiveEnvelope, checkpointing.restored);
|
|
10760
|
+
const checkpointer = checkpointing?.checkpointer ?? null;
|
|
10761
|
+
|
|
10356
10762
|
let relaySession = null;
|
|
10357
10763
|
if (agenticPlane) {
|
|
10358
10764
|
// #229: thread the correlation join keys + a lazy AgentInstance-key getter
|
|
@@ -10434,7 +10840,11 @@ async function workAgent(req, flags, ctx) {
|
|
|
10434
10840
|
// producer so its turn is appended to the engine's AgentHistory. Inert
|
|
10435
10841
|
// when no producer was minted (non-external job / off / mint failed) and
|
|
10436
10842
|
// for non-ACP harnesses (no session/updates are emitted). Best-effort.
|
|
10437
|
-
|
|
10843
|
+
// #264: the same updates pace WIP checkpoints (a completed tool call or a
|
|
10844
|
+
// plan change is a natural snapshot boundary; rate-limited inside).
|
|
10845
|
+
onAcpUpdate: (agentInstanceProducer || checkpointer)
|
|
10846
|
+
? (u) => { agentInstanceProducer?.ingest(u); checkpointer?.notify(u); }
|
|
10847
|
+
: undefined,
|
|
10438
10848
|
};
|
|
10439
10849
|
result = await runAgentJob(profile, job, runOpts);
|
|
10440
10850
|
|
|
@@ -10446,8 +10856,13 @@ async function workAgent(req, flags, ctx) {
|
|
|
10446
10856
|
// cleanup); recordJobEnd fires in the outer finally.
|
|
10447
10857
|
if (result && result.aborted) {
|
|
10448
10858
|
logger.warn(`[${jobType}] job ${job.jobKey} aborted (worker force-stop) — harness killed; yielding job for retry.`);
|
|
10859
|
+
// #264: last-chance WIP checkpoint so the retry can pick up the
|
|
10860
|
+
// uncommitted work (bounded; the lease guards a newer owner).
|
|
10861
|
+
if (checkpointer) { await checkpointer.flush('abort', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
|
|
10449
10862
|
return;
|
|
10450
10863
|
}
|
|
10864
|
+
// #264: a failed run is retried — checkpoint what it left behind.
|
|
10865
|
+
if (checkpointer && !result.ok) { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); checkpointFinalized = true; }
|
|
10451
10866
|
|
|
10452
10867
|
// Gap 2 (#678): a clean run that emitted no machine-readable result gets
|
|
10453
10868
|
// ONE bounded re-emit nudge in the same workspace, feeding back its own
|
|
@@ -10457,7 +10872,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
10457
10872
|
// the result is recoverable only via the stdout `::nano:result::`
|
|
10458
10873
|
// sentinel, which `resolveAgentResultWithNudge` handles directly.
|
|
10459
10874
|
if (result.ok) {
|
|
10460
|
-
const { stdout, nudged, truncated } = await resolveAgentResultWithNudge({
|
|
10875
|
+
const { stdout, nudged, truncated, acpOutcome } = await resolveAgentResultWithNudge({
|
|
10461
10876
|
result,
|
|
10462
10877
|
resultFile,
|
|
10463
10878
|
logger,
|
|
@@ -10482,6 +10897,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
10482
10897
|
if (nudged) {
|
|
10483
10898
|
result.nudgedForResult = true;
|
|
10484
10899
|
if (truncated) result.truncated = true;
|
|
10900
|
+
// Adopt the nudge turn's outcome so the escalation / envelope reflect
|
|
10901
|
+
// the latest continuation, not only the first turn (#263).
|
|
10902
|
+
result.acpOutcome = acpOutcome;
|
|
10485
10903
|
}
|
|
10486
10904
|
}
|
|
10487
10905
|
|
|
@@ -10493,6 +10911,20 @@ async function workAgent(req, flags, ctx) {
|
|
|
10493
10911
|
try { await agentInstanceProducer.complete(result.ok); } catch (err) { logger.warn(`[${jobType}] AgentInstance producer complete() threw (${aiCorr}) — ${oneLineLog(err?.message || err)}; job settlement unaffected.`); }
|
|
10494
10912
|
}
|
|
10495
10913
|
|
|
10914
|
+
// #264: the run is done — take a FINAL snapshot (so edits still pending in
|
|
10915
|
+
// the rate-limiter's trailing timer are not lost) and stop snapshotting
|
|
10916
|
+
// before finalizeGit touches the workspace. `flush('final')` forces one
|
|
10917
|
+
// last checkpoint past the post-stop guard, then latches the scheduler
|
|
10918
|
+
// stopped; on a successful run with no branch commit (or a crash before
|
|
10919
|
+
// job.complete is acked) the retained WIP ref then still holds the latest
|
|
10920
|
+
// work rather than only the last periodic checkpoint (advisory 10713).
|
|
10921
|
+
// `flush` is bounded by a timeout, so it can RETURN while its snapshot is
|
|
10922
|
+
// still queued/in-flight; that snapshot reads the same workspace/HEAD that
|
|
10923
|
+
// finalizeGit is about to commit and push. DRAIN the scheduler (stop()
|
|
10924
|
+
// resolves once the in-flight checkpoint chain settles) before finalizeGit
|
|
10925
|
+
// so finalization never races a late snapshot (thread 4099675063).
|
|
10926
|
+
if (checkpointer && result.ok) { await checkpointer.flush('final'); await checkpointer.stop(); checkpointFinalized = true; }
|
|
10927
|
+
|
|
10496
10928
|
// Finalize git only when the harness succeeded — never push a
|
|
10497
10929
|
// half-finished workspace.
|
|
10498
10930
|
if (provisioned && result.ok) {
|
|
@@ -10552,10 +10984,43 @@ async function workAgent(req, flags, ctx) {
|
|
|
10552
10984
|
} else if (provisioned) {
|
|
10553
10985
|
gitResult = { remote: provisioned.remote, branch: provisioned.workingBranch, baseSha: provisioned.startSha || null, commits: [], pushed: false };
|
|
10554
10986
|
}
|
|
10987
|
+
// #264: the work is durable on the pushed branch, so the WIP ref goes —
|
|
10988
|
+
// but only once the engine acks job.complete (below): until then a lost
|
|
10989
|
+
// lease or failed settle redelivers the job, and the next activation must
|
|
10990
|
+
// still be able to restore from it. Kept too when finalization failed.
|
|
10991
|
+
// REQUIRE `gitResult.pushed`: the README/AGENTS contract deletes the ref
|
|
10992
|
+
// only "for a run whose branch push SUCCEEDED". A clean run can restore a
|
|
10993
|
+
// checkpoint as uncommitted changes and make no commit (so finalizeGit
|
|
10994
|
+
// pushes nothing and `pushed` is false while none of the error flags trip)
|
|
10995
|
+
// — discarding then would drop the only durable copy of the recovered WIP.
|
|
10996
|
+
// When not pushed, keep the ref; the orphan GC reclaims it once the element
|
|
10997
|
+
// instance is terminal.
|
|
10998
|
+
//
|
|
10999
|
+
// `pushed` alone is NOT proof the WIP is durable, though: the agent can push
|
|
11000
|
+
// an UNRELATED commit while restored/working checkpoint content stays
|
|
11001
|
+
// uncommitted (or the prior ref was never restored at all). Only discard when
|
|
11002
|
+
// the workspace is clean (everything the checkpointer would snapshot is
|
|
11003
|
+
// committed, hence on the pushed branch) AND no prior checkpoint was left
|
|
11004
|
+
// unrestored — otherwise the WIP ref is the only durable copy, so keep it
|
|
11005
|
+
// (thread 4099856563).
|
|
11006
|
+
let checkpointContentUncommitted = false;
|
|
11007
|
+
if (checkpointing) {
|
|
11008
|
+
try { checkpointContentUncommitted = await checkpointing.hasUncommittedChanges(); } catch { checkpointContentUncommitted = true; }
|
|
11009
|
+
}
|
|
11010
|
+
discardCheckpointOnAck = Boolean(checkpointing && result.ok && gitResult?.pushed && !gitFinalizeFailed && !gitResult?.error && !gitResult?.pushFailed && !gitResult?.pushError && !checkpointContentUncommitted && !checkpointing.priorNotRestored);
|
|
10555
11011
|
// Reached the end of the run + finalization without throwing. A git
|
|
10556
11012
|
// finalization error keeps this false so the relay close is 'error'.
|
|
10557
11013
|
runCompleted = !gitFinalizeFailed;
|
|
10558
11014
|
} finally {
|
|
11015
|
+
if (checkpointing) {
|
|
11016
|
+
// #264: exception path — if the run threw before any terminal checkpoint
|
|
11017
|
+
// decision was taken (abort/failed flush or the success-path stop), take a
|
|
11018
|
+
// last-chance `failed` flush so the retry can recover edits made since the
|
|
11019
|
+
// last periodic checkpoint. `flush` forces past the post-stop guard and is
|
|
11020
|
+
// bounded; the lease still protects a newer owner. Then close (drain).
|
|
11021
|
+
if (checkpointer && !checkpointFinalized) { try { await checkpointer.flush('failed', { timeoutMs: checkpointing.flushTimeoutMs }); } catch { /* best effort */ } }
|
|
11022
|
+
try { await checkpointing.close(); } catch { /* best effort */ }
|
|
11023
|
+
}
|
|
10559
11024
|
if (isContainer) liveRunIds.delete(runId);
|
|
10560
11025
|
// #205: clear the in-flight job marker now the harness has stopped — the
|
|
10561
11026
|
// owning lifecycle's finally is the authoritative "job finished" signal,
|
|
@@ -10621,7 +11086,13 @@ async function workAgent(req, flags, ctx) {
|
|
|
10621
11086
|
// envelope; the sanitized (reserved-key-stripped) vars are merged into the
|
|
10622
11087
|
// job completion so the model sees `status`/`summary`/… as first-class
|
|
10623
11088
|
// outputs. Read before deleting the temp dir.
|
|
10624
|
-
|
|
11089
|
+
// Last resort: vars implied by the agent's ACP `_meta.outcome` (a `blocked`
|
|
11090
|
+
// report → an escalation) when it wrote no result of its own.
|
|
11091
|
+
const rawResult = pickAgentResult(
|
|
11092
|
+
() => readAgentResultFile(resultFile),
|
|
11093
|
+
() => parseResultFromStdout(result.stdout),
|
|
11094
|
+
() => resultVarsFromAcpOutcome(result.acpOutcome),
|
|
11095
|
+
);
|
|
10625
11096
|
if (resultDir) { try { rmSync(resultDir, { recursive: true, force: true }); } catch { /* best effort */ } liveRunDirs.delete(resultDir); }
|
|
10626
11097
|
const resultVars = sanitizeResultVars(rawResult);
|
|
10627
11098
|
|
|
@@ -10691,7 +11162,7 @@ async function workAgent(req, flags, ctx) {
|
|
|
10691
11162
|
const resultKeys = Object.keys(resultVars);
|
|
10692
11163
|
if (resultKeys.length === 0) logger.warn(`[${jobType}] job ${job.jobKey}: agent returned no usable result vars — write a JSON object of result variables to $AGENT_RESULT_FILE (or print a "${RESULT_SENTINEL} {…}" line) so downstream gateways see status/summary/etc.`);
|
|
10693
11164
|
else logger.info(`[${jobType}] job ${job.jobKey}: merged agent result vars [${resultKeys.join(', ')}]`);
|
|
10694
|
-
|
|
11165
|
+
const settled = await settleJob.complete({
|
|
10695
11166
|
...resultVars,
|
|
10696
11167
|
[AGENT_RESULT_KEY]: resultEnvelope,
|
|
10697
11168
|
output: result.stdout,
|
|
@@ -10702,6 +11173,9 @@ async function workAgent(req, flags, ctx) {
|
|
|
10702
11173
|
? { branch: gitResult.branch, commits: gitResult.commits, pushed: gitResult.pushed, pullRequest: gitResult.pr || null }
|
|
10703
11174
|
: {}),
|
|
10704
11175
|
});
|
|
11176
|
+
// The engine acknowledged completion: the WIP checkpoint is no longer needed.
|
|
11177
|
+
if (discardCheckpointOnAck) await checkpointing.discardAfterAck();
|
|
11178
|
+
return settled;
|
|
10705
11179
|
}
|
|
10706
11180
|
const retries = Math.max(0, (Number(job.retries) || 1) - 1);
|
|
10707
11181
|
const detail = result.error
|
|
@@ -16718,6 +17192,8 @@ export {
|
|
|
16718
17192
|
readAgentResultFile,
|
|
16719
17193
|
parseResultFromStdout,
|
|
16720
17194
|
sanitizeResultVars,
|
|
17195
|
+
hasEffectiveResultVars,
|
|
17196
|
+
pickAgentResult,
|
|
16721
17197
|
buildResultNudgePrompt,
|
|
16722
17198
|
resolveAgentResultWithNudge,
|
|
16723
17199
|
parseEnvPairs,
|
|
@@ -16742,6 +17218,7 @@ export {
|
|
|
16742
17218
|
runGit,
|
|
16743
17219
|
sanitizeBranchSegment,
|
|
16744
17220
|
shouldPreserveRunDir,
|
|
17221
|
+
setupWorkspaceCheckpoints,
|
|
16745
17222
|
computeRelayCloseReason,
|
|
16746
17223
|
describeGitFailure,
|
|
16747
17224
|
boundGitOutput,
|