c8ctl-plugin-nano 1.70.3 → 1.70.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -2
- package/agent-checkpoint.mjs +243 -6
- package/c8ctl-plugin.js +779 -43
- package/package.json +8 -8
- package/supervisor.dist.js +11 -11
package/README.md
CHANGED
|
@@ -826,7 +826,15 @@ the job with a decremented retry count. Profiles are stored in the plugin's
|
|
|
826
826
|
> the job, so the ref stays restorable. Orphans (cancelled instances, failed acks,
|
|
827
827
|
> exhausted retries) are reclaimed by a background sweep (at most hourly per remote per
|
|
828
828
|
> worker) that deletes WIP refs older than `NANO_AGENT_CHECKPOINT_TTL_MS` (7 days), or
|
|
829
|
-
> older than 1h
|
|
829
|
+
> older than 1h once the run is confirmed terminal. "Terminal" is **process-aware** where
|
|
830
|
+
> the host SDK exposes process-instance lookup: a ref is reclaimed only when its element
|
|
831
|
+
> instance is `COMPLETED`/`TERMINATED` **and** its process instance is also terminal
|
|
832
|
+
> (`COMPLETED`/`TERMINATED`/`CANCELED`). A terminal element inside a **still-active**
|
|
833
|
+
> process is **spared** — an SLA boundary timer can cancel the element while its process
|
|
834
|
+
> waits for a human to answer the escalation, and that spared checkpoint is exactly what
|
|
835
|
+
> the resume needs (issue #295). The sweep fails closed (any missing key / unreachable
|
|
836
|
+
> lookup spares the ref; the 7-day TTL is the backstop) and degrades to the element-only
|
|
837
|
+
> check on hosts whose SDK lacks process-instance lookup.
|
|
830
838
|
> - **Safety:** pushes use `--force-with-lease`, so a superseded worker can't clobber a
|
|
831
839
|
> newer run's checkpoint. A remote *policy* refusal (repository rules, secret push
|
|
832
840
|
> protection) or a permission error stops checkpointing for that job with a warning;
|
|
@@ -835,6 +843,26 @@ the job with a decremented retry count. Profiles are stored in the plugin's
|
|
|
835
843
|
> Caveats: WIP refs are visible to anyone who can read the repo; the worker can't pause
|
|
836
844
|
> the agent, so a snapshot may catch a half-written file. Everything is best-effort — a
|
|
837
845
|
> failure degrades to transcript-only resume.
|
|
846
|
+
>
|
|
847
|
+
> - **SLA-cancel resume + its one constraint.** When a human answers an SLA escalation
|
|
848
|
+
> the task re-enters as a *new* element instance (new key), so the predecessor's
|
|
849
|
+
> checkpoint is correlated to this task by the element's **static `elementId`** within
|
|
850
|
+
> the process instance. That id is **not unique if one agent-task element runs as
|
|
851
|
+
> parallel multi-instance within a single process instance** (the concurrent
|
|
852
|
+
> occurrences share both the id and the one process-scoped resume slot). **Parallel
|
|
853
|
+
> multi-instance of a single agent-task element is therefore unsupported for resume**
|
|
854
|
+
> (no nano-workforce process fans one agent-task element out that way — parallel
|
|
855
|
+
> fan-out uses distinct child processes or distinct element ids, and rounds/retries are
|
|
856
|
+
> sequential). Because counting **active** occurrences cannot establish occurrence
|
|
857
|
+
> identity — index lag can hide a live sibling, and a sibling that already went
|
|
858
|
+
> terminal drops out of the active set yet may have been the resume slot's last writer
|
|
859
|
+
> — no engine query makes an `elementId`-correlated self-record adoption provably safe.
|
|
860
|
+
> So a self-recorded (object) predecessor is **never auto-adopted**: the restore path
|
|
861
|
+
> fails closed to a clean start (a fresh clone, transcript-only) rather than risk a
|
|
862
|
+
> sibling occurrence's work. The only predecessor trusted is an explicit,
|
|
863
|
+
> occurrence-authoritative ref delivered as a bare ref **string**, which is trusted
|
|
864
|
+
> as-is. After a successful adoption the process-scoped pointer is advanced to this
|
|
865
|
+
> run's newly-copied ref so a no-new-checkpoint completion can't leave it stale.
|
|
838
866
|
|
|
839
867
|
> **Agent outcomes.** A harness may report an explicit end-of-task outcome in its
|
|
840
868
|
> `session/prompt` result as `_meta.outcome` — `{ "status": "completed" | "blocked",
|
|
@@ -912,7 +940,7 @@ only `latest`, so the key is the reproducibility handle).
|
|
|
912
940
|
|
|
913
941
|
On completion the plugin writes an **output envelope** back under
|
|
914
942
|
`io.nanobpm.agentResult` (`{schemaVersion, status, sandbox, image, output, truncated, stderrTruncated, exitCode, signal, error, outcome?, promptResourceKey?}`). When a repository was
|
|
915
|
-
provisioned (below) it also carries `{repository, branch, baseSha, headSha, commits[], pushed, pushError?, pushFailed?, strandedCommits?, branchMismatch?, scanError?, gitError?, pr?}`. `pushFailed` is the explicit "push failed" flag — set for a non-zero `git push` **whose result could not be confirmed as landed at the remote** (a non-fast-forward rejection, or an auth, hook, or network error), not only a server rejection. **`pushFailed` does not by itself prove a remote push was attempted or rejected:** `finalizeGit` also sets it (with `strandedCommits` and a `branchMismatch` `{expected, actual}`) when it *refuses to push at all* because the harness moved HEAD off the provisioned work branch — or left commits abandoned on another local branch — so pushing the work branch would publish stale work and strand those commits (no `git push` runs, so `pushError` is absent in that case). As a guard against false strands, a non-zero push is re-checked with `ls-remote`: if `origin/<branch>` already points at `headSha` — or the remote tip is a **descendant** of `headSha` (another actor pushed a further commit after ours landed) — the push is treated as a transport hiccup that landed after the ref was accepted (`pushed: true`, no `pushFailed`/`pushError`); otherwise `pushFailed` is set and `strandedCommits` lists the SHAs of the new commits left UNPUSHED in the throwaway workspace — together they are the recovery handle for a failed push, so consumers must not treat `pushed: false` alone as the only signal. On such a failure (a rejected push **or** a branch-mismatch refusal) the throwaway workspace is preserved **best-effort** (even under the default `--keep-runs=false`) so those SHAs stay recoverable, but this is a *recovery window, not a durable archive*: the run-directory reaper still ages it out by mtime and worker shutdown removes the namespace — copy the stranded commits out promptly (or run with `--keep-runs`). **`scanError`** flags a **best-effort/PARTIAL** `strandedCommits` list: a commit or remote-reachability scan did **not** complete, so a consumer should recover the whole preserved workspace rather than trust the list exactly. It arises two ways: (a) a **pre-push** commit-enumeration scan failed, so `finalizeGit` refused to push on an incomplete graph — **no push was attempted** (`pushError` is absent) and `strandedCommits` is a partial surfacing; or (b) a push was **ATTEMPTED and rejected** but the remote-reachability FILTER that trims already-published commits from the strand set itself failed — so `scanError` is present **alongside** `pushError`, marking the rejected-push strand list as best-effort (it may falsely include an already-landed commit). Either way `pushFailed` stays authoritative; `scanError` is how a consumer knows the strand list is inexact (recover the whole workspace) versus an exact list (a rejected push with no `scanError`).
|
|
943
|
+
provisioned (below) it also carries `{repository, branch, baseSha, headSha, commits[], pushed, pushError?, pushFailed?, strandedCommits?, branchMismatch?, scanError?, gitError?, pr?, recoveryRef?, recoverySha?}`. `pushFailed` is the explicit "push failed" flag — set for a non-zero `git push` **whose result could not be confirmed as landed at the remote** (a non-fast-forward rejection, or an auth, hook, or network error), not only a server rejection. **`pushFailed` does not by itself prove a remote push was attempted or rejected:** `finalizeGit` also sets it (with `strandedCommits` and a `branchMismatch` `{expected, actual}`) when it *refuses to push at all* because the harness moved HEAD off the provisioned work branch — or left commits abandoned on another local branch — so pushing the work branch would publish stale work and strand those commits (no `git push` runs, so `pushError` is absent in that case). As a guard against false strands, a non-zero push is re-checked with `ls-remote`: if `origin/<branch>` already points at `headSha` — or the remote tip is a **descendant** of `headSha` (another actor pushed a further commit after ours landed) — the push is treated as a transport hiccup that landed after the ref was accepted (`pushed: true`, no `pushFailed`/`pushError`); otherwise `pushFailed` is set and `strandedCommits` lists the SHAs of the new commits left UNPUSHED in the throwaway workspace — together they are the recovery handle for a failed push, so consumers must not treat `pushed: false` alone as the only signal. On such a failure (a rejected push **or** a branch-mismatch refusal) the throwaway workspace is preserved **best-effort** (even under the default `--keep-runs=false`) so those SHAs stay recoverable, but this is a *recovery window, not a durable archive*: the run-directory reaper still ages it out by mtime and worker shutdown removes the namespace — copy the stranded commits out promptly (or run with `--keep-runs`). **`scanError`** flags a **best-effort/PARTIAL** `strandedCommits` list: a commit or remote-reachability scan did **not** complete, so a consumer should recover the whole preserved workspace rather than trust the list exactly. It arises two ways: (a) a **pre-push** commit-enumeration scan failed, so `finalizeGit` refused to push on an incomplete graph — **no push was attempted** (`pushError` is absent) and `strandedCommits` is a partial surfacing; or (b) a push was **ATTEMPTED and rejected** but the remote-reachability FILTER that trims already-published commits from the strand set itself failed — so `scanError` is present **alongside** `pushError`, marking the rejected-push strand list as best-effort (it may falsely include an already-landed commit). Either way `pushFailed` stays authoritative; `scanError` is how a consumer knows the strand list is inexact (recover the whole workspace) versus an exact list (a rejected push with no `scanError`). **`recoveryRef` / `recoverySha`** name a durable recovery checkpoint (`refs/nano/recovered/<elementInstanceKey>`) the run pushed when it was interrupted (issue #295) — outside the `refs/nano/wip/*` namespace, so the snapshot survives the per-checkpoint WIP cleanup and workspace reaping. It is **not** exempt from the orphan GC, though: the sweep reclaims it under the same process-terminal / 7-day-TTL policy as a WIP ref, sparing it only while its process is still active. **These two envelope fields are present only for a *non-aborted* failed run** (a non-ok result that did not interrupt the runner). A **force-stop OR an SLA lease-loss cancellation** both take the *aborted* early return (the supervisor treats the SLA cancel's lease-loss 404 as terminal and interrupts the runner, reaching the same aborted branch as a force-stop), which yields the job and returns **before** the output envelope is built — so `recoveryRef`/`recoverySha` are **absent** there even though a recovery checkpoint WAS pushed. For those aborted paths the recovery point is reported through the process-scoped `resumeFromCheckpoint` variable (which a resumed element re-entry reads), and the pushed `refs/nano/recovered/<elementInstanceKey>` ref itself is the durable handle a resume fetches the snapshot from. A clean run omits all of these.
|
|
916
944
|
|
|
917
945
|
**Git provisioning (host).** When `--sandbox none` (the default) and the envelope
|
|
918
946
|
carries a `repository.url`, the plugin provisions a workspace on the host around
|
package/agent-checkpoint.mjs
CHANGED
|
@@ -115,6 +115,11 @@ const DEFAULTS = Object.freeze({
|
|
|
115
115
|
// timeout/abort — bounds how long the runner can stay pending past its deadline.
|
|
116
116
|
gitKillGraceMs: 5_000,
|
|
117
117
|
flushTimeoutMs: 20_000,
|
|
118
|
+
// Deadline every checkpoint-path SDK call (createElementInstanceVariables, the
|
|
119
|
+
// #295 occurrence/terminal lookups) is bounded by. A seam so a test can drive the
|
|
120
|
+
// timeout deterministically with a tiny value instead of consuming the real 10s of
|
|
121
|
+
// wall clock to observe the cancel-on-timeout behaviour (Copilot review on #296).
|
|
122
|
+
sdkTimeoutMs: 10_000,
|
|
118
123
|
ttlMs: 7 * 24 * 60 * 60 * 1000,
|
|
119
124
|
gcGraceMs: 60 * 60 * 1000,
|
|
120
125
|
gcEveryMs: 60 * 60 * 1000,
|
|
@@ -232,7 +237,16 @@ export function checkpointConfig(env = process.env) {
|
|
|
232
237
|
gcGraceMs: DEFAULTS.gcGraceMs,
|
|
233
238
|
gcEveryMs: DEFAULTS.gcEveryMs,
|
|
234
239
|
gitTimeoutMs: DEFAULTS.gitTimeoutMs,
|
|
240
|
+
// Exposed so the force-stop path can reserve a drain window: abortRecovery()
|
|
241
|
+
// only SIGTERMs checkpoint git, and createGitRunner grants this grace before
|
|
242
|
+
// SIGKILL — so the recovery race must finish early enough to leave close() a
|
|
243
|
+
// real window to drain the now-terminated git, not the 1ms an expired deadline
|
|
244
|
+
// would otherwise hand it (Copilot review on #296).
|
|
245
|
+
gitKillGraceMs: DEFAULTS.gitKillGraceMs,
|
|
235
246
|
flushTimeoutMs: DEFAULTS.flushTimeoutMs,
|
|
247
|
+
// Floor of 1ms so a test can inject a near-instant deadline; production uses the
|
|
248
|
+
// 10s default when the env var is unset (Copilot review on #296).
|
|
249
|
+
sdkTimeoutMs: Math.max(1, intOr(env.NANO_AGENT_CHECKPOINT_SDK_TIMEOUT_MS, DEFAULTS.sdkTimeoutMs)),
|
|
236
250
|
};
|
|
237
251
|
}
|
|
238
252
|
|
|
@@ -280,6 +294,213 @@ export function checkpointRef(elementInstanceKey) {
|
|
|
280
294
|
return /^[0-9A-Za-z_-]{1,128}$/.test(key) ? `${CHECKPOINT_REF_PREFIX}${key}` : null;
|
|
281
295
|
}
|
|
282
296
|
|
|
297
|
+
// Durable "recovery" refs (#295) live OUTSIDE the WIP namespace so a snapshot promoted
|
|
298
|
+
// here on an SLA-cancel or force-stop is GC-PROOF against the per-checkpoint reap — it
|
|
299
|
+
// survives the WIP cleanup so the human escalation can be answered and the run resumed.
|
|
300
|
+
// It is NOT immortal: the orphan sweep reclaims it under the SAME process-terminal /
|
|
301
|
+
// TTL policy as a WIP ref (see `sweepStaleCheckpoints`), so a recovery ref whose process
|
|
302
|
+
// has finished (or aged past the TTL) is reclaimed instead of pinning a ref forever,
|
|
303
|
+
// while an active escalation (process still ACTIVE) stays protected.
|
|
304
|
+
export const RECOVERED_REF_PREFIX = 'refs/nano/recovered/';
|
|
305
|
+
// Shared root of every nano-managed ref namespace, and the full set the orphan GC reaps.
|
|
306
|
+
export const NANO_REF_ROOT = 'refs/nano/';
|
|
307
|
+
export const SWEPT_REF_PREFIXES = Object.freeze([CHECKPOINT_REF_PREFIX, RECOVERED_REF_PREFIX]);
|
|
308
|
+
export function recoveredCheckpointRef(elementInstanceKey) {
|
|
309
|
+
const key = String(elementInstanceKey ?? '').trim();
|
|
310
|
+
return /^[0-9A-Za-z_-]{1,128}$/.test(key) ? `${RECOVERED_REF_PREFIX}${key}` : null;
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
// A resumed run re-enters the task as a NEW element instance with a new key, so the
|
|
314
|
+
// deterministic `refs/nano/wip/<newKey>` is empty even though the interrupted run
|
|
315
|
+
// pushed `refs/nano/wip/<oldKey>` (#295). The predecessor writes its ref into a
|
|
316
|
+
// PROCESS-scoped `resumeFromCheckpoint` variable (survives element re-entry); on
|
|
317
|
+
// resume the new run reads it here, fetches that ref and adopts it under the new
|
|
318
|
+
// key. Returns the predecessor WIP ref, or null when absent / malformed / the same
|
|
319
|
+
// ref this run already owns.
|
|
320
|
+
//
|
|
321
|
+
// `resumeFromCheckpoint` is ONE process-scoped slot shared by every agent task in
|
|
322
|
+
// the process, so it must be correlated to THIS task before adoption (Copilot
|
|
323
|
+
// review on #295): in a process with parallel/sequential agent tasks, whichever
|
|
324
|
+
// task checkpointed last overwrites the slot, and an un-correlated read could adopt
|
|
325
|
+
// ANOTHER task's repository state. The writer stamps the producing task's
|
|
326
|
+
// `elementId`; when the current job's `elementId` is supplied here, a recorded
|
|
327
|
+
// `elementId` that DIFFERS is refused (returns null). A recorded `elementId` that
|
|
328
|
+
// is absent/null (an older writer, or a writer whose job carried no elementId) is
|
|
329
|
+
// still adopted — refusing it would break the very resume #295 exists to support —
|
|
330
|
+
// and when the caller passes no `elementId` the correlation is simply skipped
|
|
331
|
+
// (legacy callers/tests). Task identity is `elementId`, NOT the recorded
|
|
332
|
+
// `elementInstanceKey`: a resume re-enters as a NEW element instance, so the
|
|
333
|
+
// instance key always differs and can never correlate.
|
|
334
|
+
export function predecessorCheckpointRef(variables, selfRef = null, elementId = null) {
|
|
335
|
+
const v = variables && typeof variables === 'object' ? variables.resumeFromCheckpoint : null;
|
|
336
|
+
const ref = typeof v === 'string' ? v : (v && typeof v === 'object' ? v.ref : null);
|
|
337
|
+
const s = typeof ref === 'string' ? ref.trim() : '';
|
|
338
|
+
// Accept a WIP ref (the normal per-checkpoint pointer) OR a GC-proof recovery ref
|
|
339
|
+
// (`refs/nano/recovered/<eik>`). The recovery ref is what an SLA-cancelled /
|
|
340
|
+
// force-stopped run writes here (Copilot review on #296, AC3): its element is
|
|
341
|
+
// terminated before buildResultEnvelope runs, so the process-scoped slot is the
|
|
342
|
+
// durable path that lets the resumed run discover the recovery point. Both are
|
|
343
|
+
// adopted identically below (fetch + restore are ref-agnostic).
|
|
344
|
+
const prefix = s.startsWith(CHECKPOINT_REF_PREFIX) ? CHECKPOINT_REF_PREFIX
|
|
345
|
+
: (s.startsWith(RECOVERED_REF_PREFIX) ? RECOVERED_REF_PREFIX : null);
|
|
346
|
+
if (!s || !prefix) return null;
|
|
347
|
+
const key = s.slice(prefix.length);
|
|
348
|
+
if (!/^[0-9A-Za-z_-]{1,128}$/.test(key)) return null;
|
|
349
|
+
if (s === selfRef) return null;
|
|
350
|
+
// Correlation gate: only a POSITIVE mismatch on a present recorded elementId
|
|
351
|
+
// refuses adoption. A missing recorded elementId (legacy/elementId-less writer)
|
|
352
|
+
// or a missing caller elementId (correlation unavailable) both fall through.
|
|
353
|
+
if (elementId != null && v && typeof v === 'object') {
|
|
354
|
+
const recorded = typeof v.elementId === 'string' ? v.elementId : null;
|
|
355
|
+
if (recorded != null && recorded !== elementId) return null;
|
|
356
|
+
}
|
|
357
|
+
return s;
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
// Push `sha` to `ref` on `remote`, FENCED so a slower/stale activation can never
|
|
361
|
+
// blind-overwrite a newer run's state (Copilot review on #296). A plain `+<sha>:<ref>`
|
|
362
|
+
// force let whichever writer ran LAST win, so a zombie prior activation could clobber
|
|
363
|
+
// the recovery point of the newer run that superseded it — the exact failure the WIP
|
|
364
|
+
// path already guards against with `--force-with-lease`. `expectSha` is the remote sha
|
|
365
|
+
// this run believes `ref` holds and the write is leased against it:
|
|
366
|
+
// - '' (default) demands the ref be ABSENT — "create-only". Used to ADOPT a
|
|
367
|
+
// predecessor's checkpoint under THIS run's own key (#295): if a concurrent
|
|
368
|
+
// activation already populated the key the lease is lost and we keep theirs.
|
|
369
|
+
// - a non-empty sha only overwrites while `ref` still points at exactly that sha —
|
|
370
|
+
// used to promote a GC-proof recovery ref over the value we OBSERVED, so a stale
|
|
371
|
+
// activation that observed an older/absent value loses the lease once a newer run
|
|
372
|
+
// moved the ref.
|
|
373
|
+
// - `null` keeps the legacy unconditional force (no caller uses it; kept for tests).
|
|
374
|
+
// A lease mismatch means another writer moved `ref` first: returns
|
|
375
|
+
// { ok:false, staleLease:true, error } and leaves their state intact. Other failures
|
|
376
|
+
// return { ok:false, error }.
|
|
377
|
+
export async function pushNamedRef({ git, ref, sha, remote = 'origin', expectSha = '' }) {
|
|
378
|
+
if (!ref || !sha) return { ok: false, error: 'missing ref or sha' };
|
|
379
|
+
const args = expectSha != null
|
|
380
|
+
? ['push', '--quiet', '--no-verify', `--force-with-lease=${ref}:${expectSha}`, remote, `${sha}:${ref}`]
|
|
381
|
+
: ['push', '--quiet', '--no-verify', remote, `+${sha}:${ref}`];
|
|
382
|
+
const r = await git(args);
|
|
383
|
+
if (ok(r)) return { ok: true };
|
|
384
|
+
const text = `${r.stderr}\n${r.stdout}`;
|
|
385
|
+
if (expectSha != null && classifyPushFailure(text) === 'lease') {
|
|
386
|
+
return { ok: false, staleLease: true, error: errText(r) };
|
|
387
|
+
}
|
|
388
|
+
return { ok: false, error: errText(r) };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
// Observe the sha `ref` currently holds on `remote` ('' when absent/unreadable), so a
|
|
392
|
+
// fenced `pushNamedRef` can lease against it. Best-effort: any ls-remote failure
|
|
393
|
+
// returns '' (create-only semantics — fail closed, never blind-overwrite).
|
|
394
|
+
export async function observeRemoteRefSha({ git, ref, remote = 'origin' }) {
|
|
395
|
+
try {
|
|
396
|
+
const r = await git(['ls-remote', '--quiet', remote, ref]);
|
|
397
|
+
if (!ok(r)) return '';
|
|
398
|
+
const first = String(r.stdout ?? '').trim().split(/\s+/)[0] || '';
|
|
399
|
+
return /^[0-9a-f]{7,64}$/i.test(first) ? first : '';
|
|
400
|
+
} catch { return ''; }
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
// Build the orphan predicate the GC sweep consults (#295). A WIP ref is an orphan
|
|
404
|
+
// ONLY when its element instance is terminal AND — when a process-instance reader is
|
|
405
|
+
// available — its PROCESS instance is also finished. An element cancelled by an SLA
|
|
406
|
+
// boundary timer inside a STILL-ACTIVE process is pending human escalation/resume,
|
|
407
|
+
// NOT an orphan, so it must be spared (the TTL stays the final backstop). Any
|
|
408
|
+
// unknown / unreachable lookup (404, a foreign engine, a missing field, a thrown
|
|
409
|
+
// error) returns false so only the TTL reclaims it — losing a recoverable checkpoint
|
|
410
|
+
// is never worth an eager delete. `getElementInstance`/`getProcessInstance` are
|
|
411
|
+
// injected (key → instance object with a `state`), so this is deterministically
|
|
412
|
+
// unit-testable without the SDK. Returns null when element lookups are unavailable
|
|
413
|
+
// (the sweep then reclaims purely by TTL).
|
|
414
|
+
export function makeOrphanProbe({ getElementInstance, getProcessInstance = null } = {}) {
|
|
415
|
+
if (typeof getElementInstance !== 'function') return null;
|
|
416
|
+
const terminal = (s) => s === 'COMPLETED' || s === 'TERMINATED' || s === 'CANCELED';
|
|
417
|
+
return async (key) => {
|
|
418
|
+
let ei;
|
|
419
|
+
try { ei = await getElementInstance(key); } catch { return false; }
|
|
420
|
+
if (!(ei?.state === 'COMPLETED' || ei?.state === 'TERMINATED')) return false;
|
|
421
|
+
// Element is terminal. Without a process reader, keep the legacy element-only
|
|
422
|
+
// behaviour; with one, require the process to be finished too.
|
|
423
|
+
if (typeof getProcessInstance !== 'function') return true;
|
|
424
|
+
const pik = ei?.processInstanceKey ?? ei?.processInstanceId ?? null;
|
|
425
|
+
// Can't identify / reach the process → cannot PROVE it is finished, so spare the
|
|
426
|
+
// ref (recoverable until the TTL) rather than delete a possibly-active run's WIP.
|
|
427
|
+
if (pik == null) return false;
|
|
428
|
+
let pi;
|
|
429
|
+
try { pi = await getProcessInstance(pik); } catch { return false; }
|
|
430
|
+
return terminal(pi?.state);
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
// #295 ambiguity guard (scope decided on nanobpmn/nano-workforce#891). The
|
|
435
|
+
// predecessor checkpoint is correlated to THIS task by its STATIC `elementId`, which
|
|
436
|
+
// is NOT unique when one agent-task element runs as PARALLEL multi-instance within a
|
|
437
|
+
// single process instance: N concurrent occurrences share the elementId AND the one
|
|
438
|
+
// process-scoped `resumeFromCheckpoint` slot, so a last-writer-wins read could make a
|
|
439
|
+
// resumed occurrence adopt a SIBLING occurrence's repository state. Parallel MI of a
|
|
440
|
+
// single agent-task element is OUT OF SCOPE for resume (no nano-workforce process fans
|
|
441
|
+
// one agent-task element out in parallel within a single instance — plan-fanout's
|
|
442
|
+
// parallel `implement` MI calls a separate CHILD process, delivery graphs fan out to
|
|
443
|
+
// DISTINCT elementIds, and review rounds / retries are sequential).
|
|
444
|
+
//
|
|
445
|
+
// Copilot review on #296 (r4244212483): the earlier "count ACTIVE occurrences" probe
|
|
446
|
+
// could NOT actually make a wrong adoption impossible, so it has been REMOVED as an
|
|
447
|
+
// adoption gate. Counting active instances never establishes occurrence identity:
|
|
448
|
+
// - the search is explicitly allowed to return ZERO from index lag (`consistency
|
|
449
|
+
// waitUpToMs: 0`), and that same lag can hide a genuinely-active sibling; and
|
|
450
|
+
// - even on a fresh index, a sibling that already went TERMINAL drops out of the
|
|
451
|
+
// `state:'ACTIVE'` result yet may have been the LAST WRITER of the shared
|
|
452
|
+
// process-scoped slot — so `items.length <= 1` can still adopt that sibling's
|
|
453
|
+
// checkpoint.
|
|
454
|
+
// Because a self-recorded (object) predecessor is correlated ONLY by the ambiguous
|
|
455
|
+
// static elementId, NO engine query can prove it names THIS occurrence. The guard
|
|
456
|
+
// therefore FAILS CLOSED BY CONSTRUCTION: it always reports `{ safe:false }`, so the
|
|
457
|
+
// caller never auto-adopts a self-record. The ONLY predecessor this run trusts is the
|
|
458
|
+
// EXPLICIT, occurrence-authoritative ref the process delivers as a bare STRING
|
|
459
|
+
// (`resumeRefIsExplicit`, nano-workforce#891 option 2) — the process's exact choice of
|
|
460
|
+
// predecessor for THIS occurrence. Until #891 delivers that, a resume starts clean
|
|
461
|
+
// (transcript-only) rather than risk restoring a sibling occurrence's state.
|
|
462
|
+
//
|
|
463
|
+
// Kept exported (and still returning a probe) for backward compatibility and so the
|
|
464
|
+
// diagnostics/tests can observe the refusal reason; it no longer consults the engine.
|
|
465
|
+
export function makeOccurrenceGuard() {
|
|
466
|
+
return async () => ({
|
|
467
|
+
safe: false,
|
|
468
|
+
candidates: [],
|
|
469
|
+
reason: 'an elementId-correlated self-record cannot establish occurrence identity (parallel multi-instance shares the elementId and the resume slot); only an explicit occurrence-authoritative ref is adopted',
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// The resume slot carries EITHER the predecessor's self-stamped record (a structured
|
|
474
|
+
// object `{ ref, elementId, ... }` the plugin writes at the process scope) OR — once
|
|
475
|
+
// the process delivers it (nanobpmn/nano-workforce#891, option 2) — an EXPLICIT,
|
|
476
|
+
// occurrence-authoritative ref as a bare STRING. An explicit ref is the process's exact
|
|
477
|
+
// choice of predecessor for THIS occurrence, so it bypasses the elementId correlation
|
|
478
|
+
// AND the ambiguity guard (which exist only because a self-record cannot identify the
|
|
479
|
+
// occurrence). A self-record (object) is correlated by elementId and gated by the
|
|
480
|
+
// guard. This lets #891 plug in with NO further plugin change: deliver a bare ref string
|
|
481
|
+
// and the restore path trusts it; until then the elementId-correlated self-record path
|
|
482
|
+
// (with the guard) is used. Returns true only for a bare, well-formed ref string in
|
|
483
|
+
// EITHER supported checkpoint namespace — the WIP prefix (a normal per-checkpoint
|
|
484
|
+
// pointer) OR the GC-proof recovered prefix (`refs/nano/recovered/<eik>`, the explicit
|
|
485
|
+
// pointer an SLA-cancelled run leaves). Both are occurrence-authoritative when delivered
|
|
486
|
+
// as a bare string, so both bypass the guard; restricting this to the WIP prefix would
|
|
487
|
+
// route a recovered pointer through the ambiguity guard and reject it whenever the host
|
|
488
|
+
// SDK can't prove uniqueness (Copilot review on #296). The suffix is validated against
|
|
489
|
+
// the same elementInstanceKey shape `checkpointRef`/`recoveredCheckpointRef` enforce
|
|
490
|
+
// (`/^[0-9A-Za-z_-]{1,128}$/`), so a malformed ref such as `refs/nano/wip/bad/key` is
|
|
491
|
+
// NOT occurrence-authoritative — this predicate is an authorization boundary (it decides
|
|
492
|
+
// whether the ambiguity guard is bypassed), so it must not trust a ref the canonical
|
|
493
|
+
// builders would never mint (Copilot review on #296).
|
|
494
|
+
export function resumeRefIsExplicit(variables) {
|
|
495
|
+
const v = variables && typeof variables === 'object' ? variables.resumeFromCheckpoint : null;
|
|
496
|
+
if (typeof v !== 'string') return false;
|
|
497
|
+
const s = v.trim();
|
|
498
|
+
const prefix = s.startsWith(CHECKPOINT_REF_PREFIX) ? CHECKPOINT_REF_PREFIX
|
|
499
|
+
: (s.startsWith(RECOVERED_REF_PREFIX) ? RECOVERED_REF_PREFIX : null);
|
|
500
|
+
if (!prefix) return false;
|
|
501
|
+
return /^[0-9A-Za-z_-]{1,128}$/.test(s.slice(prefix.length));
|
|
502
|
+
}
|
|
503
|
+
|
|
283
504
|
// Never let git invoke the host's configured credential helper for our remote
|
|
284
505
|
// checkpoint ops. The job token is delivered ONLY via GIT_ASKPASS; a configured
|
|
285
506
|
// helper such as `store` would otherwise reuse or PERSIST that token to disk on a
|
|
@@ -869,13 +1090,29 @@ export function shouldSweep(key, { everyMs = DEFAULTS.gcEveryMs, now = Date.now(
|
|
|
869
1090
|
return true;
|
|
870
1091
|
}
|
|
871
1092
|
|
|
872
|
-
export async function sweepStaleCheckpoints({ git, ownRef = null, ttlMs = DEFAULTS.ttlMs, graceMs = DEFAULTS.gcGraceMs, isTerminal = null, now = () => Date.now(), maxDeletes = 50, maxLookups = 20, remote = 'origin', signal = null }) {
|
|
873
|
-
|
|
1093
|
+
export async function sweepStaleCheckpoints({ git, ownRef = null, ttlMs = DEFAULTS.ttlMs, graceMs = DEFAULTS.gcGraceMs, isTerminal = null, now = () => Date.now(), maxDeletes = 50, maxLookups = 20, remote = 'origin', signal = null, prefixes = SWEPT_REF_PREFIXES }) {
|
|
1094
|
+
// Sweep EVERY nano-managed ref namespace, not just the WIP one. The GC-proof
|
|
1095
|
+
// recovery refs (`refs/nano/recovered/*`, #295) are deliberately excluded from the
|
|
1096
|
+
// WIP-only orphan reap so an SLA-cancelled escalation survives — but if nothing ever
|
|
1097
|
+
// reclaims them they grow without bound (one snapshot ref/commit pinned per element
|
|
1098
|
+
// instance, even for a transient failure later retried successfully). Reclaim both
|
|
1099
|
+
// namespaces under the SAME policy: the `isTerminal` orphan probe spares a ref whose
|
|
1100
|
+
// PROCESS is still active (the escalation is still pending), so an active escalation
|
|
1101
|
+
// stays protected while a finished one — and the TTL backstop — is reclaimed.
|
|
1102
|
+
const prefixList = (Array.isArray(prefixes) ? prefixes : [prefixes]).filter((p) => typeof p === 'string' && p);
|
|
1103
|
+
if (!prefixList.length) return { scanned: 0, deleted: [] };
|
|
1104
|
+
const prefixOf = (r) => prefixList.find((p) => r.startsWith(p)) || null;
|
|
1105
|
+
const ls = await git(['ls-remote', '--refs', remote, ...prefixList.map((p) => `${p}*`)]);
|
|
874
1106
|
if (!ok(ls)) return { error: errText(ls), deleted: [] };
|
|
875
|
-
const entries = out(ls).split('\n').filter(Boolean).map((l) => { const [sha, ref] = l.split(/\s+/); return { sha, ref }; })
|
|
1107
|
+
const entries = out(ls).split('\n').filter(Boolean).map((l) => { const [sha, ref] = l.split(/\s+/); return { sha, ref }; })
|
|
1108
|
+
.filter((e) => e.ref && e.ref !== ownRef && prefixOf(e.ref));
|
|
876
1109
|
if (!entries.length) return { scanned: 0, deleted: [] };
|
|
877
1110
|
const refs = entries.map((e) => e.ref);
|
|
878
|
-
|
|
1111
|
+
// Distinct local temp refs across namespaces: every swept prefix shares the
|
|
1112
|
+
// `refs/nano/` root, so slicing it keeps `wip/<key>` and `recovered/<key>` apart
|
|
1113
|
+
// (a bare key slice would collide a WIP ref with its recovery ref under one local).
|
|
1114
|
+
const local = (r) => `refs/nano-gc/${r.startsWith(NANO_REF_ROOT) ? r.slice(NANO_REF_ROOT.length) : r.replace(/^refs\//, '')}`;
|
|
1115
|
+
const keyOf = (r) => { const p = prefixOf(r); return p ? r.slice(p.length) : r; };
|
|
879
1116
|
const fetch = await git(['fetch', '--quiet', '--no-tags', '--no-write-fetch-head', remote, ...refs.map((r) => `+${r}:${local(r)}`)]);
|
|
880
1117
|
if (!ok(fetch)) return { error: errText(fetch), deleted: [] };
|
|
881
1118
|
const doomed = [];
|
|
@@ -897,7 +1134,7 @@ export async function sweepStaleCheckpoints({ git, ownRef = null, ttlMs = DEFAUL
|
|
|
897
1134
|
if (isTerminal && age > graceMs && lookups < maxLookups && !signal?.aborted) {
|
|
898
1135
|
lookups++;
|
|
899
1136
|
let terminal = false;
|
|
900
|
-
try { terminal = await isTerminal(ref
|
|
1137
|
+
try { terminal = await isTerminal(keyOf(ref)); } catch { terminal = false; }
|
|
901
1138
|
if (terminal) doomed.push({ ref, sha, why: 'element-terminal' });
|
|
902
1139
|
}
|
|
903
1140
|
}
|
|
@@ -909,7 +1146,7 @@ export async function sweepStaleCheckpoints({ git, ownRef = null, ttlMs = DEFAUL
|
|
|
909
1146
|
let lastError = null;
|
|
910
1147
|
for (const d of doomed) {
|
|
911
1148
|
const r = await deleteCheckpointRef({ git, ref: d.ref, remote, expectSha: d.sha });
|
|
912
|
-
if (r.ok) deleted.push({ ref: d.ref, why: d.why });
|
|
1149
|
+
if (r.ok) deleted.push({ ref: d.ref, sha: d.sha, why: d.why });
|
|
913
1150
|
else if (r.error) lastError = r.error;
|
|
914
1151
|
}
|
|
915
1152
|
const result = { scanned: refs.length, deleted };
|