@yemi33/minions 0.1.2272 → 0.1.2273
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/live-checkout-mode.md +28 -11
- package/engine/ado.js +1 -1
- package/engine/cleanup.js +19 -3
- package/engine/cli.js +2 -8
- package/engine/create-pr-worktree.js +44 -5
- package/engine/dispatch.js +1 -0
- package/engine/github.js +3 -3
- package/engine/live-checkout.js +263 -17
- package/engine/shared.js +6 -0
- package/engine/timeout.js +46 -0
- package/engine.js +96 -8
- package/package.json +1 -1
|
@@ -37,7 +37,7 @@ Before spawning, `engine/live-checkout.js#prepareLiveCheckout` runs `git status
|
|
|
37
37
|
- Work item stamped with `_pendingReason: 'live_checkout_dirty'` so the dashboard surfaces the block.
|
|
38
38
|
- Completion summary: `live-checkout refused: N dirty file(s) in <localPath>`.
|
|
39
39
|
|
|
40
|
-
The engine never calls `git reset --hard`, `git clean -fd`, `git stash`, or any other state-mutating command against the operator's checkout — not at spawn, not at cleanup, not on timeout, not on engine restart.
|
|
40
|
+
The engine never calls `git reset --hard`, `git clean -fd`, `git stash`, or any other state-mutating command against the operator's checkout — not at spawn, not at cleanup, not on timeout, not on engine restart. The dispatch-scoped cleanup paths (`worktreePool.returnToPool`, `worktree-gc.gcDispatchWorktreeIfOrphan`, `_quarantineDirtyWorktree`) are naturally no-ops because `worktreePath` stays `null` end-to-end (`engine.js:1219`). The **periodic** worktree GC, however, is NOT `worktreePath`-gated — it derives its targets from `git worktree list --porcelain`, which for a live project returns the operator's *own* primary checkout. So `engine/cleanup.js#runPeriodicWorktreeSweep` now **filters out live-checkout projects entirely** (`shared.isLiveCheckoutProject`) before handing the list to the three pruners (PL-live-checkout-reliability-hardening), keeping the operator's real checkout out of the GC decision surface rather than relying only on the pruners' path-equality + ownership-marker gates.
|
|
41
41
|
|
|
42
42
|
#### 2a. Thrown pre-spawn failures are retryable, NOT dirty (#305)
|
|
43
43
|
|
|
@@ -55,6 +55,16 @@ After the clean-tree check, `prepareLiveCheckout`:
|
|
|
55
55
|
|
|
56
56
|
PR-source live dispatches (`meta.branch` set) and shared-branch live dispatches (`meta.branchStrategy === 'shared-branch'`) flow through the same code path with no special case. If the operator already has a PR branch checked out at a different SHA than `origin/<branch>`, the engine will not fast-forward it — that is the intended contract (see Open Q5 in the source PRD). Resolve manually with `git pull --ff-only` or commit the local divergence first.
|
|
57
57
|
|
|
58
|
+
#### 3a. Partial-clone (GVFS) reliability hardening (PL-live-checkout-reliability-hardening)
|
|
59
|
+
|
|
60
|
+
Issue #226 only de-risked the *new-branch* path. The *existing-branch* `git checkout <branch>` was still a bare checkout, and on a blobless Scalar/GVFS clone, switching onto a branch whose tree differs from HEAD must hydrate the changed paths' blobs through the same auth-less GVFS cache server — so it **fails deterministically headless**. Three changes make this reliable:
|
|
61
|
+
|
|
62
|
+
- **Already-on-target-branch fast path.** Before any checkout, `prepareLiveCheckout` reads the current branch (`git rev-parse --abbrev-ref HEAD`); if HEAD already points at the target branch it returns success with `alreadyOnBranch: true` and issues **no** checkout. This eliminates the most common blob-fetch trigger — a re-dispatch onto a branch the agent is already on.
|
|
63
|
+
- **No half-switch.** `originalRef` is captured *before* the checkout, and on any existing-branch checkout failure the helper best-effort switches HEAD back to it (plain checkout, never `--force` — the tree was verified clean at the dirty gate, so nothing of the operator's can be lost). The tree is never stranded half-populated on the agent branch.
|
|
64
|
+
- **Deterministic blob-fetch failures are non-retryable.** A GVFS / missing-object checkout failure is returned as `{ ok:false, reason:'blob-fetch' }` and completed with the dedicated **`FAILURE_CLASS.LIVE_CHECKOUT_BLOB_FETCH`** (`'live-checkout-blob-fetch'`; in `dispatch.js`'s `neverRetry` set) plus an operator-actionable `live-checkout-blocked-<wi-id>` inbox alert and a `_pendingReason: 'live_checkout_blob_fetch'` stamp — instead of being classified `LIVE_CHECKOUT_FAILED` (retryable) and retry-storming an identical structural failure to `maxRetries`. Recovery: the operator hydrates the branch once with their own credentials (`git checkout <branch>` interactively, or `scalar prefetch` / `git fetch origin <branch>`), then re-dispatches — or switches the project to `checkoutMode: "worktree"`, which fetches against the authenticated git remote.
|
|
65
|
+
|
|
66
|
+
**Branch existence is checked against `refs/heads/<branch>` specifically** (not a bare `rev-parse --verify <branch>`, which DWIM-resolves a same-named tag or remote ref and would silently detach HEAD).
|
|
67
|
+
|
|
58
68
|
### 4. No worktree pool, no per-WI subdirectory isolation
|
|
59
69
|
|
|
60
70
|
Live mode shares one checkout per project. There is no pool to recycle, no quarantine directory, no per-WI subdirectory under the project root. The mutating-concurrency cap (Guarantee 1) is the only isolation mechanism: agents take turns in the same directory.
|
|
@@ -65,21 +75,22 @@ Pool short-circuits live in `engine.js:1368` and `engine/cleanup.js`; both gate
|
|
|
65
75
|
|
|
66
76
|
`prepareLiveCheckout` runs a second preflight *between* the dirty-tree check (Guarantee 2) and branch resolution (Guarantee 3). Even on a **clean** tree, the branch switch/create is refused when the operator checkout is mid-operation or sitting on a detached HEAD:
|
|
67
77
|
|
|
68
|
-
- **In-progress git operation.** The git dir is resolved robustly via `git rev-parse --git-dir` (so submodule / gitdir-file / `repo`-managed trees — the setups that motivate live mode — are covered)
|
|
69
|
-
- **Detached HEAD.** `git symbolic-ref -q HEAD` exiting
|
|
78
|
+
- **In-progress git operation.** The git dir is resolved robustly via `git rev-parse --git-dir` (so submodule / gitdir-file / `repo`-managed trees — the setups that motivate live mode — are covered; a *failure* to resolve the git dir now throws → retryable `LIVE_CHECKOUT_FAILED` rather than silently probing a fabricated `<localPath>/.git` and missing the operation). Sentinel paths under it are probed: `MERGE_HEAD` → merge, `rebase-merge/` & `rebase-apply/` → rebase, `CHERRY_PICK_HEAD` → cherry-pick, `REVERT_HEAD` → revert, **`BISECT_LOG` → bisect**. First hit returns `{ ok:false, reason:'mid-operation', op, details }`.
|
|
79
|
+
- **Detached HEAD.** `git symbolic-ref -q HEAD` exiting with **code 1** returns `{ ok:false, reason:'detached-head', sha }` (sha from `git rev-parse HEAD`). Branching off a detached HEAD would strand the operator's anonymous commits. A *transient* `symbolic-ref` failure (spawn error / timeout, no exit-1) is **not** treated as a detached HEAD — it rethrows → retryable `LIVE_CHECKOUT_FAILED` — so an on-a-branch tree that hit a hiccup is never permanently refused.
|
|
70
80
|
|
|
71
81
|
Either condition fails the dispatch non-retryably with `FAILURE_CLASS.LIVE_CHECKOUT_MID_OPERATION` (`'live-checkout-mid-operation'`; added to `engine/dispatch.js`'s neverRetry set alongside `LIVE_CHECKOUT_DIRTY`). `spawnAgent` writes a `live-checkout-blocked-<wi-id>` inbox alert and stamps the work item `_pendingReason: 'live_checkout_mid_operation'` (or `'live_checkout_detached_head'` for the detached case). The recovery guidance tells the operator to finish or abort the in-progress op with their own commands (`git <op> --continue` / `git <op> --abort`), or checkout a branch, then re-dispatch. The engine never runs `git reset`, `git clean`, `git stash`, `git rebase --abort`, or moves HEAD on the operator's behalf.
|
|
72
82
|
|
|
73
83
|
## Dispatch-end auto-restore
|
|
74
84
|
|
|
75
|
-
Live-mode agents run **in-place** in the operator's checkout, so when a dispatch ends the engine switches the tree back to the ref it was on before the agent ran. This runs on **every** terminal result — success, failure, timeout, crash — and also on the engine-restart re-attach completion path (`engine/cli.js`), so a restart
|
|
85
|
+
Live-mode agents run **in-place** in the operator's checkout, so when a dispatch ends the engine switches the tree back to the ref it was on before the agent ran. This runs on **every** terminal result — success, failure, timeout, crash — and also on **every restart-recovery reaping path**: the engine-restart re-attach orphan-completion path (`engine/cli.js`) AND the `engine/timeout.js` paths that reap a dispatch the engine lost the process handle for (`completeFromOutput` + the orphan sweep). All three route through the shared `maybeRestoreLiveCheckoutFromRecord(item, …)` helper (which fires from the *persisted* dispatch record — `originalRef`'s presence is the live-mode signal), so a live agent that spans an engine restart and finishes afterward never strands the checkout on the agent's branch. (Previously the timeout.js reaping paths had **no** restore wiring, which is exactly how a restart-spanning live dispatch left the operator stuck on `work/W-…`.)
|
|
76
86
|
|
|
77
87
|
- **Original-ref capture.** `prepareLiveCheckout` records the operator's starting ref *before* the first checkout: `git symbolic-ref --short HEAD` → `{ originalRef:<branch>, originalRefType:'branch' }`, falling back to `git rev-parse HEAD` → `{ originalRef:<sha>, originalRefType:'detached' }`. `spawnAgent` persists `originalRef` / `originalRefType` onto the dispatch record via `mutateDispatch`, so the restore survives an engine restart, where the in-memory spawn closure is gone and only the persisted record remains.
|
|
88
|
+
- **Self-healing dirty recovery (PL-live-checkout-reliability-hardening).** Before the plain checkout, if the tree is **dirty AND HEAD is on the agent branch** (and that branch differs from `originalRef`), the dirt is provably **agent-authored** — the engine created that branch and verified the tree clean before switching to it. The leftovers are committed onto the **agent branch** (`git add -A` + `git commit --no-verify -m "minions: auto-save agent WIP (dispatch …)"`), so the plain `git checkout <originalRef>` then succeeds and the operator tree returns clean. This is the fix for the *"no recovery from dirty checkout"* deadlock, where an agent's crash-leftover WIP refused the restore, stranded the tree dirty on the agent branch, and then made **every future WI** for that project fail `LIVE_CHECKOUT_DIRTY` until a human cleaned it. It only ever mutates the engine-created branch, never the operator's branch, never discards (the WIP lands as a visible, revertable commit / on its PR), and gitignored artifacts are never staged (so a tree dirty only with ignored build output never reaches here). Best-effort: a failed auto-commit falls through to the manual-recovery alert below.
|
|
78
89
|
- **AUTO-RESTORE (best-effort, never `--force`).** At dispatch-end `restoreLiveCheckoutAtDispatchEnd` issues a **plain** `git checkout <originalRef>` — no `--force`, no `-B`, no reset, no clean, no stash. It no-ops when there is nothing to restore: no captured `originalRef`, the agent branch *is* the original ref, or HEAD already sits on the original ref (matched against the branch name *or* the raw sha so the detached-HEAD case is recognized). It is strictly best-effort: every error is swallowed and logged, and a restore never alters the dispatch result.
|
|
79
|
-
- **Fallback notify (only when a safe switch is impossible).** If git declines the plain checkout — most likely because the agent left uncommitted changes a checkout would overwrite — the refusal is **honored**: the tree is left exactly as the agent left it and a deduped `live-checkout-branch-<dispatchId>` inbox alert tells the operator how to switch back manually (`git -C <localPath> checkout <originalRef>`). The engine never forces the switch.
|
|
90
|
+
- **Fallback notify (only when a safe switch is impossible).** If git declines the plain checkout — most likely because the agent left uncommitted changes a checkout would overwrite — the refusal is **honored**: the tree is left exactly as the agent left it and a deduped `live-checkout-branch-<dispatchId>` inbox alert tells the operator how to switch back manually (`git -C <localPath> checkout <originalRef>`). The engine never forces the switch. An **unexpected** restore error (git missing, repo corruption, a GVFS blob fetch on the switch-back) now also writes this alert, so a non-refusal failure never silently strands the tree.
|
|
80
91
|
- **Terminal-failure alert.** When the dispatch ends in a non-success terminal state, a deduped `live-checkout-failed-<dispatchId>` inbox alert is written so the operator knows a live-mode run failed inside their own checkout (where any partial work is visible). This is independent of the restore and fires even when the restore itself succeeds.
|
|
81
92
|
|
|
82
|
-
The core invariant holds end-to-end through restore: **the engine only ever switches branches — it never `git reset`s, `git clean`s, or `git stash`es the operator's tree, and never passes `--force`.**
|
|
93
|
+
The core invariant holds end-to-end through restore: **the engine only ever switches branches — it never `git reset`s, `git clean`s, or `git stash`es the operator's tree, and never passes `--force`.** The one addition — the self-healing auto-commit — only ever runs `git add`/`git commit` on the engine's *own* agent branch, never on operator refs, and never discards.
|
|
83
94
|
|
|
84
95
|
## Operator workflow
|
|
85
96
|
|
|
@@ -159,18 +170,24 @@ Live-checkout mode is deliberately small. These are NOT supported and will not b
|
|
|
159
170
|
|---|---|
|
|
160
171
|
| `engine/shared.js` — `CHECKOUT_MODES`, `validateCheckoutMode`, `resolveCheckoutMode`, `isLiveCheckoutProject` | Enum + validator + back-compat resolver (P-a3f9b201; consolidated W-mqiaw974). |
|
|
161
172
|
| `engine/shared.js` — `resolveSpawnPaths` | Returns `{ cwd: localPath, worktreeRootDir: null, liveMode: true }` for live projects (P-a3f9b202). |
|
|
162
|
-
| `engine/live-checkout.js` — `prepareLiveCheckout` | Pure helper: dirty check, mid-operation / detached-HEAD preflight, original-ref capture, branch resolution from HEAD (no fetch
|
|
163
|
-
| `engine/live-checkout.js` — `restoreLiveCheckoutAtDispatchEnd` | Dispatch-end auto-restore (plain `git checkout <originalRef>`, never `--force`/reset/clean/stash, best-effort) + `live-checkout-failed-<dispatchId>` terminal-failure alert + `live-checkout-branch-<dispatchId>` fallback notify (P-d9e6b2c4). |
|
|
173
|
+
| `engine/live-checkout.js` — `prepareLiveCheckout` | Pure helper: dirty check, mid-operation / detached-HEAD preflight (incl. `BISECT_LOG`; throw-on-git-dir-failure; exit-1-only detached), original-ref capture, **already-on-branch fast path**, `refs/heads/<branch>` existence check, branch resolution from HEAD (no fetch — issue #226), **no-half-switch + `blob-fetch` classification** for partial-clone hydration failures (P-a3f9b203; preflight + capture P-b2e8d4a6; hardening PL-live-checkout-reliability-hardening). |
|
|
174
|
+
| `engine/live-checkout.js` — `restoreLiveCheckoutAtDispatchEnd` | Dispatch-end auto-restore (plain `git checkout <originalRef>`, never `--force`/reset/clean/stash, best-effort) + **self-healing dirty recovery** (auto-commit agent WIP onto the agent branch) + `live-checkout-failed-<dispatchId>` terminal-failure alert + `live-checkout-branch-<dispatchId>` fallback notify (now also on unexpected restore errors) (P-d9e6b2c4; self-heal PL-live-checkout-reliability-hardening). |
|
|
175
|
+
| `engine/live-checkout.js` — `maybeRestoreLiveCheckoutFromRecord` | Shared wrapper that fires the dispatch-end restore from a persisted dispatch record; used by `cli.js` + both `timeout.js` reaping paths so a restart-spanning live dispatch is never stranded (PL-live-checkout-reliability-hardening). |
|
|
164
176
|
| `engine.js` — `spawnAgent` live-mode block | Calls `prepareLiveCheckout`, handles dirty / throw branches, gates `git worktree add` on `!liveMode` (P-a3f9b204). |
|
|
165
177
|
| `engine.js` — `spawnAgent` mid-op / detached-HEAD refusal block | Emits `LIVE_CHECKOUT_MID_OPERATION`, writes `live-checkout-blocked-<wi-id>` alert, stamps `_pendingReason: 'live_checkout_mid_operation'` / `'live_checkout_detached_head'` (P-c5a1f3b8). |
|
|
166
178
|
| `engine.js` — `spawnAgent` originalRef persistence | Persists `originalRef` / `originalRefType` onto the dispatch record via `mutateDispatch` so restore survives an engine restart (P-c5a1f3b8). |
|
|
167
179
|
| `engine.js` — `onAgentClose` live-mode restore wiring | Calls `restoreLiveCheckoutAtDispatchEnd` on every terminal result (P-d9e6b2c4). |
|
|
168
|
-
| `engine/cli.js` — orphan / reattach restore | Fires the
|
|
180
|
+
| `engine/cli.js` — orphan / reattach restore | Fires the dispatch-end restore (via `maybeRestoreLiveCheckoutFromRecord`) from the persisted record on the engine-restart completion path (P-d9e6b2c4; unified PL-live-checkout-reliability-hardening). |
|
|
181
|
+
| `engine/timeout.js` — `completeFromOutput` + orphan-sweep restore | Fires `maybeRestoreLiveCheckoutFromRecord` when a re-attached live dispatch is reaped post-restart — closes the gap where these paths had no restore wiring (PL-live-checkout-reliability-hardening). |
|
|
169
182
|
| `engine.js` — dispatcher `liveProjectsInUse` set | Per-project mutating-concurrency cap (P-a3f9b205). |
|
|
170
183
|
| `engine.js` — worktree-pool / orphan-GC short-circuits | `worktreePath===null` no-ops in live mode (P-a3f9b206). |
|
|
184
|
+
| `engine/cleanup.js` — `runPeriodicWorktreeSweep` live filter | Excludes live-checkout projects from the registry-derived periodic worktree GC so the operator's primary checkout never enters the GC decision surface (PL-live-checkout-reliability-hardening). |
|
|
185
|
+
| `engine/create-pr-worktree.js` — `prepareCreatePrWorktree` step-4 restore | `reset --hard HEAD` (not the index-leaking `checkout -- .`) + retried untracked removal + `liveTreeDirty` surfaced + `shared.removeWorktree` teardown (PL-live-checkout-reliability-hardening). |
|
|
171
186
|
| `dashboard/js/settings.js` — checkoutMode dropdown + chip | Operator-facing UI (P-a3f9b207). |
|
|
172
187
|
| `test/unit/{resolve-spawn-paths-live-mode,prepare-live-checkout,spawn-agent-live-mode-wiring}.test.js` | Wiring and contract tests (P-a3f9b208). |
|
|
173
188
|
| `engine/shared.js` — `FAILURE_CLASS.LIVE_CHECKOUT_DIRTY` | Non-retryable refusal class. |
|
|
174
|
-
| `engine/shared.js` — `FAILURE_CLASS.LIVE_CHECKOUT_MID_OPERATION` | Non-retryable refusal class for a mid-operation / detached-HEAD operator tree (in-progress merge/rebase/cherry-pick/revert or detached HEAD), distinct from the dirty-tree class. Emitted by `spawnAgent`'s mid-op / detached-HEAD refusal block (P-a7f3c1d9; wired P-c5a1f3b8). |
|
|
175
|
-
| `engine/
|
|
189
|
+
| `engine/shared.js` — `FAILURE_CLASS.LIVE_CHECKOUT_MID_OPERATION` | Non-retryable refusal class for a mid-operation / detached-HEAD operator tree (in-progress merge/rebase/cherry-pick/revert/bisect or detached HEAD), distinct from the dirty-tree class. Emitted by `spawnAgent`'s mid-op / detached-HEAD refusal block (P-a7f3c1d9; wired P-c5a1f3b8). |
|
|
190
|
+
| `engine/shared.js` — `FAILURE_CLASS.LIVE_CHECKOUT_BLOB_FETCH` | Non-retryable refusal class for an existing-branch checkout that could not hydrate the tree on a blobless GVFS partial clone (auth-less cache fetch, headless) — deterministic, so excluded from mechanical retry (PL-live-checkout-reliability-hardening). |
|
|
191
|
+
| `engine.js` — `_liveCheckoutDirtyAttempts` counter | Dedicated two-strike dirty memory (survives the discovery + retry `_pendingReason` scrubs that defeated #434); first dirty failure retries once, second fails non-retryably (PL-live-checkout-reliability-hardening). |
|
|
192
|
+
| `engine/dispatch.js` — `isRetryableFailureReason` neverRetry | Excludes `LIVE_CHECKOUT_DIRTY`, `LIVE_CHECKOUT_MID_OPERATION`, and `LIVE_CHECKOUT_BLOB_FETCH` from mechanical retry. |
|
|
176
193
|
| `engine/timeout.js` header comment | Confirms no special live-mode kill handling. |
|
package/engine/ado.js
CHANGED
|
@@ -432,7 +432,7 @@ function classifyBuildStatus(prBuilds) {
|
|
|
432
432
|
const allDone = prBuilds.every(b => b.status === 'completed');
|
|
433
433
|
const allPassed = prBuilds.every(b => b.result === 'succeeded' || b.result === 'partiallySucceeded');
|
|
434
434
|
const hasRunning = prBuilds.some(b => b.status === 'inProgress' || b.status === 'notStarted');
|
|
435
|
-
if (hasFailed
|
|
435
|
+
if (hasFailed) return BUILD_STATUS.FAILING;
|
|
436
436
|
if (allDone && allPassed) return BUILD_STATUS.PASSING;
|
|
437
437
|
if (hasRunning) return BUILD_STATUS.RUNNING;
|
|
438
438
|
return BUILD_STATUS.NONE;
|
package/engine/cleanup.js
CHANGED
|
@@ -1629,9 +1629,25 @@ function scrubStaleMetrics() {
|
|
|
1629
1629
|
*/
|
|
1630
1630
|
function runPeriodicWorktreeSweep(config) {
|
|
1631
1631
|
const worktreeGc = require('./worktree-gc');
|
|
1632
|
-
const
|
|
1632
|
+
const allProjects = getProjects(config);
|
|
1633
|
+
// PL-live-checkout-reliability-hardening — EXCLUDE live-checkout projects from
|
|
1634
|
+
// the worktree GC entirely. The three pruners derive their targets from
|
|
1635
|
+
// `git worktree list --porcelain`, which for a live project returns the
|
|
1636
|
+
// OPERATOR'S OWN primary checkout (and any secondary operator worktrees). The
|
|
1637
|
+
// engine never creates a managed worktree in live mode, so there is nothing
|
|
1638
|
+
// here to GC — and the worktree-gc module is otherwise live-mode-blind,
|
|
1639
|
+
// protected only by a path-equality skip + ownership marker (a thin, untested
|
|
1640
|
+
// gate-stack whose regression could target the operator's real repo). Filtering
|
|
1641
|
+
// here keeps the operator's checkout out of the GC decision surface completely.
|
|
1642
|
+
// (docs/live-checkout-mode.md's "naturally no-ops because worktreePath stays
|
|
1643
|
+
// null" claim is true for the dispatch-end GC but NOT for this registry-derived
|
|
1644
|
+
// periodic sweep.)
|
|
1645
|
+
const projects = allProjects.filter(p => {
|
|
1646
|
+
try { return !shared.isLiveCheckoutProject(p); } catch { return true; }
|
|
1647
|
+
});
|
|
1648
|
+
const liveSkipped = allProjects.length - projects.length;
|
|
1633
1649
|
if (projects.length === 0) {
|
|
1634
|
-
return { scanned: 0, kept: 0, evicted: 0, failed: 0, outOfRootEvicted: 0, prunedRegistry: 0, missingDirReclaimed: 0, missingDirSkippedLive: 0 };
|
|
1650
|
+
return { scanned: 0, kept: 0, evicted: 0, failed: 0, outOfRootEvicted: 0, prunedRegistry: 0, missingDirReclaimed: 0, missingDirSkippedLive: 0, liveProjectsSkipped: liveSkipped };
|
|
1635
1651
|
}
|
|
1636
1652
|
const dispatchSnap = getDispatch();
|
|
1637
1653
|
const worktreeRootRel = config?.engine?.worktreeRoot || ENGINE_DEFAULTS.worktreeRoot;
|
|
@@ -1705,7 +1721,7 @@ function runPeriodicWorktreeSweep(config) {
|
|
|
1705
1721
|
failed += r3.failed || 0;
|
|
1706
1722
|
} catch (e) { log('warn', `worktree-gc periodic missing-dir reclaim: ${e.message}`); }
|
|
1707
1723
|
|
|
1708
|
-
return { scanned, kept, evicted, failed, outOfRootEvicted, prunedRegistry, missingDirReclaimed, missingDirSkippedLive };
|
|
1724
|
+
return { scanned, kept, evicted, failed, outOfRootEvicted, prunedRegistry, missingDirReclaimed, missingDirSkippedLive, liveProjectsSkipped: liveSkipped };
|
|
1709
1725
|
}
|
|
1710
1726
|
|
|
1711
1727
|
// ─── Exports ─────────────────────────────────────────────────────────────────
|
package/engine/cli.js
CHANGED
|
@@ -868,16 +868,10 @@ const commands = {
|
|
|
868
868
|
// (never --force/reset/clean/stash) against the operator tree.
|
|
869
869
|
if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
|
|
870
870
|
try {
|
|
871
|
-
require('./live-checkout').
|
|
872
|
-
|
|
873
|
-
branchName: item.meta.branch,
|
|
874
|
-
originalRef: item.originalRef,
|
|
875
|
-
originalRefType: item.originalRefType || 'branch',
|
|
876
|
-
dispatchId: item.id,
|
|
877
|
-
projectName: item.meta.project.name,
|
|
871
|
+
require('./live-checkout').maybeRestoreLiveCheckoutFromRecord({
|
|
872
|
+
item,
|
|
878
873
|
isTerminalFailure: !isSuccess,
|
|
879
874
|
resultLabel: result,
|
|
880
|
-
gitOpts: { env: shared.gitEnv(), windowsHide: true, timeout: 30000 },
|
|
881
875
|
log: (lvl, msg) => e.log(lvl, msg),
|
|
882
876
|
writeInboxAlert: dispatchModule().writeInboxAlert,
|
|
883
877
|
}).catch(() => {});
|
|
@@ -110,7 +110,10 @@ async function prepareCreatePrWorktree({
|
|
|
110
110
|
_copyRecursive(fsm, src, dst);
|
|
111
111
|
}
|
|
112
112
|
} catch (e) {
|
|
113
|
-
|
|
113
|
+
// Tear the worktree down via shared.removeWorktree (EPERM/EBUSY retry +
|
|
114
|
+
// escalation + real-repo refusal — CLAUDE.md footgun #6). The live checkout
|
|
115
|
+
// has NOT been touched yet at this point, so the operator keeps their work.
|
|
116
|
+
try { shared.removeWorktree(wtPath, projectRoot, worktreesBase); } catch { /* leak rather than throw a second error */ }
|
|
114
117
|
throw new Error(
|
|
115
118
|
`prepareCreatePrWorktree: failed to stage changes into the worktree — ${e.message}. ` +
|
|
116
119
|
'The live checkout was left untouched.',
|
|
@@ -118,20 +121,54 @@ async function prepareCreatePrWorktree({
|
|
|
118
121
|
}
|
|
119
122
|
|
|
120
123
|
// 4. The worktree now holds the changes — restore the live checkout clean.
|
|
124
|
+
// The captured edits are SAFE in the worktree (step 3 verified), so the
|
|
125
|
+
// documented job here is to return the live checkout to a clean HEAD.
|
|
126
|
+
// Use `git reset --hard HEAD` (NOT `git checkout -- .`): the old
|
|
127
|
+
// working-tree-only revert left the STAGED INDEX intact, so any change CC
|
|
128
|
+
// had `git add`-ed stayed staged → the live checkout was reported "restored"
|
|
129
|
+
// while still dirty, wedging the next Create-PR / live dispatch (the
|
|
130
|
+
// operator's "no recovery from dirty checkout"). `reset --hard` reverts
|
|
131
|
+
// staged + unstaged tracked changes in one shot; untracked entries are
|
|
132
|
+
// removed surgically below (only the ones we captured — never ignored or
|
|
133
|
+
// pre-existing untracked files, which `git clean` would wrongly nuke).
|
|
134
|
+
// This `reset --hard` is intentional and scoped to THIS Create-PR-worktree
|
|
135
|
+
// staging flow whose explicit contract is to restore the live checkout
|
|
136
|
+
// clean — it is NOT the live-checkout DISPATCH mode (which never resets).
|
|
137
|
+
const residualUntracked = [];
|
|
121
138
|
try {
|
|
122
|
-
await git(['-C', localPath, '
|
|
139
|
+
await git(['-C', localPath, 'reset', '--hard', 'HEAD']);
|
|
123
140
|
for (const rel of untracked) {
|
|
124
|
-
|
|
141
|
+
const target = path.join(localPath, rel.replace(/\/$/, ''));
|
|
142
|
+
try {
|
|
143
|
+
// Retry transient Windows file locks (EBUSY/EPERM/ENOTEMPTY) before
|
|
144
|
+
// giving up — a single failed unlink is what left the tree dirty. A few
|
|
145
|
+
// quick retries ride out a transient AV/indexer lock without a long stall.
|
|
146
|
+
shared._retryFsOp(() => fsm.rmSync(target, { recursive: true, force: true }), `cc-create-pr rm ${rel}`, { attempts: 4, baseMs: 100 });
|
|
147
|
+
} catch (rmErr) {
|
|
148
|
+
residualUntracked.push(rel);
|
|
149
|
+
log('warn', `[cc-create-pr] could not remove untracked ${rel} from live checkout after retries: ${rmErr.message}`);
|
|
150
|
+
}
|
|
125
151
|
}
|
|
126
152
|
} catch (e) {
|
|
127
|
-
|
|
153
|
+
// shared.removeWorktree carries the EPERM/EBUSY retry + escalation +
|
|
154
|
+
// real-repo refusal (CLAUDE.md footgun #6 — don't hand-roll force-remove).
|
|
155
|
+
try { shared.removeWorktree(wtPath, projectRoot, worktreesBase); } catch { /* leak rather than double-throw */ }
|
|
128
156
|
throw new Error(
|
|
129
157
|
`prepareCreatePrWorktree: failed to restore live checkout — ${e.message}. ` +
|
|
130
158
|
`Worktree at ${wtPath} may need manual cleanup.`,
|
|
131
159
|
);
|
|
132
160
|
}
|
|
133
161
|
|
|
134
|
-
|
|
162
|
+
// If untracked removal could not fully clean the live tree, surface it in the
|
|
163
|
+
// return (liveTreeDirty) instead of silently reporting success — the caller
|
|
164
|
+
// can warn CC / the operator rather than letting the residue wedge the next
|
|
165
|
+
// dispatch with a phantom "dirty" refusal.
|
|
166
|
+
const liveTreeDirty = residualUntracked.length > 0;
|
|
167
|
+
if (liveTreeDirty) {
|
|
168
|
+
log('warn', `[cc-create-pr] live checkout ${localPath} left with ${residualUntracked.length} residual untracked path(s) after staging; surfacing liveTreeDirty`);
|
|
169
|
+
} else {
|
|
170
|
+
log('info', `[cc-create-pr] staged ${project.name} changes into isolated worktree ${wtPath} on branch ${branchName} (live checkout restored)`);
|
|
171
|
+
}
|
|
135
172
|
return {
|
|
136
173
|
ok: true,
|
|
137
174
|
worktreePath: wtPath,
|
|
@@ -140,6 +177,8 @@ async function prepareCreatePrWorktree({
|
|
|
140
177
|
baseSha,
|
|
141
178
|
trackedChanged: hasTracked,
|
|
142
179
|
untrackedCount: untracked.length,
|
|
180
|
+
liveTreeDirty,
|
|
181
|
+
residualUntracked,
|
|
143
182
|
};
|
|
144
183
|
}
|
|
145
184
|
|
package/engine/dispatch.js
CHANGED
|
@@ -534,6 +534,7 @@ function isRetryableFailureReason(reason = '', failureClass = '') {
|
|
|
534
534
|
FAILURE_CLASS.INJECTION_FLAGGED, // F5 (W-mpeklod3000we69c) — agent spotted a prompt-injection attempt in spliced untrusted content; a human must review the source before re-dispatch
|
|
535
535
|
FAILURE_CLASS.LIVE_CHECKOUT_DIRTY, // P-a3f9b204 — live-checkout refused to spawn because operator localPath is dirty; mechanical retry won't fix it (operator must commit/stash/discard)
|
|
536
536
|
FAILURE_CLASS.LIVE_CHECKOUT_MID_OPERATION, // P-a7f3c1d9 — live-checkout refused to spawn because the operator tree is mid-operation (in-progress merge/rebase/cherry-pick/bisect or detached HEAD); mechanical retry won't fix it (operator must finish/abort the op or checkout a branch)
|
|
537
|
+
FAILURE_CLASS.LIVE_CHECKOUT_BLOB_FETCH, // PL-live-checkout-reliability-hardening — live-checkout `git checkout <existing-branch>` failed hydrating the tree through the auth-less GVFS cache server (blobless partial clone, headless); deterministic, so mechanical retry just reproduces it (operator must hydrate the branch with their own creds, then re-dispatch)
|
|
537
538
|
FAILURE_CLASS.OUTPUT_TRUNCATED, // P-8e4c2a17 — agent stdout exceeded the hard capture cap before the terminal result event; mechanical retry just reproduces the overflow (agent must reduce output volume or the task must be split)
|
|
538
539
|
]);
|
|
539
540
|
if (neverRetry.has(failureClass)) return false;
|
package/engine/github.js
CHANGED
|
@@ -1031,13 +1031,13 @@ async function pollPrStatus(config) {
|
|
|
1031
1031
|
let buildFailureSignature = '';
|
|
1032
1032
|
|
|
1033
1033
|
if (runs.length > 0) {
|
|
1034
|
-
const hasFailed = runs.some(r => r.conclusion === 'failure' || r.conclusion === 'timed_out');
|
|
1034
|
+
const hasFailed = runs.some(r => r.conclusion === 'failure' || r.conclusion === 'timed_out' || r.conclusion === 'startup_failure');
|
|
1035
1035
|
const allDone = runs.every(r => r.status === 'completed');
|
|
1036
1036
|
const allPassed = runs.every(r => r.conclusion === 'success' || r.conclusion === 'skipped' || r.conclusion === 'neutral');
|
|
1037
1037
|
|
|
1038
1038
|
if (hasFailed) {
|
|
1039
1039
|
buildStatus = BUILD_STATUS.FAILING;
|
|
1040
|
-
const failed = runs.find(r => r.conclusion === 'failure' || r.conclusion === 'timed_out');
|
|
1040
|
+
const failed = runs.find(r => r.conclusion === 'failure' || r.conclusion === 'timed_out' || r.conclusion === 'startup_failure');
|
|
1041
1041
|
buildFailReason = failed?.name || 'Check failed';
|
|
1042
1042
|
buildFailureSignature = shared.safeSlugComponent([
|
|
1043
1043
|
failed?.name,
|
|
@@ -1735,7 +1735,7 @@ async function checkLiveBuildAndConflict(pr, project) {
|
|
|
1735
1735
|
if (runs.length === 0) {
|
|
1736
1736
|
buildStatus = BUILD_STATUS.NONE;
|
|
1737
1737
|
} else {
|
|
1738
|
-
const hasFailed = runs.some(r => r.conclusion === 'failure' || r.conclusion === 'timed_out');
|
|
1738
|
+
const hasFailed = runs.some(r => r.conclusion === 'failure' || r.conclusion === 'timed_out' || r.conclusion === 'startup_failure');
|
|
1739
1739
|
const allDone = runs.every(r => r.status === 'completed');
|
|
1740
1740
|
const allPassed = runs.every(r => r.conclusion === 'success' || r.conclusion === 'skipped' || r.conclusion === 'neutral');
|
|
1741
1741
|
if (hasFailed) buildStatus = BUILD_STATUS.FAILING;
|
package/engine/live-checkout.js
CHANGED
|
@@ -86,6 +86,37 @@ const fs = require('fs');
|
|
|
86
86
|
const path = require('path');
|
|
87
87
|
const shared = require('./shared');
|
|
88
88
|
|
|
89
|
+
// PL-live-checkout-reliability-hardening — partial-clone / GVFS blob-fetch
|
|
90
|
+
// signature match. On a Scalar/GVFS-managed ADO repo (blobless partial clone),
|
|
91
|
+
// switching the working tree onto a branch whose tree differs from HEAD must
|
|
92
|
+
// materialize the changed paths' blobs on demand. That hydration goes through
|
|
93
|
+
// the GVFS cache server (`*.gvfscache.dev.azure.com`) via gvfs-helper /
|
|
94
|
+
// credential.helper — a DIFFERENT endpoint than the main git HTTP remote, and
|
|
95
|
+
// one that receives NO auth in the headless engine shell (GIT_TERMINAL_PROMPT=0,
|
|
96
|
+
// credential.helper disabled). So the checkout fails DETERMINISTICALLY headless.
|
|
97
|
+
// Issue #226 only removed the new-branch-from-`origin/<mainRef>` fetch; the
|
|
98
|
+
// EXISTING-branch `git checkout <branch>` was never carved out. We can't make
|
|
99
|
+
// the auth-less fetch succeed here, but we CAN stop treating its deterministic
|
|
100
|
+
// failure as a transient retryable error (which retry-storms to the cap), and
|
|
101
|
+
// instead surface it once as an operator-actionable refusal. This matcher is
|
|
102
|
+
// deliberately conservative — it keys on partial-clone / object-hydration
|
|
103
|
+
// phrasing, not on generic git failures.
|
|
104
|
+
function _isPartialCloneBlobError(message = '') {
|
|
105
|
+
const m = String(message || '').toLowerCase();
|
|
106
|
+
return (
|
|
107
|
+
m.includes('gvfs') ||
|
|
108
|
+
m.includes('partial clone') ||
|
|
109
|
+
m.includes('missing object') ||
|
|
110
|
+
m.includes('missing blob') ||
|
|
111
|
+
m.includes('fetch-pack') ||
|
|
112
|
+
m.includes('could not read from remote') ||
|
|
113
|
+
/unable to read (tree|sha1|object|blob)/.test(m) ||
|
|
114
|
+
/could not read[^\n]*(object|blob)/.test(m) ||
|
|
115
|
+
/failed to (fetch|download)[^\n]*(object|blob|pack)/.test(m) ||
|
|
116
|
+
(m.includes('remote') && m.includes('access denied'))
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
|
|
89
120
|
async function prepareLiveCheckout(opts = {}) {
|
|
90
121
|
const {
|
|
91
122
|
localPath,
|
|
@@ -149,28 +180,37 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
149
180
|
// ── Step 2: mid-operation / detached-HEAD preflight (P-b2e8d4a6). ──────
|
|
150
181
|
// Runs AFTER the dirty bail and BEFORE branch resolution — never mutates.
|
|
151
182
|
// Resolve the git dir via rev-parse (NOT localPath/.git) so submodule /
|
|
152
|
-
// gitdir-file / repo-managed trees resolve correctly.
|
|
153
|
-
//
|
|
154
|
-
|
|
183
|
+
// gitdir-file / repo-managed trees resolve correctly. A FAILURE here is a
|
|
184
|
+
// real git-health problem (and on the very submodule / gitdir-file /
|
|
185
|
+
// repo-managed trees live mode targets, <localPath>/.git is NOT a directory,
|
|
186
|
+
// so the old fabricated-path fallback made every sentinel probe miss and
|
|
187
|
+
// silently skipped mid-operation detection). Throw so the caller classifies
|
|
188
|
+
// it as a retryable LIVE_CHECKOUT_FAILED instead of barging into a branch
|
|
189
|
+
// switch on top of a half-finished operation we couldn't see.
|
|
190
|
+
let gitDir;
|
|
155
191
|
try {
|
|
156
192
|
const gitDirRaw = await git(['rev-parse', '--git-dir'], baseOpts);
|
|
157
193
|
const gitDirStr = (typeof gitDirRaw === 'string' ? gitDirRaw : '').trim();
|
|
158
|
-
if (gitDirStr) {
|
|
159
|
-
|
|
194
|
+
if (!gitDirStr) {
|
|
195
|
+
throw new Error('git rev-parse --git-dir returned empty output');
|
|
160
196
|
}
|
|
161
|
-
|
|
162
|
-
|
|
197
|
+
gitDir = path.isAbsolute(gitDirStr) ? gitDirStr : path.join(localPath, gitDirStr);
|
|
198
|
+
} catch (e) {
|
|
199
|
+
throw new Error('prepareLiveCheckout: could not resolve git dir for ' + localPath + ' — ' + (e && e.message));
|
|
163
200
|
}
|
|
164
201
|
|
|
165
202
|
// Probe sentinel paths under the resolved git dir. First hit wins; map each
|
|
166
203
|
// to its in-progress operation. rebase-merge/ (interactive/merge rebase) and
|
|
167
|
-
// rebase-apply/ (am-based rebase) both mean a rebase is underway.
|
|
204
|
+
// rebase-apply/ (am-based rebase) both mean a rebase is underway. BISECT_LOG
|
|
205
|
+
// covers an in-progress `git bisect` (bisect state is HEAD-keyed, so a branch
|
|
206
|
+
// switch would silently destroy it — the doc + header imply bisect coverage).
|
|
168
207
|
const MID_OP_SENTINELS = [
|
|
169
208
|
{ name: 'MERGE_HEAD', op: 'merge' },
|
|
170
209
|
{ name: 'rebase-merge', op: 'rebase' },
|
|
171
210
|
{ name: 'rebase-apply', op: 'rebase' },
|
|
172
211
|
{ name: 'CHERRY_PICK_HEAD', op: 'cherry-pick' },
|
|
173
212
|
{ name: 'REVERT_HEAD', op: 'revert' },
|
|
213
|
+
{ name: 'BISECT_LOG', op: 'bisect' },
|
|
174
214
|
];
|
|
175
215
|
for (const sentinel of MID_OP_SENTINELS) {
|
|
176
216
|
const sentinelPath = path.join(gitDir, sentinel.name);
|
|
@@ -179,14 +219,22 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
179
219
|
}
|
|
180
220
|
}
|
|
181
221
|
|
|
182
|
-
// Detached HEAD: `git symbolic-ref -q HEAD` exits
|
|
183
|
-
//
|
|
184
|
-
//
|
|
222
|
+
// Detached HEAD: `git symbolic-ref -q HEAD` exits 1 when HEAD does not point
|
|
223
|
+
// at a branch. Branching off a detached HEAD would strand the operator's
|
|
224
|
+
// anonymous commits, so refuse. But distinguish a REAL detachment (git ran
|
|
225
|
+
// and returned exit code 1) from a TRANSIENT failure (spawn error / timeout /
|
|
226
|
+
// ENOENT) — the latter is not proof of detachment and must NOT permanently
|
|
227
|
+
// refuse an on-a-branch tree; rethrow so it becomes a retryable
|
|
228
|
+
// LIVE_CHECKOUT_FAILED instead.
|
|
185
229
|
let detached = false;
|
|
186
230
|
try {
|
|
187
231
|
await git(['symbolic-ref', '-q', 'HEAD'], baseOpts);
|
|
188
|
-
} catch {
|
|
189
|
-
|
|
232
|
+
} catch (symErr) {
|
|
233
|
+
if (symErr && symErr.code === 1) {
|
|
234
|
+
detached = true; // git ran, HEAD is not a symbolic ref → genuinely detached
|
|
235
|
+
} else {
|
|
236
|
+
throw symErr; // spawn failure / timeout / other → transient, not detachment
|
|
237
|
+
}
|
|
190
238
|
}
|
|
191
239
|
if (detached) {
|
|
192
240
|
let sha = '';
|
|
@@ -223,17 +271,78 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
223
271
|
// NO `git fetch` (issue #226): the operator's existing HEAD is the
|
|
224
272
|
// baseline in live mode, and a fetch against a Scalar/GVFS partial clone
|
|
225
273
|
// pulls blobs through an auth-less cache server in headless mode (fails).
|
|
274
|
+
|
|
275
|
+
// (4a) Already-on-target-branch fast path. On a re-dispatch the tree is
|
|
276
|
+
// usually ALREADY on the agent branch, and a redundant `git checkout <branch>`
|
|
277
|
+
// would still force on-demand blob hydration on a partial clone (the GVFS
|
|
278
|
+
// failure mode below). If HEAD already points at the target branch there is
|
|
279
|
+
// nothing to switch — return success without touching the working tree. This
|
|
280
|
+
// mirrors the no-op logic the restore helper already has.
|
|
281
|
+
let curBranch = '';
|
|
282
|
+
try {
|
|
283
|
+
const curRaw = await git(['rev-parse', '--abbrev-ref', 'HEAD'], baseOpts);
|
|
284
|
+
curBranch = (typeof curRaw === 'string' ? curRaw : '').trim();
|
|
285
|
+
} catch { /* best-effort; fall through to full resolution */ }
|
|
286
|
+
if (curBranch && curBranch === branchName) {
|
|
287
|
+
return { ok: true, branch: branchName, created: false, alreadyOnBranch: true, originalRef, originalRefType };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// (4b) Branch existence — check `refs/heads/<branch>` SPECIFICALLY. A bare
|
|
291
|
+
// `rev-parse --verify <branch>` DWIM-resolves a same-named tag or
|
|
292
|
+
// remote-tracking ref, which would make the plain `git checkout <branch>`
|
|
293
|
+
// below land on a NON-branch ref → a silent detached HEAD on the operator
|
|
294
|
+
// tree (exactly what the Step-2 preflight refuses). Verifying the fully
|
|
295
|
+
// qualified branch ref avoids that ambiguity.
|
|
226
296
|
let branchExists = false;
|
|
227
297
|
try {
|
|
228
|
-
await git(['rev-parse', '--verify', branchName], baseOpts);
|
|
298
|
+
await git(['rev-parse', '--verify', '--quiet', `refs/heads/${branchName}`], baseOpts);
|
|
229
299
|
branchExists = true;
|
|
230
300
|
} catch {
|
|
231
301
|
branchExists = false;
|
|
232
302
|
}
|
|
233
303
|
|
|
304
|
+
// (4c) Checkout with NO half-switch + partial-clone-aware failure surfacing.
|
|
305
|
+
// A `git checkout` can move HEAD and THEN fail materializing the tree (the
|
|
306
|
+
// GVFS auth-less blob-fetch case). If that happens on the EXISTING-branch
|
|
307
|
+
// path we best-effort switch HEAD back to the captured originalRef (PLAIN
|
|
308
|
+
// checkout, never --force — the tree was verified clean at Step 1, so there
|
|
309
|
+
// is nothing of the operator's to overwrite) so the operator tree is never
|
|
310
|
+
// stranded half-populated on the agent branch. A partial-clone/blob-fetch
|
|
311
|
+
// failure is returned as a typed { ok:false, reason:'blob-fetch' } result so
|
|
312
|
+
// the caller can refuse NON-retryably with operator-actionable guidance
|
|
313
|
+
// instead of retry-storming an identical structural failure; any OTHER error
|
|
314
|
+
// is rethrown (→ retryable LIVE_CHECKOUT_FAILED) since it may be transient.
|
|
315
|
+
const runCheckout = async (args, { creating }) => {
|
|
316
|
+
try {
|
|
317
|
+
await git(args, baseOpts);
|
|
318
|
+
return null; // success
|
|
319
|
+
} catch (checkoutErr) {
|
|
320
|
+
const msg = String((checkoutErr && checkoutErr.message) || '');
|
|
321
|
+
if (!creating && originalRef) {
|
|
322
|
+
// Undo any partial switch on the existing-branch path. (Creating a new
|
|
323
|
+
// branch off HEAD doesn't change the tree, so there's nothing to undo.)
|
|
324
|
+
try { await git(['checkout', originalRef], baseOpts); }
|
|
325
|
+
catch { /* best-effort — classification below still applies */ }
|
|
326
|
+
}
|
|
327
|
+
if (_isPartialCloneBlobError(msg)) {
|
|
328
|
+
return {
|
|
329
|
+
ok: false,
|
|
330
|
+
reason: 'blob-fetch',
|
|
331
|
+
op: creating ? 'create' : 'checkout',
|
|
332
|
+
branch: branchName,
|
|
333
|
+
message: msg.slice(0, 500),
|
|
334
|
+
originalRef,
|
|
335
|
+
originalRefType,
|
|
336
|
+
};
|
|
337
|
+
}
|
|
338
|
+
throw checkoutErr; // transient → caller retries (LIVE_CHECKOUT_FAILED)
|
|
339
|
+
}
|
|
340
|
+
};
|
|
341
|
+
|
|
234
342
|
if (branchExists) {
|
|
235
343
|
// Plain checkout — NO --force, NO -B, NO pull. Operator owns local commits.
|
|
236
|
-
await
|
|
344
|
+
const failed = await runCheckout(['checkout', branchName], { creating: false });
|
|
345
|
+
if (failed) return failed;
|
|
237
346
|
return { ok: true, branch: branchName, created: false, originalRef, originalRefType };
|
|
238
347
|
}
|
|
239
348
|
|
|
@@ -241,7 +350,8 @@ async function prepareLiveCheckout(opts = {}) {
|
|
|
241
350
|
// origin/<mainRef>: in live mode the operator's HEAD is the baseline, and
|
|
242
351
|
// seeding from a remote ref forces auth-less GVFS blob fetches that fail
|
|
243
352
|
// on partial clones in headless mode.
|
|
244
|
-
await
|
|
353
|
+
const failedCreate = await runCheckout(['checkout', '-b', branchName], { creating: true });
|
|
354
|
+
if (failedCreate) return failedCreate;
|
|
245
355
|
return { ok: true, branch: branchName, created: true, originalRef, originalRefType };
|
|
246
356
|
}
|
|
247
357
|
|
|
@@ -375,6 +485,52 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
|
|
|
375
485
|
return result;
|
|
376
486
|
}
|
|
377
487
|
|
|
488
|
+
// ── Self-healing dirty recovery (PL-live-checkout-reliability-hardening).
|
|
489
|
+
// If the tree is dirty AND HEAD is on the AGENT branch (branchName) AND that
|
|
490
|
+
// branch differs from originalRef, the dirt is provably AGENT-authored: the
|
|
491
|
+
// engine CREATED this branch and prepareLiveCheckout VERIFIED the tree clean
|
|
492
|
+
// before switching to it, so anything uncommitted here was produced by the
|
|
493
|
+
// agent (a crash mid-edit, an incomplete commit, untracked output). Commit
|
|
494
|
+
// it onto the AGENT branch so the plain `git checkout <originalRef>` below
|
|
495
|
+
// succeeds and the NEXT dispatch isn't wedged forever by LIVE_CHECKOUT_DIRTY
|
|
496
|
+
// (the operator's "no recovery from dirty checkout"). This NEVER touches the
|
|
497
|
+
// operator's branch, NEVER discards work (`git add -A` + commit is additive
|
|
498
|
+
// and fully recoverable — the WIP lands as a visible commit on the agent
|
|
499
|
+
// branch / its PR), NEVER resets/cleans/stashes, and is gated hard so it can
|
|
500
|
+
// only ever mutate the engine-created branch. Gitignored artifacts are not
|
|
501
|
+
// staged by `git add -A`, so a tree dirty only with ignored build output
|
|
502
|
+
// never reaches here (git status is empty → not dirty). Best-effort: a
|
|
503
|
+
// failed commit falls through to the plain checkout, which refuses and
|
|
504
|
+
// writes the existing manual-recovery alert — no worse than before.
|
|
505
|
+
if (branchName && branchName !== originalRef) {
|
|
506
|
+
let dirty = false;
|
|
507
|
+
try {
|
|
508
|
+
const statusRaw = await git(['status', '--porcelain'], baseOpts);
|
|
509
|
+
dirty = !!String(typeof statusRaw === 'string' ? statusRaw : '').trim();
|
|
510
|
+
} catch { /* best-effort — if we can't read status, skip the self-heal */ }
|
|
511
|
+
if (dirty) {
|
|
512
|
+
let onAgentBranch = false;
|
|
513
|
+
try {
|
|
514
|
+
const curRaw = await git(['rev-parse', '--abbrev-ref', 'HEAD'], baseOpts);
|
|
515
|
+
const cur = (typeof curRaw === 'string' ? curRaw : '').trim();
|
|
516
|
+
onAgentBranch = cur && cur === branchName;
|
|
517
|
+
} catch { /* best-effort */ }
|
|
518
|
+
if (onAgentBranch) {
|
|
519
|
+
try {
|
|
520
|
+
await git(['add', '-A'], baseOpts);
|
|
521
|
+
// --no-verify: skip operator commit hooks that may fail headless.
|
|
522
|
+
await git(['commit', '--no-verify', '-m', `minions: auto-save agent WIP (dispatch ${did})`], baseOpts);
|
|
523
|
+
result.autoCommitted = true;
|
|
524
|
+
logFn('info', `live-checkout: auto-committed agent WIP onto ${branchName} before restore (dispatch ${did})`);
|
|
525
|
+
} catch (commitErr) {
|
|
526
|
+
// Leave the tree as-is; the plain checkout below will refuse and the
|
|
527
|
+
// fallback alert fires. Never force, never discard.
|
|
528
|
+
logFn('warn', `live-checkout: could not auto-save agent WIP onto ${branchName} for ${did}: ${commitErr && commitErr.message}`);
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
378
534
|
// PLAIN checkout — NO --force. git refuses if uncommitted changes would be
|
|
379
535
|
// overwritten; that refusal is honored rather than clobbering the tree.
|
|
380
536
|
try {
|
|
@@ -413,10 +569,100 @@ async function restoreLiveCheckoutAtDispatchEnd(opts = {}) {
|
|
|
413
569
|
}
|
|
414
570
|
} catch (restoreErr) {
|
|
415
571
|
// Strictly best-effort — swallow so restore never alters the dispatch result.
|
|
572
|
+
// But unlike the inner `checkout-refused` path (which alerts), a failure
|
|
573
|
+
// HERE (git binary missing, repo corruption, a GVFS blob-fetch on the
|
|
574
|
+
// switch back, an unexpected throw) would otherwise leave the operator
|
|
575
|
+
// stranded on the agent branch with ZERO notification. Write the same
|
|
576
|
+
// deduped manual-recovery alert so the operator is never silently stranded.
|
|
416
577
|
result.reason = 'error';
|
|
417
578
|
logFn('warn', `live-checkout: restore hiccup for ${did}: ${restoreErr && restoreErr.message}`);
|
|
579
|
+
try {
|
|
580
|
+
const body = [
|
|
581
|
+
`Could not automatically switch ${proj} back to "${originalRef}" after a live-checkout`,
|
|
582
|
+
`dispatch — the restore hit an unexpected error (not a normal uncommitted-changes`,
|
|
583
|
+
`refusal). On a Scalar/GVFS partial clone this is often an auth-less blob fetch on`,
|
|
584
|
+
`the branch switch; your own (authenticated) checkout will succeed.`,
|
|
585
|
+
``,
|
|
586
|
+
`Dispatch: ${did}`,
|
|
587
|
+
`Current branch: ${branchName}`,
|
|
588
|
+
`Working tree: ${localPath}`,
|
|
589
|
+
`Error: ${restoreErr && restoreErr.message}`,
|
|
590
|
+
``,
|
|
591
|
+
`Your tree was left untouched (no force, no reset, no clean, no stash). Switch back`,
|
|
592
|
+
`manually when ready:`,
|
|
593
|
+
``,
|
|
594
|
+
` git -C "${localPath}" checkout ${originalRef}`,
|
|
595
|
+
].join('\n');
|
|
596
|
+
const wrote = alert(`live-checkout-branch-${did}`, body);
|
|
597
|
+
result.fallbackAlerted = (wrote !== false);
|
|
598
|
+
} catch (e) {
|
|
599
|
+
logFn('warn', `live-checkout: could not write restore-error alert for ${did}: ${e && e.message}`);
|
|
600
|
+
}
|
|
418
601
|
return result;
|
|
419
602
|
}
|
|
420
603
|
}
|
|
421
604
|
|
|
422
|
-
|
|
605
|
+
/**
|
|
606
|
+
* maybeRestoreLiveCheckoutFromRecord — PL-live-checkout-reliability-hardening
|
|
607
|
+
*
|
|
608
|
+
* Thin shared wrapper that fires restoreLiveCheckoutAtDispatchEnd from a
|
|
609
|
+
* persisted DISPATCH RECORD (not the in-memory spawn closure). A live-mode
|
|
610
|
+
* dispatch only ever persists `originalRef` (engine.js, P-c5a1f3b8), so its
|
|
611
|
+
* presence — together with `meta.branch` + `meta.project.localPath` — is the
|
|
612
|
+
* live-mode signal on every restart/reaping path where the slim persisted
|
|
613
|
+
* record is all that survives.
|
|
614
|
+
*
|
|
615
|
+
* This exists because the dispatch-end restore was originally wired in only TWO
|
|
616
|
+
* places: the in-memory `onAgentClose` closure (fresh spawns only) and the
|
|
617
|
+
* engine-restart orphan-COMPLETION path in cli.js (agent finished WHILE the
|
|
618
|
+
* engine was down). A live agent that SURVIVES a restart and exits LATER is
|
|
619
|
+
* re-attached with no `onAgentClose` closure and is reaped by `timeout.js`
|
|
620
|
+
* (completeFromOutput / orphan sweep) — which had NO restore wiring — so the
|
|
621
|
+
* operator's checkout was silently stranded on the agent branch. Routing all
|
|
622
|
+
* three reaping paths through this one helper closes that gap without forking a
|
|
623
|
+
* third inline copy.
|
|
624
|
+
*
|
|
625
|
+
* Best-effort and never throws: returns a result object, swallows the restore's
|
|
626
|
+
* own (already-swallowed) errors, and a no-op `{ skipped:true }` when the record
|
|
627
|
+
* isn't a live-mode dispatch. Callers should treat it as fire-and-forget.
|
|
628
|
+
*
|
|
629
|
+
* @param {object} opts
|
|
630
|
+
* @param {object} opts.item persisted dispatch record (has id, originalRef, meta.branch, meta.project.{localPath,name})
|
|
631
|
+
* @param {boolean} [opts.isTerminalFailure] true on error/timeout/crash
|
|
632
|
+
* @param {string} [opts.resultLabel] short status word for the alert body
|
|
633
|
+
* @param {function} [opts.log] (level, msg) => void
|
|
634
|
+
* @param {function} opts.writeInboxAlert (slug, body) => any
|
|
635
|
+
* @param {object} [opts.gitOpts] execFile opts; defaults to gitEnv() + 30s timeout
|
|
636
|
+
* @param {function} [opts._git] test seam forwarded to restoreLiveCheckoutAtDispatchEnd
|
|
637
|
+
* @returns {Promise<object>} restore result (or { skipped:true } for a non-live record)
|
|
638
|
+
*/
|
|
639
|
+
async function maybeRestoreLiveCheckoutFromRecord(opts = {}) {
|
|
640
|
+
const { item, isTerminalFailure, resultLabel, log, writeInboxAlert, gitOpts, _git } = opts;
|
|
641
|
+
if (!item || !item.originalRef || !item.meta?.branch || !item.meta?.project?.localPath) {
|
|
642
|
+
return { restored: false, reason: 'not-live-record', skipped: true };
|
|
643
|
+
}
|
|
644
|
+
try {
|
|
645
|
+
return await restoreLiveCheckoutAtDispatchEnd({
|
|
646
|
+
localPath: item.meta.project.localPath,
|
|
647
|
+
branchName: item.meta.branch,
|
|
648
|
+
originalRef: item.originalRef,
|
|
649
|
+
originalRefType: item.originalRefType || 'branch',
|
|
650
|
+
dispatchId: item.id,
|
|
651
|
+
projectName: item.meta.project.name,
|
|
652
|
+
isTerminalFailure: !!isTerminalFailure,
|
|
653
|
+
resultLabel,
|
|
654
|
+
gitOpts: gitOpts || { env: shared.gitEnv(), windowsHide: true, timeout: 30000 },
|
|
655
|
+
log,
|
|
656
|
+
writeInboxAlert,
|
|
657
|
+
_git,
|
|
658
|
+
});
|
|
659
|
+
} catch (err) {
|
|
660
|
+
// restoreLiveCheckoutAtDispatchEnd already swallows its own errors; this is
|
|
661
|
+
// belt-and-suspenders so a require()/argument hiccup can never break the
|
|
662
|
+
// reaping path that called us.
|
|
663
|
+
if (typeof log === 'function') log('warn', `live-checkout: maybeRestoreLiveCheckoutFromRecord threw for ${item.id}: ${err && err.message}`);
|
|
664
|
+
return { restored: false, reason: 'error', skipped: false };
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
|
|
668
|
+
module.exports = { prepareLiveCheckout, restoreLiveCheckoutAtDispatchEnd, maybeRestoreLiveCheckoutFromRecord, _isPartialCloneBlobError };
|
package/engine/shared.js
CHANGED
|
@@ -2669,6 +2669,11 @@ const ENGINE_DEFAULTS = {
|
|
|
2669
2669
|
agentTimeout: 18000000, // 5h
|
|
2670
2670
|
heartbeatTimeout: 300000, // 5min — stale-orphan grace after process tracking is lost
|
|
2671
2671
|
resumeHeartbeatTimeout: 300000, // 5min — max wait for a resumed runtime to emit its first output
|
|
2672
|
+
// M004 — minimum time a dispatch must have been alive before it can be declared
|
|
2673
|
+
// orphaned. Prevents false-positive orphan declarations for short-duration agents
|
|
2674
|
+
// (e.g. verify agents that complete naturally within ~18-19s before the engine's
|
|
2675
|
+
// process tracking catches up). Configurable via config.engine.minAliveBeforeOrphanMs.
|
|
2676
|
+
minAliveBeforeOrphanMs: 30000, // 30s
|
|
2672
2677
|
// Per-type stale-orphan overrides (merged with config.engine.heartbeatTimeouts at runtime — see timeout.js).
|
|
2673
2678
|
// Heavy work types (multi-file edits, builds, test suites, full verify cycles) routinely go quiet for
|
|
2674
2679
|
// longer than the 5-min default when the engine has lost their tracked handle (e.g. across an engine
|
|
@@ -4335,6 +4340,7 @@ const FAILURE_CLASS = {
|
|
|
4335
4340
|
LIVE_CHECKOUT_DIRTY: 'live-checkout-dirty', // P-a3f9b204 (live-checkout dispatch mode): spawnAgent ran prepareLiveCheckout against project.localPath and `git status --porcelain` reported uncommitted changes. Engine refused to spawn (it never runs `git reset`/`git clean` against the operator tree). Inbox alert lists dirty files; WI is stamped `_pendingReason: 'live_checkout_dirty'`. Non-retryable — operator must commit/stash/discard before re-dispatch. RESERVED for confirmed dirty status results only — a thrown helper/git exception is LIVE_CHECKOUT_FAILED, not this (#305).
|
|
4336
4341
|
LIVE_CHECKOUT_FAILED: 'live-checkout-failed', // #305 (live-checkout dispatch mode): prepareLiveCheckout THREW before agent spawn (helper guard, ref validation, or a transient `git status`/`rev-parse`/`checkout` failure) — distinct from the confirmed-dirty result (LIVE_CHECKOUT_DIRTY) which the helper returns, not throws. A thrown error is NOT proof the tree is dirty, so it must not be over-classified as dirty. Retryable with bounded backoff (NOT in dispatch.js neverRetry): racy branch-lock handoff, just-finished sibling dispatch, or transient git errors frequently clear on the next attempt; the engine auto-retries up to ENGINE_DEFAULTS.maxRetries before giving up. Genuinely terminal underlying reasons (auth, validation) still short-circuit via the reason-string check in isRetryableFailureReason.
|
|
4337
4342
|
LIVE_CHECKOUT_MID_OPERATION: 'live-checkout-mid-operation', // P-a7f3c1d9 (live-checkout dispatch mode): spawnAgent could not switch/create the target branch in project.localPath because the operator tree is mid-operation — an in-progress merge/rebase/cherry-pick/bisect or a detached HEAD. Distinct from LIVE_CHECKOUT_DIRTY (uncommitted changes): here the tree may be clean but the branch op cannot proceed. Engine refuses to spawn (it never runs `git reset`/`git clean`/`git rebase --abort` against the operator tree). Non-retryable — operator must finish or abort the in-progress operation, or checkout a branch, before re-dispatch.
|
|
4343
|
+
LIVE_CHECKOUT_BLOB_FETCH: 'live-checkout-blob-fetch', // PL-live-checkout-reliability-hardening (live-checkout dispatch mode): `git checkout <existing-branch>` in project.localPath failed because the tree could not be materialized — on a Scalar/GVFS-managed ADO partial (blobless) clone, switching onto a branch whose tree differs from HEAD hydrates the changed paths' blobs through the GVFS cache server (`*.gvfscache.dev.azure.com`), a different endpoint than the main git remote that receives NO auth in the headless engine shell, so the fetch fails DETERMINISTICALLY. Distinct from LIVE_CHECKOUT_FAILED (transient) because retrying reproduces it identically — surfaced once as an operator-actionable refusal instead of retry-storming to the cap. Non-retryable — the operator hydrates the branch once with their own credentials (`git checkout <branch>` interactively, or `scalar prefetch` / `git -C <repo> fetch`), then re-dispatches. The engine NEVER forces, resets, cleans, or stashes the operator tree, and best-effort switches HEAD back to the original ref so the tree is not stranded half-populated.
|
|
4338
4344
|
INVALID_WORKDIR: 'invalid-workdir', // P-714ef144: dispatch carried a meta.workdir override that failed validation — non-string, absolute path, drive-letter prefix, null byte, ".." segment, or post-resolve containment escape against project.localPath / worktree root. Engine refuses to spawn (the subpath would either be unreachable on disk or point outside the operator's allowed surface). Non-retryable — operator must fix the WI's meta.workdir before re-dispatch. Inbox alert lists the offending value + the resolved-vs-base mismatch.
|
|
4339
4345
|
MODEL_UNAVAILABLE: 'model-unavailable', // W-mpg6isvy000xca4d: requested model returned overloaded_error / 503 / service_unavailable. Retriable — engine swaps in the runtime-appropriate fallback model on next spawn (Claude leans on --fallback-model already plumbed; Copilot overrides --model with engine.copilotFallbackModel).
|
|
4340
4346
|
WORKSPACE_MANIFEST_REPO: 'workspace-manifest-repo-forbidden', // W-mq07avbk000m5543: dispatch routed an agent to a project/repo not present in its workspace_manifest.allowed_repos. Structural — never retryable until the manifest is widened or a different agent is chosen.
|
package/engine/timeout.js
CHANGED
|
@@ -607,6 +607,7 @@ function checkTimeouts(config) {
|
|
|
607
607
|
const timeout = config.engine?.agentTimeout || ENGINE_DEFAULTS.agentTimeout;
|
|
608
608
|
const defaultStaleOrphanTimeout = config.engine?.heartbeatTimeout || ENGINE_DEFAULTS.heartbeatTimeout;
|
|
609
609
|
const runtimeResumeHeartbeatTimeout = config.engine?.resumeHeartbeatTimeout || ENGINE_DEFAULTS.resumeHeartbeatTimeout || defaultStaleOrphanTimeout;
|
|
610
|
+
const minAliveBeforeOrphanMs = config.engine?.minAliveBeforeOrphanMs ?? ENGINE_DEFAULTS.minAliveBeforeOrphanMs;
|
|
610
611
|
|
|
611
612
|
// Optional per-type stale-orphan timeouts: merge ENGINE_DEFAULTS ← config overrides.
|
|
612
613
|
const perTypeStaleOrphanTimeouts = { ...ENGINE_DEFAULTS.heartbeatTimeouts, ...(config.engine?.heartbeatTimeouts || {}) };
|
|
@@ -659,11 +660,30 @@ function checkTimeouts(config) {
|
|
|
659
660
|
: null;
|
|
660
661
|
} catch (e) { log('warn', 'completion summary gate: ' + e.message); }
|
|
661
662
|
|
|
663
|
+
const _completedAsError = completionDetection || !isSuccess;
|
|
662
664
|
completeDispatch(item.id, completionDetection ? DISPATCH_RESULT.ERROR : (isSuccess ? DISPATCH_RESULT.SUCCESS : DISPATCH_RESULT.ERROR),
|
|
663
665
|
completionDetection ? completionDetection.reason : (isSuccess ? 'Completed (detected from output)' : `Exited with code ${processExitCode} (detected from output)`),
|
|
664
666
|
outputResultSummary,
|
|
665
667
|
completionDetection ? { processWorkItemFailure: false } : {});
|
|
666
668
|
|
|
669
|
+
// PL-live-checkout-reliability-hardening — restore the operator's checkout
|
|
670
|
+
// for a live-mode dispatch that finished AFTER an engine restart. This path
|
|
671
|
+
// (output-detected completion of a re-attached process) has no in-memory
|
|
672
|
+
// onAgentClose closure, so without this the tree is stranded on the agent
|
|
673
|
+
// branch. originalRef's presence is the live-mode signal; the helper no-ops
|
|
674
|
+
// for non-live records. Fire-and-forget + best-effort.
|
|
675
|
+
if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
|
|
676
|
+
try {
|
|
677
|
+
require('./live-checkout').maybeRestoreLiveCheckoutFromRecord({
|
|
678
|
+
item,
|
|
679
|
+
isTerminalFailure: _completedAsError,
|
|
680
|
+
resultLabel: _completedAsError ? 'error' : 'success',
|
|
681
|
+
log: (lvl, msg) => log(lvl, msg),
|
|
682
|
+
writeInboxAlert: dispatch().writeInboxAlert,
|
|
683
|
+
}).catch(() => {});
|
|
684
|
+
} catch (e) { log('warn', 'live-checkout restore (output completion): ' + e.message); }
|
|
685
|
+
}
|
|
686
|
+
|
|
667
687
|
// Run post-completion hooks via shared helper (async — fire and forget in timeout context).
|
|
668
688
|
// Pass the actual exit code so autoRecovery (PR-created-but-failed) still works correctly.
|
|
669
689
|
// detectPhantom: true mirrors the line 310 detectNonTerminalResultSummary call —
|
|
@@ -834,6 +854,17 @@ function checkTimeouts(config) {
|
|
|
834
854
|
const confirmedDeadAtRestart = engineRestartGraceExempt?.has(item.id);
|
|
835
855
|
const reattachedProcessEnded = !!procInfo?.reattached && !processAlive;
|
|
836
856
|
const canReapDeadProcess = confirmedDeadAtRestart || reattachedProcessEnded;
|
|
857
|
+
|
|
858
|
+
// M004 — skip orphan declaration if the dispatch has not been alive long enough.
|
|
859
|
+
// Prevents false-positive orphans for short-duration agents (e.g. verify agents
|
|
860
|
+
// that complete naturally within ~18-19s before process tracking catches up).
|
|
861
|
+
// canReapDeadProcess (confirmed-dead at restart) bypasses this guard — those
|
|
862
|
+
// PIDs were explicitly verified dead at engine startup.
|
|
863
|
+
if (!canReapDeadProcess) {
|
|
864
|
+
const aliveMs = Date.now() - (item.started_at ? new Date(item.started_at).getTime() : 0);
|
|
865
|
+
if (aliveMs < minAliveBeforeOrphanMs) continue;
|
|
866
|
+
}
|
|
867
|
+
|
|
837
868
|
if (!processAlive && (canReapDeadProcess || silentMs > staleOrphanTimeout) && (Date.now() > engineRestartGraceUntil || canReapDeadProcess)) {
|
|
838
869
|
// Last-resort PID check: lost tracked handle but OS process may still be alive.
|
|
839
870
|
if (isOsPidAliveForDispatch(item.id)) {
|
|
@@ -883,6 +914,21 @@ function checkTimeouts(config) {
|
|
|
883
914
|
// Clean up dead items
|
|
884
915
|
for (const { item, reason, failureClass } of deadItems) {
|
|
885
916
|
completeDispatch(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
|
|
917
|
+
// PL-live-checkout-reliability-hardening — a live-mode dispatch declared an
|
|
918
|
+
// orphan (lost process handle, e.g. across an engine restart) leaves the
|
|
919
|
+
// operator checkout on the agent branch. Restore it from the persisted
|
|
920
|
+
// record (originalRef present ⇒ live mode). Always a terminal failure here.
|
|
921
|
+
if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
|
|
922
|
+
try {
|
|
923
|
+
require('./live-checkout').maybeRestoreLiveCheckoutFromRecord({
|
|
924
|
+
item,
|
|
925
|
+
isTerminalFailure: true,
|
|
926
|
+
resultLabel: 'orphaned',
|
|
927
|
+
log: (lvl, msg) => log(lvl, msg),
|
|
928
|
+
writeInboxAlert: dispatch().writeInboxAlert,
|
|
929
|
+
}).catch(() => {});
|
|
930
|
+
} catch (e) { log('warn', 'live-checkout restore (orphan): ' + e.message); }
|
|
931
|
+
}
|
|
886
932
|
}
|
|
887
933
|
|
|
888
934
|
// Clear legacy blocking-tool annotations; process liveness no longer depends on tool parsing.
|
package/engine.js
CHANGED
|
@@ -2382,13 +2382,22 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2382
2382
|
if (_liveResult && _liveResult.ok === false && _liveResult.reason === 'dirty') {
|
|
2383
2383
|
const _dirtyFiles = Array.isArray(_liveResult.dirtyFiles) ? _liveResult.dirtyFiles : [];
|
|
2384
2384
|
const _branchInfo = typeof _liveResult.branchInfo === 'string' ? _liveResult.branchInfo : '';
|
|
2385
|
-
// #329
|
|
2386
|
-
//
|
|
2387
|
-
//
|
|
2388
|
-
//
|
|
2389
|
-
//
|
|
2390
|
-
//
|
|
2391
|
-
|
|
2385
|
+
// #329 + PL-live-checkout-reliability-hardening: two-strike dirty guard.
|
|
2386
|
+
// First dirty failure is retried once (a transient or engine-owned dirty
|
|
2387
|
+
// state — e.g. an agent's own leftover WIP, which dispatch-end restore now
|
|
2388
|
+
// auto-commits onto the agent branch — can clear before the next attempt);
|
|
2389
|
+
// a SECOND consecutive dirty failure is persistent (operator-owned) and
|
|
2390
|
+
// fails non-retryably. The strike memory lives on a DEDICATED counter
|
|
2391
|
+
// `_liveCheckoutDirtyAttempts`, NOT on `_pendingReason`: `_pendingReason`
|
|
2392
|
+
// is scrubbed both by the dispatch.js retry re-queue (partially fixed by
|
|
2393
|
+
// #434) AND unconditionally by discovery (`delete item._pendingReason`
|
|
2394
|
+
// before building the dispatch), so reading the strike count off it was
|
|
2395
|
+
// always falsy on the retry → the guard never fired and dirty trees
|
|
2396
|
+
// burned all maxRetries. The dedicated counter is touched ONLY here (on a
|
|
2397
|
+
// dirty failure) and cleared on a successful prepareLiveCheckout below;
|
|
2398
|
+
// neither discovery nor the retry re-queue clears it.
|
|
2399
|
+
const _priorDirtyAttempts = Number(dispatchItem.meta?.item?._liveCheckoutDirtyAttempts) || 0;
|
|
2400
|
+
const _alreadyDirtyFailed = _priorDirtyAttempts >= 1;
|
|
2392
2401
|
const _alertBody = [
|
|
2393
2402
|
'# Live-checkout refused: dirty worktree',
|
|
2394
2403
|
'',
|
|
@@ -2419,7 +2428,11 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2419
2428
|
mutateJsonFileLocked(_wiPath, (data) => {
|
|
2420
2429
|
if (!Array.isArray(data)) return data;
|
|
2421
2430
|
const wi = data.find(i => i && i.id === dispatchItem.meta.item.id);
|
|
2422
|
-
if (wi)
|
|
2431
|
+
if (wi) {
|
|
2432
|
+
wi._pendingReason = 'live_checkout_dirty';
|
|
2433
|
+
// Dedicated two-strike counter (survives discovery + retry scrubs).
|
|
2434
|
+
wi._liveCheckoutDirtyAttempts = _priorDirtyAttempts + 1;
|
|
2435
|
+
}
|
|
2423
2436
|
return data;
|
|
2424
2437
|
});
|
|
2425
2438
|
}
|
|
@@ -2507,7 +2520,82 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2507
2520
|
cleanupTempAgent(agentId);
|
|
2508
2521
|
return null;
|
|
2509
2522
|
}
|
|
2523
|
+
// PL-live-checkout-reliability-hardening: partial-clone / GVFS blob-fetch
|
|
2524
|
+
// refusal. The existing-branch `git checkout` could not materialize the tree
|
|
2525
|
+
// because the operator's checkout is a Scalar/GVFS blobless partial clone and
|
|
2526
|
+
// the on-demand blob hydration goes through the auth-less GVFS cache server
|
|
2527
|
+
// (headless → denied). This is DETERMINISTIC (not transient), so it gets its
|
|
2528
|
+
// own non-retryable FAILURE_CLASS instead of LIVE_CHECKOUT_FAILED's bounded
|
|
2529
|
+
// retry-storm. prepareLiveCheckout already best-effort switched HEAD back to
|
|
2530
|
+
// the operator's original ref, so the tree is not stranded half-populated.
|
|
2531
|
+
if (_liveResult && _liveResult.ok === false && _liveResult.reason === 'blob-fetch') {
|
|
2532
|
+
const _blobMsg = typeof _liveResult.message === 'string' ? _liveResult.message : '';
|
|
2533
|
+
const _alertBody = [
|
|
2534
|
+
'# Live-checkout blocked: could not hydrate branch (partial clone)',
|
|
2535
|
+
'',
|
|
2536
|
+
`**Project:** ${project.name || '(unknown)'}`,
|
|
2537
|
+
`**Local path:** ${cwd}`,
|
|
2538
|
+
`**Branch:** ${branchName}`,
|
|
2539
|
+
`**Work item:** ${_wiIdForAlert}`,
|
|
2540
|
+
`**Dispatch:** ${id}`,
|
|
2541
|
+
'',
|
|
2542
|
+
`The engine could not switch \`${cwd}\` onto \`${branchName}\` because the working tree could not be materialized. On a Scalar/GVFS-managed ADO repo (blobless partial clone), switching onto a branch whose tree differs from HEAD hydrates the changed files' blobs through the GVFS cache server, which receives no credentials in the headless engine — so the checkout fails. Your tree was left on its original ref (the engine never forces, resets, or cleans it).`,
|
|
2543
|
+
...(_blobMsg ? ['', '```', _blobMsg, '```'] : []),
|
|
2544
|
+
'',
|
|
2545
|
+
'## Recovery',
|
|
2546
|
+
'',
|
|
2547
|
+
`Hydrate the branch once with your own credentials, then re-dispatch:`,
|
|
2548
|
+
'',
|
|
2549
|
+
'```',
|
|
2550
|
+
`git -C "${cwd}" checkout ${branchName} # or: scalar prefetch / git -C "${cwd}" fetch origin ${branchName}`,
|
|
2551
|
+
`git -C "${cwd}" checkout - # switch back`,
|
|
2552
|
+
'```',
|
|
2553
|
+
'',
|
|
2554
|
+
'Alternatively, switch this project to `checkoutMode: "worktree"` (the default), which fetches against the authenticated git remote instead of relying on in-place GVFS hydration.',
|
|
2555
|
+
].join('\n');
|
|
2556
|
+
try { writeInboxAlert(`live-checkout-blocked-${_wiIdForAlert}`, _alertBody); }
|
|
2557
|
+
catch (e) { log('warn', `live-checkout: writeInboxAlert failed: ${e.message}`); }
|
|
2558
|
+
try {
|
|
2559
|
+
const _wiPath = resolveWorkItemPath(dispatchItem.meta);
|
|
2560
|
+
if (_wiPath && dispatchItem.meta?.item?.id) {
|
|
2561
|
+
mutateJsonFileLocked(_wiPath, (data) => {
|
|
2562
|
+
if (!Array.isArray(data)) return data;
|
|
2563
|
+
const wi = data.find(i => i && i.id === dispatchItem.meta.item.id);
|
|
2564
|
+
if (wi) wi._pendingReason = 'live_checkout_blob_fetch';
|
|
2565
|
+
return data;
|
|
2566
|
+
});
|
|
2567
|
+
}
|
|
2568
|
+
} catch (e) { log('warn', `live-checkout: failed to stamp _pendingReason: ${e.message}`); }
|
|
2569
|
+
const _shortMsg = `live-checkout blocked: could not hydrate ${branchName} in ${cwd} (partial-clone blob fetch)`;
|
|
2570
|
+
log('error', `spawnAgent: ${_shortMsg}`);
|
|
2571
|
+
_cleanupPromptFiles();
|
|
2572
|
+
completeDispatch(
|
|
2573
|
+
id,
|
|
2574
|
+
DISPATCH_RESULT.ERROR,
|
|
2575
|
+
_shortMsg.slice(0, 800),
|
|
2576
|
+
'Live-checkout could not materialize the branch tree on a blobless partial clone (auth-less GVFS cache fetch, headless). Deterministic — operator must hydrate the branch with their own credentials (or use worktree mode), then re-dispatch.',
|
|
2577
|
+
{ failureClass: FAILURE_CLASS.LIVE_CHECKOUT_BLOB_FETCH, agentRetryable: false },
|
|
2578
|
+
);
|
|
2579
|
+
cleanupTempAgent(agentId);
|
|
2580
|
+
return null;
|
|
2581
|
+
}
|
|
2510
2582
|
log('info', `live-checkout: ${_liveResult.created ? 'created' : 'switched to'} branch ${branchName} in ${cwd} (in-place; no worktree)`);
|
|
2583
|
+
// PL-live-checkout-reliability-hardening: a clean spawn clears the two-strike
|
|
2584
|
+
// dirty counter so a project that was transiently dirty once isn't treated as
|
|
2585
|
+
// second-strike on its next dirty encounter. Only write when the stamp is set.
|
|
2586
|
+
if (dispatchItem.meta?.item?._liveCheckoutDirtyAttempts) {
|
|
2587
|
+
try {
|
|
2588
|
+
const _wiPath = resolveWorkItemPath(dispatchItem.meta);
|
|
2589
|
+
if (_wiPath && dispatchItem.meta?.item?.id) {
|
|
2590
|
+
mutateJsonFileLocked(_wiPath, (data) => {
|
|
2591
|
+
if (!Array.isArray(data)) return data;
|
|
2592
|
+
const wi = data.find(i => i && i.id === dispatchItem.meta.item.id);
|
|
2593
|
+
if (wi) delete wi._liveCheckoutDirtyAttempts;
|
|
2594
|
+
return data;
|
|
2595
|
+
});
|
|
2596
|
+
}
|
|
2597
|
+
} catch (e) { log('warn', `live-checkout: failed to clear dirty-attempt counter for ${id}: ${e.message}`); }
|
|
2598
|
+
}
|
|
2511
2599
|
// P-c5a1f3b8: persist the operator's original ref on the dispatch record so
|
|
2512
2600
|
// the dispatch-end auto-restore (P-d9e6b2c4) can return the tree to where the
|
|
2513
2601
|
// operator started — even after an engine restart + re-attach, where the
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2273",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|