@yemi33/minions 0.1.2232 → 0.1.2234
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/minions.js +29 -2
- package/dashboard/js/render-other.js +37 -4
- package/dashboard/js/render-work-items.js +8 -22
- package/dashboard/js/settings.js +2 -0
- package/dashboard.js +6 -0
- package/docs/completion-reports.md +17 -0
- package/docs/worktree-lifecycle.md +9 -5
- package/engine/cleanup.js +30 -1
- package/engine/restart-health.js +54 -2
- package/engine/shared.js +51 -13
- package/engine/worktree-gc.js +313 -179
- package/engine.js +55 -37
- package/package.json +1 -1
package/bin/minions.js
CHANGED
|
@@ -373,6 +373,24 @@ function spawnSupervisor() {
|
|
|
373
373
|
// at function-definition time.
|
|
374
374
|
const _supervisorPidPath = () => path.join(MINIONS_HOME, 'engine', 'supervisor.pid');
|
|
375
375
|
|
|
376
|
+
/**
|
|
377
|
+
* Resolve the cold-start restart-health budget (ms) from config, falling back
|
|
378
|
+
* to ENGINE_DEFAULTS.restartHealthTimeoutMs. Mirrors the Settings clamp range
|
|
379
|
+
* [15000, 300000] so a config typo can't wedge `minions restart` for an hour
|
|
380
|
+
* or shrink the budget back below a real cold boot. W-mqpx0tpi.
|
|
381
|
+
*/
|
|
382
|
+
function _resolveRestartHealthTimeoutMs() {
|
|
383
|
+
const home = process.env.MINIONS_HOME || (typeof MINIONS_HOME !== 'undefined' ? MINIONS_HOME : null);
|
|
384
|
+
let cfg = null;
|
|
385
|
+
try {
|
|
386
|
+
if (home) cfg = JSON.parse(fs.readFileSync(path.join(home, 'config.json'), 'utf8'));
|
|
387
|
+
} catch { /* config may not exist yet */ }
|
|
388
|
+
const D = shared.ENGINE_DEFAULTS;
|
|
389
|
+
const raw = cfg && cfg.engine && Number(cfg.engine.restartHealthTimeoutMs);
|
|
390
|
+
const resolved = Number.isFinite(raw) && raw > 0 ? raw : D.restartHealthTimeoutMs;
|
|
391
|
+
return Math.min(300000, Math.max(15000, resolved));
|
|
392
|
+
}
|
|
393
|
+
|
|
376
394
|
/**
|
|
377
395
|
* Spawn engine + dashboard + supervisor, verify health, optionally open browser.
|
|
378
396
|
* Shared by `minions start` and `minions restart` — restart layers a kill phase
|
|
@@ -412,6 +430,7 @@ function spawnFullStackAndVerify({ rest, forceOpen, dashWasUp, restartStartMs })
|
|
|
412
430
|
minionsHome: MINIONS_HOME,
|
|
413
431
|
dashboardPid: dashProc.pid,
|
|
414
432
|
dashboardPort: actualPort,
|
|
433
|
+
timeoutMs: _resolveRestartHealthTimeoutMs(),
|
|
415
434
|
});
|
|
416
435
|
if (!result.ok) {
|
|
417
436
|
console.error(formatRestartHealthError(result));
|
|
@@ -720,7 +739,13 @@ const DASHBOARD_PORT = resolveDashboardPort(rest).port;
|
|
|
720
739
|
// spawning dashboard.js.
|
|
721
740
|
process.env.MINIONS_PORT = String(DASHBOARD_PORT);
|
|
722
741
|
const POST_UPDATE_INIT_TIMEOUT_MS = 120000;
|
|
723
|
-
|
|
742
|
+
// W-mqpx0tpi — the post-update `minions restart` runs as a child of `minions
|
|
743
|
+
// update` under this wall-clock cap. It must outlive the restart-health budget
|
|
744
|
+
// (now up to ~60s for a cold post-`npm install` dashboard boot) plus spawn +
|
|
745
|
+
// port-file-wait overhead, or the parent would SIGKILL a perfectly healthy
|
|
746
|
+
// slow boot and print a scary failure. Give comfortable headroom above the
|
|
747
|
+
// resolved health timeout.
|
|
748
|
+
const POST_UPDATE_RESTART_TIMEOUT_MS = Math.max(120000, _resolveRestartHealthTimeoutMs() + 60000);
|
|
724
749
|
|
|
725
750
|
function isSubpath(parent, child) {
|
|
726
751
|
const rel = path.relative(path.resolve(parent), path.resolve(child));
|
|
@@ -1042,7 +1067,9 @@ function runPostUpdateRestart() {
|
|
|
1042
1067
|
console.error(`\n ERROR: Post-update restart failed (${detail}).`);
|
|
1043
1068
|
}
|
|
1044
1069
|
console.error(' Runtime files were synchronized, but the engine/dashboard were not restarted.');
|
|
1045
|
-
console.error(
|
|
1070
|
+
console.error(` If this was a timeout, the dashboard may still be finishing a cold start —`);
|
|
1071
|
+
console.error(` re-check http://localhost:${DASHBOARD_PORT} or run \`minions status\` first.`);
|
|
1072
|
+
console.error(' If it is genuinely down, run this command to finish the update:');
|
|
1046
1073
|
console.error(` ${formatPackageCliCommand(args)}\n`);
|
|
1047
1074
|
process.exit(1);
|
|
1048
1075
|
}
|
|
@@ -513,14 +513,37 @@ function _scanSelectNone() {
|
|
|
513
513
|
document.querySelectorAll('[data-scan-idx]').forEach(function(cb) { if (!cb.disabled) cb.checked = false; });
|
|
514
514
|
}
|
|
515
515
|
|
|
516
|
+
// Stamp a per-repo Added/Failed status onto a scan-result row. Uses DOM
|
|
517
|
+
// construction with textContent (no innerHTML) so server-provided error text
|
|
518
|
+
// is rendered safely without tripping the no-unsanitized lint gate.
|
|
519
|
+
function _markScanRow(label, ok, text) {
|
|
520
|
+
if (!label) return;
|
|
521
|
+
var prev = label.querySelector('.scan-row-status');
|
|
522
|
+
if (prev) prev.remove();
|
|
523
|
+
label.style.borderColor = ok ? 'var(--green)' : 'var(--red)';
|
|
524
|
+
var badge = document.createElement('span');
|
|
525
|
+
badge.className = 'scan-row-status';
|
|
526
|
+
badge.style.flex = '0 0 auto';
|
|
527
|
+
badge.style.marginLeft = '8px';
|
|
528
|
+
badge.style.maxWidth = '45%';
|
|
529
|
+
badge.style.fontSize = 'var(--text-sm)';
|
|
530
|
+
badge.style.fontWeight = '600';
|
|
531
|
+
badge.style.textAlign = 'right';
|
|
532
|
+
badge.style.color = ok ? 'var(--green)' : 'var(--red)';
|
|
533
|
+
badge.textContent = (ok ? '✓ ' : '✗ ') + text;
|
|
534
|
+
label.appendChild(badge);
|
|
535
|
+
}
|
|
536
|
+
|
|
516
537
|
async function _addSelectedProjects() {
|
|
517
|
-
var checkboxes = document.querySelectorAll('[data-scan-idx]:checked:not(:disabled)');
|
|
538
|
+
var checkboxes = Array.prototype.slice.call(document.querySelectorAll('[data-scan-idx]:checked:not(:disabled)'));
|
|
518
539
|
if (checkboxes.length === 0) { alert('Select at least one repo'); return; }
|
|
519
540
|
var repos = window._scanRepos || [];
|
|
520
541
|
var added = 0;
|
|
542
|
+
var failed = 0;
|
|
521
543
|
for (var cb of checkboxes) {
|
|
522
544
|
var repo = repos[parseInt(cb.dataset.scanIdx)];
|
|
523
545
|
if (!repo) continue;
|
|
546
|
+
var label = cb.closest('label');
|
|
524
547
|
try {
|
|
525
548
|
var res = await fetch('/api/projects/add', {
|
|
526
549
|
method: 'POST', headers: { 'Content-Type': 'application/json' },
|
|
@@ -537,11 +560,21 @@ async function _addSelectedProjects() {
|
|
|
537
560
|
localPath: data.path || repo.path,
|
|
538
561
|
});
|
|
539
562
|
cb.disabled = true;
|
|
540
|
-
cb.
|
|
541
|
-
|
|
563
|
+
cb.checked = true;
|
|
564
|
+
if (label) { label.style.opacity = '0.5'; _markScanRow(label, true, 'Added'); }
|
|
565
|
+
} else {
|
|
566
|
+
failed++;
|
|
567
|
+
if (label) _markScanRow(label, false, 'Failed: ' + (data.error || ('HTTP ' + res.status)));
|
|
542
568
|
}
|
|
543
|
-
} catch {
|
|
569
|
+
} catch (e) {
|
|
570
|
+
failed++;
|
|
571
|
+
if (label) _markScanRow(label, false, 'Failed: ' + ((e && e.message) ? e.message : 'network error'));
|
|
572
|
+
}
|
|
544
573
|
}
|
|
574
|
+
// Summary toast: success styling only when everything added cleanly; any
|
|
575
|
+
// failure flips it to the error color so the count is unmistakable.
|
|
576
|
+
var summary = added + ' added' + (failed ? ', ' + failed + ' failed' : '');
|
|
577
|
+
showToast('scan-toast', summary, added > 0 && failed === 0);
|
|
545
578
|
if (added > 0) {
|
|
546
579
|
refresh();
|
|
547
580
|
}
|
|
@@ -894,13 +894,11 @@ function _wiRenderDetail(item) {
|
|
|
894
894
|
}
|
|
895
895
|
if (artPills) html += field('Artifacts', '<div style="display:flex;flex-wrap:wrap;gap:4px">' + artPills + '</div>');
|
|
896
896
|
|
|
897
|
-
// P-d5a6f7c4 (Harness Transparency, Stage 3 — surface). Render the
|
|
898
|
-
//
|
|
899
|
-
// { skills:[{name,source,path
|
|
900
|
-
// commands:[{name,scope
|
|
901
|
-
//
|
|
902
|
-
// the engine has no record of exposing (stale path, renamed skill, out-of-band
|
|
903
|
-
// tool, or hallucination) for a human to adjudicate.
|
|
897
|
+
// P-d5a6f7c4 (Harness Transparency, Stage 3 — surface). Render the harness
|
|
898
|
+
// usage the agent self-reported. Shape:
|
|
899
|
+
// { skills:[{name,source,path}], mcpServers:[{name,scope}],
|
|
900
|
+
// commands:[{name,scope}], docs:[{path,why}] }. Every entry renders as a
|
|
901
|
+
// plain solid pill; the engine's internal grounding flag is not surfaced.
|
|
904
902
|
(function() {
|
|
905
903
|
var hu = item._harnessUsed;
|
|
906
904
|
if (!hu || typeof hu !== 'object') return;
|
|
@@ -911,35 +909,23 @@ function _wiRenderDetail(item) {
|
|
|
911
909
|
{ key: 'docs', icon: '📄', label: 'Doc', primary: 'path', secondary: 'why' },
|
|
912
910
|
];
|
|
913
911
|
var pills = '';
|
|
914
|
-
var
|
|
912
|
+
var pStyle = 'display:inline-flex;align-items:center;gap:4px;padding:2px 8px;border-radius:10px;font-size:var(--text-sm);background:var(--surface2);border:1px solid var(--border);color:var(--text)';
|
|
915
913
|
kinds.forEach(function(k) {
|
|
916
914
|
var entries = Array.isArray(hu[k.key]) ? hu[k.key] : [];
|
|
917
915
|
entries.forEach(function(e) {
|
|
918
916
|
if (!e || typeof e !== 'object') return;
|
|
919
917
|
var primary = e[k.primary];
|
|
920
918
|
if (typeof primary !== 'string' || !primary) return;
|
|
921
|
-
var grounded = e.grounded === true;
|
|
922
|
-
if (!grounded) anyUngrounded = true;
|
|
923
919
|
var secondary = (typeof e[k.secondary] === 'string') ? e[k.secondary] : '';
|
|
924
|
-
|
|
925
|
-
var pStyle = grounded
|
|
926
|
-
? 'display:inline-flex;align-items:center;gap:4px;padding:2px 8px;border-radius:10px;font-size:var(--text-sm);background:var(--surface2);border:1px solid var(--border);color:var(--text)'
|
|
927
|
-
: 'display:inline-flex;align-items:center;gap:4px;padding:2px 8px;border-radius:10px;font-size:var(--text-sm);background:rgba(210,153,34,0.08);border:1px dashed var(--yellow);color:var(--text)';
|
|
928
|
-
var title = grounded
|
|
929
|
-
? k.label + ' — engine confirmed this affordance was exposed' + (secondary ? ' (' + secondary + ')' : '')
|
|
930
|
-
: k.label + ' — NOT grounded: the agent reports using this but the engine has no record of exposing it' + (secondary ? ' (' + secondary + ')' : '');
|
|
920
|
+
var title = k.label + (secondary ? ' (' + secondary + ')' : '');
|
|
931
921
|
pills += '<span style="' + pStyle + '" title="' + escapeHtml(title) + '">'
|
|
932
922
|
+ k.icon + ' ' + escapeHtml(primary)
|
|
933
923
|
+ (secondary ? ' <span style="color:var(--muted);font-size:var(--text-xs)">' + escapeHtml(secondary) + '</span>' : '')
|
|
934
|
-
+ (grounded ? '' : ' <span style="color:var(--yellow);font-weight:700" title="not grounded">⚠</span>')
|
|
935
924
|
+ '</span> ';
|
|
936
925
|
});
|
|
937
926
|
});
|
|
938
927
|
if (!pills) return;
|
|
939
|
-
|
|
940
|
-
? '<div style="font-size:var(--text-xs);color:var(--muted);margin-top:4px">⚠ dashed = agent-reported, not verified by the engine</div>'
|
|
941
|
-
: '';
|
|
942
|
-
html += field('Repo harnesses used', '<div style="display:flex;flex-wrap:wrap;gap:4px">' + pills + '</div>' + legend);
|
|
928
|
+
html += field('Repo harnesses used', '<div style="display:flex;flex-wrap:wrap;gap:4px">' + pills + '</div>');
|
|
943
929
|
})();
|
|
944
930
|
|
|
945
931
|
// P-34fa5d79 — Mentions: notes whose YAML frontmatter cites this WI
|
package/dashboard/js/settings.js
CHANGED
|
@@ -381,6 +381,7 @@ async function openSettings() {
|
|
|
381
381
|
settingsField('Worktree Create Retries', 'set-worktreeCreateRetries', e.worktreeCreateRetries || 1, '', 'Retry count for transient worktree add failures (0-3)') +
|
|
382
382
|
settingsField('Worktree Root', 'set-worktreeRoot', e.worktreeRoot || '../worktrees', '', 'Relative or absolute path for git worktrees; on Windows prefer a short path like C:\\wt') +
|
|
383
383
|
settingsField('Assert-Clean Status Probe Timeout', 'set-assertCleanStatusTimeoutMs', e.assertCleanStatusTimeoutMs || 10000, 'ms', 'Timeout for the `git status --porcelain` preflight probe in assertCleanSharedWorktree. Raise this (e.g. 60000) on GVFS/Plastic-backed monorepos where sparse hydration runs longer than the 10s default. On timeout the engine quarantines the bad worktree and emits a RETRYABLE failure so the next dispatch starts fresh. Clamped 1000–120000ms.') +
|
|
384
|
+
settingsField('Restart-health Timeout', 'set-restartHealthTimeoutMs', e.restartHealthTimeoutMs || 60000, 'ms', 'Cold-start budget for `minions restart` / post-update restart verification. The dashboard runs node:sqlite migrations + SPA HTML assembly + initial state reads synchronously BEFORE it binds its port, so right after `npm install` on a slow/loaded box the PID is alive but the port is unbound for tens of seconds. Below this budget that no longer prints a false "Restart verification failed" + exit(1) — the verifier keeps polling until the port listens. A genuinely crashed spawn (PID dies) still fails fast, well under the full budget. Clamped 15000–300000ms.') +
|
|
384
385
|
settingsField('Orphan-holder scan timeout', 'set-orphanHolderScanTimeoutMs', e.orphanHolderScanTimeoutMs || 5000, 'ms', 'Cap on the cross-platform scan (PowerShell / /proc walk / lsof) that identifies the OS process holding a stuck worktree\'s cwd. Bumps up only matter on heavily-loaded hosts where the holder scan races the sweep tick. Clamped 1000–30000ms.') +
|
|
385
386
|
settingsField('Worktree-holder reap probe timeout', 'set-statusProbeKillTimeoutMs', e.statusProbeKillTimeoutMs || 12000, 'ms', 'Windows-only. Timeout for the pre-quarantine worktree-holder reap probe (legacy git.exe-cmdline sweep + PEB-CWD scan that kills an orphaned agent process tree whose CWD pins the worktree dir, causing EBUSY on rename AND `git worktree remove --force`). The old hardcoded 2000ms was the proximate cause of the reaper being a no-op under concurrent-agent load (PowerShell cold-start + process enumeration exceeded 2s → ETIMEDOUT → reaped nothing). ETIMEDOUT is always swallowed (reap nothing; quarantine still proceeds), so raising this only widens the success window. Clamped 2000–60000ms.') +
|
|
386
387
|
'</div>';
|
|
@@ -983,6 +984,7 @@ async function saveSettings() {
|
|
|
983
984
|
worktreeCreateRetries: document.getElementById('set-worktreeCreateRetries').value,
|
|
984
985
|
worktreeRoot: document.getElementById('set-worktreeRoot').value,
|
|
985
986
|
assertCleanStatusTimeoutMs: document.getElementById('set-assertCleanStatusTimeoutMs').value,
|
|
987
|
+
restartHealthTimeoutMs: document.getElementById('set-restartHealthTimeoutMs')?.value,
|
|
986
988
|
idleAlertMinutes: document.getElementById('set-idleAlertMinutes').value,
|
|
987
989
|
shutdownTimeout: document.getElementById('set-shutdownTimeout').value,
|
|
988
990
|
restartGracePeriod: document.getElementById('set-restartGracePeriod').value,
|
package/dashboard.js
CHANGED
|
@@ -10525,6 +10525,12 @@ What would you like to discuss or change? When you're happy, say "approve" and I
|
|
|
10525
10525
|
// 2min ceiling (covers even a GVFS-backed sparse hydration without
|
|
10526
10526
|
// letting a config typo wedge dispatch for an hour).
|
|
10527
10527
|
assertCleanStatusTimeoutMs: [1000, 120000],
|
|
10528
|
+
// W-mqpx0tpi — cold-start restart-health budget. 15s floor (anything
|
|
10529
|
+
// shorter re-introduces the false-failure bug on a post-`npm install`
|
|
10530
|
+
// dashboard boot); 5min ceiling (a config typo shouldn't let a wedged
|
|
10531
|
+
// boot hang `minions restart` indefinitely — a dead PID still fails
|
|
10532
|
+
// fast well before the budget).
|
|
10533
|
+
restartHealthTimeoutMs: [15000, 300000],
|
|
10528
10534
|
// W-mq6f2fe0000557fa — cap on the cross-platform holder scan for the
|
|
10529
10535
|
// orphan-worktree GC. 1s floor (PowerShell startup alone is 0.5–1s);
|
|
10530
10536
|
// 30s ceiling (the scan runs once per orphan-sweep tick and a slow
|
|
@@ -261,10 +261,27 @@ Defined in `engine/shared.js` as `FAILURE_CLASS`. Use the canonical hyphenated s
|
|
|
261
261
|
| `out-of-context` | Context window exhausted | Flag for human review |
|
|
262
262
|
| `max-turns` | Claude CLI `error_max_turns` — work in progress | Retry same agent |
|
|
263
263
|
| `completion-nonce-mismatch` | Completion JSON missing or mismatched `nonce` (forged completion). See [Trust boundary](#trust-boundary). | Never retry (untrusted) |
|
|
264
|
+
| `worktree-preflight` | Pre-spawn worktree validation rejected the dispatch (nested-in-project, drive-root collapse, missing base). See [Pre-spawn preflight vs agent failure](#pre-spawn-preflight-vs-agent-failure). | Never retry |
|
|
265
|
+
| `live-checkout-dirty` | Live-checkout project tree had uncommitted changes; engine refused to spawn in-place. See [Pre-spawn preflight vs agent failure](#pre-spawn-preflight-vs-agent-failure). | Never retry |
|
|
266
|
+
| `workspace-manifest-repo-forbidden` | Dispatch routed an agent to a repo not in its `workspace_manifest.allowed_repos`. See [Pre-spawn preflight vs agent failure](#pre-spawn-preflight-vs-agent-failure). | Never retry |
|
|
264
267
|
| `unknown` | Unclassified failure | Default retry logic |
|
|
265
268
|
|
|
266
269
|
Use `"N/A"` when `status` is `success` or `partial` without a failure.
|
|
267
270
|
|
|
271
|
+
## Pre-spawn preflight vs agent failure
|
|
272
|
+
|
|
273
|
+
A handful of `failure_class` values are written by the **engine at dispatch time**, *before the agent process is ever spawned*. They are emitted from `spawnAgent` (`engine.js`) while it sets up the worktree, validates the checkout, or enforces the workspace manifest — the engine calls `completeDispatch(... DISPATCH_RESULT.ERROR ...)` and returns without launching any runtime. **A completion report carrying one of these classes does not imply the agent ran any work**; no prompt was sent, no edits were attempted, and there is no agent-authored `note` or output to interpret.
|
|
274
|
+
|
|
275
|
+
| `failure_class` | Pre-spawn trigger | `engine/shared.js` |
|
|
276
|
+
|---|---|---|
|
|
277
|
+
| `worktree-preflight` | Worktree placement validation failed: the worktree would nest inside the project, the resolved root collapsed to a drive root, or the base directory was missing. | `FAILURE_CLASS.WORKTREE_PREFLIGHT` (~`shared.js:4279`) |
|
|
278
|
+
| `live-checkout-dirty` | Live-checkout mode (`project.checkoutMode: 'live'`) found uncommitted changes in `project.localPath`; the engine never `reset`/`clean`s the operator tree, so it refuses to spawn. | `FAILURE_CLASS.LIVE_CHECKOUT_DIRTY` (~`shared.js:4289`) |
|
|
279
|
+
| `workspace-manifest-repo-forbidden` | The dispatch routed the agent to a repo not listed in its `workspace_manifest.allowed_repos`. Structural — widen the manifest or route the work elsewhere. | `FAILURE_CLASS.WORKSPACE_MANIFEST_REPO` (~`shared.js:4292`) |
|
|
280
|
+
|
|
281
|
+
`WORKTREE_ROOTDIR_COLLAPSED_TO_DRIVE_ROOT` is **not** a standalone enum value: it is an `Error.code` thrown by the worktree path resolver (`shared.js`) which `spawnAgent` catches and maps to the non-retryable `worktree-preflight` class (`engine.js`). It surfaces in logs/diagnostics, but the report's `failure_class` is always `worktree-preflight`.
|
|
282
|
+
|
|
283
|
+
All three classes are **never retryable** — they signal a structural/environment problem the same agent would hit again. Resolve the underlying cause (clean the live tree, fix the worktree root, widen the manifest) before re-dispatching.
|
|
284
|
+
|
|
268
285
|
## No-op semantics
|
|
269
286
|
|
|
270
287
|
A no-op completion declares that the agent correctly **declined** to do the work — the change was already shipped on master, the dispatch premise was wrong, the flagged review comment was an author-note, etc.
|
|
@@ -7,7 +7,7 @@ the **quarantine path** (dirty/divergent → quarantine dir + retry), and the
|
|
|
7
7
|
Lifecycle keeps the cross-cutting invariants; the detail lives here.
|
|
8
8
|
|
|
9
9
|
> Source of truth: `engine/worktree-pool.js`, `engine/shared.js#removeWorktree`
|
|
10
|
-
> + `_retryFsOp`, `engine.js` (`_quarantineDirtyWorktree`,
|
|
10
|
+
> + `_retryFsOp`, `engine.js` (`_quarantineDirtyWorktree`,
|
|
11
11
|
> `_killGitDescendantsForWorktree`, `pruneOrphanWorktrees*`, `gcDispatchWorktreeIfOrphan`),
|
|
12
12
|
> `engine/cleanup.js`. Last verified: 2026-06-09.
|
|
13
13
|
|
|
@@ -93,11 +93,15 @@ the `.git/index.lock` acquire around the untracked-cache refresh inside
|
|
|
93
93
|
place. **Most incidents are closed by 1b alone.**
|
|
94
94
|
Toggle: `ENGINE_DEFAULTS.statusProbeUseNoOptionalLocks` (default `true`).
|
|
95
95
|
|
|
96
|
-
### Layer 1a — `
|
|
96
|
+
### Layer 1a — quarantine rename via `shared._retryFsOp` with jittered backoff
|
|
97
97
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
98
|
+
The quarantine rename (`fs.renameSync(worktreePath, quarantinedPath)`) is wrapped
|
|
99
|
+
in `shared._retryFsOp` (P-a1c4f7e2 collapsed the former bespoke
|
|
100
|
+
`engine.js#_renameWithRetry` into the shared helper). 6 attempts × 250 ms base ×
|
|
101
|
+
2^N exponential + 200 ms random jitter (~16 s worst-case). Only retries on
|
|
102
|
+
`EBUSY|EPERM|EACCES|ENOTEMPTY` (`shared._WORKTREE_RETRYABLE_CODES`); rethrows
|
|
103
|
+
other codes immediately. `_quarantineDirtyWorktree` passes the rename-specific
|
|
104
|
+
budget explicitly. Toggles:
|
|
101
105
|
`ENGINE_DEFAULTS.quarantineRenameRetryAttempts` (6),
|
|
102
106
|
`quarantineRenameRetryBaseMs` (250).
|
|
103
107
|
|
package/engine/cleanup.js
CHANGED
|
@@ -1591,6 +1591,34 @@ function runPeriodicWorktreeSweep(config) {
|
|
|
1591
1591
|
let scanned = 0, kept = 0, evicted = 0, failed = 0, outOfRootEvicted = 0, prunedRegistry = 0;
|
|
1592
1592
|
let missingDirReclaimed = 0, missingDirSkippedLive = 0;
|
|
1593
1593
|
const _writeToInbox = (a, s, c) => { try { return shared.writeToInbox(a, s, c); } catch (_e) { return false; } };
|
|
1594
|
+
|
|
1595
|
+
// P-d8f1a3c6 — shell `git worktree list --porcelain` at most once per project
|
|
1596
|
+
// per tick and share the parsed trees with BOTH registry-based pruners
|
|
1597
|
+
// (out-of-root + missing-dir reclaim) instead of each self-shelling. Keyed on
|
|
1598
|
+
// the resolved localPath so projects sharing a parent repo only list once. A
|
|
1599
|
+
// project missing from the map (list/shell failure here) falls back to the
|
|
1600
|
+
// consumer's own self-shelling path. `pruneOrphanWorktrees` (r1, fs-readdir
|
|
1601
|
+
// based) never lists and is unaffected.
|
|
1602
|
+
const parsedTreesByProject = new Map();
|
|
1603
|
+
for (const project of projects) {
|
|
1604
|
+
if (!project || !project.localPath) continue;
|
|
1605
|
+
let rootDir;
|
|
1606
|
+
try { rootDir = path.resolve(String(project.localPath)); } catch { continue; }
|
|
1607
|
+
if (parsedTreesByProject.has(rootDir)) continue; // dedup shared-parent projects
|
|
1608
|
+
let rootExists = false;
|
|
1609
|
+
try { rootExists = fs.existsSync(rootDir); } catch { rootExists = false; }
|
|
1610
|
+
if (!rootExists) continue;
|
|
1611
|
+
try {
|
|
1612
|
+
const raw = String(shared.execSilent('git --no-optional-locks worktree list --porcelain', {
|
|
1613
|
+
cwd: rootDir, timeout: 15000, windowsHide: true,
|
|
1614
|
+
}) || '');
|
|
1615
|
+
parsedTreesByProject.set(rootDir, shared.parseWorktreePorcelain(raw));
|
|
1616
|
+
} catch (e) {
|
|
1617
|
+
// Leave unset → the consumer self-shells (existing fallback path).
|
|
1618
|
+
log('warn', `worktree-gc periodic list ${project.name || rootDir}: ${e.message}`);
|
|
1619
|
+
}
|
|
1620
|
+
}
|
|
1621
|
+
|
|
1594
1622
|
try {
|
|
1595
1623
|
const r1 = worktreeGc.pruneOrphanWorktrees({
|
|
1596
1624
|
projects, dispatchSnap, worktreeRootRel, log: _log, config, writeToInbox: _writeToInbox,
|
|
@@ -1604,6 +1632,7 @@ function runPeriodicWorktreeSweep(config) {
|
|
|
1604
1632
|
try {
|
|
1605
1633
|
const r2 = worktreeGc.pruneOrphanWorktreesFromGitRegistry({
|
|
1606
1634
|
projects, dispatchSnap, worktreeRootRel, log: _log, config, writeToInbox: _writeToInbox,
|
|
1635
|
+
parsedTreesByProject,
|
|
1607
1636
|
});
|
|
1608
1637
|
scanned += r2.scanned || 0;
|
|
1609
1638
|
kept += r2.kept || 0;
|
|
@@ -1620,7 +1649,7 @@ function runPeriodicWorktreeSweep(config) {
|
|
|
1620
1649
|
// interrupted `git worktree add` bricks the branch until a human intervenes.
|
|
1621
1650
|
try {
|
|
1622
1651
|
const r3 = worktreeGc.reclaimMissingDirWorktrees({
|
|
1623
|
-
projects, log: _log, config,
|
|
1652
|
+
projects, log: _log, config, parsedTreesByProject,
|
|
1624
1653
|
});
|
|
1625
1654
|
missingDirReclaimed += r3.reclaimed || 0;
|
|
1626
1655
|
missingDirSkippedLive += r3.skippedLive || 0;
|
package/engine/restart-health.js
CHANGED
|
@@ -3,9 +3,22 @@ const path = require('path');
|
|
|
3
3
|
const http = require('http');
|
|
4
4
|
const https = require('https');
|
|
5
5
|
const { execSync } = require('child_process');
|
|
6
|
+
const shared = require('./shared');
|
|
6
7
|
|
|
7
|
-
|
|
8
|
+
// W-mqpx0tpi — single source of truth for the cold-start health budget. The
|
|
9
|
+
// old fixed 15s was too short for a post-`npm install` dashboard boot on a
|
|
10
|
+
// slow/loaded machine: the dashboard runs node:sqlite migrations + SPA HTML
|
|
11
|
+
// assembly + initial state reads SYNCHRONOUSLY before it ever reaches
|
|
12
|
+
// server.listen(), so the PID is alive but the port is unbound for the whole
|
|
13
|
+
// window, tripping a false "Restart verification failed" + exit(1). The knob
|
|
14
|
+
// lives in ENGINE_DEFAULTS.restartHealthTimeoutMs (Settings-configurable);
|
|
15
|
+
// the literal is only a fallback if shared somehow failed to load.
|
|
16
|
+
const DEFAULT_RESTART_HEALTH_TIMEOUT_MS =
|
|
17
|
+
(shared.ENGINE_DEFAULTS && shared.ENGINE_DEFAULTS.restartHealthTimeoutMs) || 60000;
|
|
8
18
|
const DEFAULT_RESTART_HEALTH_INTERVAL_MS = 250;
|
|
19
|
+
// Consecutive dead-PID reads tolerated before fail-fast. A small guard avoids
|
|
20
|
+
// a single flaky tasklist / kill(0) false-negative aborting a healthy boot.
|
|
21
|
+
const DEFAULT_DEAD_POLLS_BEFORE_FAST_FAIL = 2;
|
|
9
22
|
|
|
10
23
|
function sleep(ms) {
|
|
11
24
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
@@ -182,9 +195,14 @@ async function waitForRestartHealth(options = {}) {
|
|
|
182
195
|
const timeoutMs = options.timeoutMs ?? DEFAULT_RESTART_HEALTH_TIMEOUT_MS;
|
|
183
196
|
const intervalMs = options.intervalMs ?? DEFAULT_RESTART_HEALTH_INTERVAL_MS;
|
|
184
197
|
const maxAttempts = normalizePid(options.maxAttempts);
|
|
198
|
+
const deadPollsBeforeFastFail =
|
|
199
|
+
Number.isInteger(options.deadPollsBeforeFastFail) && options.deadPollsBeforeFastFail > 0
|
|
200
|
+
? options.deadPollsBeforeFastFail
|
|
201
|
+
: DEFAULT_DEAD_POLLS_BEFORE_FAST_FAIL;
|
|
185
202
|
const started = Date.now();
|
|
186
203
|
let attempts = 0;
|
|
187
204
|
let last = null;
|
|
205
|
+
let consecutiveDead = 0;
|
|
188
206
|
|
|
189
207
|
while (true) {
|
|
190
208
|
attempts++;
|
|
@@ -192,6 +210,25 @@ async function waitForRestartHealth(options = {}) {
|
|
|
192
210
|
last.attempts = attempts;
|
|
193
211
|
last.elapsedMs = Date.now() - started;
|
|
194
212
|
if (last.ok) return last;
|
|
213
|
+
|
|
214
|
+
// W-mqpx0tpi — distinguish "still starting" from "dead". When the dashboard
|
|
215
|
+
// PID we just spawned is ALIVE but the port isn't listening yet, keep
|
|
216
|
+
// waiting out the (now much larger) timeout: that's the legitimate cold
|
|
217
|
+
// start. But if the PID has DIED, fail FAST — a genuinely crashed spawn
|
|
218
|
+
// should surface in seconds instead of holding the slot for the full
|
|
219
|
+
// budget. Only the process-kind check exposes `alive`; the HTTP-kind path
|
|
220
|
+
// has no PID-liveness signal, so it falls through to the timeout as before.
|
|
221
|
+
const dash = last.dashboard;
|
|
222
|
+
if (dash && dash.kind === 'process' && dash.alive === false) {
|
|
223
|
+
consecutiveDead++;
|
|
224
|
+
if (consecutiveDead >= deadPollsBeforeFastFail) {
|
|
225
|
+
last.failedFast = true;
|
|
226
|
+
return last;
|
|
227
|
+
}
|
|
228
|
+
} else {
|
|
229
|
+
consecutiveDead = 0;
|
|
230
|
+
}
|
|
231
|
+
|
|
195
232
|
if (maxAttempts && attempts >= maxAttempts) break;
|
|
196
233
|
const remainingMs = timeoutMs - (Date.now() - started);
|
|
197
234
|
if (!maxAttempts && remainingMs <= 0) break;
|
|
@@ -210,7 +247,22 @@ function formatRestartHealthError(result) {
|
|
|
210
247
|
const elapsed = typeof result.elapsedMs === 'number' ? `${result.elapsedMs}ms` : 'unknown time';
|
|
211
248
|
const attempts = result.attempts || 0;
|
|
212
249
|
const details = (result.errors || ['Unknown restart verification failure']).map(err => ` - ${err}`).join('\n');
|
|
213
|
-
|
|
250
|
+
// W-mqpx0tpi — when the timeout was hit with the dashboard PID alive but the
|
|
251
|
+
// port not yet listening, the dashboard is almost certainly still finishing a
|
|
252
|
+
// cold start rather than broken. Point the operator at a re-check instead of
|
|
253
|
+
// implying breakage.
|
|
254
|
+
let guidance = '';
|
|
255
|
+
const dash = result && result.dashboard;
|
|
256
|
+
if (dash && dash.kind === 'process' && dash.alive && !dash.listening) {
|
|
257
|
+
const port = dash.port || 7331;
|
|
258
|
+
guidance =
|
|
259
|
+
`\n The dashboard process is alive but has not yet bound port ${port}.\n` +
|
|
260
|
+
` It may still be finishing a cold start — node:sqlite migrations, SPA\n` +
|
|
261
|
+
` HTML assembly, and the first state reads all run synchronously before\n` +
|
|
262
|
+
` the dashboard reaches server.listen(). Re-check http://localhost:${port}\n` +
|
|
263
|
+
` or run \`minions status\` before assuming anything is broken.\n`;
|
|
264
|
+
}
|
|
265
|
+
return `\n ERROR: Restart verification failed after ${elapsed} (${attempts} attempt${attempts === 1 ? '' : 's'}).\n${details}\n${guidance}`;
|
|
214
266
|
}
|
|
215
267
|
|
|
216
268
|
module.exports = {
|
package/engine/shared.js
CHANGED
|
@@ -2032,7 +2032,7 @@ function writeToInbox(agentId, slug, content, _inboxDir, metadata) {
|
|
|
2032
2032
|
// ── Process Spawning ────────────────────────────────────────────────────────
|
|
2033
2033
|
// All child process calls go through these to ensure windowsHide: true
|
|
2034
2034
|
|
|
2035
|
-
const { execSync: _execSync, spawnSync: _spawnSync, spawn: _spawn, exec: _cbExec, execFile: _cbExecFile } = require('child_process');
|
|
2035
|
+
const { execSync: _execSync, execFileSync: _execFileSync, spawnSync: _spawnSync, spawn: _spawn, exec: _cbExec, execFile: _cbExecFile } = require('child_process');
|
|
2036
2036
|
const { promisify: _promisify } = require('util');
|
|
2037
2037
|
const _execFileAsync = _promisify(_cbExecFile);
|
|
2038
2038
|
|
|
@@ -2635,6 +2635,17 @@ const ENGINE_DEFAULTS = {
|
|
|
2635
2635
|
// raise this knob (e.g. 60000) before tuning anything else. Clamped to
|
|
2636
2636
|
// [1000, 120000] in dashboard.js POST /api/settings.
|
|
2637
2637
|
assertCleanStatusTimeoutMs: 10000,
|
|
2638
|
+
// W-mqpx0tpi — cold-start health budget for `minions restart` / post-update
|
|
2639
|
+
// restart verification (engine/restart-health.js#waitForRestartHealth). The
|
|
2640
|
+
// dashboard runs node:sqlite migrations + SPA HTML assembly + initial state
|
|
2641
|
+
// reads SYNCHRONOUSLY before server.listen(), so right after `npm install`
|
|
2642
|
+
// on a slow/loaded box the PID is alive but port 7331 is unbound well past
|
|
2643
|
+
// the old fixed 15s, producing a false "Restart verification failed" +
|
|
2644
|
+
// exit(1) even though the dashboard comes up healthy moments later. Default
|
|
2645
|
+
// 60s comfortably covers a cold boot; a dead PID still fails fast (the
|
|
2646
|
+
// verifier doesn't wait out the full budget for a crashed spawn). Clamped to
|
|
2647
|
+
// [15000, 300000] in dashboard.js POST /api/settings.
|
|
2648
|
+
restartHealthTimeoutMs: 60000,
|
|
2638
2649
|
workItemCreateDedupWindowMs: 15 * 60 * 1000, // 15min — collapse duplicate CC/API create races
|
|
2639
2650
|
idleAlertMinutes: 15,
|
|
2640
2651
|
restartGracePeriod: 1200000, // 20min
|
|
@@ -8296,6 +8307,13 @@ function _writeWorktreeSkipLiveInboxNote(worktreePath, callerTag) {
|
|
|
8296
8307
|
} catch { /* best-effort — never throw from the skip-note writer */ }
|
|
8297
8308
|
}
|
|
8298
8309
|
|
|
8310
|
+
// P-c7e2b405 — INTENTIONAL divergence from worktree-gc.reapAndRemoveWorktree:
|
|
8311
|
+
// this is the low-level `git worktree remove --force` → fs.rmSync → rd /s /q
|
|
8312
|
+
// removal primitive WITH its own EBUSY/_retryFsOp backoff loop. The engine-side
|
|
8313
|
+
// helper INJECTS this function as its `removeWorktree` and owns the
|
|
8314
|
+
// reap/escalate/marker-gate orchestration around it. It is deliberately NOT
|
|
8315
|
+
// folded into the helper (shared.js must not import the engine-side reaper —
|
|
8316
|
+
// layering boundary).
|
|
8299
8317
|
function removeWorktree(wtPath, gitRoot, worktreeRoot, opts = {}) {
|
|
8300
8318
|
const resolved = path.resolve(wtPath);
|
|
8301
8319
|
const resolvedRoot = path.resolve(worktreeRoot) + path.sep;
|
|
@@ -8326,14 +8344,24 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot, opts = {}) {
|
|
|
8326
8344
|
return false;
|
|
8327
8345
|
}
|
|
8328
8346
|
} catch { /* bad gitRoot — fall through to the .git probe */ }
|
|
8329
|
-
|
|
8330
|
-
|
|
8331
|
-
|
|
8332
|
-
|
|
8333
|
-
|
|
8334
|
-
|
|
8335
|
-
|
|
8336
|
-
|
|
8347
|
+
// P-b3d9a162 — the `.git`-is-a-directory refusal, extracted so it can run at
|
|
8348
|
+
// BOTH the top of the function AND again immediately before the fallback
|
|
8349
|
+
// fs.rmSync. A Windows file-lock can release between the two checks — the
|
|
8350
|
+
// primary `git worktree remove --force` failing on that lock drops us into
|
|
8351
|
+
// the catch, where a freshly-exposed real `.git` directory would otherwise be
|
|
8352
|
+
// wiped. Returns true ⇒ the caller MUST abort (refuse the delete).
|
|
8353
|
+
const _refuseIfRealRepo = (resolved) => {
|
|
8354
|
+
try {
|
|
8355
|
+
const st = fs.lstatSync(path.join(resolved, '.git'));
|
|
8356
|
+
if (st && st.isDirectory()) {
|
|
8357
|
+
log('warn', `removeWorktree: refusing to remove ${wtPath} — it is a real git repo (.git is a directory, not a linked-worktree pointer)`);
|
|
8358
|
+
try { bumpWorktreeGcMetric('refusedRealRepo'); } catch { /* metric optional */ }
|
|
8359
|
+
return true;
|
|
8360
|
+
}
|
|
8361
|
+
} catch { /* no .git, or unreadable — normal worktree husk; continue */ }
|
|
8362
|
+
return false;
|
|
8363
|
+
};
|
|
8364
|
+
if (_refuseIfRealRepo(resolved)) return false;
|
|
8337
8365
|
// W-mq5rwwss000f30a7 — never wipe a worktree while an agent is actively
|
|
8338
8366
|
// dispatched inside it. isWorktreePathLive fails OPEN (returns true) when
|
|
8339
8367
|
// the dispatches table is unreachable, so we err on the side of leaking
|
|
@@ -8363,12 +8391,19 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot, opts = {}) {
|
|
|
8363
8391
|
|
|
8364
8392
|
bumpWorktreeGcMetric('attempts');
|
|
8365
8393
|
try {
|
|
8366
|
-
|
|
8394
|
+
// P-1d7a4f80 — argv form (shell:false) so wtPath can never be re-parsed by a shell.
|
|
8395
|
+
shellSafeGitSync(['worktree', 'remove', wtPath, '--force'], { cwd: gitRoot, timeout: 15000 });
|
|
8367
8396
|
_removeWorktreeFailures.delete(resolved);
|
|
8368
8397
|
bumpWorktreeGcMetric('success');
|
|
8369
8398
|
return true;
|
|
8370
8399
|
} catch (gitErr) {
|
|
8371
8400
|
try {
|
|
8401
|
+
// P-b3d9a162 — re-run the real-repo refusal: a lock that released only
|
|
8402
|
+
// AFTER the top-of-function probe (and made the primary `git worktree
|
|
8403
|
+
// remove` throw above) can expose a real `.git` directory right here.
|
|
8404
|
+
// Refuse before the fallback rmSync so we never recurse-delete a real
|
|
8405
|
+
// repo on the fallback path.
|
|
8406
|
+
if (_refuseIfRealRepo(resolved)) return false;
|
|
8372
8407
|
// W-mq5o6bvy000x7191 (Layer 1): retry fs.rmSync with exponential backoff
|
|
8373
8408
|
// for transient Windows file-locks (EPERM/EBUSY/EACCES/ENOTEMPTY) before
|
|
8374
8409
|
// falling through to rd /s /q. A single AV/Explorer/vscode lock during
|
|
@@ -8377,7 +8412,7 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot, opts = {}) {
|
|
|
8377
8412
|
() => fs.rmSync(resolved, { recursive: true, force: true }),
|
|
8378
8413
|
`fs.rmSync(${resolved})`
|
|
8379
8414
|
);
|
|
8380
|
-
try {
|
|
8415
|
+
try { shellSafeGitSync(['worktree', 'prune'], { cwd: gitRoot, timeout: 10000 }); } catch {}
|
|
8381
8416
|
_removeWorktreeFailures.delete(resolved);
|
|
8382
8417
|
bumpWorktreeGcMetric('success');
|
|
8383
8418
|
if (attempt > 1) bumpWorktreeGcMetric('successAfterRetry');
|
|
@@ -8387,8 +8422,11 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot, opts = {}) {
|
|
|
8387
8422
|
// locked files, and partially-deleted directories (not just EPERM)
|
|
8388
8423
|
if (process.platform === 'win32') {
|
|
8389
8424
|
try {
|
|
8390
|
-
|
|
8391
|
-
|
|
8425
|
+
// P-1d7a4f80 — argv form (shell:false). `rd` is a cmd.exe builtin so it
|
|
8426
|
+
// must run via `cmd /c`, but the worktree path is passed as a discrete
|
|
8427
|
+
// argv element rather than interpolated into a shell string.
|
|
8428
|
+
_execFileSync('cmd', ['/c', 'rd', '/s', '/q', resolved], { stdio: 'pipe', timeout: 15000, windowsHide: true });
|
|
8429
|
+
try { shellSafeGitSync(['worktree', 'prune'], { cwd: gitRoot, timeout: 10000 }); } catch {}
|
|
8392
8430
|
_removeWorktreeFailures.delete(resolved);
|
|
8393
8431
|
bumpWorktreeGcMetric('success');
|
|
8394
8432
|
return true;
|