@yemi33/minions 0.1.2142 → 0.1.2144

package/engine/shared.js

@@ -85,6 +85,105 @@ const LOG_PATH = path.join(MINIONS_DIR, 'engine', 'log.json');
 const CONSTELLATION_BRIDGE_MARKER_PATH = path.join(MINIONS_DIR, 'engine', 'constellation-bridge.json');
 const CONSTELLATION_BRIDGE_MARKER_SCHEMA_VERSION = 1;
 
+// ── Dashboard port runtime file (W-mq5nwl9l) ────────────────────────────────
+// dashboard.js writes this file as soon as `server.listen` resolves so every
+// downstream consumer (CLI status/dash/restart, cli-api-client, supervisor)
+// can discover the ACTUALLY-bound port rather than guessing 7331. The
+// resolver chain (explicit > env MINIONS_DASHBOARD_PORT > config.engine.
+// dashboardPort > default 7331) is shared between dashboard.js (deciding what
+// to try first) and bin/minions.js (deciding what to display/probe when the
+// runtime file is missing or stale). The auto-fallback scan picks the first
+// free port in [requestedPort, requestedPort + DASHBOARD_PORT_SCAN_MAX) on
+// EADDRINUSE so a busy 7331 doesn't break startup.
+const DEFAULT_DASHBOARD_PORT = 7331;
+const DASHBOARD_PORT_SCAN_MAX = 100;
+const DASHBOARD_PORT_PATH = path.join(MINIONS_DIR, 'engine', 'dashboard-port.json');
+
+function _dashboardPortPath(minionsHome) {
+  return minionsHome
+    ? path.join(minionsHome, 'engine', 'dashboard-port.json')
+    : DASHBOARD_PORT_PATH;
+}
+
+function _validPort(n) {
+  const v = Number.isInteger(n) ? n : parseInt(n, 10);
+  return Number.isInteger(v) && v > 0 && v <= 65535 ? v : null;
+}
+
+/**
+ * Read the runtime dashboard-port.json beacon. Returns `{port, pid, boundAt,
+ * path}` on success, or `null` when the file is missing, malformed, or
+ * contains an out-of-range port. No throw.
+ */
+function readDashboardPortFile(minionsHome) {
+  const fp = _dashboardPortPath(minionsHome);
+  try {
+    const raw = fs.readFileSync(fp, 'utf8');
+    const data = JSON.parse(raw);
+    const port = _validPort(data && data.port);
+    if (port === null) return null;
+    return {
+      port,
+      pid: Number.isInteger(data.pid) ? data.pid : null,
+      boundAt: typeof data.boundAt === 'string' ? data.boundAt : null,
+      path: fp,
+    };
+  } catch { return null; }
+}
+
+/**
+ * Atomically write the runtime dashboard-port.json beacon. The dashboard
+ * calls this once `server.listen` resolves so every CLI/client consumer
+ * agrees on the bound port (which may differ from the requested port after
+ * an EADDRINUSE fallback). Best-effort: file IO errors do not throw.
+ */
+function writeDashboardPortFile({ port, pid, minionsHome } = {}) {
+  const validPort = _validPort(port);
+  if (validPort === null) return { ok: false, error: new Error(`invalid port: ${port}`) };
+  const fp = _dashboardPortPath(minionsHome);
+  const payload = {
+    port: validPort,
+    pid: Number.isInteger(pid) ? pid : process.pid,
+    boundAt: new Date().toISOString(),
+  };
+  try {
+    fs.mkdirSync(path.dirname(fp), { recursive: true });
+    const tmp = fp + '.tmp';
+    fs.writeFileSync(tmp, JSON.stringify(payload));
+    fs.renameSync(tmp, fp);
+    return { ok: true, path: fp };
+  } catch (err) {
+    return { ok: false, error: err };
+  }
+}
+
+/** Delete the runtime dashboard-port.json beacon. Called on graceful
+ *  dashboard shutdown so the next CLI status check doesn't probe a stale
+ *  port. Best-effort. */
+function clearDashboardPortFile(minionsHome) {
+  const fp = _dashboardPortPath(minionsHome);
+  try { fs.unlinkSync(fp); return { ok: true }; } catch { return { ok: false }; }
+}
+
+/**
+ * Resolve the *requested* dashboard port via the precedence chain:
+ *   explicit > env MINIONS_DASHBOARD_PORT > config.engine.dashboardPort > 7331
+ *
+ * Pure — performs no IO. Callers that want the actually-bound port should
+ * prefer `readDashboardPortFile()` first and fall back to this. Returns
+ * `{port, source}` so log lines can stay accurate ("from env", "from
+ * config", etc.).
+ */
+function resolveDashboardPort(opts = {}) {
+  const { explicit, env, config, defaultPort = DEFAULT_DASHBOARD_PORT } = opts;
+  let p;
+  if ((p = _validPort(explicit)) !== null) return { port: p, source: 'explicit' };
+  if ((p = _validPort(env)) !== null) return { port: p, source: 'env' };
+  if ((p = _validPort(config)) !== null) return { port: p, source: 'config' };
+  const def = _validPort(defaultPort) || DEFAULT_DASHBOARD_PORT;
+  return { port: def, source: 'default' };
+}
+
 // ── Timestamps & Logging ────────────────────────────────────────────────────
 // Extracted from engine.js so engine/* modules can import directly without
 // circular-requiring the orchestrator.
@@ -2420,6 +2519,25 @@ const ENGINE_DEFAULTS = {
   mainBranchCacheMaxEntries: 100, // bound repo/branch detection cache in long-lived dashboard/engine processes
   removeWorktreeFailureTtlMs: 24 * 60 * 60 * 1000, // stale failed paths are forgotten after a day
   removeWorktreeFailureMaxEntries: 1000, // bound failed-worktree retry suppression cache
+  // ── Worktree GC hardening (W-mq5o6bvy000x7191) ──────────────────────────
+  // Windows file-lock (EPERM/EBUSY/EACCES/ENOTEMPTY) class on worktree
+  // removal: AV scans, vscode reopen, Explorer thumbnailers, msbuild watchers
+  // hold transient locks on files inside a worktree we're trying to reap.
+  // Without retry, a single EPERM at the fs.rmSync step in removeWorktree
+  // permanently drops the worktree into the 3-attempt cooldown cache.
+  worktreeRemoveRetryAttempts: 6,           // exponential-backoff attempts on EPERM/EBUSY/EACCES/ENOTEMPTY
+  worktreeRemoveRetryBaseMs: 250,           // base delay; effective delay = baseMs * 2^i + jitter
+  // Periodic in-tick orphan-worktree sweep (Layer 2 of W-mq5o6bvy000x7191).
+  // Boot-only reconcile in cli.js doesn't catch orphans accumulated by
+  // long-lived engines; this re-runs the same pruner every N ticks.
+  worktreePruneIntervalTicks: 30,           // ~5 min at 10s tick (matches keep-processes/managed-spawn cadence factor)
+  // Stuck-dir escalation (Layer 4 of W-mq5o6bvy000x7191). After the same
+  // worktree path has failed removal N consecutive times, the engine
+  // suppresses repeat log spam, writes a dedup'd inbox note, and continues
+  // retrying at a slower cadence in the background.
+  worktreeStuckThreshold: 10,               // consecutive removal failures before escalation
+  worktreeStuckSuppressMs: 60 * 60 * 1000,  // 60min — suppress per-tick warn after escalation
+  worktreeStuckSlowRetryMs: 30 * 60 * 1000, // 30min — slow-cadence retry window after escalation
   ccMaxTurns: 50, // max tool-use turns per CC/doc-chat call before CLI stops (per response, not per session)
   ccTurnTimeoutMs: 300000, // W-mpmwxni2000c25c7-b/-d: 5min per-turn no-progress watchdog. The window resets on every liveness signal — token chunk, tool-call notification, tool-update — so an actively-streaming CC/doc-chat turn (long shell command, deep search, sub-agent loop) survives indefinitely up to the outer CC_CALL_TIMEOUT_MS (~1h) ceiling. Only true silence past this window with no progress fires the cancel: the in-flight LLM call is aborted and the handler surfaces `{code:'cc-turn-timeout', retryable:true}` via the typed error envelope so the UI can stop the spinner and offer Retry. Clamped to [10000, 3600000] in the settings POST handler. Independent of CC_CALL_TIMEOUT_MS. Non-streaming doc-chat is the lone wall-clock exception (no progress hooks); see _raceCcDocChatTimeout in dashboard.js for the dual factory/promise shape.
   docSessionMaxEntries: 200, // cap doc-chat session map/disk store by least-recent activity (LRU; sessions are non-expiring otherwise)
@@ -5429,6 +5547,32 @@ function normalizePrLinkItems(value) {
   return [...new Set(items.filter(item => typeof item === 'string' && item))];
 }
 
+/**
+ * Single source of truth for "should the engine dispatch review/fix for this PR?"
+ *
+ * Used by `discoverFromPrs` in engine.js to gate review + fix dispatch, and by
+ * `upsertPullRequestRecord` to decide whether to preserve auto-managed metadata
+ * on merge. Dispatch-loop callers MUST go through this helper — do NOT inline
+ * variants like `knownAgents.has(pr.agent) || pr.prdItems?.length || pr._manual`
+ * because they silently drop the `_autoObserve` case (PR linked with
+ * autoObserve=true but no prdItems and no configured-agent author) and the
+ * sourcePlan/itemType cases. See W-mq5rs2eq000da8a9 for the bug repro.
+ *
+ * A PR is auto-managed when ANY of the following hold (and `_contextOnly` is
+ * not explicitly true — context-only always wins):
+ *   - It carries one or more `prdItems` (linked to work items)
+ *   - It was explicitly opted-in via `_autoObserve: true` (manual link with
+ *     autoObserve, or per-row toggle via POST /api/pull-requests/observe)
+ *   - It was created from a plan or has an itemType (`sourcePlan`/`itemType`)
+ *   - It has a non-empty agent author other than the literal string `'human'`
+ *     (the helper deliberately does NOT require the agent be in
+ *     `config.agents` — once `_autoObserve` is the explicit opt-in for
+ *     non-agent authors, the loosened agent-string check has no operational
+ *     downside for managed flows and avoids the silent-skip bug.)
+ *
+ * @param {object} pr Pull request record
+ * @returns {boolean} true when the engine should drive review/fix for this PR
+ */
 function isAutoManagedPrRecord(pr) {
   if (!pr || typeof pr !== 'object' || pr._contextOnly === true) return false;
   if (normalizePrLinkItems(pr.prdItems).length > 0) return true;
@@ -6101,6 +6245,82 @@ const _WIN_RESERVED_NAMES = new Set([
   'LPT1', 'LPT2', 'LPT3', 'LPT4', 'LPT5', 'LPT6', 'LPT7', 'LPT8', 'LPT9',
 ]);
 
+// ── Worktree GC hardening (W-mq5o6bvy000x7191) ──────────────────────────────
+// Exit codes that mean "transient lock — try again later" on Windows when a
+// file inside the worktree is held by AV scan, vscode reopen, msbuild watcher,
+// or Explorer thumbnailer. Anything else (ENOENT, missing dir, real fs error)
+// is rethrown immediately.
+const _WORKTREE_RETRYABLE_CODES = new Set(['EBUSY', 'EPERM', 'EACCES', 'ENOTEMPTY']);
+
+/**
+ * Retry helper for fs ops that can hit Windows file-lock errors.
+ * Used by `removeWorktree`'s `fs.rmSync` call so a single transient EPERM
+ * doesn't drop the worktree into the 3-attempt cooldown.
+ *
+ *   - op:      thunk that performs (and returns) the fs op
+ *   - label:   short string used in the final thrown-error message
+ *   - opts.attempts: total attempts; default ENGINE_DEFAULTS.worktreeRemoveRetryAttempts
+ *   - opts.baseMs:   base delay; effective delay per attempt = baseMs * 2^i + jitter
+ *   - opts.onAttempt: optional (attempt, err) => void for tests/metrics
+ *
+ * Returns the op's result on first success. On the final attempt that fails,
+ * the underlying error is re-thrown (wrapped with attempt count in `.message`)
+ * so callers can branch on `err.code` per usual.
+ *
+ * Non-retryable errors (code not in `_WORKTREE_RETRYABLE_CODES`) are thrown
+ * immediately without sleeping or counting against the attempts budget.
+ */
+function _retryFsOp(op, label, opts = {}) {
+  const attempts = Number(opts.attempts) > 0
+    ? Math.floor(opts.attempts)
+    : (ENGINE_DEFAULTS.worktreeRemoveRetryAttempts || 6);
+  const baseMs = Number(opts.baseMs) > 0
+    ? Number(opts.baseMs)
+    : (ENGINE_DEFAULTS.worktreeRemoveRetryBaseMs || 250);
+  const onAttempt = typeof opts.onAttempt === 'function' ? opts.onAttempt : null;
+  let lastErr;
+  for (let i = 0; i < attempts; i++) {
+    try {
+      const result = op();
+      if (onAttempt) { try { onAttempt(i + 1, null); } catch { /* metric optional */ } }
+      return { result, attempt: i + 1 };
+    } catch (e) {
+      lastErr = e;
+      if (onAttempt) { try { onAttempt(i + 1, e); } catch { /* metric optional */ } }
+      if (!e || !_WORKTREE_RETRYABLE_CODES.has(e.code)) throw e;
+      if (i === attempts - 1) break;
+      const delay = baseMs * Math.pow(2, i) + Math.floor(Math.random() * 200);
+      sleepMs(delay);
+    }
+  }
+  const err = new Error(`${label} failed after ${attempts} retries: ${(lastErr && lastErr.message) || 'unknown'}`);
+  err.code = (lastErr && lastErr.code) || 'EUNKNOWN';
+  err.cause = lastErr;
+  err.attempts = attempts;
+  throw err;
+}
+
+/**
+ * Bump per-call counters on the `_worktreeGcOutcomes` metric. Best-effort;
+ * silent on lock contention or missing metrics file. The schema lives under
+ * a single key so all worktree-GC behavior is observable from one dashboard
+ * tile.
+ */
+function bumpWorktreeGcMetric(field, delta = 1) {
+  if (!field || typeof field !== 'string') return;
+  const n = Number(delta);
+  if (!Number.isFinite(n) || n === 0) return;
+  try {
+    mutateMetrics((metrics) => {
+      if (!metrics._worktreeGcOutcomes || typeof metrics._worktreeGcOutcomes !== 'object') {
+        metrics._worktreeGcOutcomes = {};
+      }
+      metrics._worktreeGcOutcomes[field] = (metrics._worktreeGcOutcomes[field] || 0) + n;
+      return metrics;
+    });
+  } catch { /* metric best-effort */ }
+}
+
 const PR_FIX_CAUSE = {
   HUMAN_FEEDBACK: 'human-feedback',
   REVIEW_FEEDBACK: 'review-feedback',
@@ -6239,15 +6459,26 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot) {
     _purgeReservedFiles(resolved);
   }
 
+  bumpWorktreeGcMetric('attempts');
   try {
     exec(`git worktree remove "${wtPath}" --force`, { cwd: gitRoot, stdio: 'pipe', timeout: 15000, windowsHide: true });
     _removeWorktreeFailures.delete(resolved);
+    bumpWorktreeGcMetric('success');
     return true;
   } catch (gitErr) {
     try {
-      fs.rmSync(resolved, { recursive: true, force: true });
+      // W-mq5o6bvy000x7191 (Layer 1): retry fs.rmSync with exponential backoff
+      // for transient Windows file-locks (EPERM/EBUSY/EACCES/ENOTEMPTY) before
+      // falling through to rd /s /q. A single AV/Explorer/vscode lock during
+      // GC must not bury the whole worktree under the 3-attempt cooldown.
+      const { attempt } = _retryFsOp(
+        () => fs.rmSync(resolved, { recursive: true, force: true }),
+        `fs.rmSync(${resolved})`
+      );
       try { exec('git worktree prune', { cwd: gitRoot, stdio: 'pipe', timeout: 10000, windowsHide: true }); } catch {}
       _removeWorktreeFailures.delete(resolved);
+      bumpWorktreeGcMetric('success');
+      if (attempt > 1) bumpWorktreeGcMetric('successAfterRetry');
       return true;
     } catch (rmErr) {
       // Windows: try cmd /c rd /s /q for any error — handles reserved device names,
@@ -6257,6 +6488,7 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot) {
           exec(`cmd /c rd /s /q "${resolved}"`, { stdio: 'pipe', timeout: 15000, windowsHide: true });
           try { exec('git worktree prune', { cwd: gitRoot, stdio: 'pipe', timeout: 10000, windowsHide: true }); } catch {}
           _removeWorktreeFailures.delete(resolved);
+          bumpWorktreeGcMetric('success');
           return true;
         } catch (rdErr) {
           log('warn', `removeWorktree: rd /s /q fallback failed for ${wtPath}: ${rdErr.message}`);
@@ -6265,8 +6497,10 @@ function removeWorktree(wtPath, gitRoot, worktreeRoot) {
       const fail = _removeWorktreeFailures.get(resolved) || { count: 0, lastAttempt: 0 };
       fail.count++;
       fail.lastAttempt = Date.now();
+      fail.lastError = rmErr && rmErr.message ? rmErr.message : 'unknown';
       _removeWorktreeFailures.set(resolved, fail);
       _pruneRemoveWorktreeFailures();
+      bumpWorktreeGcMetric('totalFailure');
       if (fail.count <= 3) log('warn', `removeWorktree: failed for ${wtPath} (attempt ${fail.count}/3): ${rmErr.message}`);
       return false;
     }
@@ -6451,6 +6685,13 @@ module.exports = {
   LOG_PATH,
   CONSTELLATION_BRIDGE_MARKER_PATH,
   CONSTELLATION_BRIDGE_MARKER_SCHEMA_VERSION,
+  DEFAULT_DASHBOARD_PORT,
+  DASHBOARD_PORT_SCAN_MAX,
+  DASHBOARD_PORT_PATH,
+  readDashboardPortFile,
+  writeDashboardPortFile,
+  clearDashboardPortFile,
+  resolveDashboardPort,
   currentLogPath: _currentLogPath,
   ts,
   normalizeIsoTimestamp, // F6 (P-f6commentedit)
@@ -6598,6 +6839,7 @@ module.exports = {
   normalizePrRecords,
   normalizePrLinkItems, // exported for testing
   mergePrLinkItems, // exported for testing
+  isAutoManagedPrRecord,
   upsertPullRequestRecord,
   nextWorkItemId,
   getProjectOrg,
@@ -6651,6 +6893,9 @@ module.exports = {
   listProcessDescendants,
   listProcessReachable,
   removeWorktree,
+  _retryFsOp, // exported for testing (W-mq5o6bvy000x7191)
+  bumpWorktreeGcMetric, // exported for testing (W-mq5o6bvy000x7191)
+  _WORKTREE_RETRYABLE_CODES, // exported for testing (W-mq5o6bvy000x7191)
   _purgeReservedFiles, // exported for testing
   _WIN_RESERVED_NAMES, // exported for testing
   LOCK_STALE_MS,

package/engine/supervisor.js

@@ -52,7 +52,24 @@ const STOP_INTENT_PATH = path.join(STATIC_ENGINE_DIR, 'stop-intent.json');
 const SUPERVISOR_PID_PATH = path.join(STATIC_ENGINE_DIR, 'supervisor.pid');
 
 const SUPERVISOR_INTERVAL_MS = Number(process.env.MINIONS_SUPERVISOR_INTERVAL_MS) || 30000;
-const DASH_PORT = Number(process.env.MINIONS_DASHBOARD_PORT) || 7331;
+// Dashboard port resolves dynamically each tick (W-mq5nwl9l). The dashboard
+// auto-falls-back to the next free port when 7331 is in use and persists the
+// actually-bound port to ~/.minions/engine/dashboard-port.json. Hard-coding
+// 7331 here would make the supervisor probe the wrong port and respawn-loop
+// the dashboard. _resolveDashPort() consults the runtime file first, falls
+// back to the env var, then to the static default.
+const DASH_PORT_DEFAULT = Number(process.env.MINIONS_DASHBOARD_PORT) || 7331;
+function _resolveDashPort() {
+  const shared = _sharedOrNull();
+  if (shared && typeof shared.readDashboardPortFile === 'function') {
+    try {
+      const home = process.env.MINIONS_HOME || path.join(os.homedir(), '.minions');
+      const entry = shared.readDashboardPortFile(home);
+      if (entry && Number.isInteger(entry.port) && entry.port > 0) return entry.port;
+    } catch { /* fall through */ }
+  }
+  return DASH_PORT_DEFAULT;
+}
 // Grace window after we (re)spawn a process — gives it time to bind its port
 // or write its PID before we re-probe. Without this we'd race the freshly
 // spawned process and double-spawn.
@@ -216,10 +233,11 @@ function checkEngine(now) {
 
 function checkDashboard(now) {
   if (now - _lastDashboardRespawnAt < POST_SPAWN_GRACE_MS) return;
-  const pids = listeningPidsForPort(DASH_PORT);
+  const dashPort = _resolveDashPort();
+  const pids = listeningPidsForPort(dashPort);
   if (pids.length > 0) return;
 
-  console.log(`[supervisor] Dashboard not listening on port ${DASH_PORT} — respawning...`);
+  console.log(`[supervisor] Dashboard not listening on port ${dashPort} — respawning...`);
   const newPid = spawnDashboard();
   _lastDashboardRespawnAt = now;
   console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
@@ -267,7 +285,7 @@ function main() {
   // its PID to control.json on cold Windows boots.
   _lastEngineRespawnAt = Date.now();
   _lastDashboardRespawnAt = Date.now();
-  console.log(`[supervisor] Started (PID ${process.pid}); interval=${SUPERVISOR_INTERVAL_MS}ms, dashboardPort=${DASH_PORT}`);
+  console.log(`[supervisor] Started (PID ${process.pid}); interval=${SUPERVISOR_INTERVAL_MS}ms, dashboardPort=${_resolveDashPort()}(dynamic)`);
 
   const interval = setInterval(tick, SUPERVISOR_INTERVAL_MS);
   // Run one tick immediately so a freshly-restarted Minions covers any early