@yemi33/minions 0.1.2424 → 0.1.2425

package/engine/kb-sweep.js

@@ -24,6 +24,7 @@ const KB_SWEEP_LOG_PATH = path.join(ENGINE_DIR, 'kb-sweep.log');
 const KB_SWEEP_RUNNER_PATH = path.join(__dirname, 'kb-sweep-runner.js');
 const SWEPT_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
 const AUTO_SWEEP_INTERVAL_MS = 4 * 60 * 60 * 1000;
+const AUTO_SWEEP_FAILURE_RETRY_MS = 15 * 60 * 1000;
 const KB_SWEPT_PATH = path.join(ENGINE_DIR, 'kb-swept.json');
 const COMPRESS_THRESHOLD_BYTES = 5000;
 const LLM_BATCH_SIZE = 30;
@@ -726,7 +727,19 @@ function _writeSweepState(state) {
   // Phase 9.3: route through shared.mutateKbSweepState (file-locked RMW) so
   // the dashboard process + the detached runner can't race each other.
   const augmented = { pid: process.pid, ...state };
-  try { shared.mutateKbSweepState(() => augmented); } catch { /* ignore */ }
+  let accepted = false;
+  try {
+    shared.mutateKbSweepState((current) => {
+      if (state.sweepToken != null
+          && current?.sweepToken != null
+          && current.sweepToken !== state.sweepToken) {
+        return undefined;
+      }
+      accepted = true;
+      return augmented;
+    });
+  } catch { return false; }
+  return accepted;
 }
 
 /**
@@ -740,7 +753,7 @@ function _writeSweepState(state) {
  * @param {number} [opts.entryCount=0] - KB entry count for staleGuardMs()
  * @param {number} [opts.now=Date.now()] - injectable clock for tests
  * @param {(pid:number)=>boolean} [opts.isPidAlive] - injectable for tests
- * @returns {{ inFlight: boolean, alive?: boolean, stale?: boolean, pid?: number,
+ * @returns {{ inFlight: boolean, alive?: boolean, stale?: boolean, overdue?: boolean, pid?: number,
  *            startedAt?: number, sweepToken?: string|number|null, guardMs?: number,
  *            status?: string }}
  */
@@ -764,16 +777,19 @@ function readSweepLiveness(opts = {}) {
   const age = startedAt ? now - startedAt : 0;
   let alive;
   if (state.status === 'starting') {
-    // No PID yet — grant a short boot-grace so spawn can complete and the
-    // runner can overwrite with status:'in-flight' + its pid.
+    // Current claims record the caller PID so slow pre-spawn I/O cannot make a
+    // live owner stealable. Legacy pid-less claims retain the short boot grace.
     const STARTING_GRACE_MS = 15000;
-    alive = age <= STARTING_GRACE_MS;
+    alive = pid > 0 ? !!isPidAlive(pid) : age <= STARTING_GRACE_MS;
   } else {
     alive = pid > 0 ? !!isPidAlive(pid) : false;
   }
-  const stale = !alive || (startedAt > 0 && age > guardMs);
+  // Sweeps regularly run for 1h+; replacing an overdue live runner would leave
+  // its detached process tree active and create overlapping sweeps.
+  const overdue = startedAt > 0 && age > guardMs;
+  const stale = !alive;
   return {
-    inFlight: true, alive, stale, pid, startedAt, guardMs,
+    inFlight: true, alive, stale, overdue, pid, startedAt, guardMs,
     sweepToken: state.sweepToken || null, status: state.status,
   };
 }
@@ -833,18 +849,6 @@ function reconcileSweepStateOnBoot(opts = {}) {
 
   const reason = pid > 0 ? 'pid-dead-on-startup-check' : 'no-pid-recorded-on-startup-check';
 
-  // CAS guard: re-read right before the write so we don't clobber a fresh
-  // `starting`/`in-flight` record that a concurrent dashboard POST or runner
-  // wrote between our decision and our write.
-  const current = safeJson(KB_SWEEP_STATE_PATH);
-  if (!current
-      || current.status !== state.status
-      || Number(current.startedAt || 0) !== Number(state.startedAt || 0)
-      || Number(current.pid || 0) !== pid
-      || (current.sweepToken || null) !== (state.sweepToken || null)) {
-    return { scanned: 1, released: 0, reason: 'state-changed-before-reconcile' };
-  }
-
   const failedState = {
     ...state,
     status: 'failed',
@@ -856,7 +860,25 @@ function reconcileSweepStateOnBoot(opts = {}) {
   // Direct mutateKbSweepState (NOT _writeSweepState) so the original pid is
   // preserved for forensics — _writeSweepState would overwrite it with this
   // process's pid, destroying the diagnostic value of the record.
-  try { shared.mutateKbSweepState(() => failedState); } catch { /* ignore */ }
+  let released = false;
+  try {
+    shared.mutateKbSweepState((current) => {
+      if (!current
+          || current.status !== state.status
+          || Number(current.startedAt || 0) !== Number(state.startedAt || 0)
+          || Number(current.pid || 0) !== pid
+          || (current.sweepToken || null) !== (state.sweepToken || null)) {
+        return undefined;
+      }
+      released = true;
+      return failedState;
+    });
+  } catch {
+    return { scanned: 1, released: 0, reason: 'reconcile-write-failed' };
+  }
+  if (!released) {
+    return { scanned: 1, released: 0, reason: 'state-changed-before-reconcile' };
+  }
   return {
     scanned: 1, released: 1, reason,
     prevStatus: state.status, prevPid: pid,
@@ -884,9 +906,18 @@ async function runKbSweep(opts = {}) {
   // still reports terminal status and the dashboard pre-write doesn't leak
   // a stale "in-flight"/"starting" record. The inner _runKbSweepImpl still
   // honors dryRun for actual file mutations.
-  _writeSweepState({ status: 'in-flight', startedAt, startedAtIso: ts(), sweepToken, dryRun });
+  const ownsSweep = _writeSweepState({
+    status: 'in-flight', startedAt, startedAtIso: ts(), sweepToken, dryRun,
+  });
+  if (sweepToken != null && !ownsSweep) {
+    throw new Error('KB sweep claim was superseded before runner start');
+  }
   try {
     const result = await _runKbSweepImpl(opts);
+    if (!dryRun) {
+      try { shared.mutateKbSwept(() => ({ timestamp: ts(), summary: result.summary, detail: result })); } catch { /* ignore */ }
+      try { queries.invalidateKnowledgeBaseCache(); } catch { /* ignore */ }
+    }
     _writeSweepState({
       status: 'completed', startedAt, completedAt: Date.now(), completedAtIso: ts(),
       durationMs: result.durationMs, summary: result.summary, lastResult: result,
@@ -1019,10 +1050,6 @@ async function _runKbSweepImpl(opts = {}) {
   summary.durationMs = Date.now() - t0;
   summary.summary = `${summary.hashDuplicatesArchived} hash-dup, ${summary.notesConsolidated} consolidated (${summary.digestsWritten} digests), ${summary.llmDuplicatesArchived} llm-dup, ${summary.staleRemoved} stale, ${summary.ageExpired} age-expired, ${summary.reclassified} reclassified, ${summary.rewritten} rewritten (${(summary.bytesBefore - summary.bytesAfter).toLocaleString()} bytes saved)`;
 
-  if (!opts.dryRun) {
-    try { shared.mutateKbSwept(() => ({ timestamp: ts(), summary: summary.summary, detail: summary })); } catch { /* ignore */ }
-    try { queries.invalidateKnowledgeBaseCache(); } catch { /* ignore */ }
-  }
   return summary;
 }
 
@@ -1032,8 +1059,8 @@ async function _runKbSweepImpl(opts = {}) {
  * tick's auto-sweep phase. Performs the same synchronous "starting" → "in-flight"
  * CAS dance the dashboard handler used to do inline.
  *
- * Callers are responsible for the in-flight / stale-guard check BEFORE calling
- * (so they can return distinct HTTP responses or log levels).
+ * The helper atomically claims the persistent sweep state before spawning, so
+ * concurrent manual and automatic callers cannot launch overlapping runners.
  *
  * @param {object} opts
  * @param {string[]} [opts.pinnedKeys] - extra pinned KB keys to skip in the sweep
@@ -1045,23 +1072,61 @@ async function _runKbSweepImpl(opts = {}) {
  *           ok=false + error on synchronous spawn failure; the "starting" claim is
  *           released so the caller can retry immediately.
  */
+function _claimSweepRunnerState({ sweepToken, startedAt, observed }) {
+  let claimed = false;
+  let currentState = null;
+  shared.mutateKbSweepState((current) => {
+    currentState = current || null;
+    const currentInFlight = current?.status === 'starting' || current?.status === 'in-flight';
+    if (currentInFlight) {
+      const matchesObserved = observed?.inFlight
+        && current.status === observed.status
+        && Number(current.startedAt || 0) === Number(observed.startedAt || 0)
+        && Number(current.pid || 0) === Number(observed.pid || 0)
+        && (current.sweepToken || null) === (observed.sweepToken || null);
+      if (!matchesObserved || observed.alive) return undefined;
+    }
+    claimed = true;
+    return {
+      status: 'starting', startedAt, startedAtIso: new Date(startedAt).toISOString(),
+      sweepToken, pid: process.pid,
+    };
+  });
+  return { claimed, current: currentState };
+}
+
 function spawnSweepRunnerDetached(opts = {}) {
   const fsLocal = require('fs');
   const { spawn: cpSpawn } = require('child_process');
+  const spawnFn = typeof opts.spawn === 'function' ? opts.spawn : cpSpawn;
   const logFn = typeof opts.log === 'function'
     ? opts.log
     : (level, msg) => { (level === 'error' ? console.error : console.log)(`[kb-sweep] ${msg}`); };
   const cwd = opts.cwd || require('./queries').MINIONS_DIR;
   const startedAt = Date.now();
   const sweepToken = `${startedAt}-${Math.random().toString(36).slice(2, 8)}`;
+  const observed = readSweepLiveness({ now: startedAt });
+  if (observed.inFlight && observed.alive) {
+    return {
+      ok: true, alreadyRunning: true, sweepToken: observed.sweepToken,
+      pid: observed.pid || null, startedAt: observed.startedAt || null, bodyFile: null,
+    };
+  }
 
   try {
-    shared.mutateKbSweepState(() => ({
-      status: 'starting', startedAt, startedAtIso: new Date().toISOString(),
-      sweepToken, pid: null,
-    }));
+    const claim = _claimSweepRunnerState({ sweepToken, startedAt, observed });
+    if (!claim.claimed) {
+      return {
+        ok: true, alreadyRunning: true,
+        sweepToken: claim.current?.sweepToken || null,
+        pid: Number(claim.current?.pid) || null,
+        startedAt: Number(claim.current?.startedAt) || null,
+        bodyFile: null,
+      };
+    }
   } catch (e) {
     logFn('error', `failed to write starting state: ${e.message}`);
+    return { ok: false, error: `claim failed: ${e.message}`, sweepToken, pid: null, bodyFile: null };
   }
 
   let bodyFile = null;
@@ -1082,9 +1147,16 @@ function spawnSweepRunnerDetached(opts = {}) {
 
   let logFdNum = null;
   let stdio = ['ignore', 'ignore', 'ignore'];
+  const closeLogFd = () => {
+    if (Number.isInteger(logFdNum)) {
+      try { fsLocal.closeSync(logFdNum); } catch { /* ignore */ }
+      logFdNum = null;
+    }
+  };
   try {
-    logFdNum = fsLocal.openSync(KB_SWEEP_LOG_PATH, 'a');
-    stdio = ['ignore', logFdNum, logFdNum];
+    const opened = shared.openAppendLogFd(path.basename(KB_SWEEP_LOG_PATH), path.dirname(KB_SWEEP_LOG_PATH));
+    logFdNum = opened.fd;
+    if (logFdNum !== 'ignore') stdio = ['ignore', logFdNum, logFdNum];
   } catch (e) {
     logFn('error', `failed to open log ${KB_SWEEP_LOG_PATH}: ${e.message}`);
   }
@@ -1092,19 +1164,61 @@ function spawnSweepRunnerDetached(opts = {}) {
   const spawnArgs = ['--sweep-token', sweepToken];
   if (bodyFile) spawnArgs.push('--body-file', bodyFile);
 
+  const claimedState = safeJson(KB_SWEEP_STATE_PATH);
+  if (!claimedState || claimedState.sweepToken !== sweepToken || claimedState.status !== 'starting') {
+    closeLogFd();
+    if (bodyFile) try { fsLocal.unlinkSync(bodyFile); } catch { /* ignore */ }
+    return {
+      ok: true, alreadyRunning: true,
+      sweepToken: claimedState?.sweepToken || null,
+      pid: Number(claimedState?.pid) || null,
+      startedAt: Number(claimedState?.startedAt) || null,
+      bodyFile: null,
+    };
+  }
+
   let proc;
   try {
-    proc = cpSpawn(process.execPath, [KB_SWEEP_RUNNER_PATH, ...spawnArgs], {
+    proc = spawnFn(process.execPath, [KB_SWEEP_RUNNER_PATH, ...spawnArgs], {
       cwd, stdio, detached: true, windowsHide: true,
       env: { ...process.env },
     });
   } catch (e) {
-    if (logFdNum != null) try { fsLocal.closeSync(logFdNum); } catch { /* ignore */ }
+    closeLogFd();
     if (bodyFile) try { fsLocal.unlinkSync(bodyFile); } catch { /* ignore */ }
-    try { shared.safeUnlink(KB_SWEEP_STATE_PATH); } catch { /* ignore */ }
+    _writeSweepState({
+      status: 'failed',
+      startedAt,
+      completedAt: Date.now(),
+      completedAtIso: new Date().toISOString(),
+      error: `spawn failed: ${e.message}`,
+      sweepToken,
+    });
     return { ok: false, error: `spawn failed: ${e.message}`, sweepToken, pid: null, bodyFile: null };
   }
-  if (logFdNum != null) try { fsLocal.closeSync(logFdNum); } catch { /* ignore */ }
+  let spawnErrorHandled = false;
+  const handleSpawnError = (error) => {
+    if (spawnErrorHandled) return;
+    spawnErrorHandled = true;
+    closeLogFd();
+    if (bodyFile) try { fsLocal.unlinkSync(bodyFile); } catch { /* ignore */ }
+    _writeSweepState({
+      status: 'failed',
+      startedAt,
+      completedAt: Date.now(),
+      completedAtIso: new Date().toISOString(),
+      error: `spawn failed: ${error?.message || error}`,
+      sweepToken,
+    });
+    logFn('error', `runner spawn failed: ${error?.message || error}`);
+  };
+  proc.once('error', handleSpawnError);
+  if (!Number.isInteger(proc.pid) || proc.pid <= 0) {
+    const error = new Error('runner process did not provide a pid');
+    handleSpawnError(error);
+    return { ok: false, error: error.message, sweepToken, pid: null, bodyFile: null };
+  }
+  closeLogFd();
 
   // Phase 9.3: atomic starting→in-flight transition under the file lock
   // (was previously a safeJson→safeWrite race).
@@ -1139,9 +1253,16 @@ function shouldAutoSweep(opts = {}) {
   const now = Number(opts.now) || Date.now();
   const intervalMs = Number(opts.intervalMs) || AUTO_SWEEP_INTERVAL_MS;
   const liveness = opts.liveness || readSweepLiveness({ entryCount: opts.entryCount || 0, now });
-  if (liveness.inFlight && liveness.alive && !liveness.stale) {
+  if (liveness.inFlight && liveness.alive) {
     return { shouldSpawn: false, reason: 'sweep-in-flight', lastCompletedAt: null };
   }
+  const sweepState = safeJson(KB_SWEEP_STATE_PATH);
+  const failedAt = sweepState?.status === 'failed'
+    ? Number(sweepState.completedAt || 0)
+    : 0;
+  if (failedAt > 0 && (now - failedAt) < AUTO_SWEEP_FAILURE_RETRY_MS) {
+    return { shouldSpawn: false, reason: 'failure-backoff', lastCompletedAt: null };
+  }
   const swept = safeJson(KB_SWEPT_PATH);
   const sweptTs = swept && swept.timestamp ? Date.parse(swept.timestamp) : NaN;
   const lastCompletedAt = Number.isFinite(sweptTs) ? sweptTs : null;
@@ -1165,6 +1286,7 @@ module.exports = {
   spawnSweepRunnerDetached,
   shouldAutoSweep,
   AUTO_SWEEP_INTERVAL_MS,
+  AUTO_SWEEP_FAILURE_RETRY_MS,
   KB_SWEPT_PATH,
   KB_SWEEP_STATE_PATH,
   KB_SWEEP_LOG_PATH,
@@ -1178,6 +1300,7 @@ module.exports = {
   _entryAgeMs,
   _resolveTtlByCategory,
   _archiveKbFile,
+  _claimSweepRunnerState, _writeSweepState, // exported for testing
   _expireByAge,
   _entryAgeMs,
   _resolveTtlByCategory,

package/engine/keep-process-sweep.js

@@ -21,11 +21,44 @@ const fs = require('fs');
 const path = require('path');
 const shared = require('./shared');
 const queries = require('./queries');
+const processUtils = require('./process-utils');
 
 const { log, ENGINE_DEFAULTS } = shared;
 
 const KEEP_PIDS_FILENAME = 'keep-pids.json';
 const INVALID_WORKDIR_REASON_PREFIX = 'invalid-workdir: ';
+const KEEP_PROCESS_AGENT_ID_MAX = 128;
+const KEEP_PROCESS_PID_MAX = 0x7fffffff;
+const KEEP_PROCESS_IDENTITY_CLOCK_SKEW_MS = 1000;
+const KEEP_PROCESS_AGENT_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/;
+
+function isSafeKeepProcessAgentId(agentId) {
+  return typeof agentId === 'string'
+    && agentId.length > 0
+    && agentId.length <= KEEP_PROCESS_AGENT_ID_MAX
+    && KEEP_PROCESS_AGENT_ID_RE.test(agentId);
+}
+
+function validateKeepProcessPid(pid) {
+  if (!Number.isInteger(pid) || pid <= 0 || pid > KEEP_PROCESS_PID_MAX) {
+    return { ok: false, reason: 'pid-invalid (' + pid + ')' };
+  }
+  return { ok: true, value: pid };
+}
+
+function getKeepProcessIdentity(pid, opts = {}) {
+  const validation = validateKeepProcessPid(pid);
+  if (!validation.ok) return null;
+  return processUtils.getProcessIdentity(pid, opts);
+}
+
+function isKeepProcessIdentityCompatible(identity, sidecarMtimeMs) {
+  return processUtils.isProcessIdentityCompatible(
+    identity,
+    sidecarMtimeMs,
+    KEEP_PROCESS_IDENTITY_CLOCK_SKEW_MS,
+  );
+}
 
 function _agentsDir() {
   return queries.AGENTS_DIR || path.join(shared.MINIONS_DIR, 'agents');
@@ -57,9 +90,9 @@ function validateKeepPidsRecord(parsed, opts) {
   if (parsed.pids.length > maxPids) return { ok: false, reason: 'pids-too-many (>' + maxPids + ')' };
   const pids = [];
   for (const raw of parsed.pids) {
-    const n = Number(raw);
-    if (!Number.isInteger(n) || n <= 0) return { ok: false, reason: 'pid-invalid (' + raw + ')' };
-    pids.push(n);
+    const validation = validateKeepProcessPid(raw);
+    if (!validation.ok) return validation;
+    pids.push(validation.value);
   }
 
   if (typeof parsed.expires_at !== 'string') return { ok: false, reason: 'expires_at-missing' };
@@ -565,6 +598,13 @@ function buildKeepProcessesHint(opts) {
 module.exports = {
   KEEP_PIDS_FILENAME: KEEP_PIDS_FILENAME,
   INVALID_WORKDIR_REASON_PREFIX: INVALID_WORKDIR_REASON_PREFIX,
+  KEEP_PROCESS_AGENT_ID_MAX: KEEP_PROCESS_AGENT_ID_MAX,
+  KEEP_PROCESS_PID_MAX: KEEP_PROCESS_PID_MAX,
+  KEEP_PROCESS_IDENTITY_CLOCK_SKEW_MS: KEEP_PROCESS_IDENTITY_CLOCK_SKEW_MS,
+  isSafeKeepProcessAgentId: isSafeKeepProcessAgentId,
+  validateKeepProcessPid: validateKeepProcessPid,
+  getKeepProcessIdentity: getKeepProcessIdentity,
+  isKeepProcessIdentityCompatible: isKeepProcessIdentityCompatible,
   validateKeepPidsRecord: validateKeepPidsRecord,
   readKeepPidsFile: readKeepPidsFile,
   listAllKeepPidsFiles: listAllKeepPidsFiles,