@askexenow/exe-os 0.9.276 → 0.9.277

package/dist/chunk-6SP45XNB.js

@@ -0,0 +1,363 @@
+// src/lib/daemon-restart-orchestrator.ts
+import {
+  existsSync,
+  readFileSync,
+  writeFileSync,
+  appendFileSync,
+  mkdirSync,
+  openSync,
+  closeSync,
+  unlinkSync,
+  statSync,
+  renameSync,
+  readdirSync
+} from "fs";
+import { execSync } from "child_process";
+import { randomUUID } from "crypto";
+import path from "path";
+import os from "os";
+var EXE_DIR = path.join(os.homedir(), ".exe-os");
+var LOG_DIR = path.join(EXE_DIR, "logs");
+var RESTART_LOG = path.join(LOG_DIR, "daemon-restarts.log");
+var RESTART_MARKER = path.join(EXE_DIR, "last-daemon-restart");
+var SPAWN_LOCK_PATH = path.join(EXE_DIR, "exed-spawn.lock");
+var PID_PATH = process.env.EXE_DAEMON_PID ?? path.join(EXE_DIR, "exed.pid");
+var SOCKET_PATH = process.env.EXE_DAEMON_SOCK ?? path.join(EXE_DIR, "exed.sock");
+var OOM_MARKER_PATH = path.join(EXE_DIR, "embedding-oom-marker");
+var WORKER_PIDS_DIR = path.join(EXE_DIR, "worker-pids");
+var MCP_VERSION_PATH = path.join(EXE_DIR, "mcp-version");
+var COOLDOWN_MS = 6e4;
+var MIN_DAEMON_AGE_MS = 3e4;
+var SPAWN_LOCK_STALE_MS = 3e4;
+var SIGTERM_GRACE_MS = 5e3;
+var MAX_LOG_SIZE = 2 * 1024 * 1024;
+var _logDirChecked = false;
+function ensureLogDir() {
+  if (_logDirChecked) return;
+  if (!existsSync(LOG_DIR)) mkdirSync(LOG_DIR, { recursive: true });
+  _logDirChecked = true;
+}
+function maybeRotateLog() {
+  try {
+    if (!existsSync(RESTART_LOG)) return;
+    const s = statSync(RESTART_LOG);
+    if (s.size > MAX_LOG_SIZE) {
+      try {
+        renameSync(RESTART_LOG, `${RESTART_LOG}.1`);
+      } catch {
+      }
+    }
+  } catch {
+  }
+}
+function logRestart(entry) {
+  try {
+    ensureLogDir();
+    maybeRotateLog();
+    appendFileSync(RESTART_LOG, JSON.stringify({ ts: (/* @__PURE__ */ new Date()).toISOString(), pid: process.pid, ...entry }) + "\n");
+  } catch {
+  }
+}
+function logRestartEvent(event, data) {
+  logRestart({ event, ...data });
+}
+function isCooldownActive() {
+  try {
+    if (!existsSync(RESTART_MARKER)) return { active: false, remainingMs: 0 };
+    const last = parseInt(readFileSync(RESTART_MARKER, "utf8").trim(), 10);
+    if (isNaN(last)) return { active: false, remainingMs: 0 };
+    const elapsed = Date.now() - last;
+    if (elapsed < COOLDOWN_MS) return { active: true, remainingMs: COOLDOWN_MS - elapsed };
+  } catch {
+  }
+  return { active: false, remainingMs: 0 };
+}
+function isDaemonTooYoung() {
+  try {
+    return Date.now() - statSync(PID_PATH).mtimeMs < MIN_DAEMON_AGE_MS;
+  } catch {
+    return false;
+  }
+}
+var RESTART_INTENT_WINDOW_MS = 3e4;
+function writeRestartIntent(reason) {
+  const marker = {
+    reason,
+    nonce: randomUUID(),
+    createdAt: (/* @__PURE__ */ new Date()).toISOString()
+  };
+  writeFileSync(RESTART_MARKER, JSON.stringify(marker));
+}
+function isRestartIntentTimestampFresh(markerTimestampMs, nowMs = Date.now()) {
+  if (markerTimestampMs === null || !Number.isFinite(markerTimestampMs)) return false;
+  return nowMs - markerTimestampMs < RESTART_INTENT_WINDOW_MS;
+}
+function consumeRestartIntent() {
+  try {
+    if (!existsSync(RESTART_MARKER)) return false;
+    const raw = readFileSync(RESTART_MARKER, "utf8").trim();
+    try {
+      unlinkSync(RESTART_MARKER);
+    } catch {
+    }
+    if (raw.startsWith("{")) {
+      const marker = JSON.parse(raw);
+      return typeof marker.reason === "string" && typeof marker.nonce === "string";
+    }
+    const last = parseInt(raw, 10);
+    return isRestartIntentTimestampFresh(isNaN(last) ? null : last);
+  } catch {
+    return false;
+  }
+}
+function isOomMarkerSet() {
+  return existsSync(OOM_MARKER_PATH);
+}
+function acquireLock() {
+  try {
+    const fd = openSync(SPAWN_LOCK_PATH, "wx");
+    closeSync(fd);
+    return true;
+  } catch {
+    try {
+      if (Date.now() - statSync(SPAWN_LOCK_PATH).mtimeMs > SPAWN_LOCK_STALE_MS) {
+        try {
+          unlinkSync(SPAWN_LOCK_PATH);
+        } catch {
+        }
+        try {
+          const fd = openSync(SPAWN_LOCK_PATH, "wx");
+          closeSync(fd);
+          return true;
+        } catch {
+        }
+      }
+    } catch {
+    }
+    return false;
+  }
+}
+function releaseLock() {
+  try {
+    unlinkSync(SPAWN_LOCK_PATH);
+  } catch {
+  }
+}
+function readDaemonPid() {
+  try {
+    if (!existsSync(PID_PATH)) return null;
+    const pid = parseInt(readFileSync(PID_PATH, "utf8").trim(), 10);
+    return pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+function isProcessAlive(pid) {
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch {
+    return false;
+  }
+}
+function killDaemonGracefully(pid) {
+  try {
+    process.kill(pid, "SIGTERM");
+  } catch {
+    return;
+  }
+  const deadline = Date.now() + SIGTERM_GRACE_MS;
+  while (Date.now() < deadline) {
+    if (!isProcessAlive(pid)) return;
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200);
+  }
+  try {
+    process.kill(pid, "SIGKILL");
+    logRestart({ event: "sigkill_escalation", target_pid: pid, message: "SIGTERM grace expired, sent SIGKILL" });
+  } catch {
+  }
+}
+function requestRestart(reason, opts) {
+  const isEmbedRelated = opts?.isEmbedRelated ?? false;
+  const cooldown = isCooldownActive();
+  if (cooldown.active) {
+    const msg = `Restart cooldown active (${Math.round(cooldown.remainingMs / 1e3)}s left)`;
+    process.stderr.write(`[restart-orchestrator] ${msg} \u2014 skipping
+`);
+    logRestart({ event: "skipped", reason, skip_reason: "cooldown" });
+    return { ok: false, skipped: msg };
+  }
+  if (isDaemonTooYoung()) {
+    const msg = `Daemon too young (< ${MIN_DAEMON_AGE_MS / 1e3}s)`;
+    process.stderr.write(`[restart-orchestrator] ${msg} \u2014 skipping
+`);
+    logRestart({ event: "skipped", reason, skip_reason: "too_young" });
+    return { ok: false, skipped: msg };
+  }
+  if (isEmbedRelated && isOomMarkerSet()) {
+    const msg = "Embeddings disabled (OOM marker)";
+    process.stderr.write(`[restart-orchestrator] ${msg} \u2014 skipping
+`);
+    logRestart({ event: "skipped", reason, skip_reason: "oom_marker" });
+    return { ok: false, skipped: msg };
+  }
+  if (!acquireLock()) {
+    const msg = "Another process is already restarting daemon";
+    process.stderr.write(`[restart-orchestrator] ${msg} \u2014 skipping
+`);
+    logRestart({ event: "skipped", reason, skip_reason: "lock_held" });
+    return { ok: false, skipped: msg };
+  }
+  try {
+    try {
+      writeRestartIntent(reason);
+    } catch {
+    }
+    const pid = readDaemonPid();
+    logRestart({ event: "restart_initiated", reason, target_pid: pid, is_embed_related: isEmbedRelated });
+    process.stderr.write(`[restart-orchestrator] Killing daemon (PID ${pid ?? "unknown"}) \u2014 reason: ${reason}
+`);
+    if (pid !== null && isProcessAlive(pid)) killDaemonGracefully(pid);
+    try {
+      unlinkSync(PID_PATH);
+    } catch {
+    }
+    try {
+      unlinkSync(SOCKET_PATH);
+    } catch {
+    }
+    logRestart({ event: "restart_complete", reason, killed_pid: pid });
+    return { ok: true, pid, reason };
+  } finally {
+    releaseLock();
+  }
+}
+function killOrphanedDaemons() {
+  try {
+    const pids = execSync("pgrep -f 'exe-daemon.js' 2>/dev/null || true", { encoding: "utf8", timeout: 5e3 }).trim().split("\n").filter(Boolean).map(Number).filter((n) => !isNaN(n) && n !== process.pid);
+    if (pids.length === 0) return 0;
+    for (const p of pids) {
+      try {
+        process.kill(p, "SIGTERM");
+      } catch {
+      }
+    }
+    try {
+      execSync("sleep 1");
+    } catch {
+    }
+    for (const p of pids) {
+      try {
+        process.kill(p, 0);
+        process.kill(p, "SIGKILL");
+      } catch {
+      }
+    }
+    process.stderr.write(`[restart-orchestrator] killed ${pids.length} orphaned daemon(s)
+`);
+    logRestart({ event: "orphan_kill", count: pids.length, pids });
+    return pids.length;
+  } catch {
+    return 0;
+  }
+}
+function killOrphanedBackfill() {
+  try {
+    const pids = execSync("pgrep -f 'backfill-vectors.js' 2>/dev/null || true", { encoding: "utf8", timeout: 5e3 }).trim().split("\n").filter(Boolean).map(Number).filter((n) => !isNaN(n) && n !== process.pid);
+    for (const p of pids) {
+      try {
+        process.kill(p, "SIGKILL");
+      } catch {
+      }
+    }
+    if (pids.length > 0) {
+      process.stderr.write(`[restart-orchestrator] killed ${pids.length} orphaned backfill worker(s)
+`);
+      logRestart({ event: "backfill_orphan_kill", count: pids.length });
+    }
+    return pids.length;
+  } catch {
+    return 0;
+  }
+}
+function cleanWorkerPidFiles() {
+  try {
+    if (!existsSync(WORKER_PIDS_DIR)) return;
+    for (const f of readdirSync(WORKER_PIDS_DIR)) {
+      try {
+        unlinkSync(path.join(WORKER_PIDS_DIR, f));
+      } catch {
+      }
+    }
+  } catch {
+  }
+}
+function updateMcpVersionMarker() {
+  try {
+    writeFileSync(MCP_VERSION_PATH, `deploy-${Date.now()}`);
+    process.stderr.write("[restart-orchestrator] MCP version marker updated \u2014 servers will hot-reload within 10s\n");
+    logRestart({ event: "mcp_marker_updated" });
+  } catch {
+  }
+}
+function requestDeployRestart(reason) {
+  const deployReason = reason ?? "deploy";
+  if (!acquireLock()) {
+    process.stderr.write("[restart-orchestrator] Lock held \u2014 waiting briefly for deploy\n");
+    try {
+      execSync("sleep 1");
+    } catch {
+    }
+    try {
+      unlinkSync(SPAWN_LOCK_PATH);
+    } catch {
+    }
+    if (!acquireLock()) {
+      logRestart({ event: "deploy_skipped", reason: deployReason, skip_reason: "lock_held" });
+      return { ok: false, skipped: "Another process is already restarting daemon" };
+    }
+  }
+  try {
+    try {
+      writeRestartIntent(deployReason);
+    } catch {
+    }
+    logRestart({ event: "deploy_restart_initiated", reason: deployReason });
+    process.stderr.write(`[restart-orchestrator] Deploy restart \u2014 reason: ${deployReason}
+`);
+    const pid = readDaemonPid();
+    if (pid !== null && isProcessAlive(pid)) killDaemonGracefully(pid);
+    try {
+      unlinkSync(PID_PATH);
+    } catch {
+    }
+    try {
+      unlinkSync(PID_PATH + ".lock");
+    } catch {
+    }
+    const orphansKilled = killOrphanedDaemons();
+    const backfillKilled = killOrphanedBackfill();
+    updateMcpVersionMarker();
+    cleanWorkerPidFiles();
+    try {
+      unlinkSync(SOCKET_PATH);
+    } catch {
+    }
+    try {
+      execSync("sleep 0.5");
+    } catch {
+    }
+    logRestart({ event: "deploy_restart_complete", reason: deployReason, killed_pid: pid, orphans_killed: orphansKilled, backfill_killed: backfillKilled });
+    return { ok: true, pid, reason: deployReason, orphansKilled, backfillKilled };
+  } finally {
+    releaseLock();
+  }
+}
+
+export {
+  logRestartEvent,
+  consumeRestartIntent,
+  requestRestart,
+  updateMcpVersionMarker,
+  requestDeployRestart
+};