agentel 0.2.0 → 0.2.3

package/src/supervisor.js

@@ -6,18 +6,21 @@ const { spawn } = require("child_process");
 const { ensureBaseDirs, paths, readJson, writeJson } = require("./paths");
 const { effectiveImportSources, loadConfig } = require("./config");
 const { startCollector } = require("./collector");
-const { reindexIfNeeded } = require("./search");
-const { hasRemoteTarget, syncArchiveIfConfigured } = require("./sync");
+const { hasRemoteTarget } = require("./sync");
 
 let tickRunning = false;
+const CHILD_MAX_OUTPUT_BYTES = 10 * 1024 * 1024;
+const CHILD_TIMEOUT_MS = 10 * 60 * 1000;
+const CHILD_KILL_GRACE_MS = 2000;
 
-function startSupervisorDetached() {
-  const p = paths();
+function startSupervisorDetached(env = process.env) {
+  const p = paths(env);
   ensureBaseDirs(p);
   const out = fs.openSync(path.join(p.logs, "supervisor.log"), "a");
   const err = fs.openSync(path.join(p.logs, "supervisor.err.log"), "a");
   const child = spawn(process.execPath, [path.resolve(__dirname, "..", "bin", "agentlog.js"), "start", "--foreground"], {
     detached: true,
+    env,
     stdio: ["ignore", out, err]
   });
   child.unref();
@@ -39,11 +42,7 @@ async function runSupervisorForeground(env = process.env) {
     if (stopping) return;
     stopping = true;
     log("supervisor stopping", env);
-    try {
-      fs.unlinkSync(p.pid);
-    } catch {
-      // Already gone.
-    }
+    removeSupervisorPidIfOwned(process.pid, env);
     process.exit(0);
   };
   process.on("SIGTERM", stop);
@@ -64,49 +63,200 @@ async function runSupervisorForeground(env = process.env) {
 async function tick(env) {
   tickRunning = true;
   try {
-    const cfg = loadConfig(env);
-    const sources = effectiveImportSources(cfg);
-    const importSources = Array.isArray(sources) && sources.length ? sources : ["all"];
-    for (const source of importSources) {
-      try {
-        const importCliHistory = loadImportCliHistory();
-        const imports = importCliHistory(supervisorImportOptionsForSource(source, cfg), env);
-        for (const result of imports) {
-          const summary = supervisorImportResultLog(result, source);
-          if (summary) log(summary, env);
-          if (result.errors?.length) log(`${result.provider} import errors from ${source}: ${result.errors.length}`, env);
+    try {
+      const cfg = loadConfig(env);
+      const sources = effectiveImportSources(cfg);
+      const importSources = Array.isArray(sources) && sources.length ? sources : ["all"];
+      for (const source of importSources) {
+        try {
+          const imports = await importSourceInChild(source, cfg, env);
+          for (const result of imports) {
+            const summary = supervisorImportResultLog(result, source);
+            if (summary) log(summary, env);
+            if (result.errors?.length) log(`${result.provider} import errors from ${source}: ${result.errors.length}`, env);
+          }
+        } catch (error) {
+          log(`${source} history import skipped: ${error.message}`, env);
         }
-      } catch (error) {
-        log(`${source} history import skipped: ${error.message}`, env);
       }
+    } catch (error) {
+      log(`history import skipped: ${error.message}`, env);
     }
-  } catch (error) {
-    log(`history import skipped: ${error.message}`, env);
-  }
 
-  try {
-    const result = reindexIfNeeded(env);
-    if (result.index?.docCount != null) log(`index ready (${result.index.docCount} chunk(s))`, env);
-  } catch (error) {
-    log(`index skipped: ${error.message}`, env);
-  }
+    try {
+      const result = await reindexInChild(env);
+      if (result.index?.docCount != null) log(`index ready (${result.index.docCount} chunk(s))`, env);
+    } catch (error) {
+      log(`index skipped: ${error.message}`, env);
+    }
 
-  try {
-    const cfg = loadConfig(env);
-    if (hasRemoteTarget(cfg, env) && shouldRunScheduledSync(cfg, env)) {
-      markScheduledSyncAttempt(env);
-      const result = await syncArchiveIfConfigured(env);
-      if (result.configured !== false && (result.uploaded || result.errors?.length)) {
-        log(`remote sync uploaded=${result.uploaded} current=${result.skipped} retried=${result.retried || 0} errors=${result.errors.length}`, env);
+    try {
+      const cfg = loadConfig(env);
+      if (hasRemoteTarget(cfg, env) && shouldRunScheduledSync(cfg, env)) {
+        markScheduledSyncAttempt(env);
+        const result = await syncInChild(env);
+        if (result.configured !== false && (result.uploaded || result.errors?.length)) {
+          log(`remote sync uploaded=${result.uploaded} current=${result.skipped} retried=${result.retried || 0} errors=${result.errors.length}`, env);
+        }
       }
+    } catch (error) {
+      log(`remote sync skipped: ${error.message}`, env);
     }
-  } catch (error) {
-    log(`remote sync skipped: ${error.message}`, env);
   } finally {
     tickRunning = false;
   }
 }
 
+function importSourceInChild(source, config, env = process.env) {
+  const options = supervisorImportOptionsForSource(source, config);
+  const title = childProcessTitle("import", source);
+  const script = `
+process.title = ${JSON.stringify(title)};
+try {
+  const { importCliHistory } = require(${JSON.stringify(path.join(__dirname, "importers"))});
+  const results = importCliHistory(${JSON.stringify(options)}, process.env);
+  process.stdout.write(JSON.stringify(results));
+} catch (error) {
+  console.error(error && error.message ? error.message : String(error));
+  process.exit(1);
+}
+`;
+  return runJsonChild(script, env, `${source} import`, title);
+}
+
+function reindexInChild(env = process.env) {
+  const title = childProcessTitle("index");
+  const script = `
+process.title = ${JSON.stringify(title)};
+try {
+  const { reindexIfNeeded } = require(${JSON.stringify(path.join(__dirname, "search"))});
+  const result = reindexIfNeeded(process.env, { rebuildInProcess: true });
+  process.stdout.write(JSON.stringify({
+    paused: Boolean(result.paused),
+    rebuilt: Boolean(result.rebuilt),
+    index: result.index
+      ? {
+          version: result.index.version,
+          builtAt: result.index.builtAt,
+          docCount: result.index.docCount,
+          avgDocLength: result.index.avgDocLength,
+          summaryOnly: true
+        }
+      : null
+  }));
+} catch (error) {
+  console.error(error && error.message ? error.message : String(error));
+  process.exit(1);
+}
+`;
+  return runJsonChild(script, env, "index", title);
+}
+
+function syncInChild(env = process.env) {
+  const title = childProcessTitle("sync");
+  const script = `
+process.title = ${JSON.stringify(title)};
+(async () => {
+  const { syncArchiveIfConfigured } = require(${JSON.stringify(path.join(__dirname, "sync"))});
+  const result = await syncArchiveIfConfigured(process.env);
+  process.stdout.write(JSON.stringify(result));
+})().catch((error) => {
+  console.error(error && error.message ? error.message : String(error));
+  process.exitCode = 1;
+});
+`;
+  return runJsonChild(script, env, "remote sync", title);
+}
+
+function runJsonChild(script, env = process.env, label = "child process", title = childProcessTitle("worker"), options = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(process.execPath, ["-e", script], {
+      argv0: title,
+      env,
+      stdio: ["ignore", "pipe", "pipe"]
+    });
+    const stdout = [];
+    const stderr = [];
+    let stdoutBytes = 0;
+    let stderrBytes = 0;
+    let settled = false;
+    let timedOut = false;
+    let killTimer = null;
+    const timeoutMs = normalizeChildTimeout(options.timeoutMs);
+    const timeoutTimer = setTimeout(() => {
+      timedOut = true;
+      child.kill("SIGTERM");
+      killTimer = setTimeout(() => child.kill("SIGKILL"), CHILD_KILL_GRACE_MS);
+      if (typeof killTimer.unref === "function") killTimer.unref();
+    }, timeoutMs);
+    if (typeof timeoutTimer.unref === "function") timeoutTimer.unref();
+
+    function clearTimers() {
+      clearTimeout(timeoutTimer);
+      if (killTimer) clearTimeout(killTimer);
+    }
+
+    function fail(error) {
+      if (settled) return;
+      settled = true;
+      clearTimers();
+      reject(error);
+    }
+
+    child.stdout.on("data", (chunk) => {
+      stdoutBytes += chunk.length;
+      if (stdoutBytes > CHILD_MAX_OUTPUT_BYTES) {
+        child.kill("SIGTERM");
+        fail(new Error(`${label} output exceeded ${CHILD_MAX_OUTPUT_BYTES} bytes`));
+        return;
+      }
+      stdout.push(chunk);
+    });
+    child.stderr.on("data", (chunk) => {
+      stderrBytes += chunk.length;
+      if (stderrBytes <= CHILD_MAX_OUTPUT_BYTES) stderr.push(chunk);
+    });
+    child.on("error", fail);
+    child.on("close", (code, signal) => {
+      if (settled) return;
+      settled = true;
+      clearTimers();
+      if (timedOut) {
+        reject(new Error(`${label} timed out after ${formatDuration(timeoutMs)}`));
+        return;
+      }
+      const output = Buffer.concat(stdout).toString("utf8").trim();
+      const errorOutput = Buffer.concat(stderr).toString("utf8").trim();
+      if (code !== 0) {
+        reject(new Error(errorOutput || `${label} exited with ${signal || `status ${code}`}`));
+        return;
+      }
+      try {
+        resolve(output ? JSON.parse(output) : null);
+      } catch (error) {
+        reject(new Error(`${label} returned invalid JSON: ${error.message}`));
+      }
+    });
+  });
+}
+
+function normalizeChildTimeout(value) {
+  const timeout = Number(value || CHILD_TIMEOUT_MS);
+  return Number.isFinite(timeout) && timeout > 0 ? timeout : CHILD_TIMEOUT_MS;
+}
+
+function formatDuration(ms) {
+  const seconds = Math.round(ms / 1000);
+  if (seconds < 60) return `${seconds}s`;
+  const minutes = Math.round(seconds / 60);
+  return `${minutes}m`;
+}
+
+function childProcessTitle(kind, detail = "") {
+  const suffix = detail ? `-${String(detail).replace(/[^A-Za-z0-9._-]+/g, "-").replace(/^-+|-+$/g, "")}` : "";
+  return `agentlog-${kind}${suffix}`.slice(0, 64);
+}
+
 function shouldRunScheduledSync(config, env = process.env, now = new Date()) {
   const intervalMinutes = Number(config?.sync?.intervalMinutes ?? 30);
   if (!Number.isFinite(intervalMinutes) || intervalMinutes <= 0) return false;
@@ -143,7 +293,6 @@ function supervisorImportResultLog(result, source) {
 }
 
 function stopSupervisor(env = process.env) {
-  const p = paths(env);
   const pid = readPid(env);
   if (!pid) return false;
   try {
@@ -151,11 +300,7 @@ function stopSupervisor(env = process.env) {
     return true;
   } catch (error) {
     if (error?.code !== "EPERM") {
-      try {
-        fs.unlinkSync(p.pid);
-      } catch {
-        // No-op.
-      }
+      removeSupervisorPidIfOwned(pid, env);
     }
     return false;
   }
@@ -178,6 +323,16 @@ function readPid(env = process.env) {
   }
 }
 
+function removeSupervisorPidIfOwned(pid, env = process.env) {
+  if (!pid || readPid(env) !== Number(pid)) return false;
+  try {
+    fs.unlinkSync(paths(env).pid);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 function isAlive(pid) {
   try {
     process.kill(pid, 0);
@@ -188,17 +343,6 @@ function isAlive(pid) {
   }
 }
 
-function loadImportCliHistory() {
-  const modulePath = require.resolve("./importers");
-  const loaded = require(modulePath);
-  if (typeof loaded.importCliHistory === "function") return loaded.importCliHistory;
-
-  delete require.cache[modulePath];
-  const reloaded = require(modulePath);
-  if (typeof reloaded.importCliHistory === "function") return reloaded.importCliHistory;
-  throw new Error("importer module did not expose importCliHistory; restart agentlog or reinstall");
-}
-
 function log(message, env = process.env) {
   const p = paths(env);
   ensureBaseDirs(p);
@@ -212,6 +356,11 @@ module.exports = {
   supervisorImportResultLog,
   supervisorImportOptionsForSource,
   supervisorStatus,
+  removeSupervisorPidIfOwned,
   shouldRunScheduledSync,
+  importSourceInChild,
+  reindexInChild,
+  runJsonChild,
+  syncInChild,
   tick
 };