@nodii/dev 0.6.28 → 0.6.29

package/dist/watchdog.js

@@ -0,0 +1,308 @@
+import { execFile as execFileCallback } from "node:child_process";
+import { mkdir, readFile, readdir, rename, rm, writeFile, chmod } from "node:fs/promises";
+import { homedir, platform } from "node:os";
+import { dirname, isAbsolute, join, resolve } from "node:path";
+import { promisify } from "node:util";
+import { CliError, configPath, devRequest, encode, readConfig } from "./core.js";
+
+const execFile = promisify(execFileCallback);
+export const WATCHDOG_LABEL = "com.nodii.watchdog";
+const RUN_ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
+
+export const watchdogHome = (home = homedir()) => join(home, ".config", "nodii-dev", "watchdog");
+export const watchdogRunDir = (home = homedir()) => join(watchdogHome(home), "runs");
+
+function validateRecord(record) {
+  if (!record || typeof record !== "object") throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run record must be an object.", 2);
+  if (!RUN_ID.test(record.runId ?? "")) throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run record needs its Dev run UUID.", 2);
+  if (!Number.isSafeInteger(record.pid) || record.pid < 1) throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run record needs a positive process id.", 2);
+  if (record.configPath !== undefined && (typeof record.configPath !== "string" || !isAbsolute(record.configPath)))
+    throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run config path must be absolute.", 2);
+  if (record.worktree !== undefined && (typeof record.worktree !== "string" || !isAbsolute(record.worktree)))
+    throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run worktree must be an absolute path.", 2);
+  return record;
+}
+
+const serializableRecord = (record) => Object.fromEntries(Object.entries(record).filter(([key]) => key !== "registryPath"));
+
+async function atomicWrite(path, contents, mode = 0o600) {
+  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+  const temporary = `${path}.tmp-${process.pid}`;
+  await writeFile(temporary, contents, { mode });
+  await rename(temporary, path);
+  await chmod(path, mode);
+}
+
+/** Persisted by lane launch / run wrappers; one file means concurrent runs never rewrite a shared array. */
+export async function recordWatchedRun(input, { home = homedir(), now = () => new Date(), processStart = processStartedAt } = {}) {
+  const record = validateRecord({ ...input, version: 1, recordedAt: now().toISOString() });
+  if (!record.processStartedAt) record.processStartedAt = await processStart(record.pid).catch(() => null);
+  const dir = watchdogRunDir(home);
+  await mkdir(dir, { recursive: true, mode: 0o700 });
+  await chmod(dir, 0o700);
+  const path = join(dir, `${record.runId}.json`);
+  await atomicWrite(path, `${JSON.stringify(serializableRecord(record), null, 2)}\n`);
+  return { ...record, registryPath: path };
+}
+
+export async function forgetWatchedRun(runId, { home = homedir() } = {}) {
+  if (!RUN_ID.test(runId ?? "")) throw new CliError("WATCHDOG_RECORD_INVALID", "A watched run id must be a UUID.", 2);
+  await rm(join(watchdogRunDir(home), `${runId}.json`), { force: true });
+}
+
+export async function readWatchedRuns({ home = homedir() } = {}) {
+  const dir = watchdogRunDir(home);
+  const names = (await readdir(dir).catch((error) => error?.code === "ENOENT" ? [] : Promise.reject(error)))
+    .filter((name) => name.endsWith(".json"));
+  const records = [], errors = [];
+  for (const name of names) {
+    try {
+      const record = validateRecord(JSON.parse(await readFile(join(dir, name), "utf8")));
+      records.push({ ...record, registryPath: join(dir, name) });
+    } catch (error) {
+      errors.push({ kind: "registry", file: name, code: error?.code ?? "WATCHDOG_RECORD_INVALID" });
+    }
+  }
+  return { records, errors };
+}
+
+export async function saveWatchedRun(record) {
+  const path = record.registryPath;
+  if (!path || !isAbsolute(path)) throw new CliError("WATCHDOG_RECORD_INVALID", "The watched run record has no registry path.", 2);
+  await atomicWrite(path, `${JSON.stringify(serializableRecord(validateRecord(record)), null, 2)}\n`);
+}
+
+export async function processStartedAt(pid, { execute = execFile } = {}) {
+  const { stdout } = await execute("ps", ["-p", String(pid), "-o", "lstart="]);
+  const value = stdout.trim();
+  if (!value) throw new Error(`process ${pid} is absent`);
+  return value;
+}
+
+export async function processStatus(pid, { execute = execFile } = {}) {
+  const { stdout } = await execute("ps", ["-p", String(pid), "-o", "stat="]);
+  return stdout.trim();
+}
+
+export async function isProcessAlive(pid, expectedStartedAt, { signal = process.kill, processStart = processStartedAt, processStatus: status = processStatus } = {}) {
+  let exists = true;
+  try { signal(pid, 0); }
+  catch (error) {
+    if (error?.code !== "EPERM") return false;
+    exists = true;
+  }
+  try { if (/^Z/.test(await status(pid))) return false; }
+  catch { return exists; }
+  if (!expectedStartedAt) return true;
+  try { return (await processStart(pid)) === expectedStartedAt; }
+  catch { return false; }
+}
+
+export async function findStaleSockets({ socketDir = "/tmp/cc-socks", isPidAlive = isProcessAlive } = {}) {
+  const entries = await readdir(socketDir, { withFileTypes: true }).catch((error) => error?.code === "ENOENT" ? [] : Promise.reject(error));
+  const stale = [];
+  for (const entry of entries) {
+    const match = /^(\d+)\.sock$/.exec(entry.name);
+    if (!match || !entry.isSocket()) continue;
+    const pid = Number(match[1]);
+    if (!Number.isSafeInteger(pid) || pid < 1) continue;
+    if (!(await isPidAlive(pid))) stale.push({ path: join(socketDir, entry.name), pid });
+  }
+  return stale;
+}
+
+const git = async (worktree, args, execute = execFile) => (await execute("git", ["-C", worktree, ...args], { encoding: "utf8", maxBuffer: 16 * 1024 * 1024 })).stdout.trim();
+
+/** Compare a worktree with the remote-tracking branch its branch actually uses. */
+export async function inspectWorktree(worktree, { execute = execFile } = {}) {
+  const root = await git(worktree, ["rev-parse", "--show-toplevel"], execute);
+  const branch = await git(worktree, ["branch", "--show-current"], execute);
+  if (!branch) return { worktree: root, branch: null, remoteBranch: null, aheadCount: 0, commits: [], error: "detached_head" };
+  let remoteBranch;
+  try { remoteBranch = await git(worktree, ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{upstream}"], execute); }
+  catch {
+    const conventional = `origin/${branch}`;
+    try { await git(worktree, ["rev-parse", "--verify", conventional], execute); remoteBranch = conventional; }
+    catch { return { worktree: root, branch, remoteBranch: null, aheadCount: 0, commits: [], error: "remote_branch_missing" }; }
+  }
+  const aheadCount = Number(await git(worktree, ["rev-list", "--count", `${remoteBranch}..HEAD`], execute));
+  const commits = aheadCount ? (await git(worktree, ["rev-list", "--max-count=100", `${remoteBranch}..HEAD`], execute)).split("\n").filter(Boolean) : [];
+  return { worktree: root, branch, remoteBranch, aheadCount, commits, ...(aheadCount > commits.length ? { commitsTruncated: true } : {}) };
+}
+
+export async function collectWatchdogFacts(records, {
+  isPidAlive = isProcessAlive,
+  listStaleSockets = () => findStaleSockets({ isPidAlive }),
+  inspect = inspectWorktree,
+  now = () => new Date(),
+  errors = [],
+} = {}) {
+  const deadRuns = [];
+  for (const record of records) {
+    if (!(await isPidAlive(record.pid, record.processStartedAt)))
+      deadRuns.push({ runId: record.runId, pid: record.pid, ...(record.worktree ? { worktree: record.worktree } : {}) });
+  }
+  const unpushedWorktrees = [];
+  for (const worktree of new Set(records.map((record) => record.worktree).filter(Boolean))) {
+    try {
+      const result = await inspect(worktree);
+      if (result.aheadCount > 0 || result.error === "remote_branch_missing") unpushedWorktrees.push(result);
+    } catch (error) {
+      errors.push({ kind: "worktree", worktree, code: error?.code ?? "GIT_INSPECTION_FAILED" });
+    }
+  }
+  let staleSockets = [];
+  try { staleSockets = await listStaleSockets(); }
+  catch (error) { errors.push({ kind: "socket", code: error?.code ?? "SOCKET_INSPECTION_FAILED" }); }
+  return { checkedAt: now().toISOString(), deadRuns, staleSockets, unpushedWorktrees, errors };
+}
+
+async function assertCoordinator(config, request) {
+  if (!config?.agentId) throw new CliError("WATCHDOG_COORDINATOR_REQUIRED", "The watchdog needs a registered machine coordinator config.", 2);
+  const agent = (await request("GET", `/api/v1/agents/${encode(config.agentId)}`)).data;
+  if (agent?.role !== "coordinator" || !agent.machineId)
+    throw new CliError("WATCHDOG_COORDINATOR_REQUIRED", "Install and run the per-Mac watchdog with that machine's coordinator agent config.", 2);
+  return agent;
+}
+
+async function coordinatorMachine({ request = devRequest, config = readConfig } = {}) {
+  return assertCoordinator(await config(), request);
+}
+
+const observationBody = ({ observation, machineId, pid, observedAt, detail }) => ({
+  observation, machineId, ...(pid === undefined ? {} : { pid }), observedAt, ...(detail ? { detail } : {}),
+});
+
+/** One launchd sweep. The machine coordinator reports every observation through Dev's run watchdog contract. */
+export async function sweepWatchdog({
+  records,
+  loadRecords = readWatchedRuns,
+  collect = collectWatchdogFacts,
+  request = devRequest,
+  saveRecord = saveWatchedRun,
+  machineId,
+  now = () => new Date(),
+} = {}) {
+  const loaded = records ? { records, errors: [] } : await loadRecords();
+  const facts = await collect(loaded.records, { errors: [...loaded.errors] });
+  const byId = new Map(loaded.records.map((record) => [record.runId, record]));
+  const effectiveMachineId = machineId ?? (await coordinatorMachine({ request })).machineId;
+  const orphanedRuns = [];
+  // Record the non-terminal observations first: process_dead transitions the
+  // run to Orphan immediately, and must not prevent its stale socket or local
+  // commits from reaching Dev in the same sweep.
+  for (const socket of facts.staleSockets) {
+    const record = loaded.records.find((candidate) => candidate.pid === socket.pid);
+    if (!record) continue;
+    try {
+      await request("POST", `/api/v1/runs/${encode(record.runId)}/watchdog`, observationBody({
+        observation: "socket_stale", machineId: effectiveMachineId, pid: socket.pid, observedAt: now().toISOString(),
+        detail: `${socket.path} has no live process.`,
+      }));
+    } catch (error) {
+      facts.errors.push({ kind: "run", runId: record.runId, observation: "socket_stale", code: error?.code ?? "RUN_WATCHDOG_REPORT_FAILED" });
+    }
+  }
+  for (const worktree of facts.unpushedWorktrees) {
+    const record = loaded.records.find((candidate) => candidate.worktree && resolve(candidate.worktree) === resolve(worktree.worktree));
+    if (!record) continue;
+    const remote = worktree.remoteBranch ?? "its missing remote branch";
+    const commits = worktree.commits.slice(0, 20).join(",");
+    const detail = worktree.error === "remote_branch_missing"
+      ? `${worktree.branch} has no remote branch.`
+      : `${worktree.branch} is ${worktree.aheadCount} commit(s) ahead of ${remote}${commits ? `: ${commits}` : ""}`;
+    try {
+      await request("POST", `/api/v1/runs/${encode(record.runId)}/watchdog`, observationBody({
+        observation: "worktree_unpushed", machineId: effectiveMachineId, observedAt: now().toISOString(), detail,
+      }));
+    } catch (error) {
+      facts.errors.push({ kind: "run", runId: record.runId, observation: "worktree_unpushed", code: error?.code ?? "RUN_WATCHDOG_REPORT_FAILED" });
+    }
+  }
+  for (const dead of facts.deadRuns) {
+    const record = byId.get(dead.runId);
+    if (!record || record.orphanReportedAt || record.terminalSeenAt) continue;
+    try {
+      const observedAt = now().toISOString();
+      const response = await request("POST", `/api/v1/runs/${encode(record.runId)}/watchdog`, observationBody({
+        observation: "process_dead", machineId: effectiveMachineId, pid: record.pid, observedAt,
+        detail: "Process exited without a terminal run state.",
+      }));
+      if (response?.data?.run?.livenessState === "orphan") {
+        record.orphanReportedAt = observedAt;
+        orphanedRuns.push({ runId: record.runId, pid: record.pid, reportedAt: record.orphanReportedAt });
+      } else if (["done", "handed_off", "failed"].includes(response?.data?.run?.status)) {
+        record.terminalSeenAt = observedAt;
+      } else {
+        throw new CliError("RUN_WATCHDOG_RESPONSE_INVALID", "Dev recorded process_dead without returning the run as orphan or terminal.", 1);
+      }
+      await saveRecord(record);
+    } catch (error) {
+      facts.errors.push({ kind: "run", runId: record.runId, observation: "process_dead", code: error?.code ?? "RUN_WATCHDOG_REPORT_FAILED" });
+    }
+  }
+  const report = { ...facts, orphanedRuns };
+  return report;
+}
+
+const xml = (value) => String(value).replaceAll("&", "&amp;").replaceAll("<", "&lt;").replaceAll(">", "&gt;").replaceAll('"', "&quot;").replaceAll("'", "&apos;");
+
+export function launchdPlist({ runtime, entry, config, interval, stdoutPath, stderrPath }) {
+  const args = [runtime, entry, "watchdog", "sweep", "--json"].map((value) => `<string>${xml(value)}</string>`).join("");
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n<plist version="1.0"><dict><key>Label</key><string>${WATCHDOG_LABEL}</string><key>ProgramArguments</key><array>${args}</array><key>EnvironmentVariables</key><dict><key>NODII_DEV_CONFIG_PATH</key><string>${xml(config)}</string></dict><key>RunAtLoad</key><true/><key>StartInterval</key><integer>${interval}</integer><key>ProcessType</key><string>Background</string><key>StandardOutPath</key><string>${xml(stdoutPath)}</string><key>StandardErrorPath</key><string>${xml(stderrPath)}</string></dict></plist>\n`;
+}
+
+export async function installWatchdog({
+  home = homedir(),
+  uid = process.getuid?.(),
+  os = platform(),
+  interval = 60,
+  runtime = process.execPath,
+  entry = resolve(process.argv[1]),
+  configFile = configPath(),
+  config = readConfig,
+  request = devRequest,
+  execute = execFile,
+} = {}) {
+  if (os !== "darwin") throw new CliError("WATCHDOG_UNSUPPORTED", "The per-Mac watchdog installs with launchd and is supported on macOS only.", 2);
+  if (!Number.isInteger(interval) || interval < 10 || interval > 3600)
+    throw new CliError("USAGE", "watchdog install --interval must be 10..3600 seconds.", 2);
+  if (!Number.isInteger(uid) || uid < 0) throw new CliError("WATCHDOG_INSTALL_FAILED", "Cannot determine the macOS user id for launchd.", 1);
+  const current = typeof config === "function" ? await config() : config;
+  await assertCoordinator(current, request);
+  const logs = join(home, "Library", "Logs", "nodii-dev");
+  const plistPath = join(home, "Library", "LaunchAgents", `${WATCHDOG_LABEL}.plist`);
+  const contents = launchdPlist({ runtime, entry, config: resolve(configFile), interval, stdoutPath: join(logs, "watchdog.log"), stderrPath: join(logs, "watchdog.error.log") });
+  await mkdir(logs, { recursive: true, mode: 0o700 });
+  const previous = await readFile(plistPath, "utf8").catch((error) => error?.code === "ENOENT" ? null : Promise.reject(error));
+  const service = `gui/${uid}/${WATCHDOG_LABEL}`;
+  const domain = `gui/${uid}`;
+  let loaded = true;
+  try { await execute("launchctl", ["print", service]); }
+  catch { loaded = false; }
+  if (previous === contents && loaded) return { changed: false, loaded: true, plistPath, label: WATCHDOG_LABEL, interval };
+  if (loaded) await execute("launchctl", ["bootout", service]);
+  try {
+    if (previous !== contents) await atomicWrite(plistPath, contents);
+    await execute("launchctl", ["bootstrap", domain, plistPath]);
+  } catch (error) {
+    if (previous === null) await rm(plistPath, { force: true });
+    else await atomicWrite(plistPath, previous);
+    if (loaded && previous !== null) await execute("launchctl", ["bootstrap", domain, plistPath]).catch(() => {});
+    throw new CliError("WATCHDOG_INSTALL_FAILED", `Could not load ${WATCHDOG_LABEL}: ${error?.message ?? error}`, 1);
+  }
+  return { changed: previous !== contents, loaded: true, plistPath, label: WATCHDOG_LABEL, interval };
+}
+
+export async function watchdogCommand(action, flags, dependencies = {}) {
+  if (action === "install") {
+    const interval = Number(flags.interval ?? 60);
+    const data = await installWatchdog({ interval, ...dependencies });
+    return { payload: { data }, human: `${data.label} is loaded from ${data.plistPath}; sweep every ${data.interval}s${data.changed ? "." : " (already current)."}` };
+  }
+  if (action === "sweep") {
+    const data = await sweepWatchdog(dependencies);
+    return { payload: { data }, human: `Watchdog sweep: ${data.orphanedRuns.length} orphaned run(s), ${data.staleSockets.length} stale socket(s), ${data.unpushedWorktrees.length} unpushed worktree(s), ${data.errors.length} error(s).` };
+  }
+  throw new CliError("USAGE", "watchdog install [--interval SECONDS] | watchdog sweep", 2);
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@nodii/dev",
-  "version": "0.6.28",
+  "version": "0.6.29",
   "description": "Agent CLI for Nodii Dev, reached through the platform edge",
   "type": "module",
   "bin": {
@@ -21,7 +21,8 @@
     "test": "node --test test/*.test.js",
     "pack:smoke": "node scripts/pack-smoke.mjs",
     "openapi:snapshot": "node scripts/openapi-snapshot.mjs",
-    "openapi:ad-era": "node scripts/ad-era-openapi-snapshot.mjs"
+    "openapi:ad-era": "node scripts/ad-era-openapi-snapshot.mjs",
+    "openapi:watchdog": "node scripts/watchdog-openapi-snapshot.mjs"
   },
   "license": "UNLICENSED",
   "publishConfig": {