@opsee/cli 0.11.9

package/src/commands/foreman-service.ts

@@ -0,0 +1,186 @@
+/**
+ * `opsee foreman service install | uninstall | status` (story 16): the daemon under the machine's
+ * service manager. `install` writes the launchd LaunchAgent or the systemd user unit for the
+ * checkout it runs in and prints the commands that load it (runs them with --start); `uninstall`
+ * removes the file (stops it first with --stop); `status` says whether the unit file is present and
+ * whether a daemon is up. The unit runs `opsee foreman up`, nothing else: reboot continuity is the
+ * daemon's own Reconcile (ADR-0009), and the unit carries no credential.
+ */
+import { chmodSync, existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import type { AccountStore } from "../foreman/account-store.js";
+import { ForemanError } from "../foreman/core/run.js";
+import { serviceUnitFor, shellLine, type ServiceCommand, type ServiceTarget, type ServiceUnit } from "../foreman/service-unit.js";
+import { liveDaemonPid, type DaemonProbe } from "./foreman.js";
+
+export type ServiceAction = "install" | "uninstall" | "status";
+
+export interface ForemanServiceArgs {
+  action: ServiceAction;
+  /** install: load and start the unit after writing it. */
+  start: boolean;
+  /** uninstall: stop and unload the unit before removing it. */
+  stop: boolean;
+}
+
+/** The result of one service-manager command: exit code and everything it printed. */
+export interface ExecResult {
+  code: number;
+  output: string;
+}
+
+export interface ForemanServiceDeps {
+  platform: NodeJS.Platform;
+  home: string;
+  uid?: number;
+  /** The node binary and the CLI entry the unit runs, both absolute. */
+  node: string;
+  bin: string;
+  /** The checkout the daemon serves: where the command runs. */
+  repoRoot: () => Promise<string | undefined>;
+  localDir: string;
+  pidFile: string;
+  /** How `status` tells a live daemon from a stale pid file (foreman.ts); the real probe by default. */
+  probe?: DaemonProbe;
+  /** The environment the unit inherits PATH from. */
+  env: NodeJS.ProcessEnv;
+  /** Registered Accounts: a key-by-name one names the variable the unit must not carry. */
+  store: AccountStore;
+  /** Runs a service-manager command (`launchctl`, `systemctl`) when asked to; never a shell. */
+  exec: (file: string, args: string[]) => Promise<ExecResult>;
+  out: (line: string) => void;
+}
+
+/** A variable name that looks like a credential, dropped from the unit whatever an Account says:
+ * the exclusion set from the store is the rule, this is the net under it. */
+export const SECRET_NAME = /TOKEN|KEY|SECRET|PASSWORD/i;
+
+/** The variables the unit must never hold: the CLI's own token, and every variable a key-by-name
+ * Account reads its key from (its keyRef is any name the developer chose, OPSEE_* included). */
+export function credentialNames(store: AccountStore): Set<string> {
+  const names = new Set(["OPSEE_API_TOKEN"]);
+  for (const account of store.load()) if (account.type === "api_key") names.add(account.keyRef);
+  return names;
+}
+
+/** The OPSEE_* overrides set where install ran (backend URL, local paths), minus every credential:
+ * the daemon must find the same backend and files the developer's shell does, and the unit must
+ * never hold a token or a key (that is the credentials file's and the Account store's job). A name
+ * in `exclude` is dropped by rule; a name that merely looks like a secret is dropped as a net. */
+export function carriedEnv(env: NodeJS.ProcessEnv, exclude: Iterable<string> = ["OPSEE_API_TOKEN"]): Record<string, string> {
+  const excluded = new Set(exclude);
+  const carried: Record<string, string> = {};
+  for (const [key, value] of Object.entries(env)) {
+    if (!key.startsWith("OPSEE_") || value === undefined) continue;
+    if (excluded.has(key) || SECRET_NAME.test(key)) continue;
+    carried[key] = value;
+  }
+  return carried;
+}
+
+export async function runForemanService(deps: ForemanServiceDeps, args: ForemanServiceArgs): Promise<number> {
+  const repoRoot = await deps.repoRoot();
+  if (!repoRoot) throw new ForemanError("Not inside a git repository; run foreman service from the checkout the daemon should serve.");
+  const target: ServiceTarget = {
+    home: deps.home,
+    node: deps.node,
+    bin: deps.bin,
+    repoRoot,
+    localDir: deps.localDir,
+    path: deps.env.PATH ?? "",
+    env: carriedEnv(deps.env, credentialNames(deps.store)),
+    uid: deps.uid,
+  };
+  const unit = serviceUnitFor(deps.platform, target);
+  if (!unit) throw new ForemanError(`No service manager is supported on ${deps.platform}; the daemon runs under launchd (macOS) or systemd (Linux).`);
+
+  switch (args.action) {
+    case "status":
+      return status(deps, unit);
+    case "install":
+      return install(deps, unit, repoRoot, args.start);
+    case "uninstall":
+      return uninstall(deps, unit, args.stop);
+  }
+}
+
+function status(deps: ForemanServiceDeps, unit: ServiceUnit): number {
+  const installed = existsSync(unit.path);
+  deps.out(installed ? `${unit.manager} unit installed (file present): ${unit.path}` : `${unit.manager} unit not installed (would be ${unit.path}); install it with: opsee foreman service install`);
+  const pid = liveDaemonPid(deps.pidFile, deps.probe);
+  deps.out(pid !== undefined ? `Foreman daemon up (pid ${pid})` : `Foreman daemon not running (no live pid in ${deps.pidFile})`);
+  return 0;
+}
+
+async function install(deps: ForemanServiceDeps, unit: ServiceUnit, repoRoot: string, start: boolean): Promise<number> {
+  const { out } = deps;
+  mkdirSync(dirname(unit.path), { recursive: true });
+  // The log the unit points at lives in the local dir, which the service manager will not create.
+  mkdirSync(deps.localDir, { recursive: true, mode: 0o700 });
+  // Owner-only: the unit names this machine's paths and backend, and `mode` only applies to a new
+  // file, so an existing one is tightened too.
+  writeFileSync(unit.path, unit.content, { mode: 0o600 });
+  chmodSync(unit.path, 0o600);
+  const replaces = unit.manager === "launchd" ? "unloads a loaded copy before loading this one" : "restarts a running copy on this one";
+  out(`Wrote ${unit.manager} unit ${unit.path}: runs "opsee foreman up" in ${repoRoot} at ${unit.manager === "launchd" ? "login" : "boot"} and restarts it after a crash; starting it ${replaces}.`);
+  for (const account of deps.store.load()) {
+    if (account.type === "api_key") {
+      out(`Note: Account "${account.name}" reads its key from $${account.keyRef}, which the unit does not carry; add it to the unit's environment yourself before starting it (the Foreman never writes a credential).`);
+    }
+  }
+  for (const note of unit.notes) out(note);
+  if (!start) {
+    out("To load and start it:");
+    for (const command of unit.start) out(`  ${shellLine(command.argv)}`);
+    out("Or rerun with --start to have this command do it.");
+    return 0;
+  }
+  return run(deps, "start", unit.start);
+}
+
+async function uninstall(deps: ForemanServiceDeps, unit: ServiceUnit, stop: boolean): Promise<number> {
+  const { out } = deps;
+  if (!existsSync(unit.path)) {
+    out(`${unit.manager} unit not installed (no ${unit.path}); nothing to remove.`);
+    return 0;
+  }
+  if (stop) {
+    const code = await run(deps, "stop", unit.stop);
+    if (code !== 0) return code;
+  } else {
+    out("If it is loaded, stop it first (or rerun with --stop):");
+    for (const command of unit.stop) out(`  ${shellLine(command.argv)}`);
+  }
+  rmSync(unit.path, { force: true });
+  out(`Removed ${unit.path}`);
+  return 0;
+}
+
+/** Whether a non-zero exit is one the command declares harmless (launchctl bootout exits 3, "No
+ * such process", when the unit was not loaded). */
+function toleratedExit(command: ServiceCommand, result: ExecResult): boolean {
+  const t = command.tolerated;
+  return t !== undefined && (t.codes.includes(result.code) || (t.output !== undefined && t.output.test(result.output)));
+}
+
+/** Runs the service manager's commands in order, stopping at the first that fails. */
+async function run(deps: ForemanServiceDeps, what: "start" | "stop", commands: ServiceCommand[]): Promise<number> {
+  for (const command of commands) {
+    const [file, ...args] = command.argv;
+    const line = shellLine(command.argv);
+    deps.out(`$ ${line}`);
+    const result = await deps.exec(file, args);
+    const output = result.output.trim();
+    if (result.code !== 0 && toleratedExit(command, result)) {
+      deps.out(`(${command.tolerated!.means}: "${line}" exited ${result.code})`);
+      continue;
+    }
+    if (output) deps.out(output);
+    if (result.code !== 0) {
+      deps.out(`${what === "start" ? "Start" : "Stop"} failed: "${line}" exited ${result.code}`);
+      return 1;
+    }
+  }
+  deps.out(what === "start" ? "Foreman service started; watch it with: opsee foreman service status" : "Foreman service stopped");
+  return 0;
+}

package/src/commands/foreman-up.ts

@@ -0,0 +1,165 @@
+/**
+ * `opsee foreman up` (story 13): the daemon that idles until given work. One per machine, guarded
+ * by the pid file: a stale file is overwritten, so a service manager (foreman service install,
+ * story 16) can start it cold. Stale is what a reboot or a kill -9 leaves: a file written before
+ * this boot (whatever pid it names, since pids restart low and last boot's number may be some other
+ * process's now), or one naming a pid that is dead or that belongs to another program (foreman.ts
+ * liveDaemonPid). A live Foreman that is not this process ends it with exit 3, not 1: not a
+ * crash, so systemd does not restart it (RestartPreventExitStatus), and the operator is told to
+ * stop the other one. Each
+ * tick it Reconciles the Process Table (story 51: every Initiative and Account it has rows for, so
+ * a Worker left by a dead daemon is resumed even when nobody asks for a Run) and drains the
+ * outbox, then serves the oldest pending Run request `foreman run` queued, in the foreground of
+ * this process, and idles otherwise. A request whose Initiative is paused (story 55) waits
+ * pending, and a Run paused midway is put back to pending, so one paused Initiative never holds
+ * the daemon. A stop signal ends the Run after the dispatch in flight (the
+ * process hooks stop the Worker itself) and removes the pid file. On a laptop it warns at start
+ * (story 17). A Run it serves fills its Account's Slots (core/scheduler.ts) like a foreground one;
+ * what this still does one at a time is the Runs themselves, one request per tick.
+ */
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { dirname } from "node:path";
+import type { AccountStore } from "../foreman/account-store.js";
+import { OutboxTracker } from "../foreman/core/outbox-tracker.js";
+import type { RunRequestRow } from "../foreman/core/process-table.js";
+import { ForemanError, PAUSE_POLL_MS, reconcileTick, runForeman } from "../foreman/core/run.js";
+import { count } from "../foreman/core/text.js";
+import { sleepWarning } from "../foreman/host.js";
+import { EXIT_ANOTHER_FOREMAN } from "../foreman/service-unit.js";
+import { errorMessage } from "../foreman/worker-process.js";
+
+/** The daemon's exit when another Foreman holds the pid file (defined beside the units that read it). */
+export { EXIT_ANOTHER_FOREMAN };
+import { checkLearningsFile, chooseAccount, liveDaemonPid, reconciled, runDepsFor, summarize, type ForemanLocal, type SharedDeps } from "./foreman.js";
+
+/** How long the daemon idles between ticks when no Run is requested; the same interval a
+ * foreground Run waits through a pause at. */
+export const DEFAULT_DAEMON_TICK_MS = PAUSE_POLL_MS;
+
+export interface ForemanUpDeps extends SharedDeps {
+  store: AccountStore;
+  /** The repository the daemon serves: the checkout the command runs in. */
+  repoRoot: () => Promise<string | undefined>;
+  local: ForemanLocal;
+  /** Resolves when the daemon should stop (SIGINT/SIGTERM); it finishes the dispatch in flight first. */
+  untilStop: Promise<void>;
+  /** The machine (story 17): a laptop that may sleep gets a warning at start, a desktop or server none. */
+  host: { platform: NodeJS.Platform; isLaptop: () => Promise<boolean> };
+  tickMs?: number;
+  sleep?: (ms: number) => Promise<void>;
+  pid?: number;
+}
+
+export async function runForemanUp(deps: ForemanUpDeps): Promise<number> {
+  const { local, out } = deps;
+  const pid = deps.pid ?? process.pid;
+  const other = liveDaemonPid(local.pidFile, local.probe);
+  if (other !== undefined && other !== pid) {
+    out(`Foreman already up (pid ${other}); stop it first, or queue work for it: opsee foreman run <initiativeId>`);
+    return EXIT_ANOTHER_FOREMAN;
+  }
+  const repoRoot = await deps.repoRoot();
+  if (!repoRoot) throw new ForemanError("Not inside a git repository; run foreman up from the checkout the Initiatives' Tasks are about.");
+  mkdirSync(dirname(local.pidFile), { recursive: true, mode: 0o700 });
+  writeFileSync(local.pidFile, `${pid}\n`, { mode: 0o600 });
+
+  let stopping = false;
+  void deps.untilStop.then(() => {
+    stopping = true;
+    out("Foreman: stopping after the dispatch in flight");
+  });
+  const sleep = deps.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+  const tickMs = deps.tickMs ?? DEFAULT_DAEMON_TICK_MS;
+  const requeued = local.table.requeueRunning();
+  out(`Foreman up (pid ${pid}): repository ${repoRoot}, Process Table ${local.table.path}${requeued ? `, ${count(requeued, "Run request", "Run requests")} put back to pending` : ""}`);
+  const learningsConfig = checkLearningsFile(repoRoot);
+  if (learningsConfig.refused) out(`learnings: ${learningsConfig.refused}`);
+  out("Waiting for Runs: opsee foreman run <initiativeId> [--account <name>] [--once] [--task <id>] from another terminal");
+  if (await deps.host.isLaptop()) for (const line of sleepWarning(deps.host.platform)) out(line);
+
+  try {
+    while (!stopping) {
+      await idleReconcile(deps, repoRoot);
+      if (stopping) break;
+      const request = local.table.takeRun();
+      if (request) {
+        await serveRun(deps, repoRoot, request, () => stopping, deps.untilStop);
+        continue;
+      }
+      await Promise.race([sleep(tickMs), deps.untilStop]);
+    }
+  } finally {
+    rmSync(local.pidFile, { force: true });
+  }
+  out("Foreman down");
+  return 0;
+}
+
+/** The idle tick: every (Initiative, Account) the Process Table has rows for gets a Reconcile, and
+ * whatever the outbox holds is offered to the backend. One group failing never stops the others. */
+async function idleReconcile(deps: ForemanUpDeps, repoRoot: string): Promise<void> {
+  const { local, out } = deps;
+  const groups = new Map<string, { initiativeId: number; account: string }>();
+  for (const row of local.table.liveWorkers()) groups.set(`${row.initiativeId}|${row.account}`, { initiativeId: row.initiativeId, account: row.account });
+  const accounts = deps.store.load();
+  for (const { initiativeId, account: name } of groups.values()) {
+    const account = accounts.find((a) => a.name === name);
+    if (!account) {
+      out(`reconcile: Initiative ${initiativeId} has Workers under Account "${name}", which is no longer registered; leaving them`);
+      continue;
+    }
+    try {
+      const outcome = await reconcileTick(await runDepsFor(deps, account, repoRoot, local, { failover: false }), initiativeId);
+      if (outcome.length) out(`reconcile: Initiative ${initiativeId}: ${summarize(reconciled(outcome)).line}`);
+    } catch (error) {
+      out(`reconcile: Initiative ${initiativeId} on Account "${name}" failed, retrying next tick: ${errorMessage(error)}`);
+    }
+  }
+  if (local.table.pending().length) {
+    try {
+      await new OutboxTracker(deps.tracker, local.table, out).drain();
+    } catch (error) {
+      out(`outbox: drain failed, retrying next tick: ${errorMessage(error)}`);
+    }
+  }
+}
+
+async function serveRun(deps: ForemanUpDeps, repoRoot: string, request: RunRequestRow, shouldStop: () => boolean, untilStop: Promise<void>): Promise<void> {
+  const { local, out } = deps;
+  const label = `Run #${request.id} (Initiative ${request.initiativeId})`;
+  if (request.repoRoot && request.repoRoot !== repoRoot) {
+    const message = `requested from ${request.repoRoot}, but this daemon serves ${repoRoot}`;
+    local.table.finishRun(request.id, "failed", message);
+    out(`${label}: refused: ${message}`);
+    return;
+  }
+  out(`${label}: starting`);
+  try {
+    const account = chooseAccount(deps.store.load(), request.account);
+    const outcome = await (deps.run ?? runForeman)(await runDepsFor(deps, account, repoRoot, local, { maxTurns: request.maxTurns, stallTimeoutMs: request.stallTimeoutMs }), {
+      initiativeId: request.initiativeId,
+      once: request.once,
+      taskId: request.taskId,
+      shouldStop,
+      // The same signal as a promise: a Reconcile turn waiting for one of the Run's Slots abandons
+      // the wait on it, so the stop is not deferred until a Worker turn ends (core/scheduler.ts).
+      untilStop,
+    });
+    const { line, needingAttention, blocked } = summarize(outcome);
+    if (outcome.paused) {
+      // The Initiative was paused mid-Run (story 55): the request goes back to pending, where
+      // `takeRun` leaves it until `foreman resume`, and this daemon is free for other Initiatives.
+      const note = `paused after ${line}; waits for foreman resume ${request.initiativeId}`;
+      local.table.requeueRun(request.id, note);
+      out(`${label}: ${note}`);
+      return;
+    }
+    const clean = needingAttention === 0 && blocked === 0;
+    local.table.finishRun(request.id, clean ? "done" : "failed", clean ? undefined : [needingAttention ? `${needingAttention} needing attention` : "", blocked ? `${count(blocked, "Task")} blocked before dispatch` : ""].filter(Boolean).join(", "));
+    out(`${label}: over: ${line}`);
+  } catch (error) {
+    const message = errorMessage(error);
+    local.table.finishRun(request.id, "failed", message);
+    out(`${label}: failed: ${message}`);
+  }
+}