@coulb/crux-cli 0.1.0

package/lib/leads.js

@@ -0,0 +1,292 @@
+'use strict';
+
+/**
+ * Reconciliation for the standing lead fleet: does every Solo project have exactly one live
+ * lead tailing the right crux stream?
+ *
+ * The fleet was once down for hours and the only symptom was silence — indistinguishable from
+ * "no items to route". Everything here exists to make that ambiguity impossible, so each check
+ * is written to be *loud* rather than convenient.
+ *
+ * ## Why coverage is read off the OS process table, not `solo processes list`
+ *
+ * A lead's Solo command is `cd /path/to/repo && claude … '<brief>'`. The slug it tails appears
+ * nowhere in that string: `crux tail --project <slug>` is started later, by the agent, as a
+ * grandchild of the Solo process. So Solo can tell us a lead *process* is running; only the OS
+ * can tell us which stream it is actually tailing — and that is the thing we care about.
+ *
+ * Names are worthless for this. The stopped `kind: agent` rows are named exactly like the new
+ * `kind: command` rows, and one of them (`lead-beacon`) is still running and still tailing.
+ * A reconciler that trusted names would call `beacon` covered by the command that is in fact
+ * stopped, and would happily start a second tail on top of the agent that is really doing the
+ * work. So: match on the running command line, attribute it to its owning Solo process by
+ * walking the process tree, and never look at a name.
+ */
+
+/** A tail is the node process running the `crux` script — never a `timeout`/`zsh -c` wrapper. */
+const TAIL_COMMAND = /^(?:\S*\/)?node(?:\.exe)?\s+(?:\S*\/)?crux(?:\.js)?\s+(.*)$/;
+
+const ERROR_STATES = ['unmapped', 'uncovered', 'duplicate', 'orphan'];
+
+/**
+ * Parse `ps -axo pid=,ppid=,command=` into a pid -> {pid, ppid, command} map.
+ *
+ * @param {string} output
+ * @returns {Map<number, {pid: number, ppid: number, command: string}>}
+ */
+function parseProcessTable(output) {
+  const table = new Map();
+  for (const line of output.split('\n')) {
+    const match = /^\s*(\d+)\s+(\d+)\s+(.*\S)\s*$/.exec(line);
+    if (!match) continue;
+    const pid = Number(match[1]);
+    table.set(pid, { pid, ppid: Number(match[2]), command: match[3] });
+  }
+  return table;
+}
+
+/** Pull `--project x` / `--interval n` / `--unrouted` out of a `crux tail` argument string. */
+function parseTailArgs(argString) {
+  const argv = argString.trim().split(/\s+/);
+  if (argv[0] !== 'tail') return null;
+
+  const flag = (name) => {
+    const i = argv.indexOf(`--${name}`);
+    if (i === -1) return undefined;
+    const next = argv[i + 1];
+    return next !== undefined && !next.startsWith('--') ? next : true;
+  };
+
+  const project = flag('project');
+  return {
+    slug: typeof project === 'string' ? project : null,
+    unrouted: flag('unrouted') === true,
+    interval: typeof flag('interval') === 'string' ? Number(flag('interval')) : 60,
+  };
+}
+
+/**
+ * Every running `crux tail`, one entry per real tail.
+ *
+ * `timeout 570 crux tail …` and `zsh -c "… eval 'crux tail …'"` both *contain* the command
+ * line, and both really are ancestors of one tail. Matching only the node process is what
+ * keeps a wrapped tail from being miscounted as two — which would report a phantom duplicate.
+ *
+ * @param {Map<number, {pid: number, ppid: number, command: string}>} table
+ */
+function findTails(table) {
+  const tails = [];
+  for (const { pid, command } of table.values()) {
+    const match = TAIL_COMMAND.exec(command);
+    if (!match) continue;
+    const args = parseTailArgs(match[1]);
+    if (!args) continue;
+    tails.push({ pid, ...args, command });
+  }
+  return tails.sort((a, b) => a.pid - b.pid);
+}
+
+/**
+ * Attribute each tail to the Solo process that owns it by walking up the process tree.
+ *
+ * A tail with no Solo ancestor is running outside the supervisor — it covers the stream right
+ * now but nothing will restart it, so it is reported rather than counted as healthy coverage.
+ *
+ * @param {Array<object>} tails
+ * @param {Map<number, {pid: number, ppid: number, command: string}>} table
+ * @param {Map<number, object>} soloByPid
+ */
+function attributeOwners(tails, table, soloByPid) {
+  return tails.map((tail) => {
+    const seen = new Set();
+    let pid = tail.pid;
+    while (pid && pid !== 1 && table.has(pid) && !seen.has(pid)) {
+      seen.add(pid);
+      if (soloByPid.has(pid)) return { ...tail, owner: soloByPid.get(pid) };
+      pid = table.get(pid).ppid;
+    }
+    return { ...tail, owner: null };
+  });
+}
+
+/**
+ * Classify every slug into exactly one state. Pure: no processes, no I/O, no clock.
+ *
+ * States:
+ *   covered    — one running tail, owned by a Solo process. The happy path.
+ *   uncovered  — Solo project expects a lead; nothing is tailing.            (failure class 1)
+ *   orphan     — something tails a slug that has no Solo project.            (failure class 2)
+ *   duplicate  — two or more tails on one slug; the stream is double-read.   (failure class 3)
+ *   unmapped   — a Solo project absent from the mapping. Never skipped silently.
+ *   leadless   — configured to expect no lead (e.g. `personal`). Expected, not an error.
+ *   unmanaged  — a crux slug with no Solo project and no tail. Registry drift; a warning.
+ *
+ * @param {{soloProjects: Array<{id: number, name: string}>, cruxSlugs: string[],
+ *          tails: Array<object>, config: object}} input
+ */
+function reconcile({ soloProjects, cruxSlugs, tails, config }) {
+  const entries = [];
+  const slugToSolo = new Map();
+
+  for (const project of soloProjects) {
+    const slug = config.map[project.name];
+    if (typeof slug !== 'string') {
+      entries.push({
+        state: 'unmapped',
+        slug: null,
+        soloProject: project,
+        tails: [],
+        detail: `Solo project "${project.name}" (id ${project.id}) is not in the name→slug map. Add it to leads.config.json.`,
+      });
+      continue;
+    }
+    slugToSolo.set(slug, project);
+  }
+
+  const projectTails = tails.filter((t) => typeof t.slug === 'string' && t.slug !== '');
+  const bySlug = new Map();
+  for (const tail of projectTails) {
+    if (!bySlug.has(tail.slug)) bySlug.set(tail.slug, []);
+    bySlug.get(tail.slug).push(tail);
+  }
+
+  const slugs = [...new Set([...slugToSolo.keys(), ...cruxSlugs, ...bySlug.keys()])].sort();
+
+  for (const slug of slugs) {
+    const own = bySlug.get(slug) ?? [];
+    const solo = slugToSolo.get(slug) ?? null;
+    const leadlessReason = config.leadless[slug];
+
+    if (own.length > 1) {
+      entries.push({
+        state: 'duplicate',
+        slug,
+        soloProject: solo,
+        tails: own,
+        detail: `${own.length} tails on "${slug}" (pids ${own.map((t) => t.pid).join(', ')}). The stream is being double-processed.`,
+      });
+      continue;
+    }
+
+    if (own.length === 1) {
+      const tail = own[0];
+      if (!solo) {
+        entries.push({
+          state: 'orphan',
+          slug,
+          soloProject: null,
+          tails: own,
+          detail: `pid ${tail.pid} tails "${slug}", which has no Solo project. Nothing will restart it.`,
+        });
+        continue;
+      }
+      entries.push({
+        state: 'covered',
+        slug,
+        soloProject: solo,
+        tails: own,
+        detail: tail.owner
+          ? `pid ${tail.pid} under Solo process ${tail.owner.id} (${tail.owner.name}).`
+          : `pid ${tail.pid} — running outside Solo; nothing will restart it if it dies.`,
+        unsupervised: !tail.owner,
+      });
+      continue;
+    }
+
+    if (typeof leadlessReason === 'string') {
+      entries.push({ state: 'leadless', slug, soloProject: solo, tails: [], detail: leadlessReason });
+      continue;
+    }
+
+    if (solo) {
+      entries.push({
+        state: 'uncovered',
+        slug,
+        soloProject: solo,
+        tails: [],
+        detail: `Solo project "${solo.name}" (id ${solo.id}) has no process tailing "${slug}".`,
+      });
+      continue;
+    }
+
+    entries.push({
+      state: 'unmanaged',
+      slug,
+      soloProject: null,
+      tails: [],
+      detail: `crux slug "${slug}" has no Solo project and no lead. Registry drift — remove it, or create the project.`,
+    });
+  }
+
+  const errors = entries.filter((e) => ERROR_STATES.includes(e.state));
+  return { entries, errors, ok: errors.length === 0, exitCode: errors.length === 0 ? 0 : 1 };
+}
+
+/** The Solo process name that starts a slug's lead. `lead-<slug>` unless overridden. */
+function leadProcessName(slug, config) {
+  return config.leadProcess[slug] ?? `lead-${slug}`;
+}
+
+/**
+ * Decide what `--fix` would start, and what it must refuse.
+ *
+ * The refusals are the point. A slug can be covered by a *stopped-looking* `kind:
+ * agent` row while its `kind: command` twin sits idle — starting the command would put two
+ * tails on one stream. So the plan is built only from `uncovered` entries, and every other
+ * state is an explicit, explained refusal rather than a silent no-op.
+ *
+ * @param {{entries: Array<object>}} report
+ * @param {Array<{id: number, name: string, kind: string, status: string, projectName: string}>} soloProcesses
+ * @param {object} config
+ */
+function planFix(report, soloProcesses, config) {
+  const starts = [];
+  const refusals = [];
+
+  for (const entry of report.entries) {
+    if (entry.state === 'covered' || entry.state === 'leadless' || entry.state === 'unmanaged') continue;
+
+    if (entry.state !== 'uncovered') {
+      refusals.push({
+        slug: entry.slug,
+        reason: `${entry.state}: ${entry.detail} --fix only starts leads for uncovered projects.`,
+      });
+      continue;
+    }
+
+    const wanted = leadProcessName(entry.slug, config);
+    const candidates = soloProcesses.filter((p) => p.projectName === config.fleetProject && p.name === wanted);
+
+    const command = candidates.find((p) => p.kind === 'command');
+    if (!command) {
+      const why = candidates.length
+        ? `only non-command rows exist (${candidates.map((p) => `${p.id}:${p.kind}`).join(', ')}); solo.yml declares no such command`
+        : `no process named "${wanted}" in Solo project "${config.fleetProject}"`;
+      refusals.push({ slug: entry.slug, reason: `cannot start ${wanted} — ${why}.` });
+      continue;
+    }
+
+    if (command.status === 'running') {
+      refusals.push({
+        slug: entry.slug,
+        reason: `${wanted} (id ${command.id}) is already running but is not tailing "${entry.slug}" — restarting it blindly would hide a real fault. Investigate.`,
+      });
+      continue;
+    }
+
+    starts.push({ slug: entry.slug, process: command });
+  }
+
+  return { starts, refusals };
+}
+
+module.exports = {
+  ERROR_STATES,
+  attributeOwners,
+  findTails,
+  leadProcessName,
+  parseProcessTable,
+  parseTailArgs,
+  planFix,
+  reconcile,
+};

package/lib/orchestrator.js

@@ -0,0 +1,237 @@
+'use strict';
+
+/**
+ * WHICH orchestrator runs the leads on this machine, and — the question nobody was asking — CAN it.
+ *
+ * `lib/backends.js` knows how to drive three orchestrators. It does not know whether any of them is
+ * installed. Nothing did, and that is the whole bug this file exists to close:
+ *
+ *     $ crux discover ~/src/mercury-billing --yes
+ *     ✓ mercury-billing is a crux project — created.
+ *       Next:  crux swarm up --project mercury-billing
+ *
+ *     $ crux swarm up --project mercury-billing
+ *     crux: the exec backend needs a command template — set `swarm.exec.command`, …
+ *
+ * The error is a good error. It names its fix. It is still a **failed first run**, because the tool
+ * printed a command it could have known would not work. Step zero exists to stop exactly that, and a
+ * five-minute quickstart that stalls at step two is not saved by the politeness of the message it
+ * stalls with.
+ *
+ * Everything here is PURE. Whether a binary exists is I/O, so it arrives as `has(bin)` — injected,
+ * exactly as `planWrite` takes `samePath` and `planUp` takes `now`. The refusals are the product, and
+ * a refusal you cannot unit-test is a refusal that rots.
+ *
+ * ## The three answers, in order
+ *
+ * 1. **What the user SAID.** `--backend`, then `CRUX_SWARM_BACKEND`, then `swarm.backend` in the
+ *    config. An explicit choice is never second-guessed — only checked.
+ * 2. **What is on the machine.** Nothing said? Then look: Solo, herdr, tmux. This is why the shipped
+ *    config no longer hardcodes `"backend": "solo"`. That line meant every reader with no Solo — the
+ *    overwhelmingly common case — got `solo` by default and hit `could not
+ *    run \`solo projects list\`` on their first run. A default that is wrong for almost everyone who
+ *    reads it is not a default, it is a trap.
+ * 3. **A refusal that names the fix.** No orchestrator at all is a real state, and the honest thing
+ *    is to say so at the moment discover would otherwise print `Next:` — with the line to add and the
+ *    file to add it to — rather than one command later.
+ *
+ * ## Why tmux gets to be a default at all, when exec is generic by design
+ *
+ * `exec` is the "bring your own orchestration" backend: it cannot know whether you run tmux, screen,
+ * systemd or nohup, and it must not pretend to. So the default is not "assume tmux" — it is **"tmux,
+ * if and only if tmux is actually installed"**. A default that assumes a binary and then fails is the
+ * same bug one layer down. {@see resolveBackend}: the tmux templates are only ever handed back when
+ * `has('tmux')` is true, and when it is false the exec backend refuses with the one line to write.
+ *
+ * And when crux does derive them, `crux discover` WRITES them into the config rather than keeping
+ * them in its head. A derived default that lives only in code is invisible and un-editable: the user
+ * cannot see what their leads are being started with, cannot change it, and cannot tell crux's
+ * opinion from their own. Written down, the derivation is a starting point they own — which is what
+ * "ship our own opinions" is supposed to mean.
+ */
+
+/**
+ * The exec templates crux derives when it finds tmux and nothing else has been said.
+ *
+ * This is the pair from the error message that this whole thread is about, promoted from advice to
+ * behaviour — and it is a *pair*, which is the part that matters. `tmux new-session -d` FORKS: the
+ * process crux spawns exits immediately while the lead lives on inside the tmux server, so a `command`
+ * without a `stop` leaves crux holding a dead pid and `swarm down` correctly refuses to claim a stop
+ * it cannot make {@see ../lib/swarm.planDown}. Shipping `command` alone would have been shipping that
+ * refusal to every first-time user. With `stop`, the lead is addressable by NAME forever
+ * (`addresses: 'handle'`), and `up`/`down` are a clean round trip.
+ *
+ * `{repo}` is deliberately absent. The exec backend already runs the line with `cwd: lead.repo`, so
+ * tmux inherits the right directory from its client — and a `-c {repo}` would put an unquoted path
+ * into a shell line, which breaks the first user whose code lives in `~/My Projects`. The safest
+ * interpolation is the one you do not do.
+ */
+const TMUX_EXEC = Object.freeze({
+  command: "tmux new-session -d -s {name} '{command}'",
+  stop: 'tmux kill-session -t {name}',
+});
+
+/** The backends `makeBackend` can build. Kept here so a typo is refused with the list, as before. */
+const BACKENDS = ['solo', 'exec', 'herdr'];
+
+/**
+ * The binary each backend cannot run without.
+ *
+ * `exec` is absent on purpose — it is the one backend with no binary of its own, because its binary
+ * is whatever the user's template names. It can only be checked through the template. {@see execFor}
+ */
+const NEEDS = { solo: 'solo', herdr: 'herdr' };
+
+/**
+ * What is on this machine, in the order crux would pick it.
+ *
+ * Solo first because it is the opinionated default and the richest: it spawns workers, so a Solo lead
+ * gets `crux spawn` and every other lead has to do its own work inline {@see ../lib/swarm.renderLeadBrief}.
+ * herdr next — a real multiplexer for agents, with named panes you can attach to. tmux last, and it
+ * is last precisely because it is the lowest common denominator: it hosts the lead and nothing more.
+ *
+ * Null means no orchestrator, which is a fact, not an error — and it is a fact worth stating out loud
+ * at step zero rather than discovering at step two.
+ */
+function detectBackend(has) {
+  if (has('solo')) return 'solo';
+  if (has('herdr')) return 'herdr';
+  if (has('tmux')) return 'exec';
+  return null;
+}
+
+/** The exec templates in force: what the user wrote, else tmux if tmux is here, else nothing. */
+function execFor(exec, has) {
+  const command = typeof exec?.command === 'string' ? exec.command.trim() : '';
+  if (command !== '') {
+    return {
+      exec: { command, stop: typeof exec.stop === 'string' && exec.stop.trim() !== '' ? exec.stop.trim() : null },
+      execSource: 'config',
+    };
+  }
+  if (has('tmux')) return { exec: { ...TMUX_EXEC }, execSource: 'tmux' };
+  return { exec: null, execSource: null };
+}
+
+/** The config block that would make the current derivation explicit. `null` when nothing is derived. */
+function derivedWrites({ name, source, exec, execSource }) {
+  const writes = {};
+  // Only a DETECTED name is written down. A `--backend` flag or a `CRUX_SWARM_BACKEND` env var is a
+  // per-run override — persisting it would turn "just this once" into "forever", silently.
+  if (source === 'detected') writes.backend = name;
+  if (execSource === 'tmux') writes.exec = { ...exec };
+  return Object.keys(writes).length ? writes : null;
+}
+
+/**
+ * Which backend, can it run, and if not — what is the one line that fixes it.
+ *
+ * @param {object} input
+ * @param {string|null} input.requested `--backend` (a per-run override)
+ * @param {string|null} input.env `CRUX_SWARM_BACKEND` (a per-shell override)
+ * @param {string|null} input.configured `swarm.backend` (may be absent — then crux looks)
+ * @param {object} input.exec `swarm.exec` — `{command, stop}`, either or both possibly absent
+ * @param {(bin: string) => boolean} input.has injected: "is this binary on PATH?" is I/O
+ * @param {string} [input.file] the config file, so a fix can say WHERE to write it
+ *
+ * @returns {{
+ *   name: string|null, source: 'flag'|'env'|'config'|'detected'|'none',
+ *   exec: {command: string, stop: string|null}|null, execSource: 'config'|'tmux'|null,
+ *   ok: boolean, problem: {reason: string, fix: string[]}|null,
+ *   writes: {backend?: string, exec?: object}|null,
+ * }}
+ */
+function resolveBackend({ requested = null, env = null, configured = null, exec = {}, has, file = 'the leads config' }) {
+  const said = (value, source) => (typeof value === 'string' && value.trim() !== '' ? { name: value.trim(), source } : null);
+
+  const chosen =
+    said(requested, 'flag') ??
+    said(env, 'env') ??
+    said(configured, 'config') ??
+    said(detectBackend(has), 'detected') ?? { name: null, source: 'none' };
+
+  const base = { ...chosen, exec: null, execSource: null, ok: false, problem: null, writes: null };
+
+  // Nothing said, and nothing found. Not a crash and not a crux bug — this machine has no way to run
+  // a long-lived agent yet, and that is a sentence a human can act on.
+  if (chosen.name === null) {
+    return {
+      ...base,
+      problem: {
+        reason: 'crux found no orchestrator on this machine — a lead is a long-running agent, so something has to host it.',
+        fix: [
+          'Install one, and crux will find it — no configuration:',
+          '    brew install tmux          # the simplest: each lead becomes a tmux session',
+          '    (or Solo, or herdr — see `crux swarm --help`)',
+          `Or bring your own, by naming the command that starts a lead in ${file}:`,
+          `    "swarm": { "backend": "exec", "exec": { "command": "${TMUX_EXEC.command}" } }`,
+        ],
+      },
+    };
+  }
+
+  // A typo in a config file is not a backend. Refuse with the list, exactly as `makeBackend` does —
+  // and refuse it HERE, before discover has printed a `Next:` that could never have worked.
+  if (!BACKENDS.includes(chosen.name)) {
+    return {
+      ...base,
+      problem: {
+        reason: `unknown backend "${chosen.name}" (available: ${BACKENDS.join(', ')})`,
+        fix: [`Set \`swarm.backend\` in ${file} to one of: ${BACKENDS.join(', ')}.`],
+      },
+    };
+  }
+
+  // An explicit choice crux cannot honour. Never silently fall back to a different one: a user who
+  // wrote `"backend": "solo"` and gets a tmux lead has been lied to about where their agents are.
+  const needs = NEEDS[chosen.name];
+  if (needs && !has(needs)) {
+    return {
+      ...base,
+      problem: {
+        reason:
+          `the ${chosen.name} backend needs \`${needs}\` on your PATH, and it is not installed` +
+          `${chosen.source === 'config' ? ` (\`swarm.backend\` in ${file} asks for it)` : ''}.`,
+        fix: [
+          `Install ${needs}, or pick a backend this machine has:`,
+          '    crux swarm up --backend exec        # any command you like — tmux, screen, nohup, systemd',
+          `Or set \`swarm.backend\` in ${file}.`,
+        ],
+      },
+    };
+  }
+
+  if (chosen.name !== 'exec') return { ...base, ok: true, writes: derivedWrites({ ...chosen, exec: null, execSource: null }) };
+
+  const { exec: resolved, execSource } = execFor(exec, has);
+
+  // exec with no template and no tmux to derive one from. This is the message that was the whole
+  // thread — kept, because it was never the wrong message, only the wrong MOMENT to say it.
+  if (!resolved) {
+    return {
+      ...base,
+      problem: {
+        reason:
+          'the exec backend needs a command template — it runs whatever you tell it to, and nothing ' +
+          'has told it. tmux, the obvious default, is not installed either, so crux has nothing to derive one from.',
+        fix: [
+          'Install tmux and crux will write the template for you:',
+          '    brew install tmux',
+          `Or say how YOUR orchestrator starts a lead, in ${file}:`,
+          '    "swarm": {',
+          '      "backend": "exec",',
+          `      "exec": { "command": "${TMUX_EXEC.command}",`,
+          `                "stop":    "${TMUX_EXEC.stop}" }`,
+          '    }',
+          'Placeholders: {slug} {name} {repo} {brief_path} {command}. Set `stop` whenever your command',
+          'forks (tmux, nohup) — without it crux is left holding a dead pid and cannot stop the lead.',
+        ],
+      },
+    };
+  }
+
+  const out = { ...base, exec: resolved, execSource, ok: true };
+  return { ...out, writes: derivedWrites(out) };
+}
+
+module.exports = { BACKENDS, TMUX_EXEC, detectBackend, execFor, resolveBackend };