@love-moon/conductor-cli 0.11.2 → 0.13.0

package/{bin/conductor-remote-exec.js → src/remote/exec.js}

@@ -1,36 +1,37 @@
-#!/usr/bin/env node
-
 /**
- * conductor remote-exec — run one command on another daemon's host.
- *
- *   conductor remote-exec --target ubuntu --workspace /home/duino/ws/holomotion ls .
- *   conductor remote-exec --target ubuntu -- bash -lc "pnpm build 2>&1 | tail -20"
+ * `conductor remote exec` — run one command on another daemon's host.
  *
  * The command is sent as argv and spawned without a shell on the target, so
  * quoting is not re-interpreted remotely. Pass `-- bash -lc "..."` when pipes
  * or globs are actually wanted.
  */
 
-import fs from "node:fs";
-import path from "node:path";
 import process from "node:process";
-import { fileURLToPath } from "node:url";
 
-import { ConductorConfig, loadConfig } from "@love-moon/conductor-sdk";
-import { envForExplicitConfigFile } from "../src/config-env.js";
-import { resolveConductorConfigPath } from "../src/conductor-paths.js";
+import {
+  EXIT,
+  UsageError,
+  callApi,
+  delay,
+  loadCliConfig,
+  parseTimeoutMs as parseSharedTimeoutMs,
+  withRetry,
+} from "./client.js";
+
+export { EXIT, UsageError };
 
-/**
- * Following ssh: the remote command's exit code is passed through verbatim
- * (0-254) and 255 is reserved for this CLI's own failures. Reusing 1/2/4 for
- * local errors would make `grep` finding nothing (1) or `ls` on a missing path
- * (2) indistinguishable from a network error or a usage mistake.
- */
-const EXIT = { OK: 0, CLI_ERROR: 255 };
 const DEFAULT_TIMEOUT_MS = 60_000;
 const POLL_INTERVAL_MS = 1_000;
 /** How long a single request may block server-side before we switch to polling. */
 const POST_WAIT_MS = 10_000;
+/**
+ * A 429 means the per-user in-flight cap (8) is full — nothing was started, so
+ * retrying is safe. Six attempts at 500ms doubling waits out ~15s, longer than
+ * one saturating POST can block (POST_WAIT_MS). Nothing else is retried here:
+ * a 5xx may have started the command already, and a second POST would run it twice.
+ */
+const POST_RETRY = { attempts: 6, retryable: (error) => error?.status === 429 };
+const INTERRUPT_SIGNALS = ["SIGINT", "SIGTERM"];
 
 const VALUE_FLAGS = new Map([
   ["--target", "target"],
@@ -50,12 +51,6 @@ const BOOL_FLAGS = new Map([
   ["-h", "help"],
 ]);
 
-const isMainModule = (() => {
-  const currentFile = fileURLToPath(import.meta.url);
-  const entryFile = process.argv[1] ? path.resolve(process.argv[1]) : "";
-  return entryFile === currentFile;
-})();
-
 /**
  * Split argv into flags and the remote command.
  *
@@ -116,81 +111,17 @@ export function parseArgs(argv) {
   return { options, command };
 }
 
-export class UsageError extends Error {}
-
 /** Accepts `500ms`, `30s`, `2m`, or a bare number of seconds. */
 export function parseTimeoutMs(value) {
-  if (value === undefined || value === null || value === "") {
-    return DEFAULT_TIMEOUT_MS;
-  }
-  const raw = String(value).trim().toLowerCase();
-  const match = raw.match(/^(\d+(?:\.\d+)?)(ms|s|m)?$/);
-  if (!match) {
-    throw new UsageError(`invalid --timeout value: ${value}`);
-  }
-  const amount = Number.parseFloat(match[1]);
-  const unit = match[2] || "s";
-  const multiplier = unit === "ms" ? 1 : unit === "m" ? 60_000 : 1_000;
-  const ms = Math.round(amount * multiplier);
-  if (!Number.isFinite(ms) || ms <= 0) {
-    throw new UsageError(`invalid --timeout value: ${value}`);
-  }
-  return ms;
+  return parseSharedTimeoutMs(value, DEFAULT_TIMEOUT_MS);
 }
 
-function loadCliConfig(configFile, env = process.env) {
-  const configPath = resolveConductorConfigPath(configFile, env);
-  const configEnv = envForExplicitConfigFile(configFile, env);
-  if (fs.existsSync(configPath)) {
-    return loadConfig(configPath, { env: configEnv });
-  }
-
-  const agentToken = typeof env.CONDUCTOR_AGENT_TOKEN === "string" ? env.CONDUCTOR_AGENT_TOKEN.trim() : "";
-  const backendUrl = typeof env.CONDUCTOR_BACKEND_URL === "string" ? env.CONDUCTOR_BACKEND_URL.trim() : "";
-  if (agentToken && backendUrl) {
-    return new ConductorConfig({ agentToken, backendUrl });
-  }
-
-  return loadConfig(configPath, { env: configEnv });
-}
-
-async function callApi(config, method, pathname, body, fetchImpl) {
-  const url = new URL(pathname, config.backendUrl);
-  const response = await fetchImpl(url.toString(), {
-    method,
-    headers: {
-      Authorization: `Bearer ${config.agentToken}`,
-      Accept: "application/json",
-      ...(body ? { "Content-Type": "application/json" } : {}),
-    },
-    ...(body ? { body: JSON.stringify(body) } : {}),
-  });
-
-  const text = await response.text();
-  let payload = null;
-  try {
-    payload = text ? JSON.parse(text) : null;
-  } catch {
-    payload = null;
-  }
-
-  if (!response.ok) {
-    const message = payload?.error || text.trim() || `HTTP ${response.status}`;
-    const error = new Error(message);
-    error.status = response.status;
-    throw error;
-  }
-  return payload;
-}
-
-const delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
-
 export function showHelp(consoleImpl = console) {
-  consoleImpl.log(`conductor remote-exec - run a command on another daemon's host
+  consoleImpl.log(`conductor remote exec - run a command on another daemon's host
 
 Usage:
-  conductor remote-exec --target <daemon> [options] <command> [args...]
-  conductor remote-exec --target <daemon> [options] -- <command> [args...]
+  conductor remote exec --target <daemon> [options] <command> [args...]
+  conductor remote exec --target <daemon> [options] -- <command> [args...]
 
 Options:
   -t, --target <daemon>   Daemon name to run on (required)
@@ -198,96 +129,46 @@ Options:
       --timeout <dur>     Overall deadline, e.g. 30s, 2m, 500ms (default: 60s)
   -e, --env KEY=VALUE     Extra environment variable (repeatable)
       --json              Print the raw run result as JSON
-      --kill-on-timeout   Stop the remote command when --timeout is reached
+      --kill-on-timeout   Stop the remote command when --timeout is reached or
+                          this CLI is interrupted (Ctrl-C, SIGTERM)
                           (default: it keeps running on the target)
       --config-file <p>   Conductor config file to authenticate with
   -h, --help              Show this help
 
 Notes:
   The command is spawned without a shell. For pipes, globs or redirection use:
-    conductor remote-exec -t ubuntu -- bash -lc "ls | wc -l"
+    conductor remote exec -t ubuntu -- bash -lc "ls | wc -l"
 
   Exit codes follow ssh: the remote command's own code is passed through, and
   255 means this CLI failed (bad usage, daemon offline, network error).
 
+  A command that outlives --timeout keeps running on the target; the run id is
+  printed and \`conductor remote wait -t <daemon> <runId>\` picks it up again.
+
 Examples:
-  conductor remote-exec --target ubuntu --workspace /home/duino/ws/holomotion ls .
-  conductor remote-exec -t ubuntu -w /srv/app -- git log --oneline -5
+  conductor remote exec --target ubuntu --workspace /home/duino/ws/holomotion ls .
+  conductor remote exec -t ubuntu -w /srv/app -- git log --oneline -5
 `);
 }
 
-export async function runRemoteExec(argv, deps = {}) {
-  const consoleImpl = deps.console || console;
-  const fetchImpl = deps.fetch || globalThis.fetch;
-  const env = deps.env || process.env;
-  const sleep = deps.sleep || delay;
-  const now = deps.now || (() => Date.now());
-
-  let parsed;
-  try {
-    parsed = parseArgs(argv);
-  } catch (error) {
-    consoleImpl.error(`Error: ${error.message}`);
-    return EXIT.CLI_ERROR;
-  }
-  const { options, command } = parsed;
-
-  if (options.help) {
-    showHelp(consoleImpl);
-    return EXIT.OK;
-  }
-
-  const target = typeof options.target === "string" ? options.target.trim() : "";
-  if (!target) {
-    consoleImpl.error("Error: --target <daemon> is required");
-    showHelp(consoleImpl);
-    return EXIT.CLI_ERROR;
-  }
-  if (command.length === 0) {
-    consoleImpl.error("Error: no command given");
-    showHelp(consoleImpl);
-    return EXIT.CLI_ERROR;
-  }
-
-  let timeoutMs;
-  try {
-    timeoutMs = parseTimeoutMs(options.timeout);
-  } catch (error) {
-    consoleImpl.error(`Error: ${error.message}`);
-    return EXIT.CLI_ERROR;
-  }
-
-  let config;
-  try {
-    config = deps.config || loadCliConfig(options.configFile, env);
-  } catch (error) {
-    consoleImpl.error(`Error: ${error.message}`);
-    return EXIT.CLI_ERROR;
-  }
-
-  const basePath = `/api/agents/${encodeURIComponent(target)}/exec`;
-  const deadline = now() + timeoutMs;
-
-  let run;
-  try {
-    run = await callApi(config, "POST", basePath, {
-      command: command[0],
-      args: command.slice(1),
-      ...(options.workspace ? { workspace: options.workspace } : {}),
-      ...(Object.keys(options.env).length > 0 ? { env: options.env } : {}),
-      // Deliberately short, and independent of `--timeout`: the overall deadline
-      // is owned by the poll loop below. Handing the daemon the full deadline
-      // would make one HTTP request block for it, and would leave the loop
-      // unreachable because the POST alone would consume the whole budget.
-      timeoutMs: Math.min(timeoutMs, POST_WAIT_MS),
-    }, fetchImpl);
-  } catch (error) {
-    consoleImpl.error(`Error: ${error.message}`);
-    return EXIT.CLI_ERROR;
-  }
-
+/**
+ * Poll `run` until it leaves `running`, the deadline passes, or `signal`
+ * aborts. With `killOnTimeout` a run that is still going at that point is
+ * cancelled on the target. Shared by `exec` (after its POST) and `wait`.
+ */
+export async function waitForRun(config, basePath, initialRun, options = {}) {
+  const {
+    deadline,
+    fetchImpl = globalThis.fetch,
+    sleep = delay,
+    now = () => Date.now(),
+    killOnTimeout = false,
+    signal,
+  } = options;
+
+  let run = initialRun;
   let pollError = null;
-  while (run?.status === "running" && now() < deadline) {
+  while (run?.status === "running" && now() < deadline && !signal?.aborted) {
     if (!run.runId) {
       pollError = new Error("daemon reported a running command but returned no runId");
       break;
@@ -310,7 +191,8 @@ export async function runRemoteExec(argv, deps = {}) {
     }
   }
 
-  if (run?.status === "running" && options.killOnTimeout && run.runId) {
+  let killError = null;
+  if (run?.status === "running" && killOnTimeout && run.runId) {
     try {
       run = await callApi(
         config,
@@ -319,13 +201,91 @@ export async function runRemoteExec(argv, deps = {}) {
         null,
         fetchImpl,
       );
-      consoleImpl.error(`[conductor] deadline reached; stopped the command on ${target}`);
     } catch (error) {
-      consoleImpl.error(`[conductor] failed to stop the run on ${target}: ${error.message}`);
+      // Failing to stop it must not cost the caller the output we already have.
+      killError = error;
     }
   }
 
-  if (options.json) {
+  return { run, pollError, killError };
+}
+
+/**
+ * Run one command on `target` and return the final run record.
+ *
+ * This is the programmatic half of `runRemoteExec`: same two-phase POST-then-poll
+ * behaviour, no argv parsing and no printing. `remote cp -r` uses it to drive
+ * `tar` on the target.
+ */
+export async function execRemote(config, target, command, options = {}) {
+  const {
+    args = [],
+    workspace,
+    env,
+    timeoutMs = DEFAULT_TIMEOUT_MS,
+    fetchImpl = globalThis.fetch,
+    sleep = delay,
+    now = () => Date.now(),
+    killOnTimeout = false,
+    signal,
+  } = options;
+
+  const basePath = `/api/agents/${encodeURIComponent(target)}/exec`;
+  const deadline = now() + timeoutMs;
+
+  const started = await withRetry(() => callApi(config, "POST", basePath, {
+    command,
+    args,
+    ...(workspace ? { workspace } : {}),
+    ...(env && Object.keys(env).length > 0 ? { env } : {}),
+    // Deliberately short, and independent of the overall deadline: that is
+    // owned by the poll loop below. Handing the daemon the full budget would
+    // make one HTTP request block for it and leave the loop unreachable.
+    timeoutMs: Math.min(timeoutMs, POST_WAIT_MS),
+  }, fetchImpl), { ...POST_RETRY, sleep });
+
+  const waited = await waitForRun(config, basePath, started, {
+    deadline, fetchImpl, sleep, now, killOnTimeout, signal,
+  });
+  return { ...waited, basePath };
+}
+
+/**
+ * `execRemote` for callers that only care whether it worked. Throws with the
+ * remote's own stderr, which is what the user needs to see.
+ */
+export async function execRemoteOrThrow(config, target, command, options = {}) {
+  const { run } = await execRemote(config, target, command, options);
+  if (run?.status === "completed" && run.exitCode === 0) return run;
+
+  const detail = (run?.stderrTail || run?.stdoutTail || run?.error || "").trim();
+  const what = run?.status === "running"
+    ? `timed out after ${options.timeoutMs ?? DEFAULT_TIMEOUT_MS}ms`
+    : `exited ${run?.exitCode ?? "?"}`;
+  throw new Error(detail ? `${what} on ${target}: ${detail}` : `${what} on ${target}`);
+}
+
+/**
+ * An AbortSignal that fires on Ctrl-C / SIGTERM. Installing a listener also
+ * stops Node from exiting on the signal, which is the point: the CLI gets to
+ * print the run id (and cancel the run with --kill-on-timeout) before it goes.
+ * Callers must `release()` so the default behaviour comes back.
+ */
+export function interruptSignal(proc = process) {
+  const controller = new AbortController();
+  const onSignal = () => controller.abort();
+  for (const name of INTERRUPT_SIGNALS) proc.on(name, onSignal);
+  return {
+    signal: controller.signal,
+    release: () => {
+      for (const name of INTERRUPT_SIGNALS) proc.off(name, onSignal);
+    },
+  };
+}
+
+/** Print a finished (or abandoned) run the way `exec` and `wait` both do, and pick the exit code. */
+export function reportRun(run, { consoleImpl, json, target, pollError, waitedMs, interrupted }) {
+  if (json) {
     consoleImpl.log(JSON.stringify(run, null, 2));
   } else {
     if (run?.stdoutTail) process.stdout.write(run.stdoutTail);
@@ -342,10 +302,11 @@ export async function runRemoteExec(argv, deps = {}) {
     if (pollError) {
       consoleImpl.error(`[conductor] last status poll failed: ${pollError.message}`);
     }
+    const why = interrupted ? "interrupted" : `still running on ${target} after ${waitedMs}ms`;
     consoleImpl.error(
-      `[conductor] still running on ${target} after ${timeoutMs}ms; ` +
-        `it keeps going there — poll GET ${basePath}/runs/${run.runId}, ` +
-        `stop it with DELETE on the same path, or use --kill-on-timeout`,
+      `[conductor] ${why}; it keeps going on ${target} — ` +
+        `resume with: conductor remote wait -t ${target} ${run.runId} ` +
+        `(add --kill-on-timeout to stop it instead)`,
     );
     return EXIT.CLI_ERROR;
   }
@@ -358,8 +319,90 @@ export async function runRemoteExec(argv, deps = {}) {
   return run?.status === "completed" ? EXIT.OK : EXIT.CLI_ERROR;
 }
 
-if (isMainModule) {
-  // `process.exitCode` rather than `process.exit()`: writes to a pipe are async,
-  // and exiting outright truncates them. Let the loop drain and end naturally.
-  process.exitCode = await runRemoteExec(process.argv.slice(2));
+export async function runRemoteExec(argv, deps = {}) {
+  const consoleImpl = deps.console || console;
+  const fetchImpl = deps.fetch || globalThis.fetch;
+  const env = deps.env || process.env;
+  const sleep = deps.sleep || delay;
+  const now = deps.now || (() => Date.now());
+
+  let parsed;
+  try {
+    parsed = parseArgs(argv);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+  const { options, command } = parsed;
+
+  if (options.help) {
+    showHelp(consoleImpl);
+    return EXIT.OK;
+  }
+
+  const target = typeof options.target === "string" ? options.target.trim() : "";
+  if (!target) {
+    consoleImpl.error("Error: --target <daemon> is required");
+    showHelp(consoleImpl);
+    return EXIT.CLI_ERROR;
+  }
+  if (command.length === 0) {
+    consoleImpl.error("Error: no command given");
+    showHelp(consoleImpl);
+    return EXIT.CLI_ERROR;
+  }
+
+  let timeoutMs;
+  try {
+    timeoutMs = parseTimeoutMs(options.timeout);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+
+  let config;
+  try {
+    config = deps.config || loadCliConfig(options.configFile, env);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+
+  const interrupt = interruptSignal(deps.process);
+  let run;
+  let pollError = null;
+  try {
+    let killError;
+    ({ run, pollError, killError } = await execRemote(config, target, command[0], {
+      args: command.slice(1),
+      workspace: options.workspace,
+      env: options.env,
+      timeoutMs,
+      fetchImpl,
+      sleep,
+      now,
+      killOnTimeout: options.killOnTimeout,
+      signal: interrupt.signal,
+    }));
+    if (killError) {
+      consoleImpl.error(`[conductor] failed to stop the run on ${target}: ${killError.message}`);
+    } else if (options.killOnTimeout && run?.status === "cancelled") {
+      const why = interrupt.signal.aborted ? "interrupted" : "deadline reached";
+      consoleImpl.error(`[conductor] ${why}; stopped the command on ${target}`);
+    }
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  } finally {
+    interrupt.release();
+  }
+
+  return reportRun(run, {
+    consoleImpl,
+    json: options.json,
+    target,
+    pollError,
+    waitedMs: timeoutMs,
+    interrupted: interrupt.signal.aborted,
+  });
 }

package/src/remote/wait.js

@@ -0,0 +1,133 @@
+/**
+ * `conductor remote wait` — pick up a command started by `remote exec` that
+ * outlived its deadline (or the process that started it).
+ *
+ * `exec` prints the run id whenever it leaves a command behind; this verb
+ * polls that run the same way `exec` would have and reports it identically,
+ * so a caller can always finish what it started.
+ */
+
+import process from "node:process";
+
+import { EXIT, callApi, delay, loadCliConfig } from "./client.js";
+import {
+  interruptSignal,
+  parseArgs,
+  parseTimeoutMs,
+  reportRun,
+  waitForRun,
+} from "./exec.js";
+
+export function showHelp(consoleImpl = console) {
+  consoleImpl.log(`conductor remote wait - wait for a command left running by \`remote exec\`
+
+Usage:
+  conductor remote wait --target <daemon> [options] <runId>
+
+Options:
+  -t, --target <daemon>   Daemon the command is running on (required)
+      --timeout <dur>     How long to wait this time, e.g. 30s, 2m (default: 60s)
+      --json              Print the raw run result as JSON
+      --kill-on-timeout   Stop the remote command when --timeout is reached or
+                          this CLI is interrupted (default: it keeps running)
+      --config-file <p>   Conductor config file to authenticate with
+  -h, --help              Show this help
+
+Output and exit codes match \`conductor remote exec\`: the command's own exit
+code is passed through, 255 means this CLI failed or the command is still running.
+
+Example:
+  conductor remote exec -t ubuntu --timeout 30s -- make build
+  # ... [conductor] still running on ubuntu after 30000ms; resume with:
+  conductor remote wait -t ubuntu --timeout 20m 3f9c1b2e-...
+`);
+}
+
+export async function runRemoteWait(argv, deps = {}) {
+  const consoleImpl = deps.console || console;
+  const fetchImpl = deps.fetch || globalThis.fetch;
+  const env = deps.env || process.env;
+  const sleep = deps.sleep || delay;
+  const now = deps.now || (() => Date.now());
+
+  let parsed;
+  try {
+    parsed = parseArgs(argv);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+  const { options, command } = parsed;
+
+  if (options.help) {
+    showHelp(consoleImpl);
+    return EXIT.OK;
+  }
+
+  const target = typeof options.target === "string" ? options.target.trim() : "";
+  if (!target) {
+    consoleImpl.error("Error: --target <daemon> is required");
+    showHelp(consoleImpl);
+    return EXIT.CLI_ERROR;
+  }
+  if (command.length !== 1) {
+    consoleImpl.error(command.length === 0 ? "Error: no runId given" : "Error: expected exactly one runId");
+    showHelp(consoleImpl);
+    return EXIT.CLI_ERROR;
+  }
+  const runId = command[0];
+
+  let timeoutMs;
+  try {
+    timeoutMs = parseTimeoutMs(options.timeout);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+
+  let config;
+  try {
+    config = deps.config || loadCliConfig(options.configFile, env);
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  }
+
+  const basePath = `/api/agents/${encodeURIComponent(target)}/exec`;
+  const interrupt = interruptSignal(deps.process);
+  let run;
+  let pollError = null;
+  try {
+    const deadline = now() + timeoutMs;
+    run = await callApi(config, "GET", `${basePath}/runs/${encodeURIComponent(runId)}`, null, fetchImpl);
+    let killError;
+    ({ run, pollError, killError } = await waitForRun(config, basePath, run, {
+      deadline,
+      fetchImpl,
+      sleep,
+      now,
+      killOnTimeout: options.killOnTimeout,
+      signal: interrupt.signal,
+    }));
+    if (killError) {
+      consoleImpl.error(`[conductor] failed to stop the run on ${target}: ${killError.message}`);
+    } else if (options.killOnTimeout && run?.status === "cancelled") {
+      const why = interrupt.signal.aborted ? "interrupted" : "deadline reached";
+      consoleImpl.error(`[conductor] ${why}; stopped the command on ${target}`);
+    }
+  } catch (error) {
+    consoleImpl.error(`Error: ${error.message}`);
+    return EXIT.CLI_ERROR;
+  } finally {
+    interrupt.release();
+  }
+
+  return reportRun(run, {
+    consoleImpl,
+    json: options.json,
+    target,
+    pollError,
+    waitedMs: timeoutMs,
+    interrupted: interrupt.signal.aborted,
+  });
+}