@effectstream/orchestrator 0.100.9

package/src/api-client.ts

@@ -0,0 +1,92 @@
+import type { ProcessConfig } from "./config.ts";
+import { readStatePort, DEFAULT_API_PORT } from "./port-check.ts";
+
+export { DEFAULT_API_PORT };
+
+export type RemoteProcess = {
+  name: string;
+  pid: number | null;
+  ports: number[];
+  status: string;
+  startedAt: string | null;
+  endedAt: string | null;
+  exitCode: number | null;
+  link: string | null;
+  critical: boolean;
+  description: string | null;
+  logFile: string | null;
+};
+
+export class OrchestratorClient {
+  private baseUrl: string;
+
+  constructor(port?: number) {
+    const p = port ?? readStatePort() ?? DEFAULT_API_PORT;
+    this.baseUrl = `http://localhost:${p}`;
+  }
+
+  async isRunning(): Promise<boolean> {
+    try {
+      const res = await fetch(`${this.baseUrl}/health`, {
+        signal: AbortSignal.timeout(1000),
+      });
+      return res.ok;
+    } catch {
+      return false;
+    }
+  }
+
+  async getProcesses(): Promise<RemoteProcess[]> {
+    const res = await fetch(`${this.baseUrl}/processes`);
+    if (!res.ok) throw new Error(`API error ${res.status}`);
+    const body = (await res.json()) as { processes: RemoteProcess[] };
+    return body.processes;
+  }
+
+  async restart(name: string): Promise<{ success: boolean; error?: string }> {
+    const res = await fetch(`${this.baseUrl}/restart`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name }),
+    });
+    return res.json() as Promise<{ success: boolean; error?: string }>;
+  }
+
+  async stop(name?: string): Promise<{ success: boolean; error?: string }> {
+    const res = await fetch(`${this.baseUrl}/stop`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ name }),
+    });
+    return res.json() as Promise<{ success: boolean; error?: string }>;
+  }
+
+  async start(processes: ProcessConfig[], opts?: { only?: string[]; except?: string[]; serial?: boolean; noDeps?: boolean }): Promise<{ success: boolean; queued?: string[]; error?: string }> {
+    const res = await fetch(`${this.baseUrl}/start`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ processes, only: opts?.only, except: opts?.except, serial: opts?.serial, noDeps: opts?.noDeps }),
+    });
+    return res.json() as Promise<{ success: boolean; queued?: string[]; error?: string }>;
+  }
+
+  async silence(names: string[], unsilence = false): Promise<{ success: boolean; silenced?: string[]; error?: string }> {
+    const res = await fetch(`${this.baseUrl}/silence`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ names, unsilence }),
+    });
+    return res.json() as Promise<{ success: boolean; silenced?: string[]; error?: string }>;
+  }
+
+  async getSilenced(): Promise<string[]> {
+    const res = await fetch(`${this.baseUrl}/silence`);
+    if (!res.ok) throw new Error(`API error ${res.status}`);
+    const body = (await res.json()) as { silenced: string[] };
+    return body.silenced;
+  }
+
+  async shutdown(): Promise<void> {
+    await fetch(`${this.baseUrl}/shutdown`, { method: "POST" }).catch(() => {});
+  }
+}

package/src/api-server.ts

@@ -0,0 +1,183 @@
+import type { ProcessConfig } from "./config.ts";
+import type { ProcessManager, ManagedProcess } from "./process-manager.ts";
+import { runProcesses, type RunOptions } from "./task-runner.ts";
+
+function json(data: unknown, status = 200): Response {
+  return new Response(JSON.stringify(data, null, 2), {
+    status,
+    headers: { "Content-Type": "application/json" },
+  });
+}
+
+function serialize(p: ManagedProcess) {
+  return {
+    name: p.name,
+    pid: p.pid,
+    ports: p.ports,
+    status: p.status,
+    startedAt: p.startedAt?.toISOString() ?? null,
+    endedAt: p.endedAt?.toISOString() ?? null,
+    exitCode: p.exitCode,
+    link: p.config.link ?? null,
+    critical: p.config.critical ?? true,
+    description: p.config.description ?? null,
+    logFile: p.logFile ?? null,
+  };
+}
+
+export type ApiServerOptions = {
+  port: number;
+  manager: ProcessManager;
+  /** Called when the /shutdown endpoint is hit. */
+  onShutdown: () => Promise<void>;
+  /** Callback hooks for task events (used by both local and remote starts). */
+  runCallbacks?: Pick<RunOptions, "onTaskStart" | "onTaskRunning" | "onTaskDone" | "onTaskFailed" | "onShutdown">;
+};
+
+/**
+ * Starts a Bun HTTP API server so that external CLI commands
+ * (status, restart, stop) can communicate with the running daemon.
+ *
+ * Routes:
+ *   GET  /health          → { ok: true }
+ *   GET  /processes       → { processes: [...] }
+ *   POST /restart         → { name } → restarts named process
+ *   POST /stop            → { name? } → stops named process or all
+ *   POST /shutdown        → graceful full shutdown
+ */
+export function startApiServer({ port, manager, onShutdown, runCallbacks }: ApiServerOptions): ReturnType<typeof Bun.serve> {
+  return Bun.serve({
+    port,
+    async fetch(req) {
+      const url = new URL(req.url);
+
+      // ── Health ────────────────────────────────────────────────────────────
+      if (req.method === "GET" && url.pathname === "/health") {
+        return json({ ok: true });
+      }
+
+      // ── Process list ──────────────────────────────────────────────────────
+      if (req.method === "GET" && url.pathname === "/processes") {
+        return json({ processes: manager.getAll().map(serialize) });
+      }
+
+      // ── Start additional processes ────────────────────────────────────────
+      if (req.method === "POST" && url.pathname === "/start") {
+        let body: { processes: ProcessConfig[]; only?: string[]; except?: string[]; serial?: boolean; noDeps?: boolean };
+        try {
+          body = (await req.json()) as typeof body;
+        } catch {
+          return json({ error: "Invalid JSON body" }, 400);
+        }
+
+        if (!Array.isArray(body.processes) || body.processes.length === 0) {
+          return json({ error: "Missing or empty field: processes" }, 400);
+        }
+
+        // Launch in the background so the HTTP response returns immediately
+        runProcesses(body.processes, manager, {
+          only: body.only,
+          except: body.except,
+          serial: body.serial,
+          noDeps: body.noDeps,
+          ...runCallbacks,
+        }).catch((err) => {
+          console.error(`[API /start] Error: ${err?.message ?? err}`);
+        });
+
+        // Report which processes will actually be considered (after filters)
+        let queued = body.processes.map((p) => p.name);
+        if (body.only) {
+          const onlySet = new Set(body.only);
+          queued = queued.filter((n) => onlySet.has(n));
+        }
+        if (body.except) {
+          const exceptSet = new Set(body.except);
+          queued = queued.filter((n) => !exceptSet.has(n));
+        }
+
+        return json({ success: true, queued });
+      }
+
+      // ── Restart ───────────────────────────────────────────────────────────
+      if (req.method === "POST" && url.pathname === "/restart") {
+        let body: { name: string };
+        try {
+          body = (await req.json()) as { name: string };
+        } catch {
+          return json({ error: "Invalid JSON body" }, 400);
+        }
+
+        if (!body.name) return json({ error: "Missing field: name" }, 400);
+
+        const result = await manager.restart(body.name);
+        if (!result) {
+          return json(
+            { error: `No process named "${body.name}" found` },
+            404,
+          );
+        }
+        return json({ success: true, name: body.name });
+      }
+
+      // ── Stop ──────────────────────────────────────────────────────────────
+      if (req.method === "POST" && url.pathname === "/stop") {
+        let body: { name?: string } = {};
+        try {
+          body = (await req.json()) as { name?: string };
+        } catch {
+          /* empty body is fine */
+        }
+
+        if (body.name) {
+          const ok = await manager.stop(body.name);
+          if (!ok) {
+            return json(
+              { error: `No running process named "${body.name}"` },
+              404,
+            );
+          }
+          return json({ success: true, name: body.name });
+        }
+
+        await manager.stopAll();
+        return json({ success: true, stopped: "all" });
+      }
+
+      // ── Silence / Unsilence ──────────────────────────────────────────────
+      if (req.method === "POST" && url.pathname === "/silence") {
+        let body: { names: string[]; unsilence?: boolean };
+        try {
+          body = (await req.json()) as typeof body;
+        } catch {
+          return json({ error: "Invalid JSON body" }, 400);
+        }
+        if (!Array.isArray(body.names) || body.names.length === 0) {
+          return json({ error: "Missing or empty field: names" }, 400);
+        }
+        for (const name of body.names) {
+          if (body.unsilence) {
+            manager.unsilence(name);
+          } else {
+            manager.silence(name);
+          }
+        }
+        return json({ success: true, silenced: manager.getSilenced() });
+      }
+
+      if (req.method === "GET" && url.pathname === "/silence") {
+        return json({ silenced: manager.getSilenced() });
+      }
+
+      // ── Shutdown ──────────────────────────────────────────────────────────
+      if (req.method === "POST" && url.pathname === "/shutdown") {
+        // Respond first, then shut down
+        const response = json({ success: true });
+        setImmediate(onShutdown);
+        return response;
+      }
+
+      return json({ error: "Not found" }, 404);
+    },
+  });
+}

package/src/cli.ts

@@ -0,0 +1,183 @@
+#!/usr/bin/env bun
+/**
+ * orchestrator CLI
+ *
+ * Commands:
+ *   start   [config]  Start processes (runs as daemon, exposes HTTP API)
+ *   status            Show process status (via daemon or port checks)
+ *   restart <name>    Restart a named process
+ *   stop    [name]    Stop a named process, or all processes
+ *   list    [config]  List processes defined in a config without starting them
+ *   silence [name...] Suppress terminal output for processes
+ *   unsilence <name...> Resume terminal output for processes
+ *   logs    [name...] Follow background daemon log files
+ *
+ * Global options:
+ *   --port <n>        Orchestrator API port (default: 4747)
+ *   --help, -h        Print help
+ */
+
+import { runStartCommand } from "./commands/start.ts";
+import { runStatusCommand } from "./commands/status.ts";
+import { runRestartCommand } from "./commands/restart.ts";
+import { runStopCommand } from "./commands/stop.ts";
+import { runListCommand } from "./commands/list.ts";
+import { runSilenceCommand } from "./commands/silence.ts";
+import { runLogsCommand } from "./commands/logs.ts";
+
+// ── Minimal arg parser ────────────────────────────────────────────────────────
+
+export type ParsedArgs = {
+  command: string;
+  positionals: string[];
+  flags: Record<string, string | boolean>;
+};
+
+const SHORT_TO_LONG: Record<string, string> = {
+  b: "background",
+  c: "config",
+  e: "except",
+  f: "follow",
+  h: "help",
+  o: "only",
+  p: "port",
+  s: "serial",
+};
+
+function parseArgs(argv: string[]): ParsedArgs {
+  const positionals: string[] = [];
+  const flags: Record<string, string | boolean> = {};
+  let i = 0;
+  while (i < argv.length) {
+    const arg = argv[i];
+    if (arg.startsWith("--")) {
+      const eq = arg.indexOf("=");
+      if (eq !== -1) {
+        flags[arg.slice(2, eq)] = arg.slice(eq + 1);
+      } else if (i + 1 < argv.length && !argv[i + 1].startsWith("-")) {
+        flags[arg.slice(2)] = argv[++i];
+      } else {
+        flags[arg.slice(2)] = true;
+      }
+    } else if (arg.startsWith("-") && arg.length === 2) {
+      const key = SHORT_TO_LONG[arg.slice(1)] ?? arg.slice(1);
+      if (i + 1 < argv.length && !argv[i + 1].startsWith("-")) {
+        flags[key] = argv[++i];
+      } else {
+        flags[key] = true;
+      }
+    } else {
+      positionals.push(arg);
+    }
+    i++;
+  }
+  const [command = "", ...rest] = positionals;
+  return { command, positionals: rest, flags };
+}
+
+// ── Main ──────────────────────────────────────────────────────────────────────
+
+const { command, positionals, flags } = parseArgs(process.argv.slice(2));
+
+if (!command || flags["help"]) {
+  printHelp();
+  process.exit(0);
+}
+
+const port = flags["port"] ? Number(flags["port"]) : undefined;
+
+try {
+  switch (command) {
+    case "start":
+      await runStartCommand({ positionals, flags, port });
+      break;
+
+    case "status":
+      await runStatusCommand({ positionals, flags, port });
+      break;
+
+    case "restart":
+      await runRestartCommand({ positionals, flags, port });
+      break;
+
+    case "stop":
+      await runStopCommand({ positionals, flags, port });
+      break;
+
+    case "list":
+      await runListCommand({ positionals, flags });
+      break;
+
+    case "silence":
+      await runSilenceCommand({ positionals, flags, port });
+      break;
+
+    case "unsilence":
+      await runSilenceCommand({ positionals, flags: { ...flags, unsilence: true }, port });
+      break;
+
+    case "logs":
+      await runLogsCommand({ positionals, flags });
+      break;
+
+    default:
+      console.error(`Unknown command: ${command}`);
+      printHelp();
+      process.exit(1);
+  }
+} catch (err: any) {
+  console.error(`\x1b[31mError:\x1b[0m ${err?.message ?? err}`);
+  process.exit(1);
+}
+
+// ── Help ──────────────────────────────────────────────────────────────────────
+
+function printHelp() {
+  console.log(`\x1b[1morchestrator\x1b[0m — Bun-based process orchestration CLI
+
+\x1b[1mUsage:\x1b[0m
+  orchestrator <command> [options]
+
+\x1b[1mCommands:\x1b[0m
+  start   [config|name...]    Start all processes, or specific ones by name
+  status                      Show status of running processes
+  restart <name>              Restart a specific process by name
+  stop    [name]              Stop a specific process (or all if no name given)
+  list    [config]            List processes in config without starting them
+  silence [name...]           Suppress terminal output for processes (no args = show list)
+  unsilence <name...>         Resume terminal output for processes
+  logs    [name...]           Follow background daemon log files (like tail -f)
+
+\x1b[1mGlobal options:\x1b[0m
+  -c, --config  <path>        Config file (default: auto-detected from daemon, package.json, or orchestrator.config.ts)
+  -h, --help                  Print help
+
+\x1b[1mOptions for \`start\`:\x1b[0m
+  -b, --background            Run as a detached background daemon; logs go to files
+  -p, --port    <n>           Orchestrator API port (default: 4747)
+  -o, --only    <p1,p2,...>   Run only these processes (+ their dependencies)
+  -e, --except  <p1,p2,...>   Skip these processes
+  -s, --serial                Launch processes one at a time instead of in parallel waves
+  --no-deps                   Launch only named processes, skip dependency resolution
+  --log-dir     <path>        Log directory when using -b (default: .orchestrator-logs)
+  --no-api                    Disable the HTTP API server
+  --silence     <p1,p2,...>   Suppress terminal output for these processes
+
+\x1b[1mOptions for \`status\`:\x1b[0m
+  -f, --follow                Continuously refresh the status table (1s interval)
+
+\x1b[1mExamples:\x1b[0m
+  orchestrator start orchestrator.config.ts
+  orchestrator start hardhat
+  orchestrator start hardhat deploy-evm-contracts
+  orchestrator start --only=midnight-node,midnight-indexer
+  orchestrator start --except=avail-client
+  orchestrator status
+  orchestrator restart midnight-node
+  orchestrator stop batcher
+  orchestrator start --silence=midnight-node,bitcoin-core
+  orchestrator silence midnight-node bitcoin-core
+  orchestrator unsilence midnight-node
+  orchestrator list orchestrator.config.ts
+`);
+}

package/src/commands/list.ts

@@ -0,0 +1,63 @@
+import type { ParsedArgs } from "../cli.ts";
+import { logError } from "../display.ts";
+import { loadConfig, findDefaultConfig, findPackageJsonConfig } from "../load-config.ts";
+import { readStateConfigPath } from "../port-check.ts";
+
+type ListOptions = Pick<ParsedArgs, "positionals" | "flags">;
+
+export async function runListCommand(opts: ListOptions): Promise<void> {
+  const configPath =
+    (opts.flags["config"] as string | undefined) ??
+    opts.positionals[0] ??
+    readStateConfigPath() ??
+    (await findPackageJsonConfig()) ??
+    (await findDefaultConfig());
+
+  if (!configPath) {
+    logError("(3) No config file found. Pass one as an argument or create orchestrator.config.ts.");
+    process.exit(1);
+  }
+
+  let config;
+  try {
+    config = await loadConfig(configPath);
+  } catch (err: any) {
+    logError(err.message);
+    process.exit(1);
+  }
+
+  const c = {
+    reset: "\x1b[0m",
+    bold: "\x1b[1m",
+    dim: "\x1b[2m",
+    cyan: "\x1b[36m",
+    gray: "\x1b[90m",
+    green: "\x1b[32m",
+  };
+
+  console.log(`${c.bold}Processes in ${configPath}${c.reset}\n`);
+
+  for (const p of config.processes) {
+    const ports = p.stopProcessAtPort?.join(", ") ?? "";
+    const deps = (p.dependsOn ?? []).join(", ");
+    const cmd = `${p.command ?? "bun"} ${p.args.join(" ")}`;
+    const wait = p.waitToExit ? "wait-to-exit" : "background";
+    const crit = p.critical === false
+      ? `${c.dim}non-critical${c.reset}`
+      : `${c.dim}critical${c.reset}`;
+
+    console.log(`  ${c.bold}${c.cyan}${p.name}${c.reset}`);
+    if (p.description) {
+      console.log(`    ${c.dim}${p.description}${c.reset}`);
+    }
+    console.log(`    cmd:   ${c.dim}${cmd}${c.reset}`);
+    if (ports) console.log(`    ports: ${ports}`);
+    if (deps)  console.log(`    deps:  ${deps}`);
+    const manual = p.autoStart === false ? `  ${c.dim}manual${c.reset}` : "";
+    console.log(`    mode:  ${wait}  ${crit}${manual}`);
+    if (p.link) console.log(`    link:  ${c.green}${p.link}${c.reset}`);
+    console.log();
+  }
+
+  console.log(`${c.dim}Total: ${config.processes.length} process(es)${c.reset}`);
+}

package/src/commands/logs.ts

@@ -0,0 +1,142 @@
+import * as fs from "fs";
+import * as path from "path";
+import type { ParsedArgs } from "../cli.ts";
+
+type LogsOptions = Pick<ParsedArgs, "positionals" | "flags">;
+
+/**
+ * `logs` command — tails all (or specific) background daemon log files.
+ *
+ *   orchestrator logs                   # follow all logs
+ *   orchestrator logs sync batcher      # follow only sync.log and batcher.log
+ *   orchestrator logs --log-dir=./logs  # custom log directory
+ */
+export async function runLogsCommand(opts: LogsOptions): Promise<void> {
+  const logDir = path.resolve(
+    (opts.flags["log-dir"] as string | undefined) ??
+      path.join(process.cwd(), ".orchestrator-logs"),
+  );
+
+  if (!fs.existsSync(logDir)) {
+    throw new Error(
+      `Log directory not found: ${logDir}\nStart the orchestrator with --background first.`,
+    );
+  }
+
+  // Determine which log files to follow
+  const filterNames = opts.positionals.length > 0 ? opts.positionals : null;
+
+  const allFiles = fs.readdirSync(logDir).filter((f) => f.endsWith(".log")).sort();
+  const files = filterNames
+    ? allFiles.filter((f) => filterNames.some((name) => f === `${name}.log`))
+    : allFiles;
+
+  if (files.length === 0) {
+    const msg = filterNames
+      ? `No log files found for: ${filterNames.join(", ")}\nAvailable: ${allFiles.map((f) => f.replace(/\.log$/, "")).join(", ") || "(none)"}`
+      : `No log files found in ${logDir}`;
+    throw new Error(msg);
+  }
+
+  const col = {
+    reset: "\x1b[0m",
+    bold: "\x1b[1m",
+    dim: "\x1b[2m",
+    cyan: "\x1b[36m",
+  };
+
+  console.log(`${col.bold}Following logs${col.reset} from ${col.cyan}${logDir}${col.reset}`);
+  console.log(`${col.dim}Files: ${files.join(", ")}${col.reset}`);
+  console.log(`${col.dim}Press Ctrl+C to stop${col.reset}\n`);
+
+  // Assign a color to each file for visual distinction
+  const palette = [
+    "\x1b[32m", // green
+    "\x1b[33m", // yellow
+    "\x1b[34m", // blue
+    "\x1b[35m", // magenta
+    "\x1b[36m", // cyan
+    "\x1b[91m", // bright red
+    "\x1b[92m", // bright green
+    "\x1b[93m", // bright yellow
+    "\x1b[94m", // bright blue
+    "\x1b[95m", // bright magenta
+  ];
+
+  const maxNameLen = Math.max(...files.map((f) => f.replace(/\.log$/, "").length));
+
+  // Track file watchers for cleanup
+  const watchers: fs.FSWatcher[] = [];
+  const filePositions = new Map<string, number>();
+
+  for (let i = 0; i < files.length; i++) {
+    const file = files[i];
+    const filePath = path.join(logDir, file);
+    const name = file.replace(/\.log$/, "");
+    const color = palette[i % palette.length];
+    const prefix = `${color}${name.padEnd(maxNameLen)}${col.reset} ${col.dim}|${col.reset} `;
+
+    // Read existing content from the end (last 50 lines) for context
+    const initialContent = readTail(filePath, 50);
+    if (initialContent) {
+      for (const line of initialContent.split("\n")) {
+        if (line) console.log(`${prefix}${line}`);
+      }
+    }
+
+    // Track current position
+    const stat = fs.statSync(filePath);
+    filePositions.set(filePath, stat.size);
+
+    // Watch for changes
+    const watcher = fs.watch(filePath, () => {
+      const prevPos = filePositions.get(filePath) ?? 0;
+      let currentSize: number;
+      try {
+        currentSize = fs.statSync(filePath).size;
+      } catch {
+        return;
+      }
+
+      if (currentSize <= prevPos) {
+        // File was truncated — reset position
+        filePositions.set(filePath, 0);
+        return;
+      }
+
+      // Read new bytes
+      const fd = fs.openSync(filePath, "r");
+      const buf = Buffer.alloc(currentSize - prevPos);
+      fs.readSync(fd, buf, 0, buf.length, prevPos);
+      fs.closeSync(fd);
+      filePositions.set(filePath, currentSize);
+
+      const text = buf.toString("utf-8");
+      for (const line of text.split("\n")) {
+        if (line) console.log(`${prefix}${line}`);
+      }
+    });
+    watchers.push(watcher);
+  }
+
+  // Keep alive until Ctrl+C
+  await new Promise<void>((resolve) => {
+    const cleanup = () => {
+      for (const w of watchers) w.close();
+      resolve();
+    };
+    process.on("SIGINT", cleanup);
+    process.on("SIGTERM", cleanup);
+  });
+}
+
+/** Read the last N lines of a file. */
+function readTail(filePath: string, lines: number): string {
+  try {
+    const content = fs.readFileSync(filePath, "utf-8");
+    const allLines = content.split("\n");
+    return allLines.slice(-lines).join("\n");
+  } catch {
+    return "";
+  }
+}

package/src/commands/restart.ts

@@ -0,0 +1,34 @@
+import type { ParsedArgs } from "../cli.ts";
+import { OrchestratorClient } from "../api-client.ts";
+import { logInfo, logError } from "../display.ts";
+
+type RestartOptions = Pick<ParsedArgs, "positionals" | "flags"> & {
+  port?: number;
+};
+
+export async function runRestartCommand(opts: RestartOptions): Promise<void> {
+  const name = opts.positionals[0];
+  if (!name) {
+    logError("Usage: orchestrator restart <process-name>");
+    process.exit(1);
+  }
+
+  const client = new OrchestratorClient(opts.port);
+
+  if (!(await client.isRunning())) {
+    logError(
+      "Orchestrator daemon is not running. Start it with: orchestrator start",
+    );
+    process.exit(1);
+  }
+
+  logInfo(`Restarting "${name}"…`);
+  const result = await client.restart(name);
+
+  if (result.error) {
+    logError(result.error);
+    process.exit(1);
+  }
+
+  logInfo(`"${name}" restarted successfully.`);
+}