@celilo/cli 0.10.0 → 0.11.0

package/CELILO_SUBSYSTEMS.md

@@ -77,6 +77,17 @@ see `design/README.md`. Companion doc: [CELILO_CORE_MODULES.md](./CELILO_CORE_MO
 
 - **Event bus** — `packages/event-bus/src/index.ts` — `Bus`, `openBus`, `defineEvents`, `defineHandler`, `runDispatcher`, pattern matching + timer ticks (`emitDueTimerTicks`, `retentionSweep`).
 
+## Remote API (drive the CLI over the wire)
+
+Run any celilo command on celilo-mgr over SSH instead of screen-scraping `ssh <host> celilo …`. Typed, streamed, per-operation authz, mid-run interviews. Design: `v2/API_COMMUNICATION.md`.
+
+- **Wire protocol** — `apps/celilo/src/api/protocol.ts` — versioned NDJSON tagged union (`command`/`progress`/`log`/`result`/`error`/`interview`/`answer`) + `translateOutputLine`.
+- **Server** — `apps/celilo/src/api/serve.ts` (`apiServeMode`); the `celilo api-serve --principal=<id>` sshd forced-command entry point (dispatched in `apps/celilo/src/cli/index.ts`). Authorizes per principal, runs the command as a protocol-mode child, streams output, audits to stderr.
+- **Client** — `apps/celilo/src/api/remote-client.ts` — `resolveRemote` (`--remote <dest>` / `CELILO_REMOTE`), `runRemoteClient` (`ssh -T`, renders progress via the local ProgressDisplay, answers interviews via clack).
+- **Access control** — `apps/celilo/src/services/api-access.ts` — `grantPrincipal`, `isAuthorized` (deny-by-default, `command:subcommand` grants), `renderAuthorizedKeys`. Table: `api_principals` (`apps/celilo/src/db/schema.ts`). CLI: `apps/celilo/src/cli/commands/api.ts` (`api grant|list|revoke|authorized-keys|key new`).
+- **Mid-run interview bridge (`kind:daemon` responder)** — `apps/celilo/src/services/remote-responder.ts` — `startRemoteResponder` bridges bus `interview.required.*` ↔ wire.
+- **Server provisioning** — the `celilo-bootstrap` deb (`packaging/celilo-bootstrap/scripts/postinst`) creates the non-root `celilo-api` landing account + sshd; membership in the `celilo` group + `/etc/sudoers.d/celilo` (`!use_pty`) gives api-serve DB access via the wrapper's sudo-drop.
+
 ## E2E simulation
 
 - **cele2e harness** — `packages/e2e/src/` — `runner.ts`, `container-manager.ts` (`startNetwork`, `reconnectNetwork`), `network-builder.ts` (`NetworkBuilder`).

package/drizzle/0014_api_principals.sql

@@ -0,0 +1,10 @@
+CREATE TABLE `api_principals` (
+	`id` text PRIMARY KEY NOT NULL,
+	`name` text NOT NULL,
+	`public_key` text NOT NULL,
+	`grants` text DEFAULT '[]' NOT NULL,
+	`created_at` integer DEFAULT (unixepoch()) NOT NULL,
+	`updated_at` integer DEFAULT (unixepoch()) NOT NULL
+);
+--> statement-breakpoint
+CREATE UNIQUE INDEX `api_principals_name_unique` ON `api_principals` (`name`);

package/drizzle/meta/_journal.json

@@ -99,6 +99,13 @@
       "when": 1782456724000,
       "tag": "0013_dns_view_overrides",
       "breakpoints": true
+    },
+    {
+      "idx": 14,
+      "version": "6",
+      "when": 1783000000000,
+      "tag": "0014_api_principals",
+      "breakpoints": true
     }
   ]
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {
@@ -16,13 +16,7 @@
     "CELILO_SUBSYSTEMS.md",
     "CELILO_CORE_MODULES.md"
   ],
-  "keywords": [
-    "celilo",
-    "homelab",
-    "orchestration",
-    "ansible",
-    "terraform"
-  ],
+  "keywords": ["celilo", "homelab", "orchestration", "ansible", "terraform"],
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -57,7 +51,7 @@
     "@aws-sdk/client-s3": "^3.1024.0",
     "@celilo/capabilities": "^0.6.0",
     "@celilo/cli-display": "^0.1.9",
-    "@celilo/event-bus": "^0.1.8",
+    "@celilo/event-bus": "^0.1.7",
     "@clack/prompts": "^1.1.0",
     "ajv": "^8.18.0",
     "drizzle-orm": "^0.36.4",

package/src/api/protocol.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, test } from 'bun:test';
+import { translateOutputLine } from './protocol';
+
+describe('translateOutputLine', () => {
+  test('start marker → progress start', () => {
+    expect(translateOutputLine('[progress:start] Deploying caddy | Deployed caddy')).toEqual({
+      type: 'progress',
+      kind: 'start',
+      doing: 'Deploying caddy',
+      done: 'Deployed caddy',
+    });
+  });
+
+  test('push marker → progress push', () => {
+    expect(translateOutputLine('[progress:push] Sub step | Sub done')).toEqual({
+      type: 'progress',
+      kind: 'push',
+      doing: 'Sub step',
+      done: 'Sub done',
+    });
+  });
+
+  test('done marker with override message', () => {
+    expect(translateOutputLine('[progress:done] all good')).toEqual({
+      type: 'progress',
+      kind: 'done',
+      message: 'all good',
+    });
+  });
+
+  test('done marker without message', () => {
+    expect(translateOutputLine('[progress:done]')).toEqual({
+      type: 'progress',
+      kind: 'done',
+      message: undefined,
+    });
+  });
+
+  test('fail marker → progress fail', () => {
+    expect(translateOutputLine('[progress:fail] it broke')).toEqual({
+      type: 'progress',
+      kind: 'fail',
+      message: 'it broke',
+    });
+  });
+
+  test('sub marker → progress sub', () => {
+    expect(translateOutputLine('[progress:sub] running terraform')).toEqual({
+      type: 'progress',
+      kind: 'sub',
+      message: 'running terraform',
+    });
+  });
+
+  test('legacy instant marker → progress message', () => {
+    expect(translateOutputLine('[progress] heads up')).toEqual({
+      type: 'progress',
+      kind: 'message',
+      message: 'heads up',
+    });
+  });
+
+  test('plain line → log', () => {
+    expect(translateOutputLine('just some output')).toEqual({
+      type: 'log',
+      message: 'just some output',
+    });
+  });
+
+  test('a line that only resembles a marker → log', () => {
+    expect(translateOutputLine('see [progress:start] in the docs')).toEqual({
+      type: 'log',
+      message: 'see [progress:start] in the docs',
+    });
+  });
+});

package/src/api/protocol.ts

@@ -0,0 +1,159 @@
+/**
+ * Remote API wire protocol (Slice 1).
+ *
+ * A versioned, Zod-validated tagged union carried as NDJSON. This is the
+ * transport-neutral message layer for driving the CLI over the wire — see
+ * `v2/API_COMMUNICATION.md`. Slice 1 covers non-interactive command execution
+ * + streamed output; `interview` / `answer` / `cancel` arrive in Slice 3.
+ *
+ * The command crosses the wire as a *structured, validated* `argv` array —
+ * never a shell string handed to `exec`.
+ */
+
+import { z } from 'zod';
+
+export const API_PROTOCOL_VERSION = 1;
+
+// ── client → server ──────────────────────────────────────────────────────
+
+export const CommandMessageSchema = z.object({
+  type: z.literal('command'),
+  /** Structured argv (e.g. ["module", "list"]) — dispatched into runCli. */
+  argv: z.array(z.string()).min(1),
+});
+export type CommandMessage = z.infer<typeof CommandMessageSchema>;
+
+/** Client's answer to a server `interview` message, correlated by `id`. */
+export const AnswerMessageSchema = z.object({
+  type: z.literal('answer'),
+  id: z.string(),
+  value: z.unknown(),
+});
+export type AnswerMessage = z.infer<typeof AnswerMessageSchema>;
+
+export const ClientMessageSchema = z.discriminatedUnion('type', [
+  CommandMessageSchema,
+  AnswerMessageSchema,
+  // Slice 3+: CancelMessageSchema
+]);
+export type ClientMessage = z.infer<typeof ClientMessageSchema>;
+
+// ── server → client ──────────────────────────────────────────────────────
+
+/** Handshake: server announces itself + the protocol version it speaks. */
+export const ReadyMessageSchema = z.object({
+  type: z.literal('ready'),
+  protocolVersion: z.literal(API_PROTOCOL_VERSION),
+});
+export type ReadyMessage = z.infer<typeof ReadyMessageSchema>;
+
+/**
+ * A forwarded `ProgressDisplay` protocol-mode marker. Mirrors the markers
+ * emitted by `packages/cli-display` in protocol mode (start/push/done/fail/sub
+ * and the legacy instant `[progress]`).
+ */
+export const ProgressMessageSchema = z.object({
+  type: z.literal('progress'),
+  kind: z.enum(['start', 'push', 'done', 'fail', 'sub', 'message']),
+  doing: z.string().optional(),
+  done: z.string().optional(),
+  message: z.string().optional(),
+});
+export type ProgressMessage = z.infer<typeof ProgressMessageSchema>;
+
+/** A plain output line from the command that isn't a progress marker. */
+export const LogMessageSchema = z.object({
+  type: z.literal('log'),
+  message: z.string(),
+});
+export type LogMessage = z.infer<typeof LogMessageSchema>;
+
+/** Terminal message for a command run. */
+export const ResultMessageSchema = z.object({
+  type: z.literal('result'),
+  success: z.boolean(),
+  exitCode: z.number(),
+});
+export type ResultMessage = z.infer<typeof ResultMessageSchema>;
+
+/** Protocol-level error (malformed message, etc.) — distinct from a failed command. */
+export const ErrorMessageSchema = z.object({
+  type: z.literal('error'),
+  error: z.string(),
+});
+export type ErrorMessage = z.infer<typeof ErrorMessageSchema>;
+
+/**
+ * A mid-run interview question forwarded from the server's event bus. The client
+ * renders it, then replies with an `answer` carrying the same `id`. Mirrors the
+ * generic `interview.required.*` payload (see bus-interview.ts).
+ */
+export const InterviewMessageSchema = z.object({
+  type: z.literal('interview'),
+  id: z.string(),
+  kind: z.enum(['text', 'confirm', 'select', 'multiselect']),
+  message: z.string(),
+  description: z.string().optional(),
+  defaultValue: z.string().optional(),
+  placeholder: z.string().optional(),
+  options: z
+    .array(z.object({ value: z.string(), label: z.string(), hint: z.string().optional() }))
+    .optional(),
+  required: z.boolean().optional(),
+});
+export type InterviewMessage = z.infer<typeof InterviewMessageSchema>;
+
+export const ServerMessageSchema = z.discriminatedUnion('type', [
+  ReadyMessageSchema,
+  ProgressMessageSchema,
+  LogMessageSchema,
+  ResultMessageSchema,
+  ErrorMessageSchema,
+  InterviewMessageSchema,
+]);
+export type ServerMessage = z.infer<typeof ServerMessageSchema>;
+
+// ── translation ───────────────────────────────────────────────────────────
+
+/**
+ * Translate one line of the command's stdout into a server message. Progress
+ * markers become structured `progress` messages; everything else is a `log`.
+ *
+ * ponytail: the marker grammar is duplicated from packages/cli-display's
+ * ProgressDisplay (which *emits* it). If a third consumer appears, extract a
+ * shared parser into @celilo/cli-display and have both sides import it.
+ */
+export function translateOutputLine(line: string): ProgressMessage | LogMessage {
+  const startPush = line.match(/^\[progress:(start|push)\] (.*?) \| (.*)$/);
+  if (startPush) {
+    return {
+      type: 'progress',
+      kind: startPush[1] as 'start' | 'push',
+      doing: startPush[2],
+      done: startPush[3],
+    };
+  }
+
+  const done = line.match(/^\[progress:done\](.*)$/);
+  if (done) {
+    const msg = done[1].trim();
+    return { type: 'progress', kind: 'done', message: msg || undefined };
+  }
+
+  const fail = line.match(/^\[progress:fail\] (.*)$/);
+  if (fail) {
+    return { type: 'progress', kind: 'fail', message: fail[1] };
+  }
+
+  const sub = line.match(/^\[progress:sub\] (.*)$/);
+  if (sub) {
+    return { type: 'progress', kind: 'sub', message: sub[1] };
+  }
+
+  const legacy = line.match(/^\[progress\] (.*)$/);
+  if (legacy) {
+    return { type: 'progress', kind: 'message', message: legacy[1] };
+  }
+
+  return { type: 'log', message: line };
+}

package/src/api/remote-client.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, test } from 'bun:test';
+import { type RemoteTransport, resolveRemote, runRemoteClient } from './remote-client';
+
+const argv = (rest: string[]) => ['bun', 'celilo', ...rest];
+
+describe('resolveRemote', () => {
+  test('leading --remote <dest>', () => {
+    expect(resolveRemote(argv(['--remote', 'host', 'module', 'list']), undefined)).toEqual({
+      dest: 'host',
+      commandArgv: ['module', 'list'],
+    });
+  });
+
+  test('leading --remote=<dest>', () => {
+    expect(resolveRemote(argv(['--remote=user@host', 'status']), undefined)).toEqual({
+      dest: 'user@host',
+      commandArgv: ['status'],
+    });
+  });
+
+  test('CELILO_REMOTE env', () => {
+    expect(resolveRemote(argv(['module', 'list']), 'envhost')).toEqual({
+      dest: 'envhost',
+      commandArgv: ['module', 'list'],
+    });
+  });
+
+  test('leading flag beats env', () => {
+    expect(resolveRemote(argv(['--remote', 'flaghost', 'status']), 'envhost')?.dest).toBe(
+      'flaghost',
+    );
+  });
+
+  test('local invocation → null', () => {
+    expect(resolveRemote(argv(['module', 'list']), undefined)).toBeNull();
+  });
+
+  test('--remote with no dest → null', () => {
+    expect(resolveRemote(argv(['--remote']), undefined)).toBeNull();
+  });
+});
+
+test('renders a forwarded interview and sends the answer back', async () => {
+  const writes: string[] = [];
+  const encoder = new TextEncoder();
+  let controller!: ReadableStreamDefaultController<Uint8Array>;
+  const stdout = new ReadableStream<Uint8Array>({
+    start(c) {
+      controller = c;
+    },
+  });
+  const push = (obj: unknown) => controller.enqueue(encoder.encode(`${JSON.stringify(obj)}\n`));
+
+  const transport: RemoteTransport = {
+    stdin: {
+      write(chunk: string) {
+        writes.push(chunk);
+        // The command's answer arrived — finish the command.
+        if (chunk.includes('"answer"')) {
+          push({ type: 'result', success: true, exitCode: 0 });
+          controller.close();
+        }
+      },
+    },
+    stdout,
+    kill() {},
+    exited: Promise.resolve(0),
+  };
+
+  // Server script: greet, then ask one interview.
+  push({ type: 'ready', protocolVersion: 1 });
+  push({ type: 'interview', id: 'q1', kind: 'text', message: 'Hostname?' });
+
+  const seen: Array<{ id: string }> = [];
+  const code = await runRemoteClient('ignored', ['module', 'deploy', 'site'], {
+    openTransport: () => transport,
+    out: { write() {} },
+    renderInterview: async (iv) => {
+      seen.push(iv);
+      return 'myhost';
+    },
+  });
+
+  expect(code).toBe(0);
+  expect(seen).toHaveLength(1);
+  expect(seen[0].id).toBe('q1');
+  expect(writes.some((w) => w.includes('"command"'))).toBe(true);
+  const answer = writes.find((w) => w.includes('"answer"'));
+  expect(answer).toBeDefined();
+  expect(JSON.parse(answer as string)).toEqual({ type: 'answer', id: 'q1', value: 'myhost' });
+});

package/src/api/remote-client.ts

@@ -0,0 +1,218 @@
+/**
+ * Remote API client — `celilo --remote <dest> <command...>` (Slice 2c).
+ *
+ * SSHes to <dest> (whose sshd forced command runs `api-serve --principal=…`),
+ * ships the command as a structured argv over the NDJSON protocol, renders the
+ * streamed `progress`/`log` through the local ProgressDisplay, and exits with
+ * the command's exit code. Reuses the system `ssh` client, so keys, known_hosts,
+ * agent, and ~/.ssh/config all apply — execution is remote, presentation local.
+ */
+
+import { type DisplayWriter, ProgressDisplay } from '@celilo/cli-display';
+import * as clack from '@clack/prompts';
+import { type InterviewMessage, type ServerMessage, ServerMessageSchema } from './protocol';
+
+const REMOTE_FLAG = '--remote';
+
+/** Render a forwarded interview via clack, returning the operator's answer. */
+async function defaultRenderInterview(iv: InterviewMessage): Promise<unknown> {
+  const options = (iv.options ?? []).map((o) => ({ value: o.value, label: o.label, hint: o.hint }));
+  let result: unknown;
+  switch (iv.kind) {
+    case 'confirm':
+      result = await clack.confirm({
+        message: iv.message,
+        initialValue: iv.defaultValue === 'true',
+      });
+      break;
+    case 'select':
+      result = await clack.select({ message: iv.message, options });
+      break;
+    case 'multiselect':
+      result = await clack.multiselect({
+        message: iv.message,
+        options,
+        required: iv.required ?? false,
+      });
+      break;
+    default:
+      result = await clack.text({
+        message: iv.message,
+        defaultValue: iv.defaultValue,
+        placeholder: iv.placeholder,
+      });
+  }
+  if (clack.isCancel(result)) {
+    clack.cancel('Cancelled.');
+    process.exit(130);
+  }
+  return result;
+}
+
+/** Minimal transport surface — satisfied by a `Bun.spawn` of `ssh`. */
+export interface RemoteTransport {
+  stdin: { write(chunk: string): void; flush?(): number | Promise<number> };
+  stdout: ReadableStream<Uint8Array>;
+  kill(): void;
+  exited: Promise<number>;
+}
+
+/**
+ * Resolve the remote target from a leading `--remote <dest>` / `--remote=<dest>`
+ * or the `CELILO_REMOTE` env. Returns null for a local invocation.
+ */
+export function resolveRemote(
+  argv: string[],
+  remoteEnv: string | undefined = process.env.CELILO_REMOTE,
+): { dest: string; commandArgv: string[] } | null {
+  const rest = argv.slice(2);
+
+  if (rest[0] === REMOTE_FLAG && rest[1]) {
+    return { dest: rest[1], commandArgv: rest.slice(2) };
+  }
+  if (rest[0]?.startsWith(`${REMOTE_FLAG}=`)) {
+    return { dest: rest[0].slice(REMOTE_FLAG.length + 1), commandArgv: rest.slice(1) };
+  }
+  if (remoteEnv) {
+    return { dest: remoteEnv, commandArgv: rest };
+  }
+  return null;
+}
+
+function openSshTransport(dest: string): RemoteTransport {
+  // -T: no pseudo-tty — we want a raw NDJSON pipe. stderr inherits so ssh's own
+  // diagnostics (host-key prompts, connection failures) reach the operator.
+  const proc = Bun.spawn(['ssh', '-T', dest], {
+    stdin: 'pipe',
+    stdout: 'pipe',
+    stderr: 'inherit',
+  });
+  return {
+    stdin: proc.stdin,
+    stdout: proc.stdout,
+    kill: () => proc.kill(),
+    exited: proc.exited,
+  };
+}
+
+/** Apply one server message to the local display; returns an exit code if terminal. */
+function applyMessage(
+  msg: ServerMessage,
+  display: ProgressDisplay,
+  out: DisplayWriter,
+): number | null {
+  switch (msg.type) {
+    case 'ready':
+      return null;
+    case 'progress':
+      switch (msg.kind) {
+        case 'start':
+          display.startStep(msg.doing ?? '', msg.done ?? '');
+          break;
+        case 'push':
+          display.pushStep(msg.doing ?? '', msg.done ?? '');
+          break;
+        case 'done':
+          display.doneStep(msg.message);
+          break;
+        case 'fail':
+          display.failStep(msg.message ?? '');
+          break;
+        case 'sub':
+          display.subEvent(msg.message ?? '');
+          break;
+        case 'message':
+          display.instantEvent(msg.message ?? '');
+          break;
+      }
+      return null;
+    case 'log':
+      // Nest under the active step so the footer isn't corrupted; otherwise print.
+      if (display.hasPending) {
+        display.subEvent(msg.message);
+      } else {
+        out.write(`${msg.message}\n`);
+      }
+      return null;
+    case 'error':
+      process.stderr.write(`${msg.error}\n`);
+      return null;
+    case 'interview':
+      // Handled by the caller (needs the transport to reply) — never reached here.
+      return null;
+    case 'result':
+      return msg.exitCode;
+  }
+}
+
+export async function runRemoteClient(
+  dest: string,
+  argv: string[],
+  opts: {
+    openTransport?: (dest: string) => RemoteTransport;
+    out?: DisplayWriter;
+    renderInterview?: (interview: InterviewMessage) => Promise<unknown>;
+  } = {},
+): Promise<number> {
+  if (argv.length === 0) {
+    process.stderr.write(
+      'celilo --remote <dest> requires a command (e.g. celilo --remote host module list)\n',
+    );
+    return 2;
+  }
+
+  const openTransport = opts.openTransport ?? openSshTransport;
+  const out = opts.out ?? process.stdout;
+  const renderInterview = opts.renderInterview ?? defaultRenderInterview;
+  const display = new ProgressDisplay({ out });
+
+  const transport = openTransport(dest);
+  transport.stdin.write(`${JSON.stringify({ type: 'command', argv })}\n`);
+  await transport.stdin.flush?.();
+
+  let resultExit: number | null = null;
+  const decoder = new TextDecoder();
+  let buffer = '';
+
+  outer: for await (const chunk of transport.stdout) {
+    buffer += decoder.decode(chunk, { stream: true });
+    let nl = buffer.indexOf('\n');
+    while (nl >= 0) {
+      const line = buffer.slice(0, nl);
+      buffer = buffer.slice(nl + 1);
+      nl = buffer.indexOf('\n');
+      if (!line.trim()) continue;
+
+      let msg: ServerMessage;
+      try {
+        msg = ServerMessageSchema.parse(JSON.parse(line));
+      } catch {
+        // Non-protocol line (e.g. an ssh notice) — pass through.
+        out.write(`${line}\n`);
+        continue;
+      }
+
+      if (msg.type === 'interview') {
+        const value = await renderInterview(msg);
+        transport.stdin.write(`${JSON.stringify({ type: 'answer', id: msg.id, value })}\n`);
+        await transport.stdin.flush?.();
+        continue;
+      }
+
+      const exit = applyMessage(msg, display, out);
+      if (exit !== null) {
+        resultExit = exit;
+        break outer;
+      }
+    }
+  }
+
+  if (resultExit !== null) {
+    // Server is still looping awaiting more input — tear it down.
+    transport.kill();
+    return resultExit;
+  }
+
+  // Stream ended without a result — surface the transport's exit (ssh failure).
+  return transport.exited;
+}