@celilo/cli 0.19.0 → 0.21.0

package/src/api/serve.ts

@@ -2,13 +2,19 @@
  * Remote API server — `celilo api-serve --principal=<id>` (Slices 1–3).
  *
  * Invoked as the sshd forced command for an enrolled key, so it starts already
- * authenticated as `principal`. Reads `command`/`answer` messages as NDJSON on
- * stdin/stdout, authorizes each command against the principal's grants
- * (deny-by-default), and — if allowed — runs it as a child `celilo` process,
- * translating the child's protocol-mode output into `progress`/`log` and a
- * terminal `result`. While a command runs, a remote responder bridges the event
- * bus to the wire so mid-run `interview`s are answered by the client. Every
- * attempt is audited to stderr.
+ * authenticated as `principal`. Reads `command`/`answer`/`unanswerable`/
+ * `attach`/`cancel` messages as NDJSON on stdin/stdout, authorizes each command
+ * against the principal's grants (deny-by-default), and — if allowed — runs it
+ * as a child `celilo` process, translating the child's protocol-mode output into
+ * `progress`/`log` and a terminal `result`. While a command runs, a remote
+ * responder bridges the event bus to the wire so mid-run `interview`s are
+ * answered by the client. Every attempt is audited to stderr.
+ *
+ * When the client cannot decide a question, the command **parks**: the query is
+ * left unanswered on the bus, the child stays alive past the end of the ssh
+ * session, its output is buffered in the session registry, and the client is
+ * told `blocked` with the session and query ids (celilo#609). Another responder
+ * answers by event id; a later `attach` collects the outcome.
  */
 
 import { createInterface } from 'node:readline';
@@ -16,16 +22,30 @@ import {
   API_PROTOCOL_VERSION,
   ClientMessageSchema,
   type ServerMessage,
+  ServerMessageSchema,
   translateOutputLine,
 } from '@celilo/core';
 import { parseArguments } from '../cli/parser';
 import { getEventBusPath } from '../config/paths';
 import { isAuthorized } from '../services/api-access';
+import { EVENT_TYPES } from '../services/bus-interview';
 import { type WireInterview, startRemoteResponder } from '../services/remote-responder';
+import {
+  SessionWriter,
+  abandonSession,
+  expiryReason,
+  readSession,
+  reapExpiredSessions,
+  replayOutput,
+  stillParked,
+} from './sessions';
 
 /** Exit code returned to the client when authz denies a command. */
 const EXIT_PERMISSION_DENIED = 126;
 
+/** How often an attached client polls a session's buffer for new output. */
+const ATTACH_POLL_MS = 250;
+
 function send(msg: ServerMessage): void {
   process.stdout.write(`${JSON.stringify(msg)}\n`);
 }
@@ -44,8 +64,20 @@ function opOf(argv: string[]): { command: string; subcommand?: string; label: st
   return { command: parsed.command, subcommand: parsed.subcommand, label };
 }
 
+/**
+ * Emit a message to the attached client (if any) AND to the session buffer, so
+ * output produced while nobody is attached is not lost.
+ */
+function emit(session: SessionWriter, msg: ServerMessage, attached: () => boolean): void {
+  session.append(msg);
+  if (attached()) send(msg);
+}
+
 /** Drain a child stream line-by-line, forwarding each line as a message. */
-async function pumpLines(stream: ReadableStream<Uint8Array>): Promise<void> {
+async function pumpLines(
+  stream: ReadableStream<Uint8Array>,
+  forward: (msg: ServerMessage) => void,
+): Promise<void> {
   const decoder = new TextDecoder();
   let buffer = '';
   for await (const chunk of stream) {
@@ -54,17 +86,17 @@ async function pumpLines(stream: ReadableStream<Uint8Array>): Promise<void> {
     while (nl >= 0) {
       const line = buffer.slice(0, nl);
       buffer = buffer.slice(nl + 1);
-      send(translateOutputLine(line));
+      forward(translateOutputLine(line));
       nl = buffer.indexOf('\n');
     }
   }
   if (buffer.length > 0) {
-    send(translateOutputLine(buffer));
+    forward(translateOutputLine(buffer));
   }
 }
 
 /** Run an authorized command as a child; returns its exit code. */
-async function runCommand(argv: string[]): Promise<number> {
+async function runCommand(argv: string[], forward: (msg: ServerMessage) => void): Promise<number> {
   // Re-invoke this same CLI as a child. The child is non-TTY (piped stdout) so
   // ProgressDisplay resolves to protocol mode and emits `[progress:*]` markers.
   const child = Bun.spawn([process.execPath, Bun.main, ...argv], {
@@ -73,49 +105,132 @@ async function runCommand(argv: string[]): Promise<number> {
     stderr: 'pipe',
   });
 
-  await Promise.all([pumpLines(child.stdout), pumpLines(child.stderr)]);
+  await Promise.all([pumpLines(child.stdout, forward), pumpLines(child.stderr, forward)]);
   const exitCode = await child.exited;
-  send({ type: 'result', success: exitCode === 0, exitCode });
+  forward({ type: 'result', success: exitCode === 0, exitCode });
   return exitCode;
 }
 
-async function handleCommand(
-  principal: string,
-  argv: string[],
-  busDbPath: string,
-  ask: (interview: WireInterview) => Promise<unknown>,
-): Promise<void> {
-  const { command, subcommand, label } = opOf(argv);
-
-  if (!(await isAuthorized(principal, command, subcommand))) {
-    send({ type: 'error', error: `permission denied: "${principal}" is not granted "${label}"` });
+/**
+ * Replay a parked session's buffered output to the attaching client, then tail
+ * it until the session goes terminal. Deny-by-default on the owning principal.
+ */
+async function handleAttach(principal: string, sessionId: string): Promise<void> {
+  const record = readSession(sessionId);
+  if (!record) {
+    send({ type: 'error', error: `no such session: ${sessionId}` });
+    send({ type: 'result', success: false, exitCode: 1 });
+    return;
+  }
+  if (record.principal !== principal) {
+    send({
+      type: 'error',
+      error: `permission denied: session ${sessionId} belongs to another principal`,
+    });
     send({ type: 'result', success: false, exitCode: EXIT_PERMISSION_DENIED });
-    audit(principal, label, 'deny');
+    audit(principal, `attach:${sessionId}`, 'deny');
     return;
   }
+  audit(principal, `attach:${sessionId}`, 'allow');
+
+  let cursor = 0;
+  for (;;) {
+    const { messages, next } = replayOutput(sessionId, cursor);
+    cursor = next;
+    for (const line of messages) {
+      const parsed = ServerMessageSchema.safeParse(JSON.parse(line));
+      if (!parsed.success) continue;
+      send(parsed.data);
+      if (parsed.data.type === 'result') return;
+    }
+    const current = readSession(sessionId);
+    if (!current) return;
+    // Still waiting on a decision → say so rather than tailing forever. Asked of
+    // the bus, not of the record: the answer arrives from a third party the
+    // owning process never hears from, so its `state` lags.
+    if (current.parkedEventId && stillParked(current, getEventBusPath())) {
+      send({
+        type: 'blocked',
+        sessionId,
+        eventId: current.parkedEventId,
+        question: current.question ?? '',
+        key: current.questionKey ?? undefined,
+      });
+      return;
+    }
+    await new Promise((r) => setTimeout(r, ATTACH_POLL_MS));
+  }
+}
 
-  // Bridge the bus to the wire so mid-run interviews reach the client.
-  const responder = startRemoteResponder({ busDbPath, ask, emittedBy: `api:${principal}` });
-  try {
-    const exitCode = await runCommand(argv);
-    audit(principal, label, 'allow', exitCode);
-  } finally {
-    responder.close();
+/** Abandon a parked session on the operator's say-so, before its TTL. */
+function handleCancel(principal: string, sessionId: string): void {
+  const record = readSession(sessionId);
+  if (!record) {
+    send({ type: 'error', error: `no such session: ${sessionId}` });
+    send({ type: 'result', success: false, exitCode: 1 });
+    return;
+  }
+  if (record.principal !== principal) {
+    send({
+      type: 'error',
+      error: `permission denied: session ${sessionId} belongs to another principal`,
+    });
+    send({ type: 'result', success: false, exitCode: EXIT_PERMISSION_DENIED });
+    audit(principal, `cancel:${sessionId}`, 'deny');
+    return;
   }
+  abandonSession(record, {
+    busDbPath: getEventBusPath(),
+    reason: `Session ${sessionId} was cancelled by ${principal} with "${record.question ?? 'the question'}" still unanswered.`,
+    emittedBy: `api:${principal}`,
+  });
+  audit(principal, `cancel:${sessionId}`, 'allow');
+  send({ type: 'result', success: true, exitCode: 0 });
 }
 
 export async function apiServeMode(principal: string): Promise<void> {
   send({ type: 'ready', protocolVersion: API_PROTOCOL_VERSION });
 
   const busDbPath = getEventBusPath();
+  // A session whose owning process died (reboot, OOM) would otherwise stay
+  // parked forever, holding whatever its command holds.
+  reapExpiredSessions({ busDbPath });
+
+  // When the ssh client goes away, sshd hangs up its forced command. Ignoring
+  // SIGHUP is what lets a parked command outlive the session that started it —
+  // the TTL reaper, not the transport, is what bounds it now.
+  process.on('SIGHUP', () => {});
+
   const pendingAnswers = new Map<
     string,
-    { resolve: (value: unknown) => void; reject: (error: Error) => void }
+    { resolve: (value: unknown) => void; reject: (error: Error) => void; interview: WireInterview }
   >();
 
+  /** Set while a command is running; the parked child outlives the transport. */
+  const live: { session: SessionWriter | null; clientAttached: boolean } = {
+    session: null,
+    clientAttached: true,
+  };
+  let commandRunning: Promise<void> | null = null;
+
   const ask = (interview: WireInterview): Promise<unknown> =>
     new Promise((resolve, reject) => {
-      pendingAnswers.set(interview.id, { resolve, reject });
+      pendingAnswers.set(interview.id, { resolve, reject, interview });
+      if (!live.clientAttached) {
+        // Nobody to ask. Re-park on THIS question: a detached command that was
+        // answered and moved on is now waiting on a *different* event, and a
+        // record still naming the previous one is actively wrong — it hides the
+        // live question from `events list-unanswered`, and it aims the TTL
+        // reaper at a question that was already decided, so the reaper retires
+        // the session while the child is still alive and blocked.
+        live.session?.park({
+          eventId: interview.id,
+          eventType: EVENT_TYPES.interviewRequired(interview.scope, interview.key),
+          question: interview.message,
+          questionKey: `${interview.scope}.${interview.key}`,
+        });
+        return;
+      }
       send({
         type: 'interview',
         id: interview.id,
@@ -131,6 +246,46 @@ export async function apiServeMode(principal: string): Promise<void> {
       });
     });
 
+  const handleCommand = async (argv: string[]): Promise<void> => {
+    const { command, subcommand, label } = opOf(argv);
+
+    if (!(await isAuthorized(principal, command, subcommand))) {
+      send({ type: 'error', error: `permission denied: "${principal}" is not granted "${label}"` });
+      send({ type: 'result', success: false, exitCode: EXIT_PERMISSION_DENIED });
+      audit(principal, label, 'deny');
+      return;
+    }
+
+    const writer = SessionWriter.create({ principal, argv });
+    live.session = writer;
+    const reaper = setTimeout(
+      () => {
+        if (writer.current.state !== 'parked') return;
+        abandonSession(writer.current, {
+          busDbPath,
+          reason: expiryReason(writer.current),
+          emittedBy: `api:${principal}`,
+        });
+        writer.finish('abandoned');
+      },
+      Math.max(0, writer.current.expiresAt - Date.now()),
+    );
+
+    // Bridge the bus to the wire so mid-run interviews reach the client.
+    const responder = startRemoteResponder({ busDbPath, ask, emittedBy: `api:${principal}` });
+    try {
+      const exitCode = await runCommand(argv, (msg) =>
+        emit(writer, msg, () => live.clientAttached),
+      );
+      writer.finish('finished');
+      audit(principal, label, 'allow', exitCode);
+    } finally {
+      clearTimeout(reaper);
+      responder.close();
+      live.session = null;
+    }
+  };
+
   const rl = createInterface({ input: process.stdin, terminal: false });
 
   for await (const line of rl) {
@@ -148,20 +303,61 @@ export async function apiServeMode(principal: string): Promise<void> {
       const pending = pendingAnswers.get(msg.id);
       if (pending) {
         pendingAnswers.delete(msg.id);
-        // `error` = the client couldn't reach a decider. Fail the waiting
-        // command rather than letting the question fall back to its default.
-        if (msg.error) pending.reject(new Error(msg.error));
-        else pending.resolve(msg.value);
+        live.session?.unpark();
+        pending.resolve(msg.value);
+      }
+      continue;
+    }
+
+    if (msg.type === 'unanswerable') {
+      // The client cannot decide. Do NOT resolve the question — park on it and
+      // tell the client where it stands. The responder, seeing `ask` reject,
+      // emits nothing on the bus, so the query is still there to be answered.
+      const pending = pendingAnswers.get(msg.id);
+      if (!pending) continue;
+      pendingAnswers.delete(msg.id);
+      const { interview } = pending;
+      pending.reject(new Error(msg.reason));
+      if (live.session) {
+        live.session.park({
+          eventId: msg.id,
+          eventType: EVENT_TYPES.interviewRequired(interview.scope, interview.key),
+          question: interview.message,
+          questionKey: `${interview.scope}.${interview.key}`,
+        });
+        send({
+          type: 'blocked',
+          sessionId: live.session.id,
+          eventId: msg.id,
+          question: interview.message,
+          key: `${interview.scope}.${interview.key}`,
+        });
       }
       continue;
     }
 
+    if (msg.type === 'attach') {
+      await handleAttach(principal, msg.sessionId);
+      continue;
+    }
+
+    if (msg.type === 'cancel') {
+      handleCancel(principal, msg.sessionId);
+      continue;
+    }
+
     if (msg.type === 'command') {
       // Fire-and-forget so the read loop keeps consuming `answer` messages while
       // the command runs — mid-run interviews are answered in-flight.
-      void handleCommand(principal, msg.argv, busDbPath, ask);
+      commandRunning = handleCommand(msg.argv);
+      void commandRunning;
     }
   }
 
+  // stdin closed: the client is gone. A parked command must NOT die with it —
+  // that is the bug. Stay alive (output goes to the session buffer) until the
+  // command finishes or the reaper abandons it.
+  live.clientAttached = false;
+  if (commandRunning) await commandRunning;
   process.exit(0);
 }

package/src/api/sessions.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Does the reaper actually fire, and does the command say the right thing?
+ *
+ * A parked session holds a live child process and possibly a module-operation
+ * lock. The TTL reaper is the only thing standing between that and the 20-day
+ * lock outage `module operations` was built for — so a reaper nobody has watched
+ * fire is decorative. This drives a real `module update` sweep into a park,
+ * expires its session deliberately, and asserts on what comes back.
+ *
+ * What it asserts is the *value*, not liveness. Three outcomes are confusable
+ * here and only one is correct:
+ *
+ *   declined   — someone said no.                    (`{value: false}`)
+ *   unanswered — nobody was listening at all.        (responder-probe throw)
+ *   abandoned  — the question stood; the deadline passed; nobody decided.
+ *
+ * Reporting an abandonment as "operator declined" is the original celilo#609
+ * bug wearing a different hat, so that string is asserted absent.
+ *
+ * WATCHED RED: with `abandonSession` emitting `{value: false}` instead of
+ * `{abandoned}` — i.e. a reaper that resolves the question the old way — this
+ * file fails on `expect(report).not.toContain('operator declined')`. With the
+ * reaper removed entirely it fails by timeout, the sweep never resuming.
+ */
+
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bus';
+import { handleModuleUpdate } from '../cli/commands/module-update';
+import { getDb } from '../db/client';
+import { modules } from '../db/schema';
+import { RESPONDER_PROBE_EVENT } from '../services/responder-probe';
+import { SessionWriter, listSessions, reapExpiredSessions } from './sessions';
+
+const NO_SCHEMAS = defineEvents({});
+const QUERY_TYPE = 'interview.required.module-upgrade:iptables.apply_breaking';
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let dir: string;
+let busDbPath: string;
+let server: ReturnType<typeof Bun.serve>;
+let registryUrl: string;
+
+/** A responder that proves liveness but never answers the question. */
+function probeOnlyResponder(dbPath: string): { bus: Bus; close: () => void } {
+  const bus = openBus({ dbPath, events: NO_SCHEMAS });
+  const watch = bus.watch(RESPONDER_PROBE_EVENT, async (event) => {
+    if (event.replyFor !== null) return;
+    bus.emitRaw(
+      `${event.type}.reply`,
+      { kind: 'daemon', emittedBy: 'reaper-test' },
+      { replyFor: event.id, emittedBy: 'reaper-test' },
+    );
+  });
+  return {
+    bus,
+    close: () => {
+      watch.close();
+      bus.close();
+    },
+  };
+}
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'celilo-reap-'));
+  busDbPath = join(dir, 'events.db');
+  process.env.CELILO_DB_PATH = join(dir, 'test.db');
+  process.env.CELILO_DATA_DIR = dir;
+  process.env.CELILO_ORIGINAL_CWD = dir;
+  process.env.EVENT_BUS_DB = busDbPath;
+
+  getDb()
+    .insert(modules)
+    .values({
+      id: 'iptables',
+      name: 'iptables',
+      sourcePath: join(dir, 'installed'),
+      version: '1.0.2+9',
+      manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
+    })
+    .run();
+
+  server = Bun.serve({
+    port: 0,
+    fetch(req) {
+      if (new URL(req.url).pathname === '/index/ip/ta/iptables') {
+        return new Response(
+          `${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
+        );
+      }
+      return new Response('not found', { status: 404 });
+    },
+  });
+  registryUrl = `http://localhost:${server.port}`;
+});
+
+afterEach(() => {
+  server.stop(true);
+  rmSync(dir, { recursive: true, force: true });
+  process.env.CELILO_DB_PATH = undefined;
+  process.env.CELILO_DATA_DIR = undefined;
+  process.env.CELILO_ORIGINAL_CWD = undefined;
+  process.env.EVENT_BUS_DB = undefined;
+});
+
+test('an expired session is reaped and the command reports abandoned, not declined', async () => {
+  const responder = probeOnlyResponder(busDbPath);
+  const observer = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+
+  try {
+    let settled = false;
+    const sweep = handleModuleUpdate([], { registry: registryUrl }).then((r) => {
+      settled = true;
+      return r;
+    });
+
+    await sleep(600);
+    expect(settled).toBe(false);
+
+    const query = observer.recentEvents({ type: QUERY_TYPE })[0] as BusEvent;
+    expect(query).toBeDefined();
+
+    // A session parked on that question, already past its deadline.
+    const session = SessionWriter.create({
+      principal: 'tester',
+      argv: ['module', 'update'],
+      ttlMs: -1,
+    });
+    session.park({
+      eventId: String(query.id),
+      eventType: QUERY_TYPE,
+      question: 'Apply breaking update for iptables (1.0.2+9 → 2.0.0+1)?',
+      questionKey: 'module-upgrade:iptables.apply_breaking',
+    });
+
+    const reaped = reapExpiredSessions({ busDbPath });
+    expect(reaped.map((r) => r.sessionId)).toEqual([session.id]);
+
+    // The command resumes — and says nobody decided, not that anyone declined.
+    const result = await sweep;
+    const report = result.success ? (result.message ?? '') : (result.error ?? '');
+    expect(report).toContain('Abandoned:');
+    expect(report).toContain('never decided');
+    expect(report).not.toContain('operator declined');
+    // ...and not the *unanswered* wording either: a responder was listening.
+    expect(report).not.toContain('could not be answered');
+    expect(report).toContain('iptables');
+    // A breaking update that silently didn't land must not read as success.
+    expect(result.success).toBe(false);
+    // The module is untouched — nothing was applied on nobody's authority.
+    expect(getDb().select().from(modules).all()[0].version).toBe('1.0.2+9');
+
+    // The record is retained, so a repeatedly-parking command is detectable.
+    const retained = listSessions();
+    expect(retained).toHaveLength(1);
+    expect(retained[0].state).toBe('abandoned');
+    expect(retained[0].question).toContain('iptables');
+  } finally {
+    observer.close();
+    responder.close();
+  }
+}, 30_000);
+
+test('a question answered before the deadline is never overwritten by the reaper', async () => {
+  const observer = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  try {
+    const query = observer.emitRaw(QUERY_TYPE, { scope: 'x', key: 'y' });
+    observer.emitRaw(
+      `${QUERY_TYPE}.reply`,
+      { value: true },
+      { replyFor: query.id, emittedBy: 'a-real-decider' },
+    );
+
+    const session = SessionWriter.create({
+      principal: 'tester',
+      argv: ['module', 'update'],
+      ttlMs: -1,
+    });
+    session.park({
+      eventId: String(query.id),
+      eventType: QUERY_TYPE,
+      question: 'Apply breaking update for iptables?',
+    });
+
+    reapExpiredSessions({ busDbPath });
+
+    // Exactly one reply, and it is the decision that was actually made.
+    const replies = observer.repliesFor(query.id);
+    expect(replies).toHaveLength(1);
+    expect((replies[0].payload as { value: unknown }).value).toBe(true);
+  } finally {
+    observer.close();
+  }
+});