@celilo/cli 0.18.0 → 0.20.0

package/src/api/sessions.test.ts

@@ -0,0 +1,196 @@
+/**
+ * Does the reaper actually fire, and does the command say the right thing?
+ *
+ * A parked session holds a live child process and possibly a module-operation
+ * lock. The TTL reaper is the only thing standing between that and the 20-day
+ * lock outage `module operations` was built for — so a reaper nobody has watched
+ * fire is decorative. This drives a real `module update` sweep into a park,
+ * expires its session deliberately, and asserts on what comes back.
+ *
+ * What it asserts is the *value*, not liveness. Three outcomes are confusable
+ * here and only one is correct:
+ *
+ *   declined   — someone said no.                    (`{value: false}`)
+ *   unanswered — nobody was listening at all.        (responder-probe throw)
+ *   abandoned  — the question stood; the deadline passed; nobody decided.
+ *
+ * Reporting an abandonment as "operator declined" is the original celilo#609
+ * bug wearing a different hat, so that string is asserted absent.
+ *
+ * WATCHED RED: with `abandonSession` emitting `{value: false}` instead of
+ * `{abandoned}` — i.e. a reaper that resolves the question the old way — this
+ * file fails on `expect(report).not.toContain('operator declined')`. With the
+ * reaper removed entirely it fails by timeout, the sweep never resuming.
+ */
+
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type Bus, type BusEvent, defineEvents, openBus } from '@celilo/event-bus';
+import { handleModuleUpdate } from '../cli/commands/module-update';
+import { getDb } from '../db/client';
+import { modules } from '../db/schema';
+import { RESPONDER_PROBE_EVENT } from '../services/responder-probe';
+import { SessionWriter, listSessions, reapExpiredSessions } from './sessions';
+
+const NO_SCHEMAS = defineEvents({});
+const QUERY_TYPE = 'interview.required.module-upgrade:iptables.apply_breaking';
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+let dir: string;
+let busDbPath: string;
+let server: ReturnType<typeof Bun.serve>;
+let registryUrl: string;
+
+/** A responder that proves liveness but never answers the question. */
+function probeOnlyResponder(dbPath: string): { bus: Bus; close: () => void } {
+  const bus = openBus({ dbPath, events: NO_SCHEMAS });
+  const watch = bus.watch(RESPONDER_PROBE_EVENT, async (event) => {
+    if (event.replyFor !== null) return;
+    bus.emitRaw(
+      `${event.type}.reply`,
+      { kind: 'daemon', emittedBy: 'reaper-test' },
+      { replyFor: event.id, emittedBy: 'reaper-test' },
+    );
+  });
+  return {
+    bus,
+    close: () => {
+      watch.close();
+      bus.close();
+    },
+  };
+}
+
+beforeEach(() => {
+  dir = mkdtempSync(join(tmpdir(), 'celilo-reap-'));
+  busDbPath = join(dir, 'events.db');
+  process.env.CELILO_DB_PATH = join(dir, 'test.db');
+  process.env.CELILO_DATA_DIR = dir;
+  process.env.CELILO_ORIGINAL_CWD = dir;
+  process.env.EVENT_BUS_DB = busDbPath;
+
+  getDb()
+    .insert(modules)
+    .values({
+      id: 'iptables',
+      name: 'iptables',
+      sourcePath: join(dir, 'installed'),
+      version: '1.0.2+9',
+      manifestData: { celilo_contract: '1.0', id: 'iptables', name: 'iptables', version: '1.0.2' },
+    })
+    .run();
+
+  server = Bun.serve({
+    port: 0,
+    fetch(req) {
+      if (new URL(req.url).pathname === '/index/ip/ta/iptables') {
+        return new Response(
+          `${JSON.stringify({ name: 'iptables', vers: '2.0.0+1', deps: [], cksum: 'x' })}\n`,
+        );
+      }
+      return new Response('not found', { status: 404 });
+    },
+  });
+  registryUrl = `http://localhost:${server.port}`;
+});
+
+afterEach(() => {
+  server.stop(true);
+  rmSync(dir, { recursive: true, force: true });
+  process.env.CELILO_DB_PATH = undefined;
+  process.env.CELILO_DATA_DIR = undefined;
+  process.env.CELILO_ORIGINAL_CWD = undefined;
+  process.env.EVENT_BUS_DB = undefined;
+});
+
+test('an expired session is reaped and the command reports abandoned, not declined', async () => {
+  const responder = probeOnlyResponder(busDbPath);
+  const observer = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+
+  try {
+    let settled = false;
+    const sweep = handleModuleUpdate([], { registry: registryUrl }).then((r) => {
+      settled = true;
+      return r;
+    });
+
+    await sleep(600);
+    expect(settled).toBe(false);
+
+    const query = observer.recentEvents({ type: QUERY_TYPE })[0] as BusEvent;
+    expect(query).toBeDefined();
+
+    // A session parked on that question, already past its deadline.
+    const session = SessionWriter.create({
+      principal: 'tester',
+      argv: ['module', 'update'],
+      ttlMs: -1,
+    });
+    session.park({
+      eventId: String(query.id),
+      eventType: QUERY_TYPE,
+      question: 'Apply breaking update for iptables (1.0.2+9 → 2.0.0+1)?',
+      questionKey: 'module-upgrade:iptables.apply_breaking',
+    });
+
+    const reaped = reapExpiredSessions({ busDbPath });
+    expect(reaped.map((r) => r.sessionId)).toEqual([session.id]);
+
+    // The command resumes — and says nobody decided, not that anyone declined.
+    const result = await sweep;
+    const report = result.success ? (result.message ?? '') : (result.error ?? '');
+    expect(report).toContain('Abandoned:');
+    expect(report).toContain('never decided');
+    expect(report).not.toContain('operator declined');
+    // ...and not the *unanswered* wording either: a responder was listening.
+    expect(report).not.toContain('could not be answered');
+    expect(report).toContain('iptables');
+    // A breaking update that silently didn't land must not read as success.
+    expect(result.success).toBe(false);
+    // The module is untouched — nothing was applied on nobody's authority.
+    expect(getDb().select().from(modules).all()[0].version).toBe('1.0.2+9');
+
+    // The record is retained, so a repeatedly-parking command is detectable.
+    const retained = listSessions();
+    expect(retained).toHaveLength(1);
+    expect(retained[0].state).toBe('abandoned');
+    expect(retained[0].question).toContain('iptables');
+  } finally {
+    observer.close();
+    responder.close();
+  }
+}, 30_000);
+
+test('a question answered before the deadline is never overwritten by the reaper', async () => {
+  const observer = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  try {
+    const query = observer.emitRaw(QUERY_TYPE, { scope: 'x', key: 'y' });
+    observer.emitRaw(
+      `${QUERY_TYPE}.reply`,
+      { value: true },
+      { replyFor: query.id, emittedBy: 'a-real-decider' },
+    );
+
+    const session = SessionWriter.create({
+      principal: 'tester',
+      argv: ['module', 'update'],
+      ttlMs: -1,
+    });
+    session.park({
+      eventId: String(query.id),
+      eventType: QUERY_TYPE,
+      question: 'Apply breaking update for iptables?',
+    });
+
+    reapExpiredSessions({ busDbPath });
+
+    // Exactly one reply, and it is the decision that was actually made.
+    const replies = observer.repliesFor(query.id);
+    expect(replies).toHaveLength(1);
+    expect((replies[0].payload as { value: unknown }).value).toBe(true);
+  } finally {
+    observer.close();
+  }
+});

package/src/api/sessions.ts

@@ -0,0 +1,278 @@
+/**
+ * Parked-session registry for `api-serve` (celilo#609).
+ *
+ * A command that parks on an unanswerable question must outlive the ssh session
+ * that started it, so a *different* responder can answer later and the caller
+ * can come back for the outcome. Two consequences shape this file:
+ *
+ *  1. **The registry is on disk, not in memory.** Each ssh connection is its own
+ *     `api-serve` process, so the process that later handles `attach` is not the
+ *     one holding the parked child. A directory under the data dir is the
+ *     handoff: metadata in `session.json`, the command's output appended to
+ *     `output.ndjson`. The attaching process replays the file and tails it.
+ *  2. **The record is retained after the session ends.** A repeatedly-parking
+ *     command is only detectable if the abandonments are still there to see —
+ *     the same reason `module operations` keeps its rows.
+ *
+ * ponytail: files + polling rather than a socket/IPC server. Same process model,
+ * a fraction of the machinery, and the buffered output has to be durable across
+ * processes anyway. Move to a socket only if attach latency becomes a problem.
+ */
+
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import type { ServerMessage } from '@celilo/core';
+import { defineEvents, openBus } from '@celilo/event-bus';
+import { z } from 'zod';
+import { getDataDir } from '../config/paths';
+
+const NO_SCHEMAS = defineEvents({});
+
+/** How long a parked session may hold its child before the reaper abandons it. */
+export const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
+
+/** Lines of buffered output an attaching client is replayed at most. */
+const REPLAY_LIMIT = 2000;
+
+export const SessionStateSchema = z.enum(['running', 'parked', 'finished', 'abandoned']);
+export type SessionState = z.infer<typeof SessionStateSchema>;
+
+export const SessionRecordSchema = z.object({
+  sessionId: z.string(),
+  principal: z.string(),
+  argv: z.array(z.string()),
+  startedAt: z.number(),
+  expiresAt: z.number(),
+  state: SessionStateSchema,
+  /** Bus event id of the question this session is parked on, when parked. */
+  parkedEventId: z.string().nullable(),
+  /** Bus event *type* of that question — what the reaper replies to. */
+  parkedEventType: z.string().nullable(),
+  question: z.string().nullable(),
+  /** The parked question's `<scope>.<key>`. */
+  questionKey: z.string().nullable(),
+});
+export type SessionRecord = z.infer<typeof SessionRecordSchema>;
+
+export function getSessionsDir(): string {
+  return join(getDataDir(), 'api-sessions');
+}
+
+function sessionDir(sessionId: string): string {
+  return join(getSessionsDir(), sessionId);
+}
+
+function recordPath(sessionId: string): string {
+  return join(sessionDir(sessionId), 'session.json');
+}
+
+function outputPath(sessionId: string): string {
+  return join(sessionDir(sessionId), 'output.ndjson');
+}
+
+export function readSession(sessionId: string): SessionRecord | null {
+  const path = recordPath(sessionId);
+  if (!existsSync(path)) return null;
+  // Untrusted only in the sense of "written by another process" — validate it
+  // rather than trusting the shape (Rule 3.7).
+  const parsed = SessionRecordSchema.safeParse(JSON.parse(readFileSync(path, 'utf-8')));
+  return parsed.success ? parsed.data : null;
+}
+
+export function listSessions(): SessionRecord[] {
+  const dir = getSessionsDir();
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .map(readSession)
+    .filter((s): s is SessionRecord => s !== null)
+    .sort((a, b) => b.startedAt - a.startedAt);
+}
+
+/** The session parked on a given bus query, if one is. */
+export function sessionParkedOn(eventId: string): SessionRecord | null {
+  return listSessions().find((s) => s.state === 'parked' && s.parkedEventId === eventId) ?? null;
+}
+
+function write(record: SessionRecord): void {
+  mkdirSync(sessionDir(record.sessionId), { recursive: true });
+  writeFileSync(recordPath(record.sessionId), `${JSON.stringify(record, null, 2)}\n`);
+}
+
+/**
+ * A live session's handle: the writer side of the registry. Held by the
+ * `api-serve` process running the command.
+ */
+export class SessionWriter {
+  private record: SessionRecord;
+
+  private constructor(record: SessionRecord) {
+    this.record = record;
+    write(record);
+  }
+
+  static create(opts: {
+    principal: string;
+    argv: string[];
+    ttlMs?: number;
+    now?: number;
+  }): SessionWriter {
+    const now = opts.now ?? Date.now();
+    return new SessionWriter({
+      sessionId: crypto.randomUUID(),
+      principal: opts.principal,
+      argv: opts.argv,
+      startedAt: now,
+      expiresAt: now + (opts.ttlMs ?? DEFAULT_SESSION_TTL_MS),
+      state: 'running',
+      parkedEventId: null,
+      parkedEventType: null,
+      question: null,
+      questionKey: null,
+    });
+  }
+
+  get id(): string {
+    return this.record.sessionId;
+  }
+
+  get current(): SessionRecord {
+    return this.record;
+  }
+
+  /** Append a message to the buffer so a client attaching later sees it. */
+  append(msg: ServerMessage): void {
+    appendFileSync(outputPath(this.record.sessionId), `${JSON.stringify(msg)}\n`);
+  }
+
+  park(opts: { eventId: string; eventType: string; question: string; questionKey?: string }): void {
+    this.record = {
+      ...this.record,
+      state: 'parked',
+      parkedEventId: opts.eventId,
+      parkedEventType: opts.eventType,
+      question: opts.question,
+      questionKey: opts.questionKey ?? null,
+    };
+    write(this.record);
+  }
+
+  /** The question was answered (by anyone) — the command is running again. */
+  unpark(): void {
+    if (this.record.state !== 'parked') return;
+    this.record = {
+      ...this.record,
+      state: 'running',
+      parkedEventId: null,
+      parkedEventType: null,
+      question: null,
+      questionKey: null,
+    };
+    write(this.record);
+  }
+
+  /**
+   * `abandoned` outranks `finished`: the command exits *because* it was
+   * abandoned, and that exit must not overwrite why. Retained either way.
+   */
+  finish(state: 'finished' | 'abandoned'): void {
+    if (this.record.state === 'abandoned') return;
+    this.record = { ...this.record, state };
+    write(this.record);
+  }
+}
+
+/**
+ * Answer a parked session's outstanding query as `abandoned` and retain the
+ * record as history.
+ *
+ * The reply is what releases everything the parked command holds: the child
+ * resumes, throws `InterviewAbandonedError`, and unwinds — releasing its own
+ * module-operation lock on the way out, exactly as any other failure does. That
+ * is the whole point of answering rather than killing.
+ *
+ * ponytail: no forced kill if the child ignores its own unwinding. Add one only
+ * once a real command is seen to survive an abandoned answer.
+ */
+export function abandonSession(
+  record: SessionRecord,
+  opts: { busDbPath: string; reason: string; emittedBy?: string },
+): boolean {
+  if (record.parkedEventId && record.parkedEventType) {
+    const bus = openBus({ dbPath: opts.busDbPath, events: NO_SCHEMAS });
+    try {
+      // Someone decided it while we were on our way to reap. Abandoning now
+      // would overwrite a real answer with "nobody decided" — the exact
+      // fabrication this whole change exists to remove, only in reverse.
+      if (bus.repliesFor(Number(record.parkedEventId)).length > 0) return false;
+      bus.emitRaw(
+        `${record.parkedEventType}.reply`,
+        { abandoned: { reason: opts.reason } },
+        {
+          replyFor: Number(record.parkedEventId),
+          emittedBy: opts.emittedBy ?? 'api-session-reaper',
+        },
+      );
+    } finally {
+      bus.close();
+    }
+  }
+  write({ ...record, state: 'abandoned' });
+  return true;
+}
+
+/**
+ * Is this session still waiting on a decision *according to the bus*?
+ *
+ * The record's `state` is the owning process's view and can lag: the answer
+ * arrives on the bus from a third party, which the owner never sees. The bus is
+ * the source of truth for whether a question stands — ask it, don't infer.
+ */
+export function stillParked(record: SessionRecord, busDbPath: string): boolean {
+  if (record.state !== 'parked' || !record.parkedEventId) return false;
+  const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  try {
+    return bus.repliesFor(Number(record.parkedEventId)).length === 0;
+  } finally {
+    bus.close();
+  }
+}
+
+/** Why a reaped session's question was never decided. */
+export function expiryReason(record: SessionRecord): string {
+  const what = record.question ?? 'the question';
+  return `Session ${record.sessionId} expired with "${what}" still unanswered — nobody decided it.`;
+}
+
+/**
+ * Abandon every parked session past its TTL. Run by the process that owns a
+ * parked child on its own timer, and again at `api-serve` startup so a session
+ * whose owner died (reboot, OOM) doesn't sit parked forever — the failure mode
+ * that left a `module deploy` holding every backup lock on the fleet for 20 days.
+ */
+export function reapExpiredSessions(opts: { busDbPath: string; now?: number }): SessionRecord[] {
+  const now = opts.now ?? Date.now();
+  const expired = listSessions().filter((s) => s.state === 'parked' && s.expiresAt <= now);
+  for (const record of expired) {
+    abandonSession(record, { busDbPath: opts.busDbPath, reason: expiryReason(record) });
+  }
+  return expired;
+}
+
+/** Buffered messages an attaching client should be replayed. */
+export function replayOutput(
+  sessionId: string,
+  fromLine = 0,
+): { messages: string[]; next: number } {
+  const path = outputPath(sessionId);
+  if (!existsSync(path)) return { messages: [], next: fromLine };
+  const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
+  const start = Math.max(fromLine, lines.length - REPLAY_LIMIT);
+  return { messages: lines.slice(start), next: lines.length };
+}

package/src/cli/commands/apt-upgrade.test.ts

@@ -2,7 +2,7 @@ import { describe, expect, test } from 'bun:test';
 import { handleAptUpgrade } from './apt-upgrade';
 
 describe('handleAptUpgrade', () => {
-  test('runs the three steps in order when each succeeds', async () => {
+  test('runs every step in order when each succeeds, ending with the dispatcher restart', async () => {
     const seen: string[][] = [];
     const result = await handleAptUpgrade([], {}, (argv) => {
       seen.push(argv);
@@ -14,9 +14,28 @@ describe('handleAptUpgrade', () => {
       ['sudo', 'apt-get', 'update'],
       ['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
       ['/usr/local/bin/celilo', 'system', 'migrate'],
+      // celilo#604: without this, apt installs new code and the running
+      // dispatcher keeps serving the old — celilo-mgr did so for 9 days.
+      ['/usr/local/bin/celilo', 'events', 'restart-daemon'],
     ]);
   });
 
+  test('fails, and names what DID complete, when the dispatcher restart fails', async () => {
+    const result = await handleAptUpgrade([], {}, (argv) => ({
+      status: argv.includes('restart-daemon') ? 1 : 0,
+    }));
+
+    // The packages ARE upgraded and the dispatcher is NOT on the new code.
+    // Saying so in the failure is the acceptance condition — silence here is
+    // what let a stale dispatcher pass for a successful upgrade.
+    expect(result.success).toBe(false);
+    if (!result.success) {
+      expect(result.error).toContain('restart the event dispatcher');
+      expect(result.error).toContain('apt-get upgrade');
+      expect(result.error).toContain('apply DB migrations');
+    }
+  });
+
   test('stops at the first failing step and does not run later ones', async () => {
     const seen: string[][] = [];
     const result = await handleAptUpgrade([], {}, (argv) => {

package/src/cli/commands/apt-upgrade.ts

@@ -34,6 +34,11 @@ const STEPS: Step[] = [
     argv: ['sudo', 'apt-get', '-y', '--only-upgrade', 'install', 'celilo', 'celilo-bootstrap'],
   },
   { label: 'apply DB migrations', argv: [CELILO_BIN, 'system', 'migrate'] },
+  // The dispatcher runs the code it LOADED, not the code on disk. Without this
+  // step celilo-mgr sat 9 days on event-bus v0.1.8 after apt installed v0.2.0,
+  // faithfully running the bug the upgrade shipped to fix (celilo#604). The
+  // upgraded binary does the restart so the verification is the new code's.
+  { label: 'restart the event dispatcher', argv: [CELILO_BIN, 'events', 'restart-daemon'] },
 ];
 
 /** Run one argv, inheriting stdio so its output streams through api-serve. */
@@ -46,18 +51,23 @@ export async function handleAptUpgrade(
   _flags: Record<string, string | boolean>,
   runStep: StepRunner = defaultRunner,
 ): Promise<CommandResult> {
+  const done: string[] = [];
   for (const step of STEPS) {
     process.stdout.write(`\n▸ ${step.label}\n`);
     const { status } = runStep(step.argv);
     if (status !== 0) {
+      // Name what DID happen. A failure on the last step means new code is
+      // installed and the dispatcher is still serving the old — the operator
+      // has to be told that in the failure itself, not left to infer it.
       return {
         success: false,
-        error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Nothing further was run.`,
+        error: `apt-upgrade failed at "${step.label}" (exit ${status ?? 'signal'}). Completed: ${done.length > 0 ? done.join(', ') : 'nothing'}. Nothing further was run.`,
       };
     }
+    done.push(step.label);
   }
   return {
     success: true,
-    message: 'celilo apt packages upgraded and migrations applied.',
+    message: 'celilo apt packages upgraded, migrations applied, dispatcher restarted on new code.',
   };
 }

package/src/cli/commands/backup-sweep.ts

@@ -17,13 +17,12 @@ import {
   findBackupEligibleModules,
   isBackupDue,
 } from '../../services/backup-create';
-import { failBackup, getBackup } from '../../services/backup-metadata';
+import { failBackup, getBackup, listInProgressBackups } from '../../services/backup-metadata';
 import { pruneBackupsForModule } from '../../services/backup-retention';
 import {
-  ABANDONED_BACKUP_MESSAGE,
   STAGING_PREFIX,
-  impliesAbandonedRecord,
   reapOrphanedStaging,
+  resolveAbandonedBackups,
 } from '../../services/backup-staging';
 import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
 import { isPidRunnable } from '../../services/module-operations';
@@ -59,18 +58,30 @@ function reapStaging() {
     now: () => Date.now(),
   });
 
-  for (const reclaimed of report.reclaimed) {
-    if (impliesAbandonedRecord(reclaimed.reason)) {
-      failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
-    }
-  }
-
   return report;
 }
 
+/**
+ * Correct every record that still claims to be running, staging or no staging.
+ *
+ * This replaces deriving the fix-up from what the reaper reclaimed. That
+ * version only ever saw records whose directory still existed, so anything
+ * cleared by a reboot or by hand stayed `in_progress` forever (#616).
+ */
+function resolveAbandonedRecords() {
+  return resolveAbandonedBackups({
+    listInProgress: () =>
+      listInProgressBackups().map((b) => ({ id: b.id, pid: b.pid, startedAt: b.startedAt })),
+    isPidRunnable,
+    fail: (id, message) => failBackup(id, message),
+    now: () => Date.now(),
+  });
+}
+
 export async function handleBackupSweep(): Promise<CommandResult> {
   const report = await runBackupSweep({
     reapStaging,
+    resolveAbandonedRecords,
     listEligible: () =>
       findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
     isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
@@ -117,6 +128,11 @@ function formatReport(report: BackupSweepReport): string {
       lines.push(`    ${reclaimed.recordId} (${reclaimed.reason})`);
     }
   }
+  if (report.records.resolved.length > 0) {
+    lines.push(
+      `  corrected ${report.records.resolved.length} record(s) that still claimed to be running`,
+    );
+  }
   for (const moduleId of report.skippedLocked) {
     lines.push(`  skipped ${moduleId}: another operation is in flight — retrying next tick`);
   }