@celilo/cli 0.19.0 → 0.20.0

package/src/api/sessions.ts

@@ -0,0 +1,278 @@
+/**
+ * Parked-session registry for `api-serve` (celilo#609).
+ *
+ * A command that parks on an unanswerable question must outlive the ssh session
+ * that started it, so a *different* responder can answer later and the caller
+ * can come back for the outcome. Two consequences shape this file:
+ *
+ *  1. **The registry is on disk, not in memory.** Each ssh connection is its own
+ *     `api-serve` process, so the process that later handles `attach` is not the
+ *     one holding the parked child. A directory under the data dir is the
+ *     handoff: metadata in `session.json`, the command's output appended to
+ *     `output.ndjson`. The attaching process replays the file and tails it.
+ *  2. **The record is retained after the session ends.** A repeatedly-parking
+ *     command is only detectable if the abandonments are still there to see —
+ *     the same reason `module operations` keeps its rows.
+ *
+ * ponytail: files + polling rather than a socket/IPC server. Same process model,
+ * a fraction of the machinery, and the buffered output has to be durable across
+ * processes anyway. Move to a socket only if attach latency becomes a problem.
+ */
+
+import {
+  appendFileSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import type { ServerMessage } from '@celilo/core';
+import { defineEvents, openBus } from '@celilo/event-bus';
+import { z } from 'zod';
+import { getDataDir } from '../config/paths';
+
+const NO_SCHEMAS = defineEvents({});
+
+/** How long a parked session may hold its child before the reaper abandons it. */
+export const DEFAULT_SESSION_TTL_MS = 30 * 60 * 1000;
+
+/** Lines of buffered output an attaching client is replayed at most. */
+const REPLAY_LIMIT = 2000;
+
+export const SessionStateSchema = z.enum(['running', 'parked', 'finished', 'abandoned']);
+export type SessionState = z.infer<typeof SessionStateSchema>;
+
+export const SessionRecordSchema = z.object({
+  sessionId: z.string(),
+  principal: z.string(),
+  argv: z.array(z.string()),
+  startedAt: z.number(),
+  expiresAt: z.number(),
+  state: SessionStateSchema,
+  /** Bus event id of the question this session is parked on, when parked. */
+  parkedEventId: z.string().nullable(),
+  /** Bus event *type* of that question — what the reaper replies to. */
+  parkedEventType: z.string().nullable(),
+  question: z.string().nullable(),
+  /** The parked question's `<scope>.<key>`. */
+  questionKey: z.string().nullable(),
+});
+export type SessionRecord = z.infer<typeof SessionRecordSchema>;
+
+export function getSessionsDir(): string {
+  return join(getDataDir(), 'api-sessions');
+}
+
+function sessionDir(sessionId: string): string {
+  return join(getSessionsDir(), sessionId);
+}
+
+function recordPath(sessionId: string): string {
+  return join(sessionDir(sessionId), 'session.json');
+}
+
+function outputPath(sessionId: string): string {
+  return join(sessionDir(sessionId), 'output.ndjson');
+}
+
+export function readSession(sessionId: string): SessionRecord | null {
+  const path = recordPath(sessionId);
+  if (!existsSync(path)) return null;
+  // Untrusted only in the sense of "written by another process" — validate it
+  // rather than trusting the shape (Rule 3.7).
+  const parsed = SessionRecordSchema.safeParse(JSON.parse(readFileSync(path, 'utf-8')));
+  return parsed.success ? parsed.data : null;
+}
+
+export function listSessions(): SessionRecord[] {
+  const dir = getSessionsDir();
+  if (!existsSync(dir)) return [];
+  return readdirSync(dir)
+    .map(readSession)
+    .filter((s): s is SessionRecord => s !== null)
+    .sort((a, b) => b.startedAt - a.startedAt);
+}
+
+/** The session parked on a given bus query, if one is. */
+export function sessionParkedOn(eventId: string): SessionRecord | null {
+  return listSessions().find((s) => s.state === 'parked' && s.parkedEventId === eventId) ?? null;
+}
+
+function write(record: SessionRecord): void {
+  mkdirSync(sessionDir(record.sessionId), { recursive: true });
+  writeFileSync(recordPath(record.sessionId), `${JSON.stringify(record, null, 2)}\n`);
+}
+
+/**
+ * A live session's handle: the writer side of the registry. Held by the
+ * `api-serve` process running the command.
+ */
+export class SessionWriter {
+  private record: SessionRecord;
+
+  private constructor(record: SessionRecord) {
+    this.record = record;
+    write(record);
+  }
+
+  static create(opts: {
+    principal: string;
+    argv: string[];
+    ttlMs?: number;
+    now?: number;
+  }): SessionWriter {
+    const now = opts.now ?? Date.now();
+    return new SessionWriter({
+      sessionId: crypto.randomUUID(),
+      principal: opts.principal,
+      argv: opts.argv,
+      startedAt: now,
+      expiresAt: now + (opts.ttlMs ?? DEFAULT_SESSION_TTL_MS),
+      state: 'running',
+      parkedEventId: null,
+      parkedEventType: null,
+      question: null,
+      questionKey: null,
+    });
+  }
+
+  get id(): string {
+    return this.record.sessionId;
+  }
+
+  get current(): SessionRecord {
+    return this.record;
+  }
+
+  /** Append a message to the buffer so a client attaching later sees it. */
+  append(msg: ServerMessage): void {
+    appendFileSync(outputPath(this.record.sessionId), `${JSON.stringify(msg)}\n`);
+  }
+
+  park(opts: { eventId: string; eventType: string; question: string; questionKey?: string }): void {
+    this.record = {
+      ...this.record,
+      state: 'parked',
+      parkedEventId: opts.eventId,
+      parkedEventType: opts.eventType,
+      question: opts.question,
+      questionKey: opts.questionKey ?? null,
+    };
+    write(this.record);
+  }
+
+  /** The question was answered (by anyone) — the command is running again. */
+  unpark(): void {
+    if (this.record.state !== 'parked') return;
+    this.record = {
+      ...this.record,
+      state: 'running',
+      parkedEventId: null,
+      parkedEventType: null,
+      question: null,
+      questionKey: null,
+    };
+    write(this.record);
+  }
+
+  /**
+   * `abandoned` outranks `finished`: the command exits *because* it was
+   * abandoned, and that exit must not overwrite why. Retained either way.
+   */
+  finish(state: 'finished' | 'abandoned'): void {
+    if (this.record.state === 'abandoned') return;
+    this.record = { ...this.record, state };
+    write(this.record);
+  }
+}
+
+/**
+ * Answer a parked session's outstanding query as `abandoned` and retain the
+ * record as history.
+ *
+ * The reply is what releases everything the parked command holds: the child
+ * resumes, throws `InterviewAbandonedError`, and unwinds — releasing its own
+ * module-operation lock on the way out, exactly as any other failure does. That
+ * is the whole point of answering rather than killing.
+ *
+ * ponytail: no forced kill if the child ignores its own unwinding. Add one only
+ * once a real command is seen to survive an abandoned answer.
+ */
+export function abandonSession(
+  record: SessionRecord,
+  opts: { busDbPath: string; reason: string; emittedBy?: string },
+): boolean {
+  if (record.parkedEventId && record.parkedEventType) {
+    const bus = openBus({ dbPath: opts.busDbPath, events: NO_SCHEMAS });
+    try {
+      // Someone decided it while we were on our way to reap. Abandoning now
+      // would overwrite a real answer with "nobody decided" — the exact
+      // fabrication this whole change exists to remove, only in reverse.
+      if (bus.repliesFor(Number(record.parkedEventId)).length > 0) return false;
+      bus.emitRaw(
+        `${record.parkedEventType}.reply`,
+        { abandoned: { reason: opts.reason } },
+        {
+          replyFor: Number(record.parkedEventId),
+          emittedBy: opts.emittedBy ?? 'api-session-reaper',
+        },
+      );
+    } finally {
+      bus.close();
+    }
+  }
+  write({ ...record, state: 'abandoned' });
+  return true;
+}
+
+/**
+ * Is this session still waiting on a decision *according to the bus*?
+ *
+ * The record's `state` is the owning process's view and can lag: the answer
+ * arrives on the bus from a third party, which the owner never sees. The bus is
+ * the source of truth for whether a question stands — ask it, don't infer.
+ */
+export function stillParked(record: SessionRecord, busDbPath: string): boolean {
+  if (record.state !== 'parked' || !record.parkedEventId) return false;
+  const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
+  try {
+    return bus.repliesFor(Number(record.parkedEventId)).length === 0;
+  } finally {
+    bus.close();
+  }
+}
+
+/** Why a reaped session's question was never decided. */
+export function expiryReason(record: SessionRecord): string {
+  const what = record.question ?? 'the question';
+  return `Session ${record.sessionId} expired with "${what}" still unanswered — nobody decided it.`;
+}
+
+/**
+ * Abandon every parked session past its TTL. Run by the process that owns a
+ * parked child on its own timer, and again at `api-serve` startup so a session
+ * whose owner died (reboot, OOM) doesn't sit parked forever — the failure mode
+ * that left a `module deploy` holding every backup lock on the fleet for 20 days.
+ */
+export function reapExpiredSessions(opts: { busDbPath: string; now?: number }): SessionRecord[] {
+  const now = opts.now ?? Date.now();
+  const expired = listSessions().filter((s) => s.state === 'parked' && s.expiresAt <= now);
+  for (const record of expired) {
+    abandonSession(record, { busDbPath: opts.busDbPath, reason: expiryReason(record) });
+  }
+  return expired;
+}
+
+/** Buffered messages an attaching client should be replayed. */
+export function replayOutput(
+  sessionId: string,
+  fromLine = 0,
+): { messages: string[]; next: number } {
+  const path = outputPath(sessionId);
+  if (!existsSync(path)) return { messages: [], next: fromLine };
+  const lines = readFileSync(path, 'utf-8').split('\n').filter(Boolean);
+  const start = Math.max(fromLine, lines.length - REPLAY_LIMIT);
+  return { messages: lines.slice(start), next: lines.length };
+}

package/src/cli/commands/backup-sweep.ts

@@ -17,13 +17,12 @@ import {
   findBackupEligibleModules,
   isBackupDue,
 } from '../../services/backup-create';
-import { failBackup, getBackup } from '../../services/backup-metadata';
+import { failBackup, getBackup, listInProgressBackups } from '../../services/backup-metadata';
 import { pruneBackupsForModule } from '../../services/backup-retention';
 import {
-  ABANDONED_BACKUP_MESSAGE,
   STAGING_PREFIX,
-  impliesAbandonedRecord,
   reapOrphanedStaging,
+  resolveAbandonedBackups,
 } from '../../services/backup-staging';
 import { type BackupSweepReport, runBackupSweep } from '../../services/backup-sweep';
 import { isPidRunnable } from '../../services/module-operations';
@@ -59,18 +58,30 @@ function reapStaging() {
     now: () => Date.now(),
   });
 
-  for (const reclaimed of report.reclaimed) {
-    if (impliesAbandonedRecord(reclaimed.reason)) {
-      failBackup(reclaimed.recordId, ABANDONED_BACKUP_MESSAGE);
-    }
-  }
-
   return report;
 }
 
+/**
+ * Correct every record that still claims to be running, staging or no staging.
+ *
+ * This replaces deriving the fix-up from what the reaper reclaimed. That
+ * version only ever saw records whose directory still existed, so anything
+ * cleared by a reboot or by hand stayed `in_progress` forever (#616).
+ */
+function resolveAbandonedRecords() {
+  return resolveAbandonedBackups({
+    listInProgress: () =>
+      listInProgressBackups().map((b) => ({ id: b.id, pid: b.pid, startedAt: b.startedAt })),
+    isPidRunnable,
+    fail: (id, message) => failBackup(id, message),
+    now: () => Date.now(),
+  });
+}
+
 export async function handleBackupSweep(): Promise<CommandResult> {
   const report = await runBackupSweep({
     reapStaging,
+    resolveAbandonedRecords,
     listEligible: () =>
       findBackupEligibleModules().map(({ module, manifest }) => ({ id: module.id, manifest })),
     isDue: (moduleId, schedule) => isBackupDue(moduleId, schedule),
@@ -117,6 +128,11 @@ function formatReport(report: BackupSweepReport): string {
       lines.push(`    ${reclaimed.recordId} (${reclaimed.reason})`);
     }
   }
+  if (report.records.resolved.length > 0) {
+    lines.push(
+      `  corrected ${report.records.resolved.length} record(s) that still claimed to be running`,
+    );
+  }
   for (const moduleId of report.skippedLocked) {
     lines.push(`  skipped ${moduleId}: another operation is in flight — retrying next tick`);
   }

package/src/cli/commands/events.ts

@@ -9,7 +9,8 @@
  *   status               bus.health() as JSON
  *   tail                 recent events
  *   list-subscribers     persistent subscribers
- *   list-pending         pending deliveries
+ *   list-pending         pending deliveries (subscriber fan-out — NOT questions)
+ *   list-unanswered      interview questions nobody has answered yet
  *   drain                process pending deliveries once
  *   run                  long-running dispatcher (foreground; SIGINT to stop)
  *   emit <type> [json]   emit an event (operator/test path; bypasses schema)
@@ -30,6 +31,7 @@ import {
   runDispatcher,
 } from '@celilo/event-bus';
 import { eq } from 'drizzle-orm';
+import { sessionParkedOn } from '../../api/sessions';
 import { getEventBusPath, shortenPath } from '../../config/paths';
 import { getDb } from '../../db/client';
 import { modules } from '../../db/schema';
@@ -241,6 +243,57 @@ export async function handleEventsListPending(
   }
 }
 
+/** One unanswered interview question, as `events list-unanswered` reports it. */
+export interface UnansweredInterview {
+  eventId: number;
+  type: string;
+  family: InterviewFamily;
+  /** `<scope>.<key>` — the identity a responder pre-stages an answer under. */
+  key: string;
+  question: string;
+  ageMs: number;
+  /** The parked api-serve session waiting on this answer, if any. */
+  sessionId: string | null;
+}
+
+/**
+ * `celilo events list-unanswered` — interview queries with no correlated reply:
+ * what is waiting on a decision right now.
+ *
+ * The instrument celilo#609 lacked. `events list-pending` was reached for and
+ * silently answered a different question (it reads subscriber *deliveries*), so
+ * a parked command looked like no command at all. Non-empty here for as long as
+ * something is parked is the recurrence gate for that whole class of bug.
+ */
+export async function handleEventsListUnanswered(
+  _args: string[],
+  flags: Record<string, string | boolean>,
+): Promise<CommandResult> {
+  const bus = openCliBus();
+  try {
+    const limit = flags.limit ? Number(flags.limit) : 50;
+    const now = Date.now();
+    const rows: UnansweredInterview[] = [];
+    for (const event of bus.unansweredQueries({ limit })) {
+      const family = interviewFamily(event.type);
+      if (!family) continue; // e.g. responder.probe — not a question for an operator.
+      const payload = (event.payload ?? {}) as { message?: string; description?: string };
+      rows.push({
+        eventId: event.id,
+        type: event.type,
+        family,
+        key: event.type.slice(`${family}.required.`.length),
+        question: payload.message ?? payload.description ?? event.type,
+        ageMs: now - event.emittedAt,
+        sessionId: sessionParkedOn(String(event.id))?.sessionId ?? null,
+      });
+    }
+    return jsonResult(rows);
+  } finally {
+    bus.close();
+  }
+}
+
 export async function handleEventsDrain(
   _args: string[],
   flags: Record<string, string | boolean>,
@@ -412,9 +465,12 @@ function interviewFamily(type: string): InterviewFamily | null {
 /**
  * `celilo events reply <query-event-id> <value-json>` — answer ONE pending
  * interview query by its event id. The one-shot reply primitive a
- * `claude-config-responder` uses: read the question with
- * `events tail --type 'config.required.*'`, ask the operator, emit the answer
- * here. Unlike `events respond` (which must be subscribed BEFORE the query is
+ * `claude-config-responder` uses: find the question with
+ * `celilo events list-unanswered`, ask the operator, emit the answer here.
+ * (It used to say "read the log with `events tail --type '…'`" — hand-scraping
+ * the log because no command listed unanswered questions. `list-unanswered` is
+ * that command; it also names the parked session, which `tail` cannot.)
+ * Unlike `events respond` (which must be subscribed BEFORE the query is
  * emitted — bus watches don't replay history) this looks the query up by id
  * and emits a correlated reply carrying `replyFor`, which plain `events emit`
  * can't set.

package/src/cli/commands/module-update.ts

@@ -23,7 +23,7 @@ import type { ModuleManifest } from '../../manifest/schema';
 import { cleanupTempDir, extractPackage } from '../../module/packaging/extract';
 import { RegistryClient } from '../../registry/client';
 import { askConfirm, withInterviewSession } from '../../services/bus-interview';
-import { InterviewUnansweredError } from '../../services/interview-errors';
+import { InterviewAbandonedError, InterviewUnansweredError } from '../../services/interview-errors';
 import { getFlag } from '../parser';
 import { log } from '../prompts';
 import type { CommandResult } from '../types';
@@ -499,6 +499,7 @@ async function runRegistrySweep(
   let appliedBreaking = 0;
   let declinedBreaking = 0;
   const unanswered: Array<{ moduleId: string; error: string }> = [];
+  const abandoned: Array<{ moduleId: string; error: string }> = [];
 
   if (breaking.length > 0) {
     log.info('\nBreaking updates available — review required (semver-major bump):');
@@ -521,15 +522,25 @@ async function runRegistrySweep(
           }),
         );
       } catch (err) {
-        // Nobody could answer. That is NOT a decline — record it as unanswered
-        // so the summary says so and the sweep exits non-zero, and keep going
-        // so the updates already applied aren't thrown away with a hard abort.
-        if (!(err instanceof InterviewUnansweredError)) throw err;
-        unanswered.push({ moduleId: plan.moduleId, error: err.message });
-        console.log(
-          `  ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion}  (major, UNANSWERED)`,
-        );
-        continue;
+        // Neither of these is a decline, and they are not each other either:
+        // UNANSWERED = no responder was listening at all (fail-fast probe);
+        // ABANDONED  = the question stood and its parked session expired.
+        // Record and keep going so updates already applied aren't thrown away.
+        if (err instanceof InterviewUnansweredError) {
+          unanswered.push({ moduleId: plan.moduleId, error: err.message });
+          console.log(
+            `  ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion}  (major, UNANSWERED)`,
+          );
+          continue;
+        }
+        if (err instanceof InterviewAbandonedError) {
+          abandoned.push({ moduleId: plan.moduleId, error: err.message });
+          console.log(
+            `  ? ${plan.moduleId.padEnd(30)} ${plan.installedVersion} → ${plan.targetVersion}  (major, ABANDONED)`,
+          );
+          continue;
+        }
+        throw err;
       }
       if (!proceed) {
         declinedBreaking++;
@@ -572,6 +583,15 @@ async function runRegistrySweep(
         )}). Re-run with a responder attached, or pre-stage the answer under "module-upgrade:<module>.apply_breaking".`,
     );
   }
+  if (abandoned.length > 0) {
+    summary.push(
+      `Skipped ${abandoned.length} breaking update(s) — NOT declined: the confirmation was never decided and its session expired (${abandoned
+        .map((a) => a.moduleId)
+        .join(
+          ', ',
+        )}). Answer it next time with "celilo events list-unanswered" + "celilo events reply <id> <value>".`,
+    );
+  }
   if (notInRegistry.length > 0) {
     summary.push(`Not in registry (${notInRegistry.length}): ${notInRegistry.join(', ')}`);
   }
@@ -580,7 +600,7 @@ async function runRegistrySweep(
       `Registry errors (${errored.length}): ${errored.map((e) => `${e.moduleId} — ${e.error}`).join('; ')}`,
     );
   }
-  if (failed.length > 0 || unanswered.length > 0) {
+  if (failed.length > 0 || unanswered.length > 0 || abandoned.length > 0) {
     const detail: string[] = [];
     if (failed.length > 0) {
       detail.push('', 'Failures:', ...failed.map((f) => `  ${f.moduleId}: ${f.error}`));
@@ -588,6 +608,9 @@ async function runRegistrySweep(
     if (unanswered.length > 0) {
       detail.push('', 'Unanswered:', ...unanswered.map((u) => `  ${u.moduleId}: ${u.error}`));
     }
+    if (abandoned.length > 0) {
+      detail.push('', 'Abandoned:', ...abandoned.map((a) => `  ${a.moduleId}: ${a.error}`));
+    }
     return { success: false, error: [...summary, ...detail].join('\n') };
   }
   return { success: true, message: summary.join('\n') };

package/src/cli/completion.ts

@@ -113,6 +113,7 @@ export async function getCompletions(words: string[], current: number): Promise<
       'list-subscribers',
       'resync-subscriptions',
       'list-pending',
+      'list-unanswered',
       'drain',
       'run',
       'run-hook',

package/src/cli/index.ts

@@ -4,7 +4,13 @@
  * Orchestration function (Rule 10.1) - routes commands to handlers
  */
 
-import { COMMANDS, type CommandDef, resolveRemote, runRemoteClient } from '@celilo/core';
+import {
+  COMMANDS,
+  type CommandDef,
+  EXIT_BLOCKED,
+  resolveRemote,
+  runRemoteClient,
+} from '@celilo/core';
 import * as p from '@clack/prompts';
 import { CLIServerRequestSchema, parseJsonWithValidation } from '../validation/schemas';
 import {
@@ -28,6 +34,7 @@ import {
   handleEventsInstallDaemon,
   handleEventsListPending,
   handleEventsListSubscribers,
+  handleEventsListUnanswered,
   handleEventsRepair,
   handleEventsReply,
   handleEventsRespond,
@@ -1418,6 +1425,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
         return handleEventsResyncSubscriptions();
       case 'list-pending':
         return handleEventsListPending(parsed.args, parsed.flags);
+      case 'list-unanswered':
+        return handleEventsListUnanswered(parsed.args, parsed.flags);
       case 'drain':
         return handleEventsDrain(parsed.args, parsed.flags);
       case 'run':
@@ -2427,7 +2436,18 @@ export async function main(): Promise<void> {
   // SSH to the remote celilo-mgr and drive its api-serve over the wire.
   const remote = resolveRemote(process.argv);
   if (remote) {
-    process.exit(await runRemoteClient(remote.dest, remote.commandArgv));
+    const outcome = await runRemoteClient(remote.dest, remote.commandArgv);
+    if (outcome.status === 'blocked') {
+      // Not a failure and not a decline: the command is parked server-side on a
+      // question this client couldn't decide, and is still alive.
+      process.stderr.write(
+        `Command is parked on an unanswered question: ${outcome.question}\n` +
+          `  answer it:  celilo events reply ${outcome.eventId} <value>\n` +
+          `  it resumes server-side under session ${outcome.sessionId}\n`,
+      );
+      process.exit(EXIT_BLOCKED);
+    }
+    process.exit(outcome.exitCode);
   }
 
   // Normal single-command execution

package/src/services/backup-metadata.ts

@@ -156,6 +156,23 @@ export function listBackups(options?: {
   return db.select().from(backups).orderBy(desc(backups.startedAt)).limit(limit).all();
 }
 
+/**
+ * Every backup record still claiming to be in progress, oldest first.
+ *
+ * Feeds the record-resolution pass (`resolveAbandonedBackups`), which corrects
+ * the ones whose process is gone. Unbounded on purpose — there is no sensible
+ * limit on "how many lies to fix", and celilo-mgr had 107 of them.
+ */
+export function listInProgressBackups(): Backup[] {
+  const db = getDb();
+  return db
+    .select()
+    .from(backups)
+    .where(eq(backups.status, 'in_progress'))
+    .orderBy(backups.startedAt)
+    .all();
+}
+
 /**
  * List completed backups for a specific module, ordered newest first
  */