@celilo/cli 0.3.4 → 0.3.10

package/src/services/ensure-interview.test.ts

@@ -162,13 +162,11 @@ describe('interviewForEnsureInputs', () => {
   // is the user's consent for the cross-module config its hooks imply.
   // See config-interview.ts and CADDY_HOSTNAME_LIST.md, Decision 6.
 
-  test('non-interactive mode applies directly', async () => {
-    // In production, non-interactive mode prints a recipe and aborts
-    // before reaching this function. The function itself supports being
-    // called non-interactively for tests / scripted use, applying inputs
-    // directly. promptOverride is still required for set_in_object inputs.
+  test('promptOverride applies inputs directly without bus or terminal', async () => {
+    // promptOverride is the test escape hatch — bypasses the bus path
+    // so unit tests don't need to set up a responder. See bus-ensure-
+    // flow.test.ts for the full bus-mediated flow.
     const result = await interviewForEnsureInputs('namecheap', ENSURE, 'celilo.computer', testDb, {
-      noInteractive: true,
       promptOverride: async () => 'pw',
     });
     expect(result.success).toBe(true);

package/src/services/module-deploy.ts

@@ -26,7 +26,6 @@ import {
   interviewForEnsureInputs,
   interviewForMissingConfig,
   interviewForMissingSecrets,
-  renderEnsureRecipe,
 } from './config-interview';
 import { getContainerService } from './container-service';
 import { executeAnsible } from './deploy-ansible';
@@ -89,7 +88,6 @@ async function updateMachineAssignment(
 }
 
 export interface DeployOptions {
-  noInteractive?: boolean;
   debug?: boolean;
   /**
    * Keep all sub-events visible during the deploy. With this off (the
@@ -99,6 +97,17 @@ export interface DeployOptions {
    * useful for debugging slow or hanging steps.
    */
   verbose?: boolean;
+  /**
+   * Exit cleanly after the initial config + secrets interview phase
+   * — before any infrastructure code generation, terraform, ansible,
+   * or hook execution. Manual validation tool: lets an operator
+   * exercise the bus-mediated interview (config.required / secret.required
+   * events fire, responders answer, values land in the encrypted store)
+   * without actually deploying anything to a machine. The cross-module
+   * `ensure` interview (which fires from hooks) is NOT exercised in
+   * this mode — that requires a real hook run.
+   */
+  stopAfterInterview?: boolean;
 }
 
 /**
@@ -189,12 +198,6 @@ async function invokeHookWithEnsureRetry(
       };
     }
 
-    if (deployOptions.noInteractive) {
-      attempt.gauge.stop(false);
-      log.error(renderEnsureRecipe(m.providerModuleId, ensure, m.value));
-      return result;
-    }
-
     // Step the gauge out of the way so prompts render on a clean line.
     attempt.gauge.stopSilent();
 
@@ -302,14 +305,15 @@ async function deployModuleImpl(
 ): Promise<DeployResult> {
   const phases: DeployResult['phases'] = {};
 
-  // --no-interactive emits structured [progress:*] markers (for cele2e
-  // to parse). --verbose forces render mode but with isTTY=false, which
-  // disables cursor magic — every sub-event stays visible after each
-  // step completes (no collapse-on-success). When neither is set, mode
-  // is 'auto' and the display picks based on the real stdout's isTTY.
-  // --no-interactive wins if both are set (protocol output is what
-  // cele2e expects regardless of verbosity).
-  const displayMode = options.noInteractive ? 'protocol' : options.verbose ? 'render' : 'auto';
+  // When stdout isn't a TTY (cele2e subprocess, CI), emit structured
+  // [progress:*] markers that cele2e parses. --verbose forces render
+  // mode but with isTTY=false, which disables cursor magic — every
+  // sub-event stays visible after each step completes (no collapse-
+  // on-success). On a TTY without --verbose, mode is 'auto' and the
+  // ProgressDisplay picks animated gauges. Non-TTY wins if both are
+  // set (protocol output is what cele2e expects regardless).
+  const nonTTY = !process.stdout.isTTY;
+  const displayMode = nonTTY ? 'protocol' : options.verbose ? 'render' : 'auto';
   const display = new ProgressDisplay({
     mode: displayMode,
     out: options.verbose
@@ -319,15 +323,14 @@ async function deployModuleImpl(
   setActiveDisplay(display);
 
   // Terminal-responder: when running on a TTY, this subscribes to
-  // `config.required.*` events and prompts via clack so the operator
-  // can answer the deploy's interview without `--no-interactive`'s
-  // brittle "everything must be pre-staged" flow. Other responders
-  // (Claude subagent, `celilo events respond`) compete; first reply
-  // wins. See infra/design/INTERACTIVE_DEPLOYS_VIA_BUS.md.
-  const terminalResponder =
-    !options.noInteractive && process.stdin.isTTY
-      ? (await import('./terminal-responder')).startTerminalResponder()
-      : null;
+  // `config.required.*` / `secret.required.*` / `ensure.required.*`
+  // events and prompts via clack. Other responder shapes (Claude
+  // subagent, `celilo events respond` from another shell, autoresponder
+  // daemon) compete on the bus; first reply wins. See
+  // infra/design/INTERACTIVE_DEPLOYS_VIA_BUS.md.
+  const terminalResponder = process.stdin.isTTY
+    ? (await import('./terminal-responder')).startTerminalResponder()
+    : null;
 
   try {
     // Check for e2e test containers — live and e2e environments are mutually exclusive.
@@ -440,18 +443,12 @@ async function deployModuleImpl(
         }
       }
 
-      if (remainingMissing.length > 0 && options.noInteractive) {
-        // Non-interactive mode: fail with error for remaining missing variables
-        const varList = remainingMissing.map((v) => `  - ${v.name} (${v.source})`).join('\n');
-        return {
-          success: false,
-          error: `Missing required configuration:\n${varList}\n\nConfigure with: celilo module config set ${moduleId} <variable> <value>\nOr run without --no-interactive to be prompted.`,
-          phases,
-        };
-      }
-
-      // Interactive mode: prompt for remaining missing config
-      // Separate secrets from regular config
+      // Bus-mediated interview: any remaining missing config produces
+      // `config.required.*` / `secret.required.*` events. A responder
+      // (terminal, Claude subagent, `events respond`) answers; the
+      // deploy waits indefinitely if none does. Operators see stuck
+      // queries via `celilo events list-pending`.
+      // Separate secrets from regular config.
       const secrets = remainingMissing.filter((v) => v.source === 'secret');
       const regularConfig = remainingMissing.filter((v) => v.source !== 'secret');
 
@@ -510,6 +507,21 @@ async function deployModuleImpl(
       }
     }
 
+    // --stop-after-interview: bail before any infrastructure work.
+    // The bus-mediated interview has fired and any responders have
+    // answered; the operator can inspect the encrypted store and
+    // module config to confirm what landed without spinning up
+    // terraform / ansible / actual hooks. Cross-module `ensure`
+    // events fire later from hook execution, so they're NOT
+    // exercised here — that requires a real run.
+    if (options.stopAfterInterview) {
+      log.info('--stop-after-interview: exiting before infrastructure phase.');
+      return {
+        success: true,
+        phases: { ...phases, validation: true },
+      };
+    }
+
     // If validation returned early (missing variables were present), generation
     // was deferred until after the interview. Run it now that all vars are set.
     if (!validation.autoGenerated) {
@@ -538,7 +550,7 @@ async function deployModuleImpl(
     if (manifest.hooks?.validate_config) {
       const hookDef = manifest.hooks.validate_config;
       const gauge = new FuelGauge('Validating configuration', {
-        skipAnimation: options.noInteractive,
+        skipAnimation: !process.stdout.isTTY,
       });
       gauge.start();
       const hookLogger = createGaugeLogger(gauge, moduleId, 'validate_config');
@@ -720,7 +732,7 @@ async function deployModuleImpl(
           installSecretMap,
           async () => {
             const gauge = new FuelGauge(`${moduleId}: on_install`, {
-              skipAnimation: options.noInteractive,
+              skipAnimation: !process.stdout.isTTY,
             });
             gauge.start();
             const logger = createGaugeLogger(gauge, moduleId, 'on_install');
@@ -756,7 +768,7 @@ async function deployModuleImpl(
         const { runModuleHealthCheck } = await import('./health-runner');
         const healthResult = await runModuleHealthCheck(moduleId, db, {
           debug: options.debug,
-          noInteractive: options.noInteractive,
+          noInteractive: !process.stdout.isTTY,
         });
         if (healthResult.status === 'error') {
           return { success: false, phases, error: `Health check failed: ${healthResult.error}` };
@@ -770,6 +782,11 @@ async function deployModuleImpl(
         }
       }
 
+      // Mirror the infrastructure-path success message at the end of
+      // a successful deploy. Without this, config-only deploys end
+      // abruptly with whatever the last hook line was — operator sees
+      // no clear "this finished cleanly" signal.
+      log.success(`Module '${moduleId}' deployed successfully`);
       return {
         success: true,
         phases: {
@@ -800,7 +817,7 @@ async function deployModuleImpl(
       }
 
       const terraformResult = await executeTerraform(generatedPath, phases, terraformEnvVars, {
-        noInteractive: options.noInteractive,
+        noInteractive: !process.stdout.isTTY,
       });
 
       if (!terraformResult.success) {
@@ -917,7 +934,7 @@ async function deployModuleImpl(
         );
 
         const gauge = new FuelGauge(`${capRecord.moduleId}: container_created`, {
-          skipAnimation: options.noInteractive,
+          skipAnimation: !process.stdout.isTTY,
         });
         gauge.start();
         const hookLogger = createGaugeLogger(gauge, capRecord.moduleId, 'container_created');
@@ -1050,7 +1067,7 @@ async function deployModuleImpl(
 
     try {
       const ansibleResult = await executeAnsible(generatedPath, {
-        noInteractive: options.noInteractive,
+        noInteractive: !process.stdout.isTTY,
       });
       phases.ansible = ansibleResult.success;
       if (!ansibleResult.success) {
@@ -1139,7 +1156,7 @@ async function deployModuleImpl(
           installSecretMap,
           async () => {
             const gauge = new FuelGauge(`${moduleId}: on_install`, {
-              skipAnimation: options.noInteractive,
+              skipAnimation: !process.stdout.isTTY,
             });
             gauge.start();
             const logger = createGaugeLogger(gauge, moduleId, 'on_install');
@@ -1179,7 +1196,7 @@ async function deployModuleImpl(
         const { runModuleHealthCheck } = await import('./health-runner');
         const healthResult = await runModuleHealthCheck(moduleId, db, {
           debug: options.debug,
-          noInteractive: options.noInteractive,
+          noInteractive: !process.stdout.isTTY,
         });
 
         if (healthResult.status === 'error') {

package/src/services/programmatic-responder.ts

@@ -0,0 +1,294 @@
+/**
+ * Programmatic bus responder — watches `config.required.*`,
+ * `secret.required.*`, and `ensure.required.*` events and replies
+ * from a pre-baked values map. Drives the `celilo events respond
+ * --values` CLI mode and the `bus-responder` test fixture.
+ *
+ * The responder runs in-process: it owns its own Bus + DbClient
+ * against the celilo data dir, so it can write secrets to the
+ * encrypted store out-of-band (per the `secret.required` flow) and
+ * the deploy re-reads after acks.
+ *
+ * Two consumers, two missing-value policies:
+ *   - tests use `onMissing: 'throw'` so a forgotten value fails
+ *     loudly instead of hanging.
+ *   - the CLI uses `onMissing: 'skip'` so the responder lets other
+ *     responders win the race instead of erroring out.
+ */
+
+import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
+import type { DbClient } from '../db/client';
+import { generateSecret } from '../secrets/generators';
+import { getOrCreateMasterKey } from '../secrets/master-key';
+import type {
+  ConfigRequiredPayload,
+  EnsureRequiredPayload,
+  SecretRequiredPayload,
+} from './bus-interview';
+import { readModuleSecretKey, writeModuleSecretKey } from './config-interview';
+
+const NO_SCHEMAS = defineEvents({});
+
+export interface ResponderValues {
+  /**
+   * Config values keyed by `<module>.<key>`. When the deploy emits
+   * `config.required.<m>.<k>`, the responder replies with `{ value }`.
+   */
+  config?: Record<string, unknown>;
+  /**
+   * Secret values keyed by `<module>.<key>`. When the deploy emits
+   * `secret.required.<m>.<k>`, the responder writes the value to
+   * the encrypted store and replies with `{ acknowledged: true }`.
+   * If the entry is missing AND the payload's `style` is
+   * `generated_optional`, the responder auto-generates per the
+   * payload's `generate` hint instead of raising / skipping.
+   */
+  secrets?: Record<string, string>;
+  /**
+   * Ensure values keyed by `<provider>.<ensureId>`. For each
+   * `set_in_object` input in the payload, the responder either puts
+   * the value in `reply.values[target]` (config target) or writes
+   * the value out-of-band (secret target). `append_to_array` inputs
+   * never reach the responder — the deploy applies them deterministic.
+   */
+  ensures?: Record<
+    string,
+    {
+      configValues?: Record<string, unknown>;
+      secretValues?: Record<string, string>;
+    }
+  >;
+}
+
+export interface ProgrammaticResponderOptions {
+  /** Path to the bus sqlite db (the deploy and responder share it). */
+  busDbPath: string;
+  /** Open db client used for out-of-band secret writes. */
+  db: DbClient;
+  /** Values to reply with. */
+  values: ResponderValues;
+  /**
+   * What to do when an event arrives that has no matching value:
+   * 'throw' — fail the responder loudly (tests use this).
+   * 'skip'  — log and don't reply (let other responders win).
+   */
+  onMissing: 'throw' | 'skip';
+  /**
+   * Identifier for the `emittedBy` audit field on replies. Operators
+   * see this in `celilo events tail` to correlate which responder
+   * answered which query. Defaults to `programmatic`.
+   */
+  emittedBy?: string;
+}
+
+export interface AnsweredEvent {
+  /** Full event type (e.g. `config.required.lunacycle.domain`). */
+  type: string;
+  /** `<module>.<key>` or `<provider>.<ensureId>` lookup key. */
+  key: string;
+}
+
+export interface MissedEvent extends AnsweredEvent {
+  reason: string;
+}
+
+export interface ProgrammaticResponderHandle {
+  /** Activity timestamp (ms) — last time we saw an event. */
+  lastActivityAt(): number;
+  /** Total events answered + skipped since start. */
+  eventCount(): number;
+  /** Snapshot of replied events. */
+  answered(): AnsweredEvent[];
+  /** Snapshot of skipped events (reason populated). */
+  missed(): MissedEvent[];
+  /**
+   * Full payload snapshots, indexed by event family. Tests use these
+   * to assert on what the deploy emitted; the CLI ignores them.
+   */
+  seenConfigPayloads(): ConfigRequiredPayload[];
+  seenSecretPayloads(): SecretRequiredPayload[];
+  seenEnsurePayloads(): EnsureRequiredPayload[];
+  /** Stop watching. Caller still owns the db client. */
+  close(): void;
+}
+
+/**
+ * Open the bus, register watches on the three interview event
+ * families, reply per the values map. Returns a handle the caller
+ * uses to drive idle-timeout exit logic; the caller owns the db
+ * client lifecycle.
+ */
+export function startProgrammaticResponder(
+  opts: ProgrammaticResponderOptions,
+): ProgrammaticResponderHandle {
+  const answered: AnsweredEvent[] = [];
+  const missed: MissedEvent[] = [];
+  const seenConfig: ConfigRequiredPayload[] = [];
+  const seenSecret: SecretRequiredPayload[] = [];
+  const seenEnsure: EnsureRequiredPayload[] = [];
+  let lastActivityAt = Date.now();
+
+  const me = opts.emittedBy ?? 'programmatic';
+  const bus: Bus = openBus({ dbPath: opts.busDbPath, events: NO_SCHEMAS });
+
+  const handleMissing = (type: string, key: string, reason: string): boolean => {
+    if (opts.onMissing === 'throw') {
+      throw new Error(`programmatic-responder: ${reason} (event: ${type}, key: "${key}")`);
+    }
+    missed.push({ type, key, reason });
+    return false;
+  };
+
+  const configWatch = bus.watch('config.required.*.*', async (event) => {
+    if (event.replyFor !== null) return;
+    lastActivityAt = Date.now();
+
+    const payload = event.payload as ConfigRequiredPayload;
+    if (!payload || typeof payload.module !== 'string' || typeof payload.key !== 'string') {
+      missed.push({ type: event.type, key: '?', reason: 'malformed payload' });
+      return;
+    }
+    seenConfig.push(payload);
+
+    const lookupKey = `${payload.module}.${payload.key}`;
+    const value = opts.values.config?.[lookupKey];
+    if (value === undefined) {
+      handleMissing(event.type, lookupKey, `no config value for "${lookupKey}"`);
+      return;
+    }
+
+    bus.emitRaw(`${event.type}.reply`, { value }, { replyFor: event.id, emittedBy: me });
+    answered.push({ type: event.type, key: lookupKey });
+  });
+
+  const secretWatch = bus.watch('secret.required.*.*', async (event) => {
+    if (event.replyFor !== null) return;
+    lastActivityAt = Date.now();
+
+    const payload = event.payload as SecretRequiredPayload;
+    if (!payload || typeof payload.module !== 'string' || typeof payload.key !== 'string') {
+      missed.push({ type: event.type, key: '?', reason: 'malformed payload' });
+      return;
+    }
+    seenSecret.push(payload);
+
+    const lookupKey = `${payload.module}.${payload.key}`;
+    let value = opts.values.secrets?.[lookupKey];
+    if (value === undefined) {
+      // generated_optional fallback: use the payload's hint instead
+      // of asking the operator. Same UX as the terminal-responder's
+      // empty-input branch.
+      if (payload.style === 'generated_optional' && payload.generate) {
+        value = generateSecret(payload.generate);
+      } else {
+        handleMissing(event.type, lookupKey, `no secret value for "${lookupKey}"`);
+        return;
+      }
+    }
+
+    const masterKey = await getOrCreateMasterKey();
+    await writeModuleSecretKey(payload.module, payload.key, value, opts.db, masterKey);
+
+    bus.emitRaw(
+      `${event.type}.reply`,
+      { acknowledged: true },
+      { replyFor: event.id, emittedBy: me },
+    );
+    answered.push({ type: event.type, key: lookupKey });
+  });
+
+  const ensureWatch = bus.watch('ensure.required.*.*', async (event) => {
+    if (event.replyFor !== null) return;
+    lastActivityAt = Date.now();
+
+    const payload = event.payload as EnsureRequiredPayload;
+    if (!payload || typeof payload.provider !== 'string' || !Array.isArray(payload.inputs)) {
+      missed.push({ type: event.type, key: '?', reason: 'malformed payload' });
+      return;
+    }
+    seenEnsure.push(payload);
+
+    const lookupKey = `${payload.provider}.${payload.ensureId}`;
+    const fixture = opts.values.ensures?.[lookupKey];
+    if (!fixture) {
+      handleMissing(event.type, lookupKey, `no ensure values for "${lookupKey}"`);
+      return;
+    }
+
+    const values: Record<string, unknown> = {};
+    let acknowledged = false;
+    let masterKey: Buffer | null = null;
+    let perInputMissing = false;
+
+    for (const input of payload.inputs) {
+      if (input.target.startsWith('config.')) {
+        const v = fixture.configValues?.[input.target];
+        if (v === undefined) {
+          handleMissing(
+            event.type,
+            lookupKey,
+            `ensure "${lookupKey}" missing config value for "${input.target}"`,
+          );
+          perInputMissing = true;
+          break;
+        }
+        values[input.target] = v;
+        continue;
+      }
+
+      // Secret target — read-merge-write the JSON-encoded secret.
+      const name = input.target.slice('secret.'.length);
+      const v = fixture.secretValues?.[input.target];
+      if (v === undefined) {
+        handleMissing(
+          event.type,
+          lookupKey,
+          `ensure "${lookupKey}" missing secret value for "${input.target}"`,
+        );
+        perInputMissing = true;
+        break;
+      }
+      if (!masterKey) masterKey = await getOrCreateMasterKey();
+
+      let obj: Record<string, unknown> = {};
+      const currentRaw = await readModuleSecretKey(payload.provider, name, opts.db, masterKey);
+      if (currentRaw) {
+        try {
+          const parsed = JSON.parse(currentRaw);
+          if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            obj = parsed as Record<string, unknown>;
+          }
+        } catch {
+          /* overwrite */
+        }
+      }
+      obj[input.objectKey] = v;
+      await writeModuleSecretKey(payload.provider, name, JSON.stringify(obj), opts.db, masterKey);
+      acknowledged = true;
+    }
+
+    if (perInputMissing) return;
+
+    bus.emitRaw(`${event.type}.reply`, acknowledged ? { values, acknowledged: true } : { values }, {
+      replyFor: event.id,
+      emittedBy: me,
+    });
+    answered.push({ type: event.type, key: lookupKey });
+  });
+
+  return {
+    lastActivityAt: () => lastActivityAt,
+    eventCount: () => answered.length + missed.length,
+    answered: () => [...answered],
+    missed: () => [...missed],
+    seenConfigPayloads: () => [...seenConfig],
+    seenSecretPayloads: () => [...seenSecret],
+    seenEnsurePayloads: () => [...seenEnsure],
+    close: () => {
+      configWatch.close();
+      secretWatch.close();
+      ensureWatch.close();
+      bus.close();
+    },
+  };
+}