@celilo/cli 0.3.4 → 0.3.9

package/src/services/terminal-responder.ts

@@ -1,7 +1,8 @@
 /**
  * Terminal-responder: when a deploy runs on a TTY, this subscribes
- * to `config.required.*` events and prompts the operator via clack
- * for each one, replying on the bus with the user's input.
+ * to `config.required.*`, `secret.required.*`, and `ensure.required.*`
+ * events and prompts the operator via clack for each one, replying
+ * on the bus.
  *
  * Just one of several responder shapes — the bus query/reply path is
  * a race, and the terminal is one racer. Other racers (Claude
@@ -14,14 +15,31 @@
  * deploy already moved on. The user sees their prompt close and the
  * deploy continues with the winning value (logged by the deploy in
  * its normal output). Race-during-typing detection is a follow-up.
+ *
+ * Secret + ensure values never cross the bus. The responder writes
+ * them directly to the encrypted store via the same helpers the
+ * deploy uses, then replies with `{ acknowledged: true }`. Other
+ * responder shapes (Claude subagent, separate-shell `events respond`)
+ * achieve the same out-of-band injection by shelling out to
+ * `celilo module secret set`; the in-process terminal-responder takes
+ * the shorter path of calling the helpers directly.
  */
 
 import { hostname } from 'node:os';
 import { type Bus, openBus } from '@celilo/event-bus';
 import { defineEvents } from '@celilo/event-bus';
-import { log, promptText } from '../cli/prompts';
+import * as p from '@clack/prompts';
+import { log, promptPassword, promptText } from '../cli/prompts';
 import { getEventBusPath } from '../config/paths';
-import type { ConfigRequiredPayload } from './bus-interview';
+import { getDb } from '../db/client';
+import { generateSecret } from '../secrets/generators';
+import { getOrCreateMasterKey } from '../secrets/master-key';
+import type {
+  ConfigRequiredPayload,
+  EnsureRequiredPayload,
+  SecretRequiredPayload,
+} from './bus-interview';
+import { readModuleSecretKey, writeModuleSecretKey } from './config-interview';
 
 const NO_SCHEMAS = defineEvents({});
 
@@ -41,8 +59,9 @@ export interface TerminalResponderHandle {
 }
 
 /**
- * Register a transient subscription on `config.required.*` events.
- * For each event, prompt the operator via clack and reply on the bus.
+ * Register transient subscriptions on the three interview event
+ * families. For each event, prompt the operator via clack, optionally
+ * inject the secret value out-of-band, and reply on the bus.
  *
  * Returns a handle the caller can close() when the deploy completes.
  */
@@ -51,7 +70,9 @@ export function startTerminalResponder(): TerminalResponderHandle {
   const me = responderId();
 
   // Track which queries we've already started a prompt for, so a
-  // re-fire (e.g. validation re-emit) doesn't double-prompt.
+  // re-fire (e.g. validation re-emit) doesn't double-prompt. Shared
+  // across the three watches by event id (ids are globally unique
+  // within the events table).
   const handled = new Set<number>();
 
   // Pattern matches exactly `config.required.<module>.<key>` (4 dot
@@ -60,16 +81,12 @@ export function startTerminalResponder(): TerminalResponderHandle {
   // we'd recursively try to prompt for our own reply event. The
   // explicit replyFor === null check below is defense in depth in
   // case any responder ever emits a reply with the same shape.
-  const watch = bus.watch('config.required.*.*', async (event) => {
-    // Reply events carry replyFor; queries don't. Skip anything that
-    // looks like a reply, even if it accidentally matches the pattern.
+  const configWatch = bus.watch('config.required.*.*', async (event) => {
     if (event.replyFor !== null) return;
     if (handled.has(event.id)) return;
     handled.add(event.id);
 
     const payload = event.payload as ConfigRequiredPayload;
-    // Defensive: if the payload is malformed (no module/key), don't
-    // render a "undefined.undefined:" prompt — log and skip.
     if (!payload || typeof payload.module !== 'string' || typeof payload.key !== 'string') {
       log.warn(
         `Terminal responder skipped malformed event ${event.type} (id ${event.id}): missing module/key`,
@@ -81,38 +98,99 @@ export function startTerminalResponder(): TerminalResponderHandle {
       ? `${payload.module}.${payload.key} — ${payload.description}:`
       : `${payload.module}.${payload.key}:`;
 
-    // Hint shown next to the prompt. For non-string types, tell the
-    // user what shape we're expecting so they don't get a coercion
-    // failure mid-deploy.
-    const typeHint = describeTypeHint(payload.type);
+    let coerced: unknown;
 
-    const value = await promptText({
-      message: typeHint ? `${message}  (${typeHint})` : message,
-      validate: (val) => {
-        if (payload.required && (!val || val.trim() === '')) {
-          return 'This field is required';
-        }
-        if (payload.pattern && val) {
-          const re = new RegExp(payload.pattern);
-          if (!re.test(val)) return `Value must match: ${payload.pattern}`;
-        }
-        // Type-shape validation runs HERE, at submit time, so clack
-        // re-prompts on bad input rather than letting a malformed
-        // value through that fails post-submit.
-        try {
-          coerceValue(val, payload.type);
-        } catch (err) {
-          return err instanceof Error ? err.message : String(err);
-        }
+    if (payload.options && payload.options.length > 0) {
+      // Multi-select prompt for vars with options[] declared in the
+      // manifest. clack's multiselect returns an array of selected
+      // values directly — no JSON-typing to coerce.
+      const selected = await p.multiselect({
+        message,
+        options: payload.options.map((opt) => ({
+          value: opt.value,
+          label: opt.label,
+          hint: opt.hint,
+        })),
+        required: payload.required,
+      });
+      if (p.isCancel(selected)) {
+        log.warn(
+          `Terminal responder: cancelled prompt for ${payload.module}.${payload.key}; no reply emitted`,
+        );
+        return;
+      }
+      coerced = selected as unknown[];
+    } else {
+      // Free-text prompt with type-shape validation at submit-time.
+      const typeHint = describeTypeHint(payload.type);
+      const value = await promptText({
+        message: typeHint ? `${message}  (${typeHint})` : message,
+        validate: (val) => {
+          if (payload.required && (!val || val.trim() === '')) {
+            return 'This field is required';
+          }
+          if (payload.pattern && val) {
+            const re = new RegExp(payload.pattern);
+            if (!re.test(val)) return `Value must match: ${payload.pattern}`;
+          }
+          try {
+            coerceValue(val, payload.type);
+          } catch (err) {
+            return err instanceof Error ? err.message : String(err);
+          }
+        },
+      });
+      coerced = coerceValue(value, payload.type);
+    }
+
+    bus.emitRaw(
+      `${event.type}.reply`,
+      { value: coerced },
+      {
+        replyFor: event.id,
+        emittedBy: me,
       },
-    });
+    );
+  });
+
+  const secretWatch = bus.watch('secret.required.*.*', async (event) => {
+    if (event.replyFor !== null) return;
+    if (handled.has(event.id)) return;
+    handled.add(event.id);
 
-    // Already validated above; coerce can't throw here.
-    const coerced = coerceValue(value, payload.type);
+    const payload = event.payload as SecretRequiredPayload;
+    if (!payload || typeof payload.module !== 'string' || typeof payload.key !== 'string') {
+      log.warn(
+        `Terminal responder skipped malformed event ${event.type} (id ${event.id}): missing module/key`,
+      );
+      return;
+    }
+
+    const value = await promptForSecret(payload);
+    if (value === undefined) {
+      // User cancelled; don't reply.
+      log.warn(
+        `Terminal responder: cancelled prompt for ${payload.module}.${payload.key}; no reply emitted`,
+      );
+      return;
+    }
+
+    try {
+      const db = getDb();
+      const masterKey = await getOrCreateMasterKey();
+      await writeModuleSecretKey(payload.module, payload.key, value, db, masterKey);
+    } catch (err) {
+      log.error(
+        `Terminal responder failed to write secret ${payload.module}.${payload.key}: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+      return;
+    }
 
     bus.emitRaw(
       `${event.type}.reply`,
-      { value: coerced },
+      { acknowledged: true },
       {
         replyFor: event.id,
         emittedBy: me,
@@ -120,14 +198,163 @@ export function startTerminalResponder(): TerminalResponderHandle {
     );
   });
 
+  const ensureWatch = bus.watch('ensure.required.*.*', async (event) => {
+    if (event.replyFor !== null) return;
+    if (handled.has(event.id)) return;
+    handled.add(event.id);
+
+    const payload = event.payload as EnsureRequiredPayload;
+    if (!payload || typeof payload.provider !== 'string' || !Array.isArray(payload.inputs)) {
+      log.warn(`Terminal responder skipped malformed ensure event ${event.type} (id ${event.id})`);
+      return;
+    }
+
+    const values: Record<string, unknown> = {};
+    let acknowledged = false;
+    let cancelled = false;
+
+    let masterKey: Buffer | null = null;
+
+    for (const input of payload.inputs) {
+      const message = input.hint ? `${input.prompt}\n  ${input.hint}` : input.prompt;
+      const userValue = await promptText({ message });
+      if (userValue === undefined || userValue.trim() === '') {
+        // Treat empty/cancel as cancellation; don't reply with a
+        // half-filled values object.
+        cancelled = true;
+        break;
+      }
+
+      if (input.target.startsWith('config.')) {
+        // The deploy applies the read-merge-write on the config side
+        // using the value we hand back here. Keyed by full target;
+        // the deploy already knows objectKey from the input payload
+        // it sent us, so we don't echo it.
+        values[input.target] = userValue;
+        continue;
+      }
+
+      // Secret target: read-merge-write the secret in-process so the
+      // value never crosses the bus. The deploy will re-read after
+      // the ack, so it sees the merged object.
+      const name = input.target.slice('secret.'.length);
+      const db = getDb();
+      if (!masterKey) masterKey = await getOrCreateMasterKey();
+
+      let obj: Record<string, unknown> = {};
+      const currentRaw = await readModuleSecretKey(payload.provider, name, db, masterKey);
+      if (currentRaw) {
+        try {
+          const parsed = JSON.parse(currentRaw);
+          if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+            obj = parsed as Record<string, unknown>;
+          }
+        } catch {
+          // Existing secret isn't a JSON object — overwrite (the
+          // schema declares this secret as an object, so any other
+          // shape is stale).
+        }
+      }
+      obj[input.objectKey] = userValue;
+      try {
+        await writeModuleSecretKey(payload.provider, name, JSON.stringify(obj), db, masterKey);
+        acknowledged = true;
+      } catch (err) {
+        log.error(
+          `Terminal responder failed to write secret ${payload.provider}.${name}: ${
+            err instanceof Error ? err.message : String(err)
+          }`,
+        );
+        cancelled = true;
+        break;
+      }
+    }
+
+    if (cancelled) {
+      log.warn(
+        `Terminal responder: cancelled ensure ${payload.provider}.${payload.ensureId}; no reply emitted`,
+      );
+      return;
+    }
+
+    bus.emitRaw(`${event.type}.reply`, acknowledged ? { values, acknowledged: true } : { values }, {
+      replyFor: event.id,
+      emittedBy: me,
+    });
+  });
+
   return {
     close() {
-      watch.close();
+      configWatch.close();
+      secretWatch.close();
+      ensureWatch.close();
       bus.close();
     },
   };
 }
 
+/**
+ * Prompt the operator for a secret value, handling all three
+ * `style` variants. Returns the value to store, or `undefined` if
+ * the user cancelled.
+ *
+ * For `user_password`: re-prompts on mismatch (loop until match or
+ * cancel) — a much friendlier UX than the old "fail and re-run
+ * deploy" behavior.
+ *
+ * For `generated_optional`: empty input falls back to
+ * `generateSecret(payload.generate)` so the operator can hit Enter
+ * to skip and let the deploy auto-generate.
+ */
+async function promptForSecret(payload: SecretRequiredPayload): Promise<string | undefined> {
+  const message = payload.description
+    ? `${payload.module}.${payload.key} — ${payload.description}:`
+    : `${payload.module}.${payload.key}:`;
+  const style = payload.style ?? 'user_provided';
+
+  if (style === 'user_password') {
+    while (true) {
+      const value = await promptPassword({
+        message,
+        validate: (val) => (!val || val.trim() === '' ? 'This field is required' : undefined),
+      });
+      if (value === undefined) return undefined;
+      const confirm = await promptPassword({
+        message: `Confirm ${payload.module}.${payload.key}:`,
+        validate: (val) => (!val || val.trim() === '' ? 'This field is required' : undefined),
+      });
+      if (confirm === undefined) return undefined;
+      if (value === confirm) return value;
+      log.error('Passwords do not match. Try again.');
+    }
+  }
+
+  if (style === 'generated_optional') {
+    const optMessage = `${message} (Enter = auto-generate)`;
+    const value = await promptPassword({
+      message: optMessage,
+      validate: () => undefined,
+    });
+    if (value === undefined) return undefined;
+    if (value.trim() === '') {
+      const format = payload.generate?.format ?? 'base64';
+      const length = payload.generate?.length ?? 32;
+      const generated = generateSecret({ format, length });
+      log.message(`Auto-generated ${format} secret: ${payload.module}.${payload.key}`);
+      return generated;
+    }
+    return value;
+  }
+
+  // Default: user_provided — single prompt, required.
+  const value = await promptPassword({
+    message,
+    validate: (val) =>
+      payload.required && (!val || val.trim() === '') ? 'This field is required' : undefined,
+  });
+  return value;
+}
+
 /**
  * Short hint shown in the prompt for non-string types so the operator
  * isn't guessing what shape we want. Returns null for plain strings —

package/src/test-utils/bus-responder.ts

@@ -0,0 +1,126 @@
+/**
+ * Bus responder test fixture.
+ *
+ * Stage 4 dropped the `--no-interactive` deploy flag, so missing
+ * config now fires bus events instead of aborting the deploy.
+ * Integration tests that previously asserted on the abort behavior
+ * need to attach a responder to the same bus DB the celilo CLI
+ * subprocess uses, then assert on what the responder saw + the
+ * deploy's final state.
+ *
+ * The fixture is in-process, the deploy runs in a subprocess —
+ * they share the bus via the file path passed in.
+ *
+ * Implementation: thin wrapper around `services/programmatic-
+ * responder.ts`, which is the production-grade responder that also
+ * powers `celilo events respond --values <file>`. The fixture sets
+ * `onMissing: 'throw'` so a forgotten value fails the test loudly
+ * instead of letting the deploy hang waiting for a reply.
+ *
+ * Usage:
+ *
+ *   const responder = startBusResponderFixture({
+ *     busDbPath,
+ *     config: { 'mymod.host': 'example.com' },
+ *     secrets: { 'mymod.api_key': 'sk-fake' },
+ *     ensures: {
+ *       'namecheap.managed_domain': {
+ *         configValues: { 'config.zone_ids': 'zone-1' },
+ *         secretValues: { 'secret.ddns_passwords': 'pw' },
+ *       },
+ *     },
+ *   });
+ *   try {
+ *     ...
+ *     expect(responder.seenConfigKeys()).toContain('mymod.host');
+ *   } finally {
+ *     responder.close();
+ *   }
+ */
+
+import { type DbClient, createDbClient } from '../db/client';
+import type {
+  ConfigRequiredPayload,
+  EnsureRequiredPayload,
+  SecretRequiredPayload,
+} from '../services/bus-interview';
+import {
+  type ProgrammaticResponderHandle,
+  type ResponderValues,
+  startProgrammaticResponder,
+} from '../services/programmatic-responder';
+
+export interface BusResponderFixtureOptions extends ResponderValues {
+  /** Path to the bus sqlite db (shared with the celilo subprocess). */
+  busDbPath: string;
+  /**
+   * Path to the celilo sqlite db. The responder reaches into this DB
+   * directly to write secret values out-of-band — same pattern the
+   * terminal-responder uses when running in-process inside the deploy.
+   */
+  celiloDbPath: string;
+  /**
+   * Path to the celilo data dir (where master.key lives). The
+   * responder needs the master key to encrypt secret values.
+   */
+  dataDir: string;
+}
+
+export interface BusResponderFixture {
+  /** Module.key strings the responder fielded on `config.required`. */
+  seenConfigKeys(): string[];
+  /** Module.key strings the responder fielded on `secret.required`. */
+  seenSecretKeys(): string[];
+  /** `<provider>.<ensureId>` strings the responder fielded. */
+  seenEnsureKeys(): string[];
+  /** Snapshot of every config.required payload the responder saw. */
+  seenConfigPayloads(): ConfigRequiredPayload[];
+  /** Snapshot of every secret.required payload the responder saw. */
+  seenSecretPayloads(): SecretRequiredPayload[];
+  /** Snapshot of every ensure.required payload the responder saw. */
+  seenEnsurePayloads(): EnsureRequiredPayload[];
+  /** Stop watching and close the bus + db connections. */
+  close(): void;
+}
+
+/**
+ * Start a programmatic responder against the given bus DB. Opens
+ * its own bus + db client in the test process so the celilo
+ * subprocess and the responder share state via file storage.
+ */
+export function startBusResponderFixture(opts: BusResponderFixtureOptions): BusResponderFixture {
+  // Set CELILO_DATA_DIR for getOrCreateMasterKey so the responder
+  // reads the subprocess's master.key. The fixture's process inherits
+  // these env vars; restore on close so other tests aren't affected.
+  const prevDataDir = process.env.CELILO_DATA_DIR;
+  process.env.CELILO_DATA_DIR = opts.dataDir;
+
+  const prevDbPath = process.env.CELILO_DB_PATH;
+  process.env.CELILO_DB_PATH = opts.celiloDbPath;
+  const db: DbClient = createDbClient();
+
+  const handle: ProgrammaticResponderHandle = startProgrammaticResponder({
+    busDbPath: opts.busDbPath,
+    db,
+    values: { config: opts.config, secrets: opts.secrets, ensures: opts.ensures },
+    onMissing: 'throw',
+    emittedBy: 'test-bus-responder',
+  });
+
+  return {
+    seenConfigKeys: () => handle.seenConfigPayloads().map((p) => `${p.module}.${p.key}`),
+    seenSecretKeys: () => handle.seenSecretPayloads().map((p) => `${p.module}.${p.key}`),
+    seenEnsureKeys: () => handle.seenEnsurePayloads().map((p) => `${p.provider}.${p.ensureId}`),
+    seenConfigPayloads: () => handle.seenConfigPayloads(),
+    seenSecretPayloads: () => handle.seenSecretPayloads(),
+    seenEnsurePayloads: () => handle.seenEnsurePayloads(),
+    close: () => {
+      handle.close();
+      db.$client.close();
+      if (prevDataDir === undefined) process.env.CELILO_DATA_DIR = undefined;
+      else process.env.CELILO_DATA_DIR = prevDataDir;
+      if (prevDbPath === undefined) process.env.CELILO_DB_PATH = undefined;
+      else process.env.CELILO_DB_PATH = prevDbPath;
+    },
+  };
+}

package/src/test-utils/index.ts

@@ -21,6 +21,13 @@ export { runCli, runCliExpectingFailure } from './cli';
 // Integration test setup
 export { setupIntegrationTest, type IntegrationTestContext } from './integration';
 
+// Bus responder fixture for stage-4+ tests
+export {
+  startBusResponderFixture,
+  type BusResponderFixture,
+  type BusResponderFixtureOptions,
+} from './bus-responder';
+
 // Database utilities
 export {
   setupTestDatabase,

package/src/test-utils/integration.ts

@@ -12,6 +12,12 @@ export interface IntegrationTestContext {
   dbPath: string;
   /** Path to test data directory */
   dataDir: string;
+  /**
+   * Path to the bus event sqlite db. Tests that need to attach a
+   * bus responder fixture can pass this to `startBusResponderFixture`
+   * so the responder shares the bus with the celilo CLI subprocess.
+   */
+  busDbPath: string;
   /** CLI command prefix with environment variables */
   cli: string;
   /** Cleanup function (closes DB and removes temp directories) */
@@ -56,6 +62,11 @@ export async function setupIntegrationTest(): Promise<IntegrationTestContext> {
   // Setup isolated data directory (use mkdtemp to avoid timestamp collisions in parallel tests)
   const dataDir = await mkdtemp(join(tmpdir(), 'celilo-test-'));
 
+  // The celilo CLI's getEventBusPath() falls back to <dataDir>/events.db
+  // when EVENT_BUS_DB isn't set. Tests that attach a responder need
+  // the same path so subprocess + responder share the bus DB.
+  const busDbPath = join(dataDir, 'events.db');
+
   // CLI with isolated environment
   const cli = `CELILO_DB_PATH="${dbPath}" CELILO_DATA_DIR="${dataDir}" bun run src/cli/index.ts`;
 
@@ -75,6 +86,7 @@ export async function setupIntegrationTest(): Promise<IntegrationTestContext> {
   return {
     dbPath,
     dataDir,
+    busDbPath,
     cli,
     cleanup,
   };