@celilo/cli 0.1.4 → 0.1.6

package/src/services/config-interview.ts

@@ -8,8 +8,9 @@ import * as p from '@clack/prompts';
 import { and, eq } from 'drizzle-orm';
 import { log, promptPassword, promptText } from '../cli/prompts';
 import type { DbClient } from '../db/client';
-import { moduleConfigs, secrets } from '../db/schema';
-import { encryptSecret } from '../secrets/encryption';
+import { moduleConfigs, modules, secrets } from '../db/schema';
+import type { Ensure } from '../manifest/schema';
+import { decryptSecret, encryptSecret } from '../secrets/encryption';
 import { deriveSecret, generateSecret } from '../secrets/generators';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import type { Machine } from '../types/infrastructure';
@@ -81,6 +82,80 @@ function resolveMachineZoneIp(machine: Machine, zone: string): string | null {
   return iface?.ipAddress ?? null;
 }
 
+/**
+ * Auto-derive $machine: variables from a machine without prompting.
+ * Called in both interactive and non-interactive deployments so machine-
+ * catalogued data (zones, zone IPs, etc.) is always applied automatically.
+ *
+ * @returns InterviewResult with the keys that were successfully derived
+ */
+export async function autoDeriveMachineConfig(
+  moduleId: string,
+  missingVariables: MissingVariable[],
+  db: DbClient,
+  machine: Machine,
+): Promise<InterviewResult> {
+  const configured: string[] = [];
+
+  for (const variable of missingVariables) {
+    if (!variable.derive_from?.startsWith('$machine:')) continue;
+
+    const derived = resolveMachineDerivation(variable.derive_from, machine);
+    if (derived === null) continue;
+
+    log.info(`✓ ${variable.name} = ${derived} (auto-derived from ${machine.hostname})`);
+
+    await db
+      .insert(moduleConfigs)
+      .values({
+        moduleId,
+        key: variable.name,
+        value: derived,
+        valueJson: null,
+        createdAt: new Date(),
+        updatedAt: new Date(),
+      })
+      .onConflictDoUpdate({
+        target: [moduleConfigs.moduleId, moduleConfigs.key],
+        set: { value: derived, updatedAt: new Date() },
+      });
+    configured.push(variable.name);
+
+    // Handle per_selection follow-ups (e.g., zone_ip_* from zone list)
+    if (variable.options && variable.per_selection) {
+      for (const selectedVal of derived.split(',')) {
+        const followUpKey = variable.per_selection.key_pattern.replace('{value}', selectedVal);
+        let followUpValue: string | null = null;
+
+        if (variable.per_selection.derive_from === '$machine:zone_ip') {
+          followUpValue = resolveMachineZoneIp(machine, selectedVal);
+        }
+
+        if (followUpValue !== null) {
+          log.info(`✓ ${followUpKey} = ${followUpValue} (auto-derived from ${machine.hostname})`);
+          await db
+            .insert(moduleConfigs)
+            .values({
+              moduleId,
+              key: followUpKey,
+              value: followUpValue,
+              valueJson: null,
+              createdAt: new Date(),
+              updatedAt: new Date(),
+            })
+            .onConflictDoUpdate({
+              target: [moduleConfigs.moduleId, moduleConfigs.key],
+              set: { value: followUpValue, updatedAt: new Date() },
+            });
+          configured.push(followUpKey);
+        }
+      }
+    }
+  }
+
+  return { success: true, configured };
+}
+
 /**
  * Interview user for missing required configuration
  *
@@ -692,3 +767,333 @@ export async function interviewForMissingSecrets(
     configured,
   };
 }
+
+// ─────────────────────────────────────────────────────────────────────────
+// Cross-module "ensure" interview
+//
+// When a hook's capability call detects that a value isn't covered by a
+// provider module's config, the framework runs this interview against
+// the provider's `ensures` block to extend its config in place. See
+// `apps/celilo/designs/CROSS_MODULE_CONFIG_INTERVIEW.md`.
+// ─────────────────────────────────────────────────────────────────────────
+
+interface EnsureTargetParts {
+  /** Always one of "config" | "secret" — derived from `target:` prefix. */
+  scope: 'config' | 'secret';
+  /** Variable / secret name on the provider module. */
+  name: string;
+}
+
+function parseEnsureTarget(target: string): EnsureTargetParts {
+  const [scope, name] = target.split('.', 2);
+  if ((scope !== 'config' && scope !== 'secret') || !name) {
+    throw new Error(
+      `Invalid ensure target: "${target}" (expected "config.<name>" or "secret.<name>")`,
+    );
+  }
+  return { scope, name };
+}
+
+function renderEnsureTemplate(template: string, value: string): string {
+  return template.replace(/\{\{\s*value\s*\}\}/g, value);
+}
+
+async function readModuleConfigKey(moduleId: string, key: string, db: DbClient): Promise<unknown> {
+  const row = db
+    .select()
+    .from(moduleConfigs)
+    .where(and(eq(moduleConfigs.moduleId, moduleId), eq(moduleConfigs.key, key)))
+    .get();
+  if (!row) return undefined;
+  if (row.valueJson) {
+    try {
+      return JSON.parse(row.valueJson);
+    } catch {
+      return row.value;
+    }
+  }
+  return row.value;
+}
+
+async function writeModuleConfigKey(
+  moduleId: string,
+  key: string,
+  value: unknown,
+  db: DbClient,
+): Promise<void> {
+  const valueJson = JSON.stringify(value);
+  const stringValue = typeof value === 'string' ? value : valueJson;
+  await db
+    .insert(moduleConfigs)
+    .values({
+      moduleId,
+      key,
+      value: stringValue,
+      valueJson,
+      createdAt: new Date(),
+      updatedAt: new Date(),
+    })
+    .onConflictDoUpdate({
+      target: [moduleConfigs.moduleId, moduleConfigs.key],
+      set: { value: stringValue, valueJson, updatedAt: new Date() },
+    })
+    .run();
+}
+
+async function readModuleSecretKey(
+  moduleId: string,
+  name: string,
+  db: DbClient,
+  masterKey: Buffer,
+): Promise<string | undefined> {
+  const row = db
+    .select()
+    .from(secrets)
+    .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, name)))
+    .get();
+  if (!row) return undefined;
+  return decryptSecret(
+    { encryptedValue: row.encryptedValue, iv: row.iv, authTag: row.authTag },
+    masterKey,
+  );
+}
+
+async function writeModuleSecretKey(
+  moduleId: string,
+  name: string,
+  plaintext: string,
+  db: DbClient,
+  masterKey: Buffer,
+): Promise<void> {
+  const encrypted = encryptSecret(plaintext, masterKey);
+  // No unique index on (module_id, name) for secrets; manual upsert.
+  const existing = db
+    .select()
+    .from(secrets)
+    .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, name)))
+    .get();
+  if (existing) {
+    await db
+      .update(secrets)
+      .set({
+        encryptedValue: encrypted.encryptedValue,
+        iv: encrypted.iv,
+        authTag: encrypted.authTag,
+        updatedAt: new Date(),
+      })
+      .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, name)))
+      .run();
+  } else {
+    await db
+      .insert(secrets)
+      .values({
+        moduleId,
+        name,
+        encryptedValue: encrypted.encryptedValue,
+        iv: encrypted.iv,
+        authTag: encrypted.authTag,
+      })
+      .run();
+  }
+}
+
+export interface EnsureInterviewOptions {
+  /**
+   * In non-interactive mode the framework prints the recipe and aborts
+   * before reaching this function — but tests / scripted callers still
+   * use this flag to apply changes without prompting for any text input.
+   */
+  noInteractive?: boolean;
+  /** Override prompts for testing — return string answers in order. */
+  promptOverride?: (prompt: string, hint?: string) => Promise<string>;
+}
+
+export interface EnsureInterviewResult {
+  success: boolean;
+  /** True when every input was already satisfied (idempotent retry). */
+  alreadyApplied?: boolean;
+  error?: string;
+  /** Human-readable lines describing what changed (for the deploy log). */
+  applied: string[];
+}
+
+/**
+ * Render the CLI recipe shown in `--no-interactive` mode. Generated
+ * from the manifest's `ensures` block so it stays correct as modules
+ * evolve.
+ */
+export function renderEnsureRecipe(
+  providerModuleId: string,
+  ensure: Ensure,
+  value: string,
+): string {
+  const lines: string[] = [
+    `Provider module "${providerModuleId}" doesn't yet ensure "${ensure.id}" for "${value}".`,
+    'Resolve before re-running:',
+    '',
+  ];
+  for (const input of ensure.inputs) {
+    const { scope, name } = parseEnsureTarget(input.target);
+    if (input.kind === 'append_to_array') {
+      lines.push(
+        `  # append "${value}" to ${scope}.${name}:`,
+        `  celilo module config get ${providerModuleId} ${name}   # read existing array`,
+        `  celilo module config set ${providerModuleId} ${name} '<existing + "${value}">'`,
+      );
+    } else {
+      const renderedKey = renderEnsureTemplate(input.key, value);
+      const renderedPrompt = renderEnsureTemplate(input.prompt, value);
+      const cmd = scope === 'secret' ? 'secret' : 'config';
+      lines.push(
+        `  # ${renderedPrompt}:`,
+        `  celilo module ${cmd} set ${providerModuleId} ${name} '<JSON object with "${renderedKey}":"<value>">'`,
+      );
+    }
+    lines.push('');
+  }
+  if (ensure.post === 'redeploy_self') {
+    lines.push(`  celilo module deploy ${providerModuleId}`);
+  }
+  return lines.join('\n');
+}
+
+/**
+ * Apply one `ensures` block to the provider module's config + secrets.
+ *
+ * - `append_to_array`: idempotent — re-running with an already-present
+ *   value is a no-op.
+ * - `set_in_object`: prompts only when the key isn't already set on the
+ *   target object. Re-running with the same value is a no-op.
+ *
+ * Returns success even when every input was already satisfied — the
+ * caller (module-deploy) decides whether to retry the hook regardless.
+ */
+export async function interviewForEnsureInputs(
+  providerModuleId: string,
+  ensure: Ensure,
+  value: string,
+  db: DbClient,
+  options: EnsureInterviewOptions = {},
+): Promise<EnsureInterviewResult> {
+  const applied: string[] = [];
+  let allNoop = true;
+
+  // No confirm prompt: running `module deploy <consumer>` is itself the
+  // user's consent for whatever cross-module config its hooks imply. A
+  // confirm here would be redundant — the user just typed the deploy
+  // command. Prompts that *gather information* (e.g. a DDNS password)
+  // still appear, because the framework genuinely doesn't know the
+  // value. See apps/celilo/designs/CADDY_HOSTNAME_LIST.md, Decision 6.
+
+  const masterKey = await getOrCreateMasterKey();
+
+  for (const input of ensure.inputs) {
+    const { scope, name } = parseEnsureTarget(input.target);
+
+    if (input.kind === 'append_to_array') {
+      if (scope !== 'config') {
+        return {
+          success: false,
+          applied,
+          error: `append_to_array only supports config targets (got ${input.target})`,
+        };
+      }
+      const current = await readModuleConfigKey(providerModuleId, name, db);
+      const arr = Array.isArray(current) ? [...current] : [];
+      if (arr.includes(value)) {
+        applied.push(`${input.target} already contains "${value}" — skipped`);
+        continue;
+      }
+      arr.push(value);
+      await writeModuleConfigKey(providerModuleId, name, arr, db);
+      applied.push(`${input.target} ← appended "${value}"`);
+      allNoop = false;
+      continue;
+    }
+
+    // input.kind === 'set_in_object'
+    const objectKey = renderEnsureTemplate(input.key, value);
+    const promptMsg = renderEnsureTemplate(input.prompt, value);
+
+    if (scope === 'config') {
+      const current = await readModuleConfigKey(providerModuleId, name, db);
+      const obj =
+        current && typeof current === 'object' && !Array.isArray(current)
+          ? { ...(current as Record<string, unknown>) }
+          : {};
+      if (objectKey in obj) {
+        applied.push(`${input.target}["${objectKey}"] already set — skipped`);
+        continue;
+      }
+      const promptFn =
+        options.promptOverride ??
+        (async () => promptText({ message: promptMsg, placeholder: input.hint }));
+      const userValue = await promptFn(promptMsg, input.hint);
+      obj[objectKey] = userValue;
+      await writeModuleConfigKey(providerModuleId, name, obj, db);
+      applied.push(`${input.target}["${objectKey}"] set`);
+      allNoop = false;
+      continue;
+    }
+
+    // Secret object: stored as a JSON-encoded string secret. Round-trip
+    // through parse/serialize so other keys are preserved.
+    const currentRaw = await readModuleSecretKey(providerModuleId, name, db, masterKey);
+    let obj: Record<string, unknown> = {};
+    if (currentRaw) {
+      try {
+        const parsed = JSON.parse(currentRaw);
+        if (parsed && typeof parsed === 'object' && !Array.isArray(parsed)) {
+          obj = parsed as Record<string, unknown>;
+        }
+      } catch {
+        // Existing secret isn't a JSON object — overwrite (the schema
+        // declares this secret as an object, so any other shape is stale).
+      }
+    }
+    if (objectKey in obj) {
+      applied.push(`${input.target}["${objectKey}"] already set — skipped`);
+      continue;
+    }
+    const promptFn =
+      options.promptOverride ??
+      // promptPassword doesn't take a placeholder, so fold the hint
+      // into the message — the user only sees the message before typing.
+      (async () =>
+        promptPassword({
+          message: input.hint ? `${promptMsg}\n  ${input.hint}` : promptMsg,
+        }));
+    const userValue = await promptFn(promptMsg, input.hint);
+    obj[objectKey] = userValue;
+    await writeModuleSecretKey(providerModuleId, name, JSON.stringify(obj), db, masterKey);
+    applied.push(`${input.target}["${objectKey}"] set`);
+    allNoop = false;
+  }
+
+  return { success: true, alreadyApplied: allNoop, applied };
+}
+
+/**
+ * Look up an `ensures` block on a provider module's manifest by id.
+ * Returns null when the module doesn't exist or doesn't declare a
+ * matching ensure — callers should treat that as a hard failure since
+ * the consumer's capability call asked for something the provider
+ * doesn't know how to satisfy.
+ */
+export function findEnsureOnProvider(
+  providerModuleId: string,
+  ensureId: string,
+  db: DbClient,
+): Ensure | null {
+  const row = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
+  if (!row?.manifestData) return null;
+  const manifest = row.manifestData as {
+    provides?: { capabilities?: Array<{ ensures?: Ensure[] }> };
+  };
+  for (const cap of manifest.provides?.capabilities ?? []) {
+    for (const ensure of cap.ensures ?? []) {
+      if (ensure.id === ensureId) return ensure;
+    }
+  }
+  return null;
+}

package/src/services/deploy-ansible.ts

@@ -18,6 +18,69 @@ export interface AnsibleResult {
   error?: string;
 }
 
+/**
+ * Parse raw Ansible output lines into concise human-readable status.
+ * Returns null for lines that should be suppressed (decorative separators, etc.)
+ */
+// biome-ignore lint/suspicious/noControlCharactersInRegex: intentionally stripping ANSI escape codes
+const ANSI_ESCAPE = /\x1b\[[0-9;]*m/g;
+
+/**
+ * Emit structured progress markers to stdout for each Ansible play/task.
+ * Uses console.log (same channel as log.info) so the e2e runner sees them
+ * in real-time. The runner matches [ansible:play] / [ansible:task] patterns.
+ */
+function emitAnsibleProgress(rawChunk: string): void {
+  // In non-TTY mode (e.g. inside a Docker exec) write to stdout, which is
+  // line-buffered by stdbuf. stderr may arrive batched or out-of-order through
+  // docker exec -T, so stdout gives the e2e runner reliable real-time delivery.
+  const out = process.stdout.isTTY ? process.stderr : process.stdout;
+  for (const line of rawChunk.split('\n')) {
+    const stripped = line.replace(ANSI_ESCAPE, '').trim();
+    if (/^PLAY \[/.test(stripped)) {
+      const name = stripped.replace(/^PLAY \[/, '').replace(/\].*$/, '');
+      out.write(`[ansible:play] ${name}\n`);
+    } else if (/^TASK \[/.test(stripped)) {
+      const name = stripped.replace(/^TASK \[/, '').replace(/\].*$/, '');
+      out.write(`[ansible:task] ${name}\n`);
+    } else if (/^RUNNING HANDLER \[/.test(stripped)) {
+      const name = stripped.replace(/^RUNNING HANDLER \[/, '').replace(/\].*$/, '');
+      out.write(`[ansible:handler] ${name}\n`);
+    } else if (/^\w[\w.-]+ *:/.test(stripped) && stripped.includes('ok=')) {
+      out.write(`[ansible:recap] ${stripped}\n`);
+    }
+  }
+}
+
+function parseAnsibleLine(line: string): string | null {
+  const stripped = line.replace(ANSI_ESCAPE, '').trim();
+
+  if (/^PLAY \[/.test(stripped)) {
+    const name = stripped.replace(/^PLAY \[/, '').replace(/\].*$/, '');
+    return `▶ ${name}`;
+  }
+  if (/^TASK \[/.test(stripped)) {
+    const name = stripped.replace(/^TASK \[/, '').replace(/\].*$/, '');
+    return `  • ${name}`;
+  }
+  if (/^RUNNING HANDLER \[/.test(stripped)) {
+    const name = stripped.replace(/^RUNNING HANDLER \[/, '').replace(/\].*$/, '');
+    return `  ↺ handler: ${name}`;
+  }
+  if (/^ok:/.test(stripped)) return '    ok';
+  if (/^changed:/.test(stripped)) return '    changed';
+  if (/^skipping:/.test(stripped)) return '    skipped';
+  if (/^failed:/.test(stripped)) return '    FAILED';
+  if (/^fatal:/.test(stripped)) return `    FATAL: ${stripped.slice(7)}`;
+  if (/^PLAY RECAP/.test(stripped)) return '  recap:';
+  if (/^\w[\w.-]+ *:/.test(stripped) && stripped.includes('ok=')) return `    ${stripped}`;
+
+  // Suppress decorative separator lines (all * or = chars)
+  if (/^[*=\s]+$/.test(stripped)) return null;
+
+  return null;
+}
+
 /**
  * Execute Ansible playbook with streaming progress
  * Execution function - performs software deployment
@@ -33,12 +96,8 @@ export async function executeAnsible(
   const inventoryPath = join(ansibleDir, 'inventory', 'hosts.ini');
   const playbookPath = join(ansibleDir, 'playbook.yml');
 
-  // Get Ansible Vault password
   const vaultPassword = await getVaultPassword();
 
-  // Write vault password to temp file (same approach as ansible/secrets.ts)
-  // Using /dev/stdin doesn't work reliably — Ansible tries to execute it as a
-  // script and hits permission errors on some platforms.
   const tempDir = await mkdtemp(join(tmpdir(), 'celilo-vault-'));
   const passwordPath = join(tempDir, 'vault-pass');
   await writeFile(passwordPath, vaultPassword, { mode: 0o600 });
@@ -47,8 +106,6 @@ export async function executeAnsible(
   log.info('Deploying software...');
 
   try {
-    // Execute ansible-playbook with streaming progress
-    // Note: Paths escaped for shell execution (macOS paths contain spaces)
     const result = await executeBuildWithProgress({
       command: 'ansible-playbook',
       args: [
@@ -59,11 +116,20 @@ export async function executeAnsible(
         shellEscape(playbookPath),
       ],
       cwd: ansibleDir,
+      // PYTHONUNBUFFERED=1: forces Python (Ansible) to flush stdout immediately
+      // instead of buffering when writing to a pipe — makes progress stream in
+      // real-time rather than appearing in one burst at the end.
+      // ANSIBLE_SSH_PIPELINING: enables SSH pipelining for performance without ControlMaster
+      env: {
+        PYTHONUNBUFFERED: '1',
+        ANSIBLE_SSH_PIPELINING: 'True',
+      },
       title: 'Deploying software',
       noInteractive: options?.noInteractive,
+      filterOutput: parseAnsibleLine,
+      onOutput: emitAnsibleProgress,
     });
 
-    // Persist full output to log file for debugging
     const timestamp = new Date().toISOString();
     const logHeader = `\n--- Ansible deploy ${timestamp} ---\n`;
     await appendFile(logPath, logHeader + result.output, 'utf-8');

package/src/services/deploy-planner.ts

@@ -126,15 +126,15 @@ export async function extractTargetHost(
   const hostname = configMap.get('hostname') || moduleId;
 
   // Extract IP - support infrastructure-derived variables
-  // Priority: container_ip > ip.primary > vps_ip (backward compatibility)
+  // Priority: target_ip > ip.primary > vps_ip (backward compatibility)
   let ip = '';
-  const containerIp = configMap.get('container_ip');
+  const targetIp = configMap.get('target_ip');
   const ipPrimary = configMap.get('ip.primary');
   const vpsIp = configMap.get('vps_ip');
 
-  if (containerIp) {
-    // Container IP format: "10.0.10.10/24" - extract just IP
-    ip = containerIp.split('/')[0];
+  if (targetIp) {
+    // Target IP format can be "10.0.10.10/24" - extract just IP
+    ip = targetIp.split('/')[0];
   } else if (ipPrimary) {
     // Infrastructure-derived IP (already plain format)
     ip = ipPrimary;

package/src/services/deploy-preflight.ts

@@ -21,6 +21,7 @@
  * - Modify any state
  */
 
+import { compareConsumerToProvider } from '@celilo/capabilities';
 import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import { getDb } from '../db/client';
@@ -39,6 +40,7 @@ export interface PreflightError {
   category:
     | 'missing-config'
     | 'missing-capability'
+    | 'capability-version-mismatch'
     | 'unresolved-template'
     | 'no-infrastructure'
     | 'missing-secret';
@@ -84,21 +86,58 @@ export async function runPreflight(
 
   const manifest = module.manifestData as ModuleManifest;
 
-  // 2. Required capabilities have providers?
+  // 2. Required capabilities have providers, and at least one
+  //    provider's version is compatible. Multi-provider scenarios
+  //    (e.g., zone-scoped dns_registrar with separate internal +
+  //    external providers) pass when ANY installed provider matches
+  //    the consumer's required major — that's what the runtime
+  //    zone-aware lookup actually picks. The audit catches the same
+  //    drift after deploy; this surface refuses up front so the
+  //    operator gets an actionable error instead of a silent
+  //    template-generation against an ABI the provider can't fulfil.
   if (manifest.requires?.capabilities) {
     for (const cap of manifest.requires.capabilities) {
-      const provider = db
+      const providers = db
         .select()
         .from(capabilities)
         .where(eq(capabilities.capabilityName, cap.name))
-        .get();
-      if (!provider) {
+        .all();
+      if (providers.length === 0) {
         errors.push({
           category: 'missing-capability',
           message: `Required capability '${cap.name}' has no provider installed`,
           suggestion: `Deploy a module that provides '${cap.name}' first`,
         });
+        continue;
       }
+      let exampleMismatch: {
+        provider: (typeof providers)[number];
+        reason: string;
+      } | null = null;
+      let anyCompatible = false;
+      for (const p of providers) {
+        const result = compareConsumerToProvider(cap.version, p.version);
+        if (result.compatible) {
+          anyCompatible = true;
+          break;
+        }
+        if (!exampleMismatch) {
+          exampleMismatch = { provider: p, reason: result.reason };
+        }
+      }
+      if (anyCompatible || !exampleMismatch) continue;
+
+      const { provider: example, reason } = exampleMismatch;
+      errors.push({
+        category: 'capability-version-mismatch',
+        message:
+          `Module '${moduleId}' requires ${cap.name}@${cap.version} but ` +
+          `provider '${example.moduleId}' offers ${cap.name}@${example.version}`,
+        suggestion:
+          reason === 'caller_minor_too_old'
+            ? `Upgrade provider '${example.moduleId}' to a version that provides ${cap.name}@${cap.version} or newer`
+            : `The interface major version differs. Update '${moduleId}' to require ${cap.name}@${example.version} (matching the provider's major), or rebuild '${example.moduleId}' against ${cap.name}@${cap.version}.`,
+      });
     }
   }
 
@@ -264,6 +303,8 @@ function errorIcon(category: PreflightError['category']): string {
       return '✗';
     case 'missing-capability':
       return '◯';
+    case 'capability-version-mismatch':
+      return '⚠';
     case 'unresolved-template':
       return '⟲';
     case 'no-infrastructure':