@celilo/cli 0.3.20 → 0.3.21

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.3.20",
+  "version": "0.3.21",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {

package/src/cli/commands/module-upgrade.test.ts

@@ -1,10 +1,16 @@
 /**
  * Unit tests for the version-change classifier used by `module update`'s
- * registry-sweep mode.
+ * registry-sweep mode, plus integration tests for `upgradeOne`'s
+ * `displayVersion` / `quiet` opts that drive the cleaner sweep output.
  */
 
-import { describe, expect, test } from 'bun:test';
-import { classifyVersionChange } from './module-upgrade';
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type DbClient, getDb } from '../../db/client';
+import { modules } from '../../db/schema';
+import { classifyVersionChange, upgradeOne } from './module-upgrade';
 
 describe('classifyVersionChange', () => {
   test('identical versions are up-to-date', () => {
@@ -56,3 +62,95 @@ describe('classifyVersionChange', () => {
     expect(classifyVersionChange('1.0.0', '1.0.0+1')).toBe('patch');
   });
 });
+
+describe('upgradeOne — displayVersion and quiet', () => {
+  let tempDir: string;
+  let srcDir: string;
+  let installedDir: string;
+  let db: DbClient;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'celilo-upgrade-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+    process.env.CELILO_ORIGINAL_CWD = tempDir;
+
+    // Pre-installed module landing zone (where files get copied to).
+    installedDir = join(tempDir, 'installed', 'testmod');
+    mkdirSync(installedDir, { recursive: true });
+
+    // "New" source dir we're upgrading from (mimics a registry extract).
+    srcDir = join(tempDir, 'src');
+    mkdirSync(srcDir, { recursive: true });
+    writeFileSync(
+      join(srcDir, 'manifest.yml'),
+      `celilo_contract: "1.0"
+id: testmod
+name: Test Module
+version: 1.0.0
+description: fixture
+`,
+    );
+
+    db = getDb();
+    // Seed the modules table with the "currently installed" record.
+    // We deliberately put a registry-style version string in here so
+    // displayVersion-less upgrades preserve the previousVersion field.
+    db.insert(modules)
+      .values({
+        id: 'testmod',
+        name: 'Test Module',
+        sourcePath: installedDir,
+        version: '1.0.0+5',
+        manifestData: {
+          celilo_contract: '1.0',
+          id: 'testmod',
+          name: 'Test Module',
+          version: '1.0.0',
+        },
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_ORIGINAL_CWD = undefined;
+  });
+
+  test('returns previousVersion + newVersion in the success outcome', async () => {
+    const result = await upgradeOne(srcDir, db, {}, { quiet: true });
+    expect(result.status).toBe('success');
+    if (result.status !== 'success') return; // narrow for ts
+    expect(result.moduleId).toBe('testmod');
+    expect(result.previousVersion).toBe('1.0.0+5');
+    // No displayVersion supplied → falls back to the new manifest's
+    // semver core (no +N visible to the operator from a path upgrade).
+    expect(result.newVersion).toBe('1.0.0');
+  });
+
+  test('displayVersion is persisted to modules.version (so +N survives)', async () => {
+    // The bug case: registry says 1.0.0+6, manifest says 1.0.0. Without
+    // displayVersion, the +6 was dropped on the floor and `module list`
+    // rolled back to "1.0.0", masking the actual installed revision.
+    const result = await upgradeOne(srcDir, db, {}, { quiet: true, displayVersion: '1.0.0+6' });
+    expect(result.status).toBe('success');
+    if (result.status !== 'success') return;
+    expect(result.newVersion).toBe('1.0.0+6');
+
+    const row = db.select().from(modules).all()[0];
+    expect(row.version).toBe('1.0.0+6');
+  });
+
+  test('quiet=true suppresses the per-call log lines (caller renders its own)', async () => {
+    // We can't easily intercept @clack's log output without adding test
+    // hooks, so instead we exercise that the call simply succeeds and
+    // returns the structured outcome — the sweep relies on this to
+    // render its own output without duplicates. A non-quiet call
+    // exercises the same code path with the log lines enabled; both
+    // return the same shape.
+    const quietResult = await upgradeOne(srcDir, db, {}, { quiet: true });
+    expect(quietResult.status).toBe('success');
+    if (quietResult.status !== 'success') return;
+    expect(quietResult.newVersion).toBe('1.0.0');
+  });
+});

package/src/cli/commands/module-upgrade.ts

@@ -151,7 +151,7 @@ async function fetchAndUpgrade(
 /**
  * Upgrade a single module from a source path
  */
-async function upgradeOne(
+export async function upgradeOne(
   sourcePath: string,
   db: ReturnType<typeof getDb>,
   flags: Record<string, string | boolean> = {},

package/src/services/config-interview.ts

@@ -17,51 +17,124 @@ import {
   type SecretRequiredPayload,
   busInterview,
 } from './bus-interview';
-import { getSecretMetadata, loadSecretsSchema } from './secret-schema-loader';
+import { getSecretMetadata } from './secret-schema-loader';
 
 /**
- * Pull the manifest's `secrets.declares[]` entries keyed by name. Best-
- * effort — returns an empty Map on any read/parse failure so the caller
- * can fall back to JSON-schema-only interview behavior.
+ * Read a module's manifest from disk. Returns null on any read/parse
+ * failure (so callers can degrade gracefully instead of crashing the
+ * deploy/generate flow on a malformed install).
+ *
+ * Used by `validateModuleSecrets` to discover the canonical secret
+ * declarations. Deploy-side callers already have the manifest loaded
+ * and call `findMissingSecrets` directly without going through here.
  */
-async function loadManifestSecretDeclares(
+async function loadInstalledManifest(
   moduleId: string,
   db: DbClient,
-): Promise<
-  Map<string, { type?: string; description?: string; key_label?: string; value_label?: string }>
-> {
-  const out = new Map<
-    string,
-    { type?: string; description?: string; key_label?: string; value_label?: string }
-  >();
+): Promise<{ secrets?: { declares?: unknown[] } } | null> {
   const moduleRow = db.select().from(modules).where(eq(modules.id, moduleId)).get();
-  if (!moduleRow?.sourcePath) return out;
+  if (!moduleRow?.sourcePath) return null;
 
   try {
     const { readFile } = await import('node:fs/promises');
     const { join } = await import('node:path');
     const { parse: parseYaml } = await import('yaml');
     const yamlContent = await readFile(join(moduleRow.sourcePath, 'manifest.yml'), 'utf-8');
-    const parsed = parseYaml(yamlContent) as { secrets?: { declares?: unknown[] } } | undefined;
-    const declares = parsed?.secrets?.declares;
-    if (!Array.isArray(declares)) return out;
-    for (const d of declares) {
-      if (typeof d !== 'object' || d === null) continue;
-      const decl = d as Record<string, unknown>;
-      if (typeof decl.name !== 'string') continue;
-      out.set(decl.name, {
-        type: typeof decl.type === 'string' ? decl.type : undefined,
-        description: typeof decl.description === 'string' ? decl.description : undefined,
-        key_label: typeof decl.key_label === 'string' ? decl.key_label : undefined,
-        value_label: typeof decl.value_label === 'string' ? decl.value_label : undefined,
-      });
-    }
+    return parseYaml(yamlContent) as { secrets?: { declares?: unknown[] } };
   } catch {
-    // Manifest unreadable / malformed — fall back to schema-only.
+    return null;
+  }
+}
+
+/**
+ * Canonical "find missing secrets" entry point. Walks
+ * `manifest.secrets.declares[]`, filters to required + uninstalled,
+ * and projects each one into a MissingVariable with all the fields
+ * downstream consumers (config-interview, terminal-responder) need:
+ * type (incl. 'string-map'), key_label/value_label (for the add-loop
+ * prompt), and the auto-generation block.
+ *
+ * Single source of truth for both the deploy path
+ * (`findMissingRequiredVariables` in deploy-validation.ts) and the
+ * generate path (`validateModuleSecrets` below). Previously these
+ * had two separate implementations that diverged on what fields they
+ * carried — a regression slipped through where deploy was dropping
+ * type/key_label/value_label, leaving namecheap on the old JSON-blob
+ * prompt UX even after the manifest declared `string-map`. One impl,
+ * one set of tests.
+ */
+interface SecretDeclareLike {
+  name: string;
+  type?: string;
+  required?: boolean;
+  description?: string;
+  key_label?: string;
+  value_label?: string;
+  generate?: { method: string; length: number; encoding: string };
+}
+
+/**
+ * Coerce one entry from `manifest.secrets.declares[]` into the strict
+ * shape `findMissingSecrets` consumes. Returns null on any item that's
+ * unrecognizable (not an object, missing name, etc.) — best-effort
+ * parsing matches the deploy/generate paths' historical leniency.
+ */
+function coerceSecretDeclare(entry: unknown): SecretDeclareLike | null {
+  if (typeof entry !== 'object' || entry === null) return null;
+  const e = entry as Record<string, unknown>;
+  if (typeof e.name !== 'string') return null;
+  const out: SecretDeclareLike = { name: e.name };
+  if (typeof e.type === 'string') out.type = e.type;
+  if (typeof e.required === 'boolean') out.required = e.required;
+  if (typeof e.description === 'string') out.description = e.description;
+  if (typeof e.key_label === 'string') out.key_label = e.key_label;
+  if (typeof e.value_label === 'string') out.value_label = e.value_label;
+  if (typeof e.generate === 'object' && e.generate !== null) {
+    const g = e.generate as Record<string, unknown>;
+    if (
+      typeof g.method === 'string' &&
+      typeof g.length === 'number' &&
+      typeof g.encoding === 'string'
+    ) {
+      out.generate = { method: g.method, length: g.length, encoding: g.encoding };
+    }
   }
   return out;
 }
 
+export async function findMissingSecrets(
+  moduleId: string,
+  manifest: { secrets?: { declares?: unknown[] } },
+  db: DbClient,
+): Promise<MissingVariable[]> {
+  const missing: MissingVariable[] = [];
+  if (!manifest.secrets?.declares) return missing;
+
+  for (const raw of manifest.secrets.declares) {
+    const secret = coerceSecretDeclare(raw);
+    if (!secret) continue;
+    if (!secret.required) continue;
+
+    const existing = db
+      .select()
+      .from(secrets)
+      .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, secret.name)))
+      .get();
+    if (existing) continue;
+
+    missing.push({
+      name: secret.name,
+      source: 'secret',
+      description: secret.description,
+      type: secret.type,
+      key_label: secret.key_label,
+      value_label: secret.value_label,
+      generate: secret.generate,
+    });
+  }
+  return missing;
+}
+
 export interface MissingVariable {
   name: string;
   source: 'user' | 'secret' | 'capability' | 'system';
@@ -482,42 +555,13 @@ export async function validateModuleSecrets(
   moduleId: string,
   db: DbClient,
 ): Promise<MissingVariable[]> {
-  const missingSecrets: MissingVariable[] = [];
-
-  // Load secrets schema
-  const schema = await loadSecretsSchema(moduleId, db);
-  if (!schema) {
-    // No schema file = no secrets declared
-    return [];
-  }
-
-  // Best-effort: load manifest declarations so we can enrich each missing
-  // secret with `type` (e.g. 'string-map'), `key_label`, `value_label`.
-  // The JSON schema knows shape; the manifest knows interview UX. If the
-  // manifest can't be read, we degrade to plain prompts.
-  const manifestDeclares = await loadManifestSecretDeclares(moduleId, db);
-
-  // Check each declared secret
-  for (const [secretName, propertySchema] of Object.entries(schema.properties)) {
-    // Check if secret exists in database
-    const existing = db
-      .select()
-      .from(secrets)
-      .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, secretName)))
-      .get();
-
-    if (!existing) {
-      const declare = manifestDeclares.get(secretName);
-      missingSecrets.push({
-        name: secretName,
-        source: 'secret',
-        description: declare?.description || propertySchema.description || propertySchema.title,
-        type: declare?.type,
-        key_label: declare?.key_label,
-        value_label: declare?.value_label,
-      });
-    }
-  }
+  // The manifest is the canonical declaration of which secrets a
+  // module needs (every module in the codebase has one; the
+  // schema/secrets.json file is supplementary, used for getSecretMetadata
+  // lookups elsewhere). Read it directly and delegate to findMissingSecrets.
+  const manifest = await loadInstalledManifest(moduleId, db);
+  if (!manifest) return [];
+  const missingSecrets = await findMissingSecrets(moduleId, manifest, db);
 
   return missingSecrets;
 }

package/src/services/deploy-validation.test.ts

@@ -0,0 +1,263 @@
+/**
+ * Regression tests for the deploy-side missing-secret discovery path
+ * AND the shared `findMissingSecrets` it now delegates to.
+ *
+ * Background: there used to be two implementations of "find missing
+ * secrets" — `validateModuleSecrets` (used by `module generate`) and
+ * `findMissingRequiredVariables` (used by `module deploy`). They
+ * diverged on what fields they carried; deploy was silently dropping
+ * `type` / `key_label` / `value_label`, which routed string-map
+ * secrets through the wrong responder UX.
+ *
+ * Both call sites now delegate to `findMissingSecrets` in
+ * config-interview.ts. These tests cover both layers: the shared
+ * function directly, and the public deploy entry point that wraps it.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { type DbClient, getDb } from '../db/client';
+import { modules, secrets } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import { findMissingSecrets } from './config-interview';
+import { findMissingRequiredVariables } from './deploy-validation';
+
+function manifestWithStringMapSecret(): ModuleManifest {
+  return {
+    celilo_contract: '1.0',
+    id: 'testmod',
+    name: 'Test Module',
+    version: '1.0.0',
+    description: 'fixture',
+    secrets: {
+      declares: [
+        {
+          name: 'ddns_passwords',
+          type: 'string-map',
+          required: true,
+          description: 'DDNS password per managed domain',
+          sensitive: true,
+          key_label: 'Domain',
+          value_label: 'Password',
+        },
+      ],
+    },
+  } as unknown as ModuleManifest;
+}
+
+function manifestWithPlainStringSecret(): ModuleManifest {
+  return {
+    celilo_contract: '1.0',
+    id: 'testmod',
+    name: 'Test Module',
+    version: '1.0.0',
+    description: 'fixture',
+    secrets: {
+      declares: [
+        {
+          name: 'api_key',
+          type: 'string',
+          required: true,
+          description: 'Upstream API key',
+          sensitive: true,
+        },
+      ],
+    },
+  } as unknown as ModuleManifest;
+}
+
+describe('findMissingRequiredVariables (deploy path)', () => {
+  let tempDir: string;
+  let db: DbClient;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'celilo-deploy-validation-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+    db = getDb();
+    db.insert(modules)
+      .values({
+        id: 'testmod',
+        name: 'Test Module',
+        sourcePath: tempDir,
+        version: '1.0.0',
+        manifestData: {},
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    process.env.CELILO_DB_PATH = undefined;
+  });
+
+  // The bug we're guarding against: the deploy used to drop type +
+  // labels on the floor here, so the bus payload arrived with type:
+  // 'string' and the responder defaulted to single-line input.
+  test('propagates type / key_label / value_label for string-map secrets', async () => {
+    const missing = await findMissingRequiredVariables(
+      'testmod',
+      manifestWithStringMapSecret(),
+      db,
+    );
+    expect(missing).toHaveLength(1);
+    expect(missing[0]).toMatchObject({
+      name: 'ddns_passwords',
+      source: 'secret',
+      type: 'string-map',
+      key_label: 'Domain',
+      value_label: 'Password',
+      description: 'DDNS password per managed domain',
+    });
+  });
+
+  test('plain string secrets still arrive with type=string and no labels', async () => {
+    const missing = await findMissingRequiredVariables(
+      'testmod',
+      manifestWithPlainStringSecret(),
+      db,
+    );
+    expect(missing).toHaveLength(1);
+    expect(missing[0]).toMatchObject({
+      name: 'api_key',
+      source: 'secret',
+      type: 'string',
+      description: 'Upstream API key',
+    });
+    expect(missing[0].key_label).toBeUndefined();
+    expect(missing[0].value_label).toBeUndefined();
+  });
+
+  test('configured secrets are not reported as missing', async () => {
+    db.insert(secrets)
+      .values({
+        moduleId: 'testmod',
+        name: 'ddns_passwords',
+        encryptedValue: 'dummy',
+        iv: 'dummy',
+        authTag: 'dummy',
+      })
+      .run();
+    const missing = await findMissingRequiredVariables(
+      'testmod',
+      manifestWithStringMapSecret(),
+      db,
+    );
+    expect(missing).toHaveLength(0);
+  });
+
+  test('optional secrets are skipped (only required show up)', async () => {
+    const manifest = {
+      ...manifestWithStringMapSecret(),
+      secrets: {
+        declares: [
+          {
+            name: 'ddns_passwords',
+            type: 'string-map',
+            required: false,
+            description: 'optional',
+            sensitive: true,
+          },
+        ],
+      },
+    } as unknown as ModuleManifest;
+    const missing = await findMissingRequiredVariables('testmod', manifest, db);
+    expect(missing).toHaveLength(0);
+  });
+});
+
+describe('findMissingSecrets (shared)', () => {
+  let tempDir: string;
+  let db: DbClient;
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'celilo-find-missing-secrets-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+    db = getDb();
+    db.insert(modules)
+      .values({
+        id: 'testmod',
+        name: 'Test Module',
+        sourcePath: tempDir,
+        version: '1.0.0',
+        manifestData: {},
+      })
+      .run();
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    process.env.CELILO_DB_PATH = undefined;
+  });
+
+  test('accepts loose unknown[] declares (manifest read off disk)', async () => {
+    // The validateModuleSecrets path reads manifest.yml off disk and
+    // hands us a loose-typed shape. findMissingSecrets coerces each
+    // entry per-field; malformed entries are skipped, well-formed ones
+    // pass through with their fields preserved.
+    const looseManifest = {
+      secrets: {
+        declares: [
+          {
+            name: 'ok_secret',
+            type: 'string-map',
+            required: true,
+            key_label: 'K',
+            value_label: 'V',
+          },
+          { name: 'no_required_field', type: 'string' }, // required missing → skipped
+          { name: 'optional', type: 'string', required: false }, // required:false → skipped
+          'this is not an object', // skipped
+          { /* no name */ required: true }, // skipped
+          {
+            name: 'with_generate',
+            required: true,
+            generate: { method: 'random', length: 32, encoding: 'base64' },
+          },
+        ],
+      },
+    };
+    const missing = await findMissingSecrets('testmod', looseManifest, db);
+    expect(missing.map((m) => m.name)).toEqual(['ok_secret', 'with_generate']);
+
+    const okEntry = missing.find((m) => m.name === 'ok_secret');
+    expect(okEntry?.type).toBe('string-map');
+    expect(okEntry?.key_label).toBe('K');
+    expect(okEntry?.value_label).toBe('V');
+
+    const genEntry = missing.find((m) => m.name === 'with_generate');
+    expect(genEntry?.generate).toEqual({ method: 'random', length: 32, encoding: 'base64' });
+  });
+
+  test('returns empty array when manifest has no secrets section', async () => {
+    expect(await findMissingSecrets('testmod', {}, db)).toEqual([]);
+    expect(await findMissingSecrets('testmod', { secrets: {} }, db)).toEqual([]);
+    expect(await findMissingSecrets('testmod', { secrets: { declares: [] } }, db)).toEqual([]);
+  });
+
+  test('drops entries whose secrets are already in the DB', async () => {
+    db.insert(secrets)
+      .values({
+        moduleId: 'testmod',
+        name: 'already_set',
+        encryptedValue: 'x',
+        iv: 'x',
+        authTag: 'x',
+      })
+      .run();
+    const missing = await findMissingSecrets(
+      'testmod',
+      {
+        secrets: {
+          declares: [
+            { name: 'already_set', required: true },
+            { name: 'still_missing', required: true },
+          ],
+        },
+      },
+      db,
+    );
+    expect(missing.map((m) => m.name)).toEqual(['still_missing']);
+  });
+});

package/src/services/deploy-validation.ts

@@ -9,9 +9,10 @@ import { join } from 'node:path';
 import { compareConsumerToProvider } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
+import { capabilities, moduleConfigs, modules } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
 import { generateTemplates } from '../templates/generator';
+import { findMissingSecrets } from './config-interview';
 import { buildModuleFromSource, getModuleBuildStatus, verifyArtifactsExist } from './module-build';
 
 export interface ValidationResult {
@@ -373,7 +374,7 @@ function getSuggestedDeploymentOrder(
  * @param db - Database connection
  * @returns Array of missing required variables
  */
-async function findMissingRequiredVariables(
+export async function findMissingRequiredVariables(
   moduleId: string,
   manifest: ModuleManifest,
   db: DbClient,
@@ -450,33 +451,22 @@ async function findMissingRequiredVariables(
     }
   }
 
-  // Check declared secrets
-  if (manifest.secrets?.declares) {
-    for (const secret of manifest.secrets.declares) {
-      if (!secret.required) continue; // Skip optional secrets
-
-      // Check if secret is configured
-      const secretRecord = await db
-        .select()
-        .from(secrets)
-        .where(and(eq(secrets.moduleId, moduleId), eq(secrets.name, secret.name)))
-        .get();
-
-      if (!secretRecord) {
-        missing.push({
-          name: secret.name,
-          source: 'secret',
-          description: secret.description,
-          generate: secret.generate,
-          // Pass through manifest-declared type + add-loop labels so the
-          // bus payload can drive the right responder UX (e.g. string-map
-          // gets the per-key add-loop instead of the JSON-blob prompt).
-          type: secret.type,
-          key_label: secret.key_label,
-          value_label: secret.value_label,
-        });
-      }
-    }
+  // Delegate the secret-discovery half to the canonical implementation
+  // in config-interview.ts. Previously the secrets section was inlined
+  // here and diverged from validateModuleSecrets — most recently
+  // dropping `type` / `key_label` / `value_label` on the floor, which
+  // routed string-map secrets through the wrong responder UX.
+  const missingSecrets = await findMissingSecrets(moduleId, manifest, db);
+  for (const s of missingSecrets) {
+    missing.push({
+      name: s.name,
+      source: s.source,
+      description: s.description,
+      type: s.type,
+      generate: s.generate,
+      key_label: s.key_label,
+      value_label: s.value_label,
+    });
   }
 
   return missing;