@celilo/cli 1.6.0 → 1.8.0

package/src/cli/commands/module-update.ts

@@ -9,7 +9,15 @@
  * The module ID is read from the manifest at the given path.
  */
 
-import { cpSync, existsSync, readFileSync, readdirSync, rmSync } from 'node:fs';
+import {
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  readdirSync,
+  renameSync,
+  rmSync,
+} from 'node:fs';
 import { unlink } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join, relative, resolve } from 'node:path';
@@ -163,6 +171,13 @@ export async function fetchAndUpdate(
  * `generated/`, the hook runtime closure, `screenshots/`, `cookies.json` — and
  * `generated/` alone carries terraform state and provider binaries.
  */
+/**
+ * What an update leaves alone: celilo's own output and the operator's state,
+ * never the package's. The set the in-place update skipped, kept verbatim so
+ * the staging swap changes only WHEN files move, not WHICH ones.
+ */
+const PRESERVED_ENTRIES = new Set(['generated', 'screenshots', 'cookies.json']);
+
 function listPrunableFiles(root: string, dir = root): string[] {
   const found: string[] = [];
   for (const entry of readdirSync(dir, { withFileTypes: true })) {
@@ -178,6 +193,88 @@ function listPrunableFiles(root: string, dir = root): string[] {
   return found;
 }
 
+/**
+ * Replace an installed module tree atomically (celilo#1008).
+ *
+ * The install used to be built in place: copy the new files over the old,
+ * then prune what the new version dropped. A failure anywhere in that
+ * sequence left a module half old and half new, with nothing on disk or in
+ * the DB recording which files were which. The comment on the in-place copy
+ * already named the hazard — "the copy dies partway, after the prune has
+ * already run".
+ *
+ * So the new tree is assembled in a sibling directory and swapped in with two
+ * renames. Everything that can fail — the copy, the prune, a bad source —
+ * fails while the live install is still untouched, because nothing has moved
+ * yet. If the second rename throws, the first is undone and the original is
+ * back. A sibling is deliberate: it is on the same filesystem, so the renames
+ * are atomic rather than a copy in disguise.
+ *
+ * The staging tree is removed on both paths, so a failure leaves no debris
+ * beside the module for the next reader to wonder about.
+ *
+ * `prune` receives the STAGED root, so the surviving-path rule is the one
+ * `updateOne` has always applied — only where it runs has changed.
+ *
+ * Adapted from `replaceInstalledModule` in Jeremy Banka's `f9a57f1b`. The rest
+ * of that commit is superseded by celilo#925; this half is not. It uses
+ * celilo's `classifyModulePath` rather than that commit's hand-rolled
+ * preserve/skip sets, which is what makes the `celilo/types.d.ts` special case
+ * it carried unnecessary — the classifier already calls that file `derived`.
+ */
+function replaceInstalledModule(
+  actualPath: string,
+  installedPath: string,
+  prune: (stagedRoot: string) => void,
+): void {
+  const suffix = `${process.pid}-${Date.now()}`;
+  const stagedPath = `${installedPath}.update-${suffix}`;
+  const previousPath = `${installedPath}.previous-${suffix}`;
+
+  rmSync(stagedPath, { recursive: true, force: true });
+  mkdirSync(stagedPath, { recursive: true });
+
+  try {
+    // The incoming version, filtered by the same classifier the in-place copy
+    // used, so an update from a directory still cannot plant `e2e/`,
+    // `*.test.ts` or `scripts/tsconfig.json` in the install.
+    for (const entry of readdirSync(actualPath)) {
+      if (PRESERVED_ENTRIES.has(entry)) continue;
+      cpSync(join(actualPath, entry), join(stagedPath, entry), {
+        recursive: true,
+        force: true,
+        filter: (from) => classifyModulePath(relative(actualPath, from)) !== 'unknown',
+      });
+    }
+
+    // Carry the live tree's own copies across. Same three entries the in-place
+    // update left alone, for the same reason: they are celilo's or the
+    // operator's, not the package's. Every other `derived` path — the hook
+    // runtime closure under `scripts/node_modules`, `celilo/types.d.ts` —
+    // still comes from the incoming version exactly as it did before, so a new
+    // module version can still deliver new hook dependencies.
+    for (const entry of PRESERVED_ENTRIES) {
+      const from = join(installedPath, entry);
+      if (!existsSync(from)) continue;
+      cpSync(from, join(stagedPath, entry), { recursive: true, force: true });
+    }
+
+    prune(stagedPath);
+
+    renameSync(installedPath, previousPath);
+    try {
+      renameSync(stagedPath, installedPath);
+    } catch (error) {
+      renameSync(previousPath, installedPath);
+      throw error;
+    }
+    rmSync(previousPath, { recursive: true, force: true });
+  } catch (error) {
+    rmSync(stagedPath, { recursive: true, force: true });
+    throw error;
+  }
+}
+
 export async function updateOne(
   sourcePath: string,
   db: ReturnType<typeof getDb>,
@@ -293,27 +390,7 @@ export async function updateOne(
     log.info(`Upgrading ${moduleId}: ${previousVersion} → ${newVersion}`);
   }
 
-  // Copy new module files, preserving generated output and state
   const installedPath = module.sourcePath;
-  const preserveDirs = new Set(['generated', 'screenshots', 'cookies.json']);
-
-  // Route the copy through the one classifier, the way `module import` does.
-  // `updateOne` used to copy the source tree wholesale, so updating from a
-  // directory planted `e2e/`, `*.test.ts` and `scripts/tsconfig.json` in the
-  // install — files no package ships and no target runs. Import never did,
-  // because a directory import goes through the packager; update is the path
-  // that skipped it.
-  const entries = readdirSync(actualPath);
-  for (const entry of entries) {
-    if (preserveDirs.has(entry)) continue;
-    const src = join(actualPath, entry);
-    const dest = join(installedPath, entry);
-    cpSync(src, dest, {
-      recursive: true,
-      force: true,
-      filter: (from) => classifyModulePath(relative(actualPath, from)) !== 'unknown',
-    });
-  }
 
   // The integrity baseline for the version just installed. Prefer the package's
   // own signed `checksums.json`; a directory update has none, so compute over
@@ -343,14 +420,25 @@ export async function updateOne(
   // pruned: `generated/`, the hook runtime closure, `screenshots/` and
   // `cookies.json` are celilo's or the operator's, and survive an update by
   // design.
+  //
+  // Unchanged except for WHERE it runs. It now prunes the staged tree, before
+  // anything is swapped in, so a prune that throws cannot leave the live
+  // install short of files (celilo#1008).
   const survivingPaths = new Set(
     Object.keys(baselineChecksums).filter((p) => classifyModulePath(p) === 'package'),
   );
-  for (const relPath of listPrunableFiles(installedPath)) {
-    if (!survivingPaths.has(relPath)) {
-      rmSync(join(installedPath, relPath));
+  const pruneDropped = (root: string) => {
+    for (const relPath of listPrunableFiles(root)) {
+      if (!survivingPaths.has(relPath)) {
+        rmSync(join(root, relPath));
+      }
     }
-  }
+  };
+
+  // Build the new tree beside the install and swap it in with two renames.
+  // Everything above this line is reversible by doing nothing; everything
+  // below it runs only once the files are really in place.
+  replaceInstalledModule(actualPath, installedPath, pruneDropped);
 
   // Record it. `updateOne` never touched this table, so the baseline stayed
   // frozen at the module's FIRST import no matter how many times it was

package/src/cli/commands/service-set-credentials.test.ts

@@ -0,0 +1,108 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { closeDb } from '../../db/client';
+import { runMigrations } from '../../db/migrate';
+import {
+  type ProxmoxCredentials,
+  addContainerService,
+  getContainerService,
+  getServiceCredentials,
+  updateVerificationStatus,
+} from '../../services/container-service';
+import { handleServiceSetCredentials } from './service-set-credentials';
+
+describe('service set-credentials', () => {
+  let testDir: string;
+
+  beforeEach(async () => {
+    testDir = mkdtempSync(join(tmpdir(), 'celilo-service-credentials-test-'));
+    process.env.CELILO_DB_PATH = join(testDir, 'test.db');
+    process.env.CELILO_MASTER_KEY_PATH = join(testDir, 'master.key');
+    writeFileSync(process.env.CELILO_MASTER_KEY_PATH, 'a'.repeat(64), 'utf8');
+    await runMigrations(process.env.CELILO_DB_PATH);
+  });
+
+  afterEach(() => {
+    closeDb();
+    delete process.env.CELILO_DB_PATH;
+    delete process.env.CELILO_MASTER_KEY_PATH;
+    delete process.env.PROXMOX_API_URL;
+    delete process.env.PROXMOX_API_TOKEN_ID;
+    delete process.env.PROXMOX_API_TOKEN_SECRET;
+    delete process.env.DIGITALOCEAN_API_TOKEN;
+    rmSync(testDir, { recursive: true, force: true });
+  });
+
+  it('updates only the Proxmox endpoint and retains the token', async () => {
+    const service = await addContainerService({
+      name: 'Chubs',
+      providerName: 'proxmox',
+      zones: ['internal'],
+      providerConfig: {},
+      apiCredentials: {
+        api_url: 'https://192.168.0.50:8006',
+        api_token_id: 'root@pam!celilo',
+        api_token_secret: 'existing-secret',
+      },
+    });
+    await updateVerificationStatus(service.id, { success: true, message: 'Connected' });
+
+    const result = await handleServiceSetCredentials(['chubs'], {
+      'api-url': 'https://10.77.20.50:8006',
+    });
+
+    expect(result.success).toBe(true);
+    expect(await getServiceCredentials(service.id)).toEqual({
+      api_url: 'https://10.77.20.50:8006',
+      api_token_id: 'root@pam!celilo',
+      api_token_secret: 'existing-secret',
+    });
+    expect((await getContainerService(service.id))?.verified).toBe(false);
+    if (!result.success) throw new Error(result.error);
+    expect(result.message).not.toContain('existing-secret');
+  });
+
+  it('accepts the endpoint through the documented environment variable', async () => {
+    const service = await addContainerService({
+      name: 'Nubs',
+      providerName: 'proxmox',
+      zones: ['internal'],
+      providerConfig: {},
+      apiCredentials: {
+        api_url: 'https://192.168.0.51:8006',
+        api_token_id: 'root@pam!celilo',
+        api_token_secret: 'existing-secret',
+      },
+    });
+    process.env.PROXMOX_API_URL = 'https://10.77.20.51:8006';
+
+    const result = await handleServiceSetCredentials(['nubs']);
+
+    expect(result.success).toBe(true);
+    expect(((await getServiceCredentials(service.id)) as ProxmoxCredentials).api_url).toBe(
+      'https://10.77.20.51:8006',
+    );
+  });
+
+  it('refuses a no-op that would only churn encrypted state', async () => {
+    await addContainerService({
+      name: 'Chubs',
+      providerName: 'proxmox',
+      zones: ['internal'],
+      providerConfig: {},
+      apiCredentials: {
+        api_url: 'https://192.168.0.50:8006',
+        api_token_id: 'root@pam!celilo',
+        api_token_secret: 'existing-secret',
+      },
+    });
+
+    const result = await handleServiceSetCredentials(['chubs']);
+
+    expect(result.success).toBe(false);
+    if (result.success) throw new Error('Expected set-credentials to reject a no-op');
+    expect(result.error).toContain('No credential changes supplied');
+  });
+});

package/src/cli/commands/service-set-credentials.ts

@@ -0,0 +1,115 @@
+/**
+ * Service set-credentials command.
+ *
+ * Credential values travel only by flag or environment variable (D7). Fields
+ * omitted from the invocation retain their existing encrypted values, which
+ * lets an operator move a provider endpoint without rotating its API token.
+ */
+
+import {
+  type DigitalOceanCredentials,
+  type ProxmoxCredentials,
+  getContainerServiceByServiceId,
+  getServiceCredentials,
+  updateServiceCredentials,
+} from '../../services/container-service';
+import type { CommandResult } from '../types';
+
+type Flags = Record<string, boolean | string>;
+
+function credentialUpdateValue(flags: Flags, flag: string, envVar: string): string | undefined {
+  const flagValue = flags[flag];
+  if (flagValue === true) {
+    throw new Error(`--${flag} requires a value`);
+  }
+  if (typeof flagValue === 'string') {
+    const value = flagValue.trim();
+    if (!value) throw new Error(`--${flag} requires a non-empty value`);
+    return value;
+  }
+
+  const envValue = process.env[envVar]?.trim();
+  return envValue || undefined;
+}
+
+export async function handleServiceSetCredentials(
+  args: string[],
+  flags: Flags = {},
+): Promise<CommandResult> {
+  const serviceId = args[0];
+  if (!serviceId) {
+    return {
+      success: false,
+      error:
+        'Service ID is required\n\nUsage: celilo service set-credentials <service-id> [options]',
+    };
+  }
+
+  try {
+    const service = await getContainerServiceByServiceId(serviceId);
+    if (!service) {
+      return {
+        success: false,
+        error: `Service not found: ${serviceId}\n\nRun 'celilo service list' to see available services.`,
+      };
+    }
+
+    const current = await getServiceCredentials(service.id);
+    let changed = false;
+
+    if (service.providerName === 'proxmox') {
+      const existing = current as ProxmoxCredentials;
+      const apiUrl = credentialUpdateValue(flags, 'api-url', 'PROXMOX_API_URL');
+      const apiTokenId = credentialUpdateValue(flags, 'api-token-id', 'PROXMOX_API_TOKEN_ID');
+      const apiTokenSecret = credentialUpdateValue(
+        flags,
+        'api-token-secret',
+        'PROXMOX_API_TOKEN_SECRET',
+      );
+      changed = Boolean(apiUrl || apiTokenId || apiTokenSecret);
+
+      if (changed) {
+        await updateServiceCredentials(service.id, {
+          api_url: apiUrl ?? existing.api_url,
+          api_token_id: apiTokenId ?? existing.api_token_id,
+          api_token_secret: apiTokenSecret ?? existing.api_token_secret,
+        });
+      }
+    } else if (service.providerName === 'digitalocean') {
+      const existing = current as DigitalOceanCredentials;
+      const apiToken = credentialUpdateValue(flags, 'api-token', 'DIGITALOCEAN_API_TOKEN');
+      changed = Boolean(apiToken);
+
+      if (changed) {
+        await updateServiceCredentials(service.id, {
+          api_token: apiToken ?? existing.api_token,
+        });
+      }
+    } else {
+      return {
+        success: false,
+        error: `Credential updates are not supported for provider: ${service.providerName}`,
+      };
+    }
+
+    if (!changed) {
+      return {
+        success: false,
+        error:
+          service.providerName === 'proxmox'
+            ? 'No credential changes supplied. Pass --api-url, --api-token-id, or --api-token-secret (or set the corresponding PROXMOX_API_* environment variable).'
+            : 'No credential changes supplied. Pass --api-token or set $DIGITALOCEAN_API_TOKEN.',
+      };
+    }
+
+    return {
+      success: true,
+      message: `Updated credentials for service '${serviceId}'. Verification status cleared; run: celilo service verify ${serviceId}`,
+    };
+  } catch (error) {
+    return {
+      success: false,
+      error: `Failed to update service credentials: ${error instanceof Error ? error.message : String(error)}`,
+    };
+  }
+}

package/src/cli/commands/system-migrate.ts

@@ -108,9 +108,11 @@ export async function handleSystemMigrate(
     }
   }
 
-  // getDb() auto-migrates on open; do it inside try so an existing DB that
-  // predates the drizzle-authoritative change fails with an actionable message
-  // instead of a raw migrator error.
+  // getDb() auto-migrates on open, and repairs a frozen `__drizzle_migrations`
+  // watermark itself when the declared schema is already complete. What reaches
+  // this catch is the case it will not guess at: schema that is only PARTLY
+  // there, where stamping would record migrations that never ran. Caught so it
+  // says what to do instead of surfacing a raw migrator error.
   let db: ReturnType<typeof getDb>;
   try {
     db = getDb();
@@ -118,7 +120,7 @@ export async function handleSystemMigrate(
     const msg = error instanceof Error ? error.message : String(error);
     return {
       success: false,
-      error: `Migration failed: ${msg}\n\nIf this DB predates the drizzle-authoritative migration change, it needs a one-time remediation (stamp \`__drizzle_migrations\` to the latest migration + create any missing table) before the migrator can run cleanly — see ISS-0100.`,
+      error: `Migration failed: ${msg}\n\nThis DB's \`__drizzle_migrations\` watermark disagrees with a schema that is only partly applied, which celilo will not resolve on its own. It needs a one-time remediation by hand (create the genuinely missing objects from their migration .sql, then stamp the watermark to the latest migration) — runbook in celilo#169.`,
     };
   }
 

package/src/cli/completion.ts

@@ -325,7 +325,15 @@ export async function getCompletions(words: string[], current: number): Promise<
   }
 
   if (command === 'service' && currentIndex === 1) {
-    const subcommands = ['add', 'list', 'verify', 'reconfigure', 'remove', 'config'];
+    const subcommands = [
+      'add',
+      'list',
+      'verify',
+      'reconfigure',
+      'remove',
+      'config',
+      'set-credentials',
+    ];
     return filterSuggestions(subcommands, args[1] || '');
   }
 
@@ -356,6 +364,13 @@ export async function getCompletions(words: string[], current: number): Promise<
     return filterSuggestions(serviceIds, args[2] || '');
   }
 
+  // Service set-credentials - complete with service IDs
+  if (command === 'service' && args[1] === 'set-credentials' && currentIndex === 2) {
+    const services = await listContainerServices();
+    const serviceIds = services.map((s) => s.serviceId);
+    return filterSuggestions(serviceIds, args[2] || '');
+  }
+
   // Service config operations
   if (command === 'service' && args[1] === 'config' && currentIndex === 2) {
     const operations = ['get', 'set'];

package/src/cli/index.ts

@@ -109,6 +109,7 @@ import { handleServiceConfigSet } from './commands/service-config-set';
 import { handleServiceList } from './commands/service-list';
 import { handleServiceReconfigure } from './commands/service-reconfigure';
 import { handleServiceRemove } from './commands/service-remove';
+import { handleServiceSetCredentials } from './commands/service-set-credentials';
 import { handleServiceVerify } from './commands/service-verify';
 import { handleStatus } from './commands/status';
 import { handleSubscribersAdd } from './commands/subscribers-add';
@@ -721,6 +722,7 @@ Subcommands:
     Options:
       --zone <zone>     Filter by network zone
   verify <service-id>   Re-verify a container service connection
+  set-credentials <service-id>  Update a provider endpoint or API credential
   reconfigure <service-id>  Re-run configuration interview (change template, storage, etc.)
   remove <id>           Remove a container service
     Options:
@@ -751,6 +753,9 @@ Examples:
   # Verify a service connection
   celilo service verify proxmox-home-lab
 
+  # Move a Proxmox endpoint while retaining its existing token
+  celilo service set-credentials proxmox-home-lab --api-url https://10.77.20.50:8006
+
   # Get service configuration
   celilo service config get proxmox-home-lab
   celilo service config get proxmox-home-lab name
@@ -1750,6 +1755,10 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
       return handleServiceVerify(parsed.args, parsed.flags);
     }
 
+    if (parsed.subcommand === 'set-credentials') {
+      return handleServiceSetCredentials(parsed.args, parsed.flags);
+    }
+
     if (parsed.subcommand === 'reconfigure') {
       return handleServiceReconfigure(parsed.args, parsed.flags);
     }

package/src/db/client.ts

@@ -4,8 +4,8 @@ import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { BUSY_TIMEOUT_MS, ensureWalMode } from '@celilo/event-bus/wal';
 import { drizzle } from 'drizzle-orm/bun-sqlite';
-import { migrate } from 'drizzle-orm/bun-sqlite/migrator';
 import { getDbPath } from '../config/paths';
+import { runMigrationsOn } from './migrate';
 import * as schema from './schema';
 
 /**
@@ -83,15 +83,17 @@ export function createDbClient(config?: Partial<DatabaseConfig>) {
   // idempotent: it applies every migration newer than the latest recorded in
   // `__drizzle_migrations` and no-ops once current.
   //
-  // One-time caveat (ISS-0100): an existing DB from the hand-list era has a
-  // frozen `__drizzle_migrations` watermark; it must be remediated by hand
-  // (stamp the watermark to the latest migration + create any missing table)
-  // BEFORE this code opens it, or migrate() re-runs already-applied migrations
-  // and throws. `celilo system doctor` (checkSchemaDrift) detects the drift.
+  // A DB from the hand-list era has a frozen `__drizzle_migrations` watermark,
+  // so drizzle re-runs already-applied migrations and throws (celilo#169).
+  // runMigrationsOn repairs that itself where the schema is already complete.
+  // It has to happen HERE and not only in `celilo system migrate`, because
+  // that command reaches its own repair through getDb() — this line — and so
+  // would die before getting there. A PARTIALLY applied schema still throws
+  // and still needs a human. `celilo system doctor` (checkSchemaDrift) detects
+  // the drift.
   if (!readonly) {
     try {
-      const migrationsFolder = findMigrationsFolder();
-      migrate(db, { migrationsFolder });
+      runMigrationsOn(db);
     } catch (error) {
       console.error('Failed to run migrations:', error);
       throw error;