@celilo/cli 0.14.2 → 0.15.0

package/src/services/machine-probe.ts

@@ -0,0 +1,73 @@
+/**
+ * Reachability probe for the machine pool.
+ *
+ * One implementation, deliberately. This SSH probe previously existed twice —
+ * once in `alerting/builtin-source.ts` for the monitor sweep and once in
+ * `cli/commands/system-audit.ts` for `celilo system audit` — as identical
+ * copy-pasted blocks. Both carried the same bug, and fixing one would have left
+ * the other reporting the management server as unreachable forever.
+ */
+
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import type { MachineReachableResult } from './audit/machines-reachable';
+import { listMachines } from './machine-pool';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Probe every machine in the pool.
+ *
+ * `BatchMode=yes` prevents a password prompt from hanging the probe forever,
+ * and `ConnectTimeout` bounds the wait on an unresponsive host — the exact
+ * condition this check exists to detect must not be the one that wedges it.
+ *
+ * The local management box is reported reachable WITHOUT probing it. celilo
+ * runs there as the `celilo` user and deliberately does not materialize an SSH
+ * key for itself, so `ssh root@127.0.0.1` fails with `Permission denied
+ * (publickey)` on a perfectly healthy host. Left unhandled that produced a
+ * permanently firing `machines_reachable` alert against celilo-mgr — and since
+ * that monitor is unsuppressible by design, nothing could explain it away.
+ *
+ * The check is also meaningless there: if the box running this code were
+ * unreachable, this code would not be running.
+ */
+export async function probeMachines(): Promise<MachineReachableResult[]> {
+  const machines = await listMachines();
+  return Promise.all(
+    machines.map(async (m): Promise<MachineReachableResult> => {
+      if (m.ipAddress === LOCAL_MACHINE_IP) {
+        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+      }
+      try {
+        await execFileAsync(
+          'ssh',
+          [
+            '-o',
+            'BatchMode=yes',
+            '-o',
+            'ConnectTimeout=5',
+            '-o',
+            'StrictHostKeyChecking=no',
+            '-o',
+            'UserKnownHostsFile=/dev/null',
+            `${m.sshUser}@${m.ipAddress}`,
+            'true',
+          ],
+          { timeout: 8000 },
+        );
+        return { id: m.id, hostname: m.hostname, ipAddress: m.ipAddress, reachable: true };
+      } catch (err) {
+        const e = err as { stderr?: string; message?: string };
+        return {
+          id: m.id,
+          hostname: m.hostname,
+          ipAddress: m.ipAddress,
+          reachable: false,
+          message: e.stderr?.trim() || e.message || 'SSH probe failed',
+        };
+      }
+    }),
+  );
+}

package/src/services/module-operations.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
-import { spawnSync } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 import { mkdtempSync, rmSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
@@ -7,10 +7,11 @@ import { closeDb } from '../db/client';
 import { runMigrations } from '../db/migrate';
 import {
   InFlightError,
+  OPERATION_TTL_MS,
   checkInFlight,
   completeOperation,
   failOperation,
-  isPidAlive,
+  isPidRunnable,
   refuseIfInFlight,
   startOperation,
 } from './module-operations';
@@ -95,7 +96,7 @@ describe('module-operations', () => {
       const child = spawnSync('node', ['-e', 'process.exit(0)']);
       const deadPid = child.pid;
       expect(deadPid).toBeGreaterThan(0);
-      expect(isPidAlive(deadPid)).toBe(false);
+      expect(isPidRunnable(deadPid)).toBe(false);
 
       // Insert a fake row with the dead pid via raw SQL (bypasses pid=process.pid in startOperation).
       const { getDb } = require('../db/client');
@@ -141,14 +142,74 @@ describe('module-operations', () => {
     });
   });
 
-  describe('isPidAlive', () => {
+  describe('isPidRunnable', () => {
     it('returns true for the current process', () => {
-      expect(isPidAlive(process.pid)).toBe(true);
+      expect(isPidRunnable(process.pid)).toBe(true);
     });
 
     it('returns false for a dead pid', () => {
       const child = spawnSync('node', ['-e', 'process.exit(0)']);
-      expect(isPidAlive(child.pid as number)).toBe(false);
+      expect(isPidRunnable(child.pid as number)).toBe(false);
+    });
+
+    // The bug this whole module exists to prevent: a Ctrl-Z'd `module deploy`
+    // is still "alive" by kill(pid, 0) and held the backup lock for 20 days.
+    it('returns false for a STOPPED process, which kill(pid, 0) calls alive', () => {
+      const child = spawn('sleep', ['60'], { stdio: 'ignore' });
+      const pid = child.pid as number;
+      try {
+        expect(isPidRunnable(pid)).toBe(true);
+
+        child.kill('SIGSTOP');
+        // Wait for the state change to land in the process table.
+        for (let i = 0; i < 100 && isPidRunnable(pid); i++) spawnSync('sleep', ['0.01']);
+
+        // Still passes the old liveness test...
+        let existsByKill = true;
+        try {
+          process.kill(pid, 0);
+        } catch {
+          existsByKill = false;
+        }
+        expect(existsByKill).toBe(true);
+
+        // ...but is correctly reported as unable to make progress.
+        expect(isPidRunnable(pid)).toBe(false);
+      } finally {
+        child.kill('SIGCONT');
+        child.kill('SIGKILL');
+      }
+    });
+  });
+
+  describe('abandonment by age', () => {
+    function insertRow(id: string, pid: number, startedAt: Date): void {
+      const { getDb } = require('../db/client');
+      const { moduleOperations } = require('../db/schema');
+      getDb()
+        .insert(moduleOperations)
+        .values({
+          id,
+          moduleId: 'ancient',
+          operation: 'deploy',
+          status: 'in_progress',
+          pid,
+          startedAt,
+        })
+        .run();
+    }
+
+    it('ignores a row older than the TTL even though its process is alive', () => {
+      // process.pid is unquestionably running, so age is the only thing that
+      // can release this row. This is the pid-reuse case: an old row whose
+      // number now belongs to some unrelated healthy process.
+      insertRow('ancient-row', process.pid, new Date(Date.now() - OPERATION_TTL_MS - 60_000));
+      expect(checkInFlight()).toHaveLength(0);
+    });
+
+    it('still blocks on a young row whose process is alive', () => {
+      insertRow('fresh-row', process.pid, new Date(Date.now() - 60_000));
+      expect(checkInFlight()).toHaveLength(1);
     });
   });
 });

package/src/services/module-operations.ts

@@ -13,13 +13,23 @@
  *     throw err;
  *   }
  *
- * Rows with status='in_progress' whose pid is no longer alive are
- * treated as abandoned (the process crashed before the completion
- * update landed) and ignored by `checkInFlight()`. This keeps a single
- * Ctrl-C from wedging the system, at the cost of leaving stale rows in
- * the table; a future cleanup command can sweep them.
+ * A row with status='in_progress' stops holding the lock once it looks
+ * abandoned, which is three different things:
+ *
+ *   - the process is GONE      — it crashed before writing completion
+ *   - the process is STOPPED   — suspended (Ctrl-Z) or a zombie, so it
+ *                                will never reach the completion write
+ *   - the row is OLD           — past `OPERATION_TTL_MS`
+ *
+ * Only the first was originally handled, and the other two are not
+ * hypothetical: a `module deploy` Ctrl-Z'd on a lost terminal held the
+ * lock for 20 days and blocked every backup on the fleet.
+ *
+ * Stale rows are ignored rather than deleted; `celilo module operations`
+ * lists them and `... clear` sweeps them.
  */
 
+import { spawnSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { eq } from 'drizzle-orm';
 import { getDb } from '../db/client';
@@ -62,17 +72,51 @@ export function failOperation(operationId: string, error: unknown): void {
 }
 
 /**
- * True if the OS still has a process with the given pid. `kill(pid, 0)`
- * sends no signal but throws ESRCH if the process is gone — the standard
- * idiom for liveness on POSIX.
+ * How long an in_progress row may hold the lock before it is treated as
+ * abandoned regardless of what its process appears to be doing.
+ *
+ * This is not belt-and-braces on the liveness check — it is the only
+ * check that survives pid reuse. A pid is a recycled number, not a
+ * stable identity: a busy host wraps the whole pid space in days, after
+ * which an old row's pid names an unrelated live process and the
+ * liveness check happily reports "still running" forever. Ageing the row
+ * out is the only thing that ends that.
+ *
+ * Two hours is longer than any real deploy and short enough that a wedge
+ * is an inconvenience rather than an outage.
+ */
+export const OPERATION_TTL_MS = 2 * 60 * 60 * 1000;
+
+/**
+ * True if the process can still make progress on its operation.
+ *
+ * `kill(pid, 0)` answers only "does this pid exist". A STOPPED process —
+ * SIGTSTP from a Ctrl-Z, or a lost controlling terminal — passes that
+ * test while being permanently unable to finish, which is exactly how
+ * the 20-day wedge happened. `ps -o state=` reports the state itself and
+ * is spelled the same on Linux and macOS.
  */
-export function isPidAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
+export function isPidRunnable(pid: number): boolean {
+  const result = spawnSync('ps', ['-o', 'state=', '-p', String(pid)], { encoding: 'utf-8' });
+
+  // No usable `ps`. Fall back to bare existence: a stopped process will
+  // still block, which is the old behavior, but we never wrongly release
+  // a lock that a live operation is holding.
+  if (result.error) {
+    try {
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
   }
+
+  if (result.status !== 0) return false; // no such process
+
+  // Linux reports multi-character states ("Tl", "Ss"); the first
+  // character is the state proper. T = stopped, Z = zombie.
+  const state = result.stdout.trim()[0] ?? '';
+  return state !== 'T' && state !== 'Z';
 }
 
 export interface InFlightConflict {
@@ -82,9 +126,10 @@ export interface InFlightConflict {
 }
 
 /**
- * Returns rows that genuinely look in-flight: status='in_progress' AND
- * the originating process is still alive. Abandoned rows (process gone)
- * are excluded so a stale Ctrl-C doesn't block future operations.
+ * Returns rows that genuinely look in-flight: status='in_progress', the
+ * row is younger than `OPERATION_TTL_MS`, AND the originating process is
+ * still able to make progress. Everything else is abandoned and excluded,
+ * so a crashed, suspended, or forgotten operation cannot wedge the fleet.
  *
  * @param excludeOperationId - operation id to exclude from the check
  *   (so an operation doesn't see itself as a conflict).
@@ -97,10 +142,13 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
     .where(eq(moduleOperations.status, 'in_progress'))
     .all();
 
+  const now = Date.now();
   const conflicts: InFlightConflict[] = [];
   for (const row of rows) {
     if (excludeOperationId && row.id === excludeOperationId) continue;
-    if (!isPidAlive(row.pid)) continue;
+    // Age first: it costs nothing, where the liveness probe spawns `ps`.
+    if (now - row.startedAt.getTime() > OPERATION_TTL_MS) continue;
+    if (!isPidRunnable(row.pid)) continue;
     conflicts.push({
       operation: row,
       describe: `${row.operation} of ${row.moduleId} (pid ${row.pid})`,
@@ -117,8 +165,10 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
 export class InFlightError extends Error {
   constructor(public readonly conflicts: InFlightConflict[]) {
     const list = conflicts.map((c) => `  • ${c.describe}`).join('\n');
+    const hint =
+      'If it is not really running: "celilo module operations" to inspect, "celilo module operations clear" to release.';
     super(
-      `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.`,
+      `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.\n${hint}`,
     );
     this.name = 'InFlightError';
   }

package/src/services/module-subscriptions.test.ts

@@ -158,6 +158,34 @@ describe('register / unregister roundtrip', () => {
     expect(result.registered).toBe(0);
   });
 
+  it('a module with an on_backup hook arms the scheduled backup sweep', () => {
+    // The sweep is a system-level subscriber, not a module one, so it does not
+    // count toward `registered`. It appears the moment the fleet has something
+    // to back up — including on `module update`, which is how a manifest that
+    // newly declares a cadence reaches an already-deployed fleet.
+    const result = registerModuleSubscriptions(
+      baseManifest({ hooks: { on_backup: { script: 'backup.ts' } } }),
+      '/p',
+    );
+    expect(result.registered).toBe(0);
+
+    const bus = openBus({ dbPath, events: defineEvents({}) });
+    try {
+      const row = bus.db
+        .query<{ name: string; pattern: string; handler: string }, []>(
+          'SELECT name, pattern, handler FROM subscribers',
+        )
+        .get();
+      expect(row).toEqual({
+        name: 'celilo-backup-sweep',
+        pattern: 'timer.tick.1h',
+        handler: 'celilo backup sweep',
+      });
+    } finally {
+      bus.close();
+    }
+  });
+
   it('registers each subscription as a row, names scoped to module id', () => {
     const result = registerModuleSubscriptions(
       baseManifest({
@@ -358,7 +386,10 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
     expect(poll).toBeDefined();
     expect(poll?.pattern).toBe('timer.tick.15m');
     // A literal handler command (the CLI poll), not a hook — see manifest comment.
-    expect(poll?.handler).toBe('celilo module upgrade');
+    // `--poll` is load-bearing: the dispatcher appends the event id to a
+    // subprocess handler, and without the flag it lands in the module-name slot
+    // ("Module not found: <event_id>" every tick).
+    expect(poll?.handler).toBe('celilo module upgrade --poll');
     expect(poll?.hook).toBeUndefined();
 
     const resolved = resolveSubscription(
@@ -368,6 +399,6 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
       '/modules/celilo-mgmt',
     );
     expect(resolved.name).toBe('celilo-mgmt.registry-poll');
-    expect(resolved.handler).toBe('celilo module upgrade');
+    expect(resolved.handler).toBe('celilo module upgrade --poll');
   });
 });

package/src/services/module-subscriptions.ts

@@ -21,6 +21,7 @@ import { getEventBusPath, getModuleStoragePath } from '../config/paths';
 import { getDb } from '../db/client';
 import { modules } from '../db/schema';
 import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
+import { ensureBackupSweepSubscriber } from './backup-sweep';
 
 /**
  * The bus is opened by the celilo CLI without an event registry — the
@@ -90,10 +91,18 @@ export function registerModuleSubscriptions(
   modulePath: string,
 ): { registered: number } {
   const subs = manifest.subscriptions ?? [];
-  if (subs.length === 0) return { registered: 0 };
+  const backupSweep = Boolean(manifest.hooks?.on_backup);
+  if (subs.length === 0 && !backupSweep) return { registered: 0 };
 
   const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
   try {
+    // A module that can be backed up is also what switches the scheduled
+    // backup sweep on. Registering here rather than at system init means the
+    // sweep appears the moment the fleet has something to back up, and — since
+    // `module update` comes through here too — a manifest that newly declares a
+    // cadence arms the sweep on the same update that declares it. Idempotent.
+    if (backupSweep) ensureBackupSweepSubscriber(bus);
+
     for (const sub of subs) {
       const resolved = resolveSubscription(sub, manifest.id, modulePath);
       bus.subscribe(resolved);

package/src/services/module-validator/typescript-build.test.ts

@@ -43,11 +43,30 @@ describe('checkTypeScriptBuild', () => {
     }
   });
 
-  test('fail with helpful message when tsconfig present but no node_modules', async () => {
+  // A tsconfig at the module ROOT is not the one we run: it lives in scripts/,
+  // next to the package.json and node_modules that make @celilo/capabilities
+  // resolve. Looking at the wrong level is why this check never fired.
+  test('ignores a tsconfig at the module root', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
     try {
+      mkdirSync(join(dir, 'scripts'));
+      writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
       writeFileSync(join(dir, 'tsconfig.json'), '{}');
       const r = await checkTypeScriptBuild(dir);
+      expect(r.status).toBe('warn');
+      expect(r.message).toContain('scripts/tsconfig.json');
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  test('fail with helpful message when tsconfig present but no node_modules', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
+    try {
+      mkdirSync(join(dir, 'scripts'));
+      writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
+      writeFileSync(join(dir, 'scripts', 'tsconfig.json'), '{}');
+      const r = await checkTypeScriptBuild(dir);
       expect(r.status).toBe('fail');
       expect(r.message).toContain('node_modules');
       expect(r.message).toContain('bun install');

package/src/services/module-validator/typescript-build.ts

@@ -59,7 +59,11 @@ export async function checkTypeScriptBuild(
     };
   }
 
-  const hasTsConfig = existsSync(join(modulePath, 'tsconfig.json'));
+  // tsconfig.json, package.json and node_modules all live in scripts/, not at
+  // the module root: scripts/ is the standalone package that resolves
+  // @celilo/capabilities the way the deployed module does.
+  const scriptsDir = join(modulePath, 'scripts');
+  const hasTsConfig = existsSync(join(scriptsDir, 'tsconfig.json'));
   const hasTsFiles = await hasTypeScriptSources(modulePath);
 
   if (!hasTsConfig && !hasTsFiles) {
@@ -77,21 +81,21 @@ export async function checkTypeScriptBuild(
       name: 'tsc --noEmit',
       status: 'warn',
       message:
-        'module has .ts files but no tsconfig.json; add one so we can typecheck against current @celilo types',
+        'module has .ts files but no scripts/tsconfig.json; add one (extending modules/tsconfig.scripts.base.json) so we can typecheck against current @celilo types',
     };
   }
 
-  if (!existsSync(join(modulePath, 'node_modules'))) {
+  if (!existsSync(join(scriptsDir, 'node_modules'))) {
     return {
       category: 'typescript_build',
       name: 'tsc --noEmit',
       status: 'fail',
-      message: 'no node_modules — run `bun install` in the module directory first',
+      message: 'no node_modules — run `bun install` in the module scripts/ directory first',
     };
   }
 
   const r = spawnSync('bunx', ['tsc', '--noEmit'], {
-    cwd: modulePath,
+    cwd: scriptsDir,
     encoding: 'utf-8',
   });
 

package/src/services/restore-from-file.ts

@@ -30,7 +30,6 @@ import {
   readSync,
   readdirSync,
   rmSync,
-  writeFileSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
@@ -42,9 +41,9 @@ import { modules } from '../db/schema';
 import { invokeHook } from '../hooks/executor';
 import { createConsoleLogger } from '../hooks/logger';
 import type { ModuleManifest } from '../manifest/schema';
-import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
+import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
 import { getModuleSystems } from './deployed-systems';
@@ -98,8 +97,8 @@ export async function restoreFromArtifactFile(
     return { success: false, error: `Artifact not found: ${filePath}` };
   }
 
-  // 1. Decrypt the encrypted envelope (JSON wrapper, identical to the
-  //    shape backup-create.ts writes).
+  // 1. Decrypt the artifact into the inner tar (see backup-cipher.ts —
+  //    handles both the streamed format and the legacy JSON envelope).
   let masterKey: Buffer;
   try {
     masterKey = await getOrCreateMasterKey();
@@ -110,37 +109,23 @@ export async function restoreFromArtifactFile(
     };
   }
 
-  let encryptedJson: unknown;
-  try {
-    encryptedJson = JSON.parse(readFileSync(filePath, 'utf-8'));
-  } catch (err) {
-    return {
-      success: false,
-      error: `Artifact is not valid JSON: ${err instanceof Error ? err.message : String(err)}. The file may be corrupted or not a celilo backup artifact.`,
-    };
-  }
-
   const tempDir = join(tmpdir(), `celilo-restore-from-file-${Date.now()}`);
   const envelopeDir = join(tempDir, 'envelope');
 
   try {
     mkdirSync(envelopeDir, { recursive: true });
 
-    // Decrypt the JSON wrapper into the inner tar bytes.
-    let tarData: Buffer;
+    const tarPath = join(tempDir, 'envelope.tar');
     try {
-      const base64 = decryptSecret(encryptedJson as Parameters<typeof decryptSecret>[0], masterKey);
-      tarData = Buffer.from(base64, 'base64');
+      await decryptFileToFile(filePath, tarPath, masterKey);
     } catch (err) {
       return {
         success: false,
-        error: `Decryption failed: ${err instanceof Error ? err.message : String(err)}. The master key may not match the one used at backup time.`,
+        error: `Decryption failed: ${err instanceof Error ? err.message : String(err)}. Either the master key does not match the one used at backup time, or the file is corrupted / not a celilo backup artifact.`,
       };
     }
 
     // Extract the envelope tar.
-    const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // 2. Read + validate the envelope manifest.

package/src/templates/generator.test.ts

@@ -6,6 +6,7 @@ import { type DbClient, createDbClient } from '../db/client';
 import { capabilities } from '../db/schema';
 import { upsertModuleConfig } from '../services/module-config';
 import {
+  decideStorage,
   decideTargetNode,
   discoverTemplateFiles,
   generateTemplates,
@@ -13,6 +14,7 @@ import {
   injectProxmoxDns,
   isTemplateFile,
   readTemplateFiles,
+  storageFromTfState,
   targetNodeFromTfState,
   writeGeneratedFiles,
 } from './generator';
@@ -845,3 +847,89 @@ describe('decideTargetNode (ISS-0090 — deploy follows reality: Proxmox > state
     ).toEqual({ node: 'node2', source: 'proxmox' });
   });
 });
+
+describe("storageFromTfState (terraform state is celilo's storage record)", () => {
+  test('reads storage from the proxmox_lxc rootfs block', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_lxc',
+          instances: [
+            {
+              attributes: {
+                rootfs: [{ storage: 'local-lvm', volume: 'local-lvm:vm-204-disk-0' }],
+              },
+            },
+          ],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('falls back to the volume prefix when storage is absent', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_lxc',
+          instances: [{ attributes: { rootfs: [{ volume: 'local-lvm:vm-204-disk-0' }] } }],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('accepts a bare rootfs object as well as a block list', () => {
+    const state = {
+      resources: [
+        { type: 'proxmox_lxc', instances: [{ attributes: { rootfs: { storage: 'datacenter' } } }] },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('datacenter');
+  });
+
+  test('reads a VM data disk, ignoring the cloudinit disk (type:vm)', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_vm_qemu',
+          instances: [
+            {
+              attributes: {
+                disk: [
+                  { type: 'cloudinit', storage: 'datacenter' },
+                  { type: 'disk', storage: 'local-lvm' },
+                ],
+              },
+            },
+          ],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('returns null on a fresh/empty state', () => {
+    expect(storageFromTfState({ resources: [] })).toBeNull();
+    expect(storageFromTfState({})).toBeNull();
+    expect(storageFromTfState({ resources: [{ type: 'proxmox_lxc', instances: [] }] })).toBeNull();
+  });
+});
+
+describe('decideStorage (storage is sticky — a changed default must not force replacement)', () => {
+  test('an existing container keeps the storage it was created on', () => {
+    // The celilo-registry case: lxc 204 lives on local-lvm, the service default
+    // later became datacenter. Emitting the default would plan 1-to-destroy.
+    expect(decideStorage({ stateStorage: 'local-lvm', defaultStorage: 'datacenter' })).toEqual({
+      storage: 'local-lvm',
+      source: 'state',
+    });
+  });
+
+  test('first create (no state) uses the service default', () => {
+    expect(decideStorage({ stateStorage: null, defaultStorage: 'datacenter' })).toEqual({
+      storage: 'datacenter',
+      source: 'default',
+    });
+  });
+});