@celilo/cli 0.14.4 → 0.16.1

package/src/services/module-operations.ts

@@ -13,13 +13,23 @@
  *     throw err;
  *   }
  *
- * Rows with status='in_progress' whose pid is no longer alive are
- * treated as abandoned (the process crashed before the completion
- * update landed) and ignored by `checkInFlight()`. This keeps a single
- * Ctrl-C from wedging the system, at the cost of leaving stale rows in
- * the table; a future cleanup command can sweep them.
+ * A row with status='in_progress' stops holding the lock once it looks
+ * abandoned, which is three different things:
+ *
+ *   - the process is GONE      — it crashed before writing completion
+ *   - the process is STOPPED   — suspended (Ctrl-Z) or a zombie, so it
+ *                                will never reach the completion write
+ *   - the row is OLD           — past `OPERATION_TTL_MS`
+ *
+ * Only the first was originally handled, and the other two are not
+ * hypothetical: a `module deploy` Ctrl-Z'd on a lost terminal held the
+ * lock for 20 days and blocked every backup on the fleet.
+ *
+ * Stale rows are ignored rather than deleted; `celilo module operations`
+ * lists them and `... clear` sweeps them.
  */
 
+import { spawnSync } from 'node:child_process';
 import { randomUUID } from 'node:crypto';
 import { eq } from 'drizzle-orm';
 import { getDb } from '../db/client';
@@ -62,17 +72,51 @@ export function failOperation(operationId: string, error: unknown): void {
 }
 
 /**
- * True if the OS still has a process with the given pid. `kill(pid, 0)`
- * sends no signal but throws ESRCH if the process is gone — the standard
- * idiom for liveness on POSIX.
+ * How long an in_progress row may hold the lock before it is treated as
+ * abandoned regardless of what its process appears to be doing.
+ *
+ * This is not belt-and-braces on the liveness check — it is the only
+ * check that survives pid reuse. A pid is a recycled number, not a
+ * stable identity: a busy host wraps the whole pid space in days, after
+ * which an old row's pid names an unrelated live process and the
+ * liveness check happily reports "still running" forever. Ageing the row
+ * out is the only thing that ends that.
+ *
+ * Two hours is longer than any real deploy and short enough that a wedge
+ * is an inconvenience rather than an outage.
+ */
+export const OPERATION_TTL_MS = 2 * 60 * 60 * 1000;
+
+/**
+ * True if the process can still make progress on its operation.
+ *
+ * `kill(pid, 0)` answers only "does this pid exist". A STOPPED process —
+ * SIGTSTP from a Ctrl-Z, or a lost controlling terminal — passes that
+ * test while being permanently unable to finish, which is exactly how
+ * the 20-day wedge happened. `ps -o state=` reports the state itself and
+ * is spelled the same on Linux and macOS.
  */
-export function isPidAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
+export function isPidRunnable(pid: number): boolean {
+  const result = spawnSync('ps', ['-o', 'state=', '-p', String(pid)], { encoding: 'utf-8' });
+
+  // No usable `ps`. Fall back to bare existence: a stopped process will
+  // still block, which is the old behavior, but we never wrongly release
+  // a lock that a live operation is holding.
+  if (result.error) {
+    try {
+      process.kill(pid, 0);
+      return true;
+    } catch {
+      return false;
+    }
   }
+
+  if (result.status !== 0) return false; // no such process
+
+  // Linux reports multi-character states ("Tl", "Ss"); the first
+  // character is the state proper. T = stopped, Z = zombie.
+  const state = result.stdout.trim()[0] ?? '';
+  return state !== 'T' && state !== 'Z';
 }
 
 export interface InFlightConflict {
@@ -82,9 +126,10 @@ export interface InFlightConflict {
 }
 
 /**
- * Returns rows that genuinely look in-flight: status='in_progress' AND
- * the originating process is still alive. Abandoned rows (process gone)
- * are excluded so a stale Ctrl-C doesn't block future operations.
+ * Returns rows that genuinely look in-flight: status='in_progress', the
+ * row is younger than `OPERATION_TTL_MS`, AND the originating process is
+ * still able to make progress. Everything else is abandoned and excluded,
+ * so a crashed, suspended, or forgotten operation cannot wedge the fleet.
  *
  * @param excludeOperationId - operation id to exclude from the check
  *   (so an operation doesn't see itself as a conflict).
@@ -97,10 +142,13 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
     .where(eq(moduleOperations.status, 'in_progress'))
     .all();
 
+  const now = Date.now();
   const conflicts: InFlightConflict[] = [];
   for (const row of rows) {
     if (excludeOperationId && row.id === excludeOperationId) continue;
-    if (!isPidAlive(row.pid)) continue;
+    // Age first: it costs nothing, where the liveness probe spawns `ps`.
+    if (now - row.startedAt.getTime() > OPERATION_TTL_MS) continue;
+    if (!isPidRunnable(row.pid)) continue;
     conflicts.push({
       operation: row,
       describe: `${row.operation} of ${row.moduleId} (pid ${row.pid})`,
@@ -117,8 +165,10 @@ export function checkInFlight(excludeOperationId?: string): InFlightConflict[] {
 export class InFlightError extends Error {
   constructor(public readonly conflicts: InFlightConflict[]) {
     const list = conflicts.map((c) => `  • ${c.describe}`).join('\n');
+    const hint =
+      'If it is not really running: "celilo module operations" to inspect, "celilo module operations clear" to release.';
     super(
-      `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.`,
+      `Cannot start: another module operation is in progress.\n${list}\nWait for it to complete (or fail) and re-run.\n${hint}`,
     );
     this.name = 'InFlightError';
   }

package/src/services/module-subscriptions.test.ts

@@ -158,6 +158,34 @@ describe('register / unregister roundtrip', () => {
     expect(result.registered).toBe(0);
   });
 
+  it('a module with an on_backup hook arms the scheduled backup sweep', () => {
+    // The sweep is a system-level subscriber, not a module one, so it does not
+    // count toward `registered`. It appears the moment the fleet has something
+    // to back up — including on `module update`, which is how a manifest that
+    // newly declares a cadence reaches an already-deployed fleet.
+    const result = registerModuleSubscriptions(
+      baseManifest({ hooks: { on_backup: { script: 'backup.ts' } } }),
+      '/p',
+    );
+    expect(result.registered).toBe(0);
+
+    const bus = openBus({ dbPath, events: defineEvents({}) });
+    try {
+      const row = bus.db
+        .query<{ name: string; pattern: string; handler: string }, []>(
+          'SELECT name, pattern, handler FROM subscribers',
+        )
+        .get();
+      expect(row).toEqual({
+        name: 'celilo-backup-sweep',
+        pattern: 'timer.tick.1h',
+        handler: 'celilo backup sweep',
+      });
+    } finally {
+      bus.close();
+    }
+  });
+
   it('registers each subscription as a row, names scoped to module id', () => {
     const result = registerModuleSubscriptions(
       baseManifest({
@@ -358,7 +386,10 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
     expect(poll).toBeDefined();
     expect(poll?.pattern).toBe('timer.tick.15m');
     // A literal handler command (the CLI poll), not a hook — see manifest comment.
-    expect(poll?.handler).toBe('celilo module upgrade');
+    // `--poll` is load-bearing: the dispatcher appends the event id to a
+    // subprocess handler, and without the flag it lands in the module-name slot
+    // ("Module not found: <event_id>" every tick).
+    expect(poll?.handler).toBe('celilo module upgrade --poll');
     expect(poll?.hook).toBeUndefined();
 
     const resolved = resolveSubscription(
@@ -368,6 +399,6 @@ describe('build-bus registry-poll wiring (ISS-0139)', () => {
       '/modules/celilo-mgmt',
     );
     expect(resolved.name).toBe('celilo-mgmt.registry-poll');
-    expect(resolved.handler).toBe('celilo module upgrade');
+    expect(resolved.handler).toBe('celilo module upgrade --poll');
   });
 });

package/src/services/module-subscriptions.ts

@@ -21,6 +21,7 @@ import { getEventBusPath, getModuleStoragePath } from '../config/paths';
 import { getDb } from '../db/client';
 import { modules } from '../db/schema';
 import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
+import { ensureBackupSweepSubscriber } from './backup-sweep';
 
 /**
  * The bus is opened by the celilo CLI without an event registry — the
@@ -90,10 +91,18 @@ export function registerModuleSubscriptions(
   modulePath: string,
 ): { registered: number } {
   const subs = manifest.subscriptions ?? [];
-  if (subs.length === 0) return { registered: 0 };
+  const backupSweep = Boolean(manifest.hooks?.on_backup);
+  if (subs.length === 0 && !backupSweep) return { registered: 0 };
 
   const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
   try {
+    // A module that can be backed up is also what switches the scheduled
+    // backup sweep on. Registering here rather than at system init means the
+    // sweep appears the moment the fleet has something to back up, and — since
+    // `module update` comes through here too — a manifest that newly declares a
+    // cadence arms the sweep on the same update that declares it. Idempotent.
+    if (backupSweep) ensureBackupSweepSubscriber(bus);
+
     for (const sub of subs) {
       const resolved = resolveSubscription(sub, manifest.id, modulePath);
       bus.subscribe(resolved);

package/src/services/module-validator/typescript-build.test.ts

@@ -43,11 +43,30 @@ describe('checkTypeScriptBuild', () => {
     }
   });
 
-  test('fail with helpful message when tsconfig present but no node_modules', async () => {
+  // A tsconfig at the module ROOT is not the one we run: it lives in scripts/,
+  // next to the package.json and node_modules that make @celilo/capabilities
+  // resolve. Looking at the wrong level is why this check never fired.
+  test('ignores a tsconfig at the module root', async () => {
     const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
     try {
+      mkdirSync(join(dir, 'scripts'));
+      writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
       writeFileSync(join(dir, 'tsconfig.json'), '{}');
       const r = await checkTypeScriptBuild(dir);
+      expect(r.status).toBe('warn');
+      expect(r.message).toContain('scripts/tsconfig.json');
+    } finally {
+      rmSync(dir, { recursive: true, force: true });
+    }
+  });
+
+  test('fail with helpful message when tsconfig present but no node_modules', async () => {
+    const dir = mkdtempSync(join(tmpdir(), 'celilo-tsc-'));
+    try {
+      mkdirSync(join(dir, 'scripts'));
+      writeFileSync(join(dir, 'scripts', 'install.ts'), 'export const x = 1;\n');
+      writeFileSync(join(dir, 'scripts', 'tsconfig.json'), '{}');
+      const r = await checkTypeScriptBuild(dir);
       expect(r.status).toBe('fail');
       expect(r.message).toContain('node_modules');
       expect(r.message).toContain('bun install');

package/src/services/module-validator/typescript-build.ts

@@ -59,7 +59,11 @@ export async function checkTypeScriptBuild(
     };
   }
 
-  const hasTsConfig = existsSync(join(modulePath, 'tsconfig.json'));
+  // tsconfig.json, package.json and node_modules all live in scripts/, not at
+  // the module root: scripts/ is the standalone package that resolves
+  // @celilo/capabilities the way the deployed module does.
+  const scriptsDir = join(modulePath, 'scripts');
+  const hasTsConfig = existsSync(join(scriptsDir, 'tsconfig.json'));
   const hasTsFiles = await hasTypeScriptSources(modulePath);
 
   if (!hasTsConfig && !hasTsFiles) {
@@ -77,21 +81,21 @@ export async function checkTypeScriptBuild(
       name: 'tsc --noEmit',
       status: 'warn',
       message:
-        'module has .ts files but no tsconfig.json; add one so we can typecheck against current @celilo types',
+        'module has .ts files but no scripts/tsconfig.json; add one (extending modules/tsconfig.scripts.base.json) so we can typecheck against current @celilo types',
     };
   }
 
-  if (!existsSync(join(modulePath, 'node_modules'))) {
+  if (!existsSync(join(scriptsDir, 'node_modules'))) {
     return {
       category: 'typescript_build',
       name: 'tsc --noEmit',
       status: 'fail',
-      message: 'no node_modules — run `bun install` in the module directory first',
+      message: 'no node_modules — run `bun install` in the module scripts/ directory first',
     };
   }
 
   const r = spawnSync('bunx', ['tsc', '--noEmit'], {
-    cwd: modulePath,
+    cwd: scriptsDir,
     encoding: 'utf-8',
   });
 

package/src/services/restore-from-file.ts

@@ -30,7 +30,6 @@ import {
   readSync,
   readdirSync,
   rmSync,
-  writeFileSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
@@ -42,9 +41,9 @@ import { modules } from '../db/schema';
 import { invokeHook } from '../hooks/executor';
 import { createConsoleLogger } from '../hooks/logger';
 import type { ModuleManifest } from '../manifest/schema';
-import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { shellEscape } from '../utils/shell';
+import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
 import { getModuleSystems } from './deployed-systems';
@@ -98,8 +97,8 @@ export async function restoreFromArtifactFile(
     return { success: false, error: `Artifact not found: ${filePath}` };
   }
 
-  // 1. Decrypt the encrypted envelope (JSON wrapper, identical to the
-  //    shape backup-create.ts writes).
+  // 1. Decrypt the artifact into the inner tar (see backup-cipher.ts —
+  //    handles both the streamed format and the legacy JSON envelope).
   let masterKey: Buffer;
   try {
     masterKey = await getOrCreateMasterKey();
@@ -110,37 +109,23 @@ export async function restoreFromArtifactFile(
     };
   }
 
-  let encryptedJson: unknown;
-  try {
-    encryptedJson = JSON.parse(readFileSync(filePath, 'utf-8'));
-  } catch (err) {
-    return {
-      success: false,
-      error: `Artifact is not valid JSON: ${err instanceof Error ? err.message : String(err)}. The file may be corrupted or not a celilo backup artifact.`,
-    };
-  }
-
   const tempDir = join(tmpdir(), `celilo-restore-from-file-${Date.now()}`);
   const envelopeDir = join(tempDir, 'envelope');
 
   try {
     mkdirSync(envelopeDir, { recursive: true });
 
-    // Decrypt the JSON wrapper into the inner tar bytes.
-    let tarData: Buffer;
+    const tarPath = join(tempDir, 'envelope.tar');
     try {
-      const base64 = decryptSecret(encryptedJson as Parameters<typeof decryptSecret>[0], masterKey);
-      tarData = Buffer.from(base64, 'base64');
+      await decryptFileToFile(filePath, tarPath, masterKey);
     } catch (err) {
       return {
         success: false,
-        error: `Decryption failed: ${err instanceof Error ? err.message : String(err)}. The master key may not match the one used at backup time.`,
+        error: `Decryption failed: ${err instanceof Error ? err.message : String(err)}. Either the master key does not match the one used at backup time, or the file is corrupted / not a celilo backup artifact.`,
       };
     }
 
     // Extract the envelope tar.
-    const tarPath = join(tempDir, 'envelope.tar');
-    writeFileSync(tarPath, tarData);
     execSync(`tar -xf ${shellEscape(tarPath)} -C ${shellEscape(envelopeDir)}`);
 
     // 2. Read + validate the envelope manifest.

package/src/templates/generator.test.ts

@@ -6,6 +6,7 @@ import { type DbClient, createDbClient } from '../db/client';
 import { capabilities } from '../db/schema';
 import { upsertModuleConfig } from '../services/module-config';
 import {
+  decideStorage,
   decideTargetNode,
   discoverTemplateFiles,
   generateTemplates,
@@ -13,6 +14,7 @@ import {
   injectProxmoxDns,
   isTemplateFile,
   readTemplateFiles,
+  storageFromTfState,
   targetNodeFromTfState,
   writeGeneratedFiles,
 } from './generator';
@@ -845,3 +847,89 @@ describe('decideTargetNode (ISS-0090 — deploy follows reality: Proxmox > state
     ).toEqual({ node: 'node2', source: 'proxmox' });
   });
 });
+
+describe("storageFromTfState (terraform state is celilo's storage record)", () => {
+  test('reads storage from the proxmox_lxc rootfs block', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_lxc',
+          instances: [
+            {
+              attributes: {
+                rootfs: [{ storage: 'local-lvm', volume: 'local-lvm:vm-204-disk-0' }],
+              },
+            },
+          ],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('falls back to the volume prefix when storage is absent', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_lxc',
+          instances: [{ attributes: { rootfs: [{ volume: 'local-lvm:vm-204-disk-0' }] } }],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('accepts a bare rootfs object as well as a block list', () => {
+    const state = {
+      resources: [
+        { type: 'proxmox_lxc', instances: [{ attributes: { rootfs: { storage: 'datacenter' } } }] },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('datacenter');
+  });
+
+  test('reads a VM data disk, ignoring the cloudinit disk (type:vm)', () => {
+    const state = {
+      resources: [
+        {
+          type: 'proxmox_vm_qemu',
+          instances: [
+            {
+              attributes: {
+                disk: [
+                  { type: 'cloudinit', storage: 'datacenter' },
+                  { type: 'disk', storage: 'local-lvm' },
+                ],
+              },
+            },
+          ],
+        },
+      ],
+    };
+    expect(storageFromTfState(state)).toBe('local-lvm');
+  });
+
+  test('returns null on a fresh/empty state', () => {
+    expect(storageFromTfState({ resources: [] })).toBeNull();
+    expect(storageFromTfState({})).toBeNull();
+    expect(storageFromTfState({ resources: [{ type: 'proxmox_lxc', instances: [] }] })).toBeNull();
+  });
+});
+
+describe('decideStorage (storage is sticky — a changed default must not force replacement)', () => {
+  test('an existing container keeps the storage it was created on', () => {
+    // The celilo-registry case: lxc 204 lives on local-lvm, the service default
+    // later became datacenter. Emitting the default would plan 1-to-destroy.
+    expect(decideStorage({ stateStorage: 'local-lvm', defaultStorage: 'datacenter' })).toEqual({
+      storage: 'local-lvm',
+      source: 'state',
+    });
+  });
+
+  test('first create (no state) uses the service default', () => {
+    expect(decideStorage({ stateStorage: null, defaultStorage: 'datacenter' })).toEqual({
+      storage: 'datacenter',
+      source: 'default',
+    });
+  });
+});

package/src/templates/generator.ts

@@ -248,12 +248,51 @@ export function targetNodeFromTfState(state: {
 }
 
 /**
- * Read the node a module's container is currently deployed on, from its terraform
- * state file. This is celilo's authoritative record of placement (ISS-0090).
- * Returns null when no state exists yet (a first deploy) or it can't be read —
- * the caller then falls back to the service `default_target_node`.
+ * The storage an existing container's root disk actually lives on, from
+ * terraform state. Prefers the LXC `rootfs.storage` attribute, falls back to the
+ * `<storage>:vm-<vmid>-disk-0` volume prefix, then to a VM's data `disk` block.
+ * Returns null when there's no such resource (e.g. an empty/fresh state).
  */
-async function readDeployedTargetNode(moduleId: string): Promise<string | null> {
+export function storageFromTfState(state: {
+  resources?: Array<{
+    type?: string;
+    instances?: Array<{
+      attributes?: {
+        rootfs?:
+          | Array<{ storage?: string; volume?: string }>
+          | { storage?: string; volume?: string };
+        disk?: Array<{ storage?: string; type?: string }>;
+      };
+    }>;
+  }>;
+}): string | null {
+  const guest = state.resources?.find(
+    (r) => r.type === 'proxmox_lxc' || r.type === 'proxmox_vm_qemu',
+  );
+  const attrs = guest?.instances?.[0]?.attributes;
+  if (!attrs) {
+    return null;
+  }
+  // The provider models `rootfs` as a single-element block list; older state
+  // shapes record it as a bare object.
+  const rootfs = Array.isArray(attrs.rootfs) ? attrs.rootfs[0] : attrs.rootfs;
+  if (rootfs?.storage) {
+    return rootfs.storage;
+  }
+  // volume format: "<storage>:vm-<vmid>-disk-0"
+  const volumeStorage = rootfs?.volume?.split(':')[0];
+  if (volumeStorage) {
+    return volumeStorage;
+  }
+  // VMs: the data disk carries the storage that matters (the cloudinit disk
+  // rides on the same storage — see the `disk` block ordering note in the
+  // vm templates).
+  const disks = attrs.disk ?? [];
+  return (disks.find((d) => d.type === 'disk') ?? disks[0])?.storage || null;
+}
+
+/** Parse a module's terraform state, or null when absent/unreadable. */
+async function readTfState(moduleId: string): Promise<Record<string, unknown> | null> {
   const statePath = join(
     getModuleStoragePath(),
     moduleId,
@@ -265,12 +304,33 @@ async function readDeployedTargetNode(moduleId: string): Promise<string | null>
     return null;
   }
   try {
-    return targetNodeFromTfState(JSON.parse(await readFile(statePath, 'utf-8')));
+    return JSON.parse(await readFile(statePath, 'utf-8'));
   } catch {
     return null;
   }
 }
 
+/**
+ * Read the node a module's container is currently deployed on, from its terraform
+ * state file. This is celilo's authoritative record of placement (ISS-0090).
+ * Returns null when no state exists yet (a first deploy) or it can't be read —
+ * the caller then falls back to the service `default_target_node`.
+ */
+async function readDeployedTargetNode(moduleId: string): Promise<string | null> {
+  const state = await readTfState(moduleId);
+  return state ? targetNodeFromTfState(state) : null;
+}
+
+/**
+ * Read the storage a module's container is currently deployed on, from its
+ * terraform state file. Returns null on a first deploy — the caller then falls
+ * back to the service default storage.
+ */
+async function readDeployedStorage(moduleId: string): Promise<string | null> {
+  const state = await readTfState(moduleId);
+  return state ? storageFromTfState(state) : null;
+}
+
 /**
  * The node a module's container ACTUALLY lives on, from Proxmox (ISS-0090).
  * Proxmox is the ultimate source of truth for current location — it sees a
@@ -314,6 +374,27 @@ export function decideTargetNode(opts: {
   return { node: opts.defaultNode, source: 'default' };
 }
 
+/**
+ * Decide which storage to target for a deploy. Pure (Rule 10). The sibling of
+ * `decideTargetNode`: the service's `storage` governs only a FIRST placement.
+ * A container already living on `local-lvm` must keep living there when the
+ * service default later changes to `datacenter` — emitting the new default
+ * makes terraform plan a destroy-and-recreate ("forces replacement") of a
+ * running container and lose its volume. Relocating storage is a deliberate
+ * migration, never a side effect of a default drifting.
+ *
+ * ponytail: terraform state only, no Proxmox-reality tier like decideTargetNode
+ * has. A hand `pct move-volume` is invisible until the next apply refreshes
+ * state; add a Proxmox read if that ever bites.
+ */
+export function decideStorage(opts: {
+  stateStorage: string | null;
+  defaultStorage: string;
+}): { storage: string; source: 'state' | 'default' } {
+  if (opts.stateStorage) return { storage: opts.stateStorage, source: 'state' };
+  return { storage: opts.defaultStorage, source: 'default' };
+}
+
 /**
  * Discover template files in directory recursively
  *
@@ -784,6 +865,7 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
   // Resolved live each generate (ISS-0090) and injected into the context below,
   // never cached in the DB. undefined for non-Proxmox / machine deploys.
   let resolvedTargetNode: string | undefined;
+  let resolvedStorage: string | undefined;
   if (isContainerService && isProxmoxService && infrastructureSelection?.serviceId) {
     const service = await db
       .select()
@@ -824,11 +906,24 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
         );
       }
 
-      // Persist only the non-drift provider values. target_node is reality — it's
-      // injected into the resolution context below, never cached (ISS-0090); drop
-      // any stale __infra_target_node a prior generate left behind.
+      // Storage is sticky for the same reason placement is: the service default
+      // governs only a FIRST create. Emitting a changed default at an existing
+      // container makes terraform plan a destroy-and-recreate.
+      const storageDecision = decideStorage({
+        stateStorage: await readDeployedStorage(moduleId),
+        defaultStorage: providerConfig.storage,
+      });
+      resolvedStorage = storageDecision.storage;
+      if (storageDecision.source === 'state' && resolvedStorage !== providerConfig.storage) {
+        log.info(
+          `${moduleId} → storage '${resolvedStorage}' (from terraform state; service default is '${providerConfig.storage}'). Moving storage requires a deliberate migration.`,
+        );
+      }
+
+      // Persist only the non-drift provider values. target_node and storage are
+      // reality — they're injected into the resolution context below, never
+      // cached (ISS-0090); drop any stale __infra_* rows a prior generate left.
       upsertModuleConfig(db, moduleId, '__infra_lxc_template', providerConfig.lxc_template);
-      upsertModuleConfig(db, moduleId, '__infra_storage', providerConfig.storage);
       // vm_template is only present once a VM template exists for the service.
       // Persist it when set so `type: vm` modules can resolve $self:vm_template;
       // a `type: vm` deploy against a service without one then fails loudly at
@@ -837,8 +932,12 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
         upsertModuleConfig(db, moduleId, '__infra_vm_template', providerConfig.vm_template);
       }
       deleteModuleConfig(db, moduleId, '__infra_target_node');
+      deleteModuleConfig(db, moduleId, '__infra_storage');
 
-      log.success(`Infrastructure resolved: target_node=${decision.node} (${decision.source})`);
+      log.success(
+        `Infrastructure resolved: target_node=${decision.node} (${decision.source}), ` +
+          `storage=${storageDecision.storage} (${storageDecision.source})`,
+      );
     }
   }
 
@@ -870,15 +969,19 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
       context.selfConfig.target_ip = ipConfig.value!;
     }
 
-    // target_node is the live-resolved reality (ISS-0090) — inject it directly,
-    // never from a cached __infra_target_node row (which drifts).
+    // target_node and storage are the live-resolved reality of an existing
+    // container (ISS-0090) — inject them directly, never from a cached
+    // __infra_* row (which drifts against the service default).
     if (resolvedTargetNode) {
       context.selfConfig.target_node = resolvedTargetNode;
     }
+    if (resolvedStorage) {
+      context.selfConfig.storage = resolvedStorage;
+    }
 
-    // lxc_template / storage are provider config (intent, not drift-prone) — read
-    // them back from the __infra_* rows persisted above.
-    const infraKeys = ['lxc_template', 'storage', 'vm_template'];
+    // lxc_template / vm_template are provider config (intent, not drift-prone) —
+    // read them back from the __infra_* rows persisted above.
+    const infraKeys = ['lxc_template', 'vm_template'];
     for (const key of infraKeys) {
       const infraConfig = db
         .select()