@celilo/cli 0.4.0-alpha.1 → 0.4.0

package/src/services/celilo-events.test.ts

@@ -13,6 +13,10 @@ import {
   emitUninstallCompleted,
   emitUninstallFailed,
   emitUninstallStarted,
+  emitWebRoutesChanged,
+  emitWebRoutesChangedAndWait,
+  routesChangedHighWater,
+  waitForRouteReconcile,
 } from './celilo-events';
 
 describe('celilo lifecycle events', () => {
@@ -165,4 +169,122 @@ describe('celilo lifecycle events', () => {
     process.env.EVENT_BUS_DB = '/proc/no/such/place/events.db';
     expect(() => emitDeployStarted({ module: 'x', startedAt: 0 })).not.toThrow();
   });
+
+  describe('deploy-waits for web-route reconcile (ISS-0035)', () => {
+    function subscribeReconciler(): void {
+      const bus = openBus({ dbPath, events: defineEvents({}) });
+      bus.subscribe({
+        name: 'caddy.reconcile-web-routes',
+        pattern: 'public_web.routes_changed',
+        handler: 'unused',
+      });
+      bus.close();
+    }
+
+    /** Stamp a fresh dispatcher heartbeat so health() reports a live dispatcher. */
+    function markDispatcherAlive(): void {
+      const bus = openBus({ dbPath, events: defineEvents({}) });
+      bus.db.run(
+        `INSERT OR REPLACE INTO dispatcher_heartbeat
+           (dispatcher_id, last_heartbeat, started_at, pid, version)
+         VALUES ('test', ?, ?, 1, '0.1.0')`,
+        [Date.now(), Date.now()],
+      );
+      bus.close();
+    }
+
+    function settleDelivery(how: 'succeed' | 'abandon'): void {
+      const bus = openBus({ dbPath, events: defineEvents({}) });
+      const sub = bus.getSubscriberByName('caddy.reconcile-web-routes');
+      const event = bus.recentEvents({ type: 'public_web.routes_changed', limit: 1 })[0];
+      if (!sub || !event) {
+        throw new Error('expected a subscriber and an emitted event');
+      }
+      if (how === 'succeed') {
+        bus.markSucceeded({ eventId: event.id, subscriberId: sub.id });
+      } else {
+        bus.markFailed({ eventId: event.id, subscriberId: sub.id }, 'boom', { abandoned: true });
+      }
+      bus.close();
+    }
+
+    it('routesChangedHighWater is 0 with no events, then the latest id', () => {
+      expect(routesChangedHighWater()).toBe(0);
+      emitWebRoutesChanged('celilo-website');
+      expect(routesChangedHighWater()).toBeGreaterThan(0);
+    });
+
+    it('returns immediately when the deploy changed no routes', async () => {
+      const since = routesChangedHighWater();
+      const result = await waitForRouteReconcile(since, { timeoutMs: 1000 });
+      expect(result).toEqual({ events: 0, succeeded: 0, failed: 0, timedOut: false });
+    });
+
+    it('skips the wait (no time-out) when no dispatcher is running', async () => {
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      // No heartbeat stamped → health() is no_dispatcher.
+      const result = await waitForRouteReconcile(since, { timeoutMs: 5000, pollMs: 50 });
+      expect(result).toEqual({ events: 1, succeeded: 0, failed: 0, timedOut: false });
+    });
+
+    it('settles immediately when no provider subscribes (zero deliveries)', async () => {
+      markDispatcherAlive();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
+      expect(result).toEqual({ events: 1, succeeded: 0, failed: 0, timedOut: false });
+    });
+
+    it('times out while the provider delivery is still pending', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      const result = await waitForRouteReconcile(since, { timeoutMs: 300, pollMs: 50 });
+      expect(result.events).toBe(1);
+      expect(result.timedOut).toBe(true);
+    });
+
+    it('completes once the provider delivery succeeds', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      settleDelivery('succeed');
+
+      const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
+      expect(result).toEqual({ events: 1, succeeded: 1, failed: 0, timedOut: false });
+    });
+
+    it('reports a failed delivery without timing out', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+      const since = routesChangedHighWater();
+      emitWebRoutesChanged('celilo-website');
+      settleDelivery('abandon');
+
+      const result = await waitForRouteReconcile(since, { timeoutMs: 1000, pollMs: 50 });
+      expect(result.events).toBe(1);
+      expect(result.failed).toBe(1);
+      expect(result.timedOut).toBe(false);
+    });
+
+    it('emitWebRoutesChangedAndWait emits then waits for the reconcile to settle', async () => {
+      markDispatcherAlive();
+      subscribeReconciler();
+
+      // Settle the delivery concurrently, mid-wait, the way the dispatcher would.
+      const pending = emitWebRoutesChangedAndWait('celilo-website', {
+        timeoutMs: 2000,
+        pollMs: 50,
+      });
+      await new Promise((r) => setTimeout(r, 120));
+      settleDelivery('succeed');
+
+      const result = await pending;
+      expect(result).toEqual({ events: 1, succeeded: 1, failed: 0, timedOut: false });
+    });
+  });
 });

package/src/services/celilo-events.ts

@@ -176,6 +176,150 @@ export function emitSystemDestroyed(payload: SystemDestroyedPayload): void {
   emitBest(`system.destroyed.${payload.module}`, payload);
 }
 
+/**
+ * Emit `public_web.routes_changed` (ISS-0035). A coarse "the route table
+ * changed, re-read it" signal: a consumer registered or unregistered a route,
+ * so the public_web PROVIDER (caddy) should reconcile its running config from
+ * web_routes. The provider subscribes via a manifest `subscriptions:` entry.
+ * Best-effort — a failed emit never breaks register_route; the route is already
+ * persisted in web_routes and the provider's next deploy reconciles it anyway.
+ */
+export function emitWebRoutesChanged(triggeredBy: string): void {
+  emitBest('public_web.routes_changed', { triggeredBy });
+}
+
+const ROUTES_CHANGED_TYPE = 'public_web.routes_changed';
+
+/**
+ * Emit `public_web.routes_changed` and block until the provider's reconcile
+ * delivery settles (ISS-0035). This is what `register_route` / `unregister_routes`
+ * await so they return only once the route is actually live — restoring the
+ * synchronous guarantee the old SSH path gave, provider-agnostically. A
+ * consuming module's `health_check` runs right after it registers its route, so
+ * without this wait it races the ~1s async reconcile and sees the placeholder.
+ *
+ * Captures the high-water id BEFORE emitting so the wait targets exactly the
+ * event this call produced. Degrades safely: no dispatcher → returns without
+ * waiting; no provider subscribed → no deliveries → returns immediately.
+ */
+export async function emitWebRoutesChangedAndWait(
+  triggeredBy: string,
+  opts?: { timeoutMs?: number; pollMs?: number },
+): Promise<RouteReconcileWaitResult> {
+  const since = routesChangedHighWater();
+  emitWebRoutesChanged(triggeredBy);
+  return waitForRouteReconcile(since, opts);
+}
+
+/**
+ * Outcome of {@link waitForRouteReconcile}. `events` is how many
+ * `public_web.routes_changed` events the deploy emitted; `succeeded`/`failed`
+ * count their settled deliveries (one per subscribing provider). `timedOut`
+ * is true when the deadline hit with deliveries still pending/running.
+ */
+export interface RouteReconcileWaitResult {
+  events: number;
+  succeeded: number;
+  failed: number;
+  timedOut: boolean;
+}
+
+/**
+ * Highest `public_web.routes_changed` event id at this instant, or 0 if none
+ * exist yet. A deploy captures this before it runs so {@link waitForRouteReconcile}
+ * can tell which route-change events the deploy itself produced.
+ */
+export function routesChangedHighWater(): number {
+  let bus: ReturnType<typeof openBus> | undefined;
+  try {
+    bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
+    return bus.recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 1 })[0]?.id ?? 0;
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn(`[celilo] failed to read routes-changed high-water: ${msg}`);
+    return 0;
+  } finally {
+    bus?.close();
+  }
+}
+
+function sleep(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/**
+ * Block until every `public_web.routes_changed` event emitted after
+ * `sinceEventId` has had all its deliveries settle (succeeded / failed /
+ * abandoned), or until `timeoutMs` elapses. This is what makes
+ * `module deploy` return only once the public_web PROVIDER (caddy) has
+ * reconciled its running config from web_routes — closing the race where
+ * deploy returned while the route was still the placeholder (ISS-0035).
+ *
+ * When the deploy registered no routes, or no provider subscribes to the
+ * event, there are zero deliveries and this returns immediately. With no
+ * dispatcher running, nothing will ever deliver the reconcile, so the wait is
+ * skipped rather than burning the whole deadline (the deploy-time dispatcher
+ * gate, ISS-0042, is the upstream guard). The wait is advisory: it never turns
+ * a bus error into a deploy failure. The caller inspects the result and
+ * decides what to surface.
+ */
+export async function waitForRouteReconcile(
+  sinceEventId: number,
+  opts: { timeoutMs?: number; pollMs?: number } = {},
+): Promise<RouteReconcileWaitResult> {
+  const timeoutMs = opts.timeoutMs ?? 90_000;
+  const pollMs = opts.pollMs ?? 250;
+  const deadline = Date.now() + timeoutMs;
+
+  let bus: ReturnType<typeof openBus> | undefined;
+  try {
+    bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
+    const b = bus;
+
+    // The deploy's emits already happened, so the set of route-change events
+    // is fixed; only their deliveries transition as the dispatcher works.
+    const events = b
+      .recentEvents({ type: ROUTES_CHANGED_TYPE, limit: 100 })
+      .filter((e) => e.id > sinceEventId);
+
+    if (events.length === 0) {
+      return { events: 0, succeeded: 0, failed: 0, timedOut: false };
+    }
+
+    if (b.health().status === 'no_dispatcher') {
+      console.warn(
+        `[celilo] route-reconcile: no dispatcher running — not waiting; ${events.length} change event(s) persisted, the provider will reconcile on its next run`,
+      );
+      return { events: events.length, succeeded: 0, failed: 0, timedOut: false };
+    }
+
+    while (true) {
+      const deliveries = events.flatMap((e) => b.deliveriesForEvent(e.id));
+      const settled = (status: string) => deliveries.filter((d) => d.status === status).length;
+      const pending = deliveries.filter(
+        (d) => d.status === 'pending' || d.status === 'running',
+      ).length;
+
+      if (pending === 0 || Date.now() >= deadline) {
+        return {
+          events: events.length,
+          succeeded: settled('succeeded'),
+          failed: settled('failed') + settled('abandoned'),
+          timedOut: pending > 0,
+        };
+      }
+
+      await sleep(pollMs);
+    }
+  } catch (err) {
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn(`[celilo] route-reconcile wait errored: ${msg}`);
+    return { events: 0, succeeded: 0, failed: 0, timedOut: false };
+  } finally {
+    bus?.close();
+  }
+}
+
 /**
  * Open the bus, emit, close. Errors are caught and logged so a
  * misbehaving bus never wedges the caller. The empty-registry mode

package/src/services/celilo-mgmt-hooks.test.ts

@@ -13,6 +13,7 @@
  * and (eventually) a fresh-box e2e.
  */
 
+import { Database } from 'bun:sqlite';
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
@@ -103,7 +104,14 @@ describe('celilo-mgmt on_backup', () => {
     );
 
     dbPath = join(dir, 'celilo.db');
-    writeFileSync(dbPath, 'fake-sqlite-bytes-for-test');
+    // A REAL (minimal) SQLite DB. on_backup snapshots it via bun:sqlite
+    // serialize(), which requires a valid database file — a plain byte blob
+    // makes the serializer read a garbage page count and OOM. One table with a
+    // row is enough to exercise the snapshot path.
+    const seed = new Database(dbPath);
+    seed.run('CREATE TABLE backup_probe (id INTEGER PRIMARY KEY, v TEXT)');
+    seed.run("INSERT INTO backup_probe (v) VALUES ('hello')");
+    seed.close();
     process.env.CELILO_DB_PATH = dbPath;
 
     keyPath = join(dir, 'master.key');

package/src/services/config-interview.ts

@@ -4,7 +4,7 @@ import type { DbClient } from '../db/client';
 import { moduleConfigs, modules, secrets } from '../db/schema';
 import type { Ensure } from '../manifest/schema';
 import { decryptSecret, encryptSecret } from '../secrets/encryption';
-import { deriveSecret, generateSecret } from '../secrets/generators';
+import { deriveSecret, generateGpgPrivateKey, generateSecret } from '../secrets/generators';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import type { Machine } from '../types/infrastructure';
 import {
@@ -15,7 +15,7 @@ import {
   type EnsureRequiredPayload,
   type SecretAck,
   type SecretRequiredPayload,
-  busInterview,
+  busInterviewGuarded,
 } from './bus-interview';
 import { parseStoredConfigValue, upsertModuleConfig } from './module-config';
 import { getSecretMetadata } from './secret-schema-loader';
@@ -75,7 +75,7 @@ interface SecretDeclareLike {
   key_pattern_message?: string;
   value_pattern?: string;
   value_pattern_message?: string;
-  generate?: { method: string; length: number; encoding: string };
+  generate?: { method: string; length: number; encoding: string; identity?: string };
 }
 
 /**
@@ -106,7 +106,13 @@ function coerceSecretDeclare(entry: unknown): SecretDeclareLike | null {
       typeof g.length === 'number' &&
       typeof g.encoding === 'string'
     ) {
-      out.generate = { method: g.method, length: g.length, encoding: g.encoding };
+      const gen: { method: string; length: number; encoding: string; identity?: string } = {
+        method: g.method as string,
+        length: g.length as number,
+        encoding: g.encoding as string,
+      };
+      if (typeof g.identity === 'string') gen.identity = g.identity;
+      out.generate = gen;
     }
   }
   return out;
@@ -123,7 +129,11 @@ export async function findMissingSecrets(
   for (const raw of manifest.secrets.declares) {
     const secret = coerceSecretDeclare(raw);
     if (!secret) continue;
-    if (!secret.required) continue;
+    // Process a secret if it's required OR has a `generate` directive — an
+    // auto-generated secret (e.g. a GPG signing key) is needed for the module
+    // to function and must never be silently skipped just because it isn't
+    // marked required. Optional, non-generated secrets are still skipped.
+    if (!secret.required && !secret.generate) continue;
 
     const existing = db
       .select()
@@ -171,6 +181,7 @@ export interface MissingVariable {
     method: string;
     length: number;
     encoding: string;
+    identity?: string;
   };
   /** For `type: string-map` only — labels shown in the add-loop prompt. */
   key_label?: string;
@@ -340,7 +351,7 @@ export async function interviewForMissingConfig(
                   required: true,
                   description: followUpPrompt,
                 };
-                const followUpReply = await busInterview<ConfigReply>(
+                const followUpReply = await busInterviewGuarded<ConfigReply>(
                   EVENT_TYPES.configRequired(moduleId, followUpKey),
                   followUpPayload,
                 );
@@ -405,7 +416,7 @@ export async function interviewForMissingConfig(
           description: variable.description,
           options: variable.options,
         };
-        const reply = await busInterview<ConfigReply>(
+        const reply = await busInterviewGuarded<ConfigReply>(
           EVENT_TYPES.configRequired(moduleId, variable.name),
           payload,
         );
@@ -440,7 +451,7 @@ export async function interviewForMissingConfig(
                 required: true,
                 description: followUpPrompt,
               };
-              const followUpReply = await busInterview<ConfigReply>(
+              const followUpReply = await busInterviewGuarded<ConfigReply>(
                 EVENT_TYPES.configRequired(moduleId, followUpKey),
                 followUpPayload,
               );
@@ -466,7 +477,7 @@ export async function interviewForMissingConfig(
           required: true,
           description: variable.description,
         };
-        const reply = await busInterview<ConfigReply>(
+        const reply = await busInterviewGuarded<ConfigReply>(
           EVENT_TYPES.configRequired(moduleId, variable.name),
           payload,
         );
@@ -649,14 +660,19 @@ export async function interviewForMissingSecrets(
 
         log.message(`Derived ${variable.name} from ${metadata.deriveFrom}`);
       } else if (source === 'generated') {
-        // Auto-generate without prompting
-        // Manifest generate field takes priority over schema metadata
-        const format = variable.generate?.encoding || metadata?.format || 'base64';
-        const length = variable.generate?.length || metadata?.length || 32;
-
-        value = generateSecret({ format, length });
-
-        log.message(`Auto-generated ${format} secret: ${variable.name}`);
+        // Auto-generate without prompting. method: gpg mints a real signing
+        // key (celilo owns it as infrastructure); otherwise random bytes.
+        if (variable.generate?.method === 'gpg') {
+          const identity = variable.generate.identity || moduleId;
+          value = generateGpgPrivateKey(identity);
+          log.message(`Auto-generated GPG signing key for ${variable.name} (${identity})`);
+        } else {
+          // Manifest generate field takes priority over schema metadata
+          const format = variable.generate?.encoding || metadata?.format || 'base64';
+          const length = variable.generate?.length || metadata?.length || 32;
+          value = generateSecret({ format, length });
+          log.message(`Auto-generated ${format} secret: ${variable.name}`);
+        }
       } else if (
         source === 'user_provided' ||
         source === 'user_password' ||
@@ -694,7 +710,10 @@ export async function interviewForMissingSecrets(
           value_pattern: variable.value_pattern,
           value_pattern_message: variable.value_pattern_message,
         };
-        await busInterview<SecretAck>(EVENT_TYPES.secretRequired(moduleId, variable.name), payload);
+        await busInterviewGuarded<SecretAck>(
+          EVENT_TYPES.secretRequired(moduleId, variable.name),
+          payload,
+        );
         log.success(`Saved ${variable.name}`);
         configured.push(`${variable.name} (secret)`);
         // Responder already wrote to the encrypted store; skip the
@@ -1071,7 +1090,7 @@ export async function interviewForEnsureInputs(
       objectKey: i.objectKey,
     })),
   };
-  const reply = await busInterview<EnsureReply>(
+  const reply = await busInterviewGuarded<EnsureReply>(
     EVENT_TYPES.ensureRequired(providerModuleId, ensure.id),
     payload,
   );

package/src/services/deploy-planner.test.ts

@@ -0,0 +1,66 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync } from 'node:fs';
+import { rm } from 'node:fs/promises';
+import { type DbClient, createDbClient } from '../db/client';
+import { moduleConfigs, modules } from '../db/schema';
+import { extractTargetHost } from './deploy-planner';
+
+const TEST_DB_PATH = './test-deploy-planner.db';
+
+/**
+ * Regression for the malformed SSH target bug (ISS-0019): extractTargetHost
+ * read module_configs.valueJson RAW. valueJson is JSON-ENCODED — a string is
+ * stored as `"10.0.20.13/24"` (with quotes) — so splitting on '/' kept the
+ * leading quote, producing `root@"10.0.20.13` and a 180s SSH-wait timeout
+ * against a container that was actually up. It must parse valueJson.
+ */
+describe('extractTargetHost', () => {
+  let db: DbClient;
+
+  beforeEach(() => {
+    db = createDbClient({ path: TEST_DB_PATH });
+    db.insert(modules)
+      .values({
+        id: 'apt-repo',
+        name: 'apt-repo',
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: '/tmp/apt-repo',
+      })
+      .run();
+  });
+
+  afterEach(async () => {
+    db.$client.close();
+    for (const suffix of ['', '-shm', '-wal']) {
+      const p = `${TEST_DB_PATH}${suffix}`;
+      if (existsSync(p)) await rm(p);
+    }
+  });
+
+  function setConfig(key: string, value: string): void {
+    db.insert(moduleConfigs)
+      .values({ moduleId: 'apt-repo', key, value, valueJson: JSON.stringify(value) })
+      .run();
+  }
+
+  test('strips the CIDR from a JSON-encoded target_ip without keeping the quote', async () => {
+    setConfig('hostname', 'apt.celilo.computer');
+    setConfig('target_ip', '10.0.20.13/24');
+
+    const host = await extractTargetHost('apt-repo', db);
+    expect(host.ip).toBe('10.0.20.13'); // not `"10.0.20.13`
+    expect(host.ip).not.toContain('"');
+    expect(host.hostname).toBe('apt.celilo.computer');
+    expect(host.user).toBe('root');
+  });
+
+  test('honors ansible_user and falls back through ip.primary', async () => {
+    setConfig('ip.primary', '203.0.113.7');
+    setConfig('ansible_user', 'peba');
+
+    const host = await extractTargetHost('apt-repo', db);
+    expect(host.ip).toBe('203.0.113.7');
+    expect(host.user).toBe('peba');
+  });
+});

package/src/services/deploy-planner.ts

@@ -113,10 +113,24 @@ export async function extractTargetHost(
     .where(eq(moduleConfigs.moduleId, moduleId))
     .all();
 
-  // Build config map
+  // Build config map. valueJson is JSON-ENCODED — a string value is stored as
+  // `"10.0.20.13/24"` (with quotes), so using it raw and then splitting on '/'
+  // kept the leading quote (`"10.0.20.13`), producing the malformed SSH target
+  // `root@"10.0.20.13`. Parse it (the pattern used everywhere else); fall back
+  // to the plain `value` column.
   const configMap = new Map<string, string>();
   for (const config of configs) {
-    const value = config.valueJson || config.value;
+    let value: string | undefined;
+    if (config.valueJson != null) {
+      try {
+        const parsed: unknown = JSON.parse(config.valueJson);
+        value = typeof parsed === 'string' ? parsed : String(parsed);
+      } catch {
+        value = config.value ?? undefined;
+      }
+    } else {
+      value = config.value ?? undefined;
+    }
     if (value) {
       configMap.set(config.key, value);
     }

package/src/services/deploy-preflight.ts

@@ -28,6 +28,7 @@ import { getDb } from '../db/client';
 import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
 import { buildResolutionContext } from '../variables/context';
+import { E2E_CONFLICT_FIX, runningE2eContainers } from './e2e-guard';
 
 export interface PreflightResult {
   success: boolean;
@@ -43,7 +44,8 @@ export interface PreflightError {
     | 'capability-version-mismatch'
     | 'unresolved-template'
     | 'no-infrastructure'
-    | 'missing-secret';
+    | 'missing-secret'
+    | 'e2e-conflict';
   message: string;
   variable?: string;
   suggestion?: string;
@@ -86,6 +88,19 @@ export async function runPreflight(
 
   const manifest = module.manifestData as ModuleManifest;
 
+  // 1b. Environment: live deploys and the e2e simulator are mutually
+  //     exclusive. The deploy refuses this same condition; surfacing it
+  //     here means `--preflight` answers "can I deploy right now?" honestly
+  //     instead of a false green that dies seconds into the real deploy.
+  const e2eContainers = runningE2eContainers();
+  if (e2eContainers.length > 0) {
+    errors.push({
+      category: 'e2e-conflict',
+      message: `e2e test containers are running (${e2eContainers.length}) — live and e2e environments are mutually exclusive`,
+      suggestion: E2E_CONFLICT_FIX,
+    });
+  }
+
   // 2. Required capabilities have providers, and at least one
   //    provider's version is compatible. Multi-provider scenarios
   //    (e.g., zone-scoped dns_registrar with separate internal +
@@ -311,5 +326,7 @@ function errorIcon(category: PreflightError['category']): string {
       return '▢';
     case 'missing-secret':
       return '🔑';
+    case 'e2e-conflict':
+      return '🐳';
   }
 }

package/src/services/deployed-systems.ts

@@ -1,5 +1,5 @@
 import type { DeployedSystem } from '@celilo/capabilities';
-import { and, eq } from 'drizzle-orm';
+import { and, eq, inArray } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import {
   type NetworkZone,
@@ -52,6 +52,35 @@ export function getModuleSystems(moduleId: string, db: DbClient): DeployedSystem
   return rows.map(rowToSystem).sort((a, b) => a.name.localeCompare(b.name));
 }
 
+/**
+ * All container_service systems (Proxmox LXCs, droplets, …) whose zone is in
+ * `zones`, across every module — the LXC complement to machine-pool's
+ * `getSystemsByZone`. Used by the aspect fan-out to reach LXCs, not just
+ * machine-pool boxes (ISS-0028). Deduped by hostname (one LXC = one module = one
+ * row, but defensive). Ordered by hostname for determinism.
+ */
+export function getContainerSystemsInZones(zones: string[], db: DbClient): DeployedSystem[] {
+  if (zones.length === 0) return [];
+  const rows = db
+    .select()
+    .from(moduleSystems)
+    .where(
+      and(
+        inArray(moduleSystems.zone, zones as NetworkZone[]),
+        eq(moduleSystems.infraType, 'container_service'),
+      ),
+    )
+    .all();
+  const seen = new Set<string>();
+  const out: DeployedSystem[] = [];
+  for (const row of rows) {
+    if (seen.has(row.hostname)) continue;
+    seen.add(row.hostname);
+    out.push(rowToSystem(row));
+  }
+  return out.sort((a, b) => a.hostname.localeCompare(b.hostname));
+}
+
 /** Input to {@link upsertDeployedSystem} — the realized facts about one host. */
 export interface DeployedSystemInput {
   /** Stable handle from requires.systems[].name — the per-system key. */