@celilo/cli 0.11.0-alpha.0 → 0.12.0

package/src/services/port-forwards.test.ts

@@ -0,0 +1,92 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { PortForwardStore } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { buildPortForwardStore } from './port-forwards';
+
+const FW = '192.168.0.254';
+
+describe('port-forward store', () => {
+  let dir: string;
+  let db: DbClient;
+  let store: PortForwardStore;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'pf-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    store = buildPortForwardStore(db);
+  });
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('adds and lists forwards for a firewall', () => {
+    store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'caddy' });
+    store.add(FW, {
+      internalIp: '10.0.20.42',
+      port: 2222,
+      protocol: 'TCP',
+      description: 'forgejo',
+    });
+    const forwards = store.list(FW);
+    expect(forwards).toHaveLength(2);
+    expect(forwards.map((f) => f.port).sort((a, b) => a - b)).toEqual([443, 2222]);
+  });
+
+  it('add is an idempotent upsert — same tuple → one row, description updated', () => {
+    store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'v1' });
+    store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'v2' });
+    const forwards = store.list(FW);
+    expect(forwards).toHaveLength(1);
+    expect(forwards[0].description).toBe('v2');
+  });
+
+  it('scopes forwards by firewallIp', () => {
+    store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'a' });
+    store.add('10.0.30.254', {
+      internalIp: '10.0.30.5',
+      port: 53,
+      protocol: 'UDP',
+      description: 'b',
+    });
+    expect(store.list(FW)).toHaveLength(1);
+    expect(store.list('10.0.30.254')).toHaveLength(1);
+  });
+
+  it('remove deletes exactly the tuple and leaves others', () => {
+    store.add(FW, { internalIp: '10.0.20.5', port: 443, protocol: 'TCP', description: 'x' });
+    store.add(FW, { internalIp: '10.0.20.5', port: 80, protocol: 'TCP', description: 'y' });
+    store.remove(FW, '10.0.20.5', 443, 'TCP');
+    const forwards = store.list(FW);
+    expect(forwards).toHaveLength(1);
+    expect(forwards[0].port).toBe(80);
+  });
+
+  it('ingressIp forwards are distinct from the public (NULL) ones', () => {
+    store.add(FW, { internalIp: '10.0.20.5', port: 53, protocol: 'UDP', description: 'public' });
+    store.add(FW, {
+      internalIp: '10.0.20.5',
+      port: 53,
+      protocol: 'UDP',
+      ingressIp: '10.0.10.53',
+      description: 'ingress',
+    });
+    expect(store.list(FW)).toHaveLength(2);
+    // removing the public (NULL-ingress) one leaves the ingress one intact
+    store.remove(FW, '10.0.20.5', 53, 'UDP');
+    const forwards = store.list(FW);
+    expect(forwards).toHaveLength(1);
+    expect(forwards[0].ingressIp).toBe('10.0.10.53');
+  });
+});

package/src/services/port-forwards.ts

@@ -0,0 +1,86 @@
+/**
+ * Port-forward registry — the DB-backed `PortForwardStore` (openspec/changes/unified-management-no-ssh/proposal.md).
+ *
+ * The shared-core desired-state store for the `firewall` capability. The
+ * capability-loader constructs one of these and injects it into the firewall
+ * provider factory, so `exposeService`/`unexposeService` become register/
+ * unregister and the provider's converge renders the complete ruleset from
+ * `list(firewallIp)` and applies it atomically (`iptables-restore`). Replaces
+ * "read the box back with `iptables -L`" as the source of truth.
+ */
+
+import type { PortForward, PortForwardStore, Protocol } from '@celilo/capabilities';
+import { and, eq, isNull } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { portForwards } from '../db/schema';
+
+/** NULL-aware match on the optional ingress IP (SQLite treats NULL as distinct). */
+function ingressMatch(ingressIp: string | undefined) {
+  return ingressIp ? eq(portForwards.ingressIp, ingressIp) : isNull(portForwards.ingressIp);
+}
+
+export function buildPortForwardStore(db: DbClient): PortForwardStore {
+  return {
+    list(firewallIp: string): PortForward[] {
+      return db
+        .select()
+        .from(portForwards)
+        .where(eq(portForwards.firewallIp, firewallIp))
+        .all()
+        .map((r) => ({
+          internalIp: r.internalIp,
+          port: r.port,
+          protocol: r.protocol,
+          ingressIp: r.ingressIp ?? undefined,
+          description: r.description,
+        }));
+    },
+
+    add(firewallIp: string, forward: PortForward): void {
+      // Idempotent upsert: delete any existing row for this exact tuple, then
+      // insert. Matches buildRouteOps' delete-then-insert (keeps the operation
+      // atomic w.r.t. the unique index, and NULL-aware on ingress_ip).
+      db.delete(portForwards)
+        .where(
+          and(
+            eq(portForwards.firewallIp, firewallIp),
+            eq(portForwards.internalIp, forward.internalIp),
+            eq(portForwards.port, forward.port),
+            eq(portForwards.protocol, forward.protocol),
+            ingressMatch(forward.ingressIp),
+          ),
+        )
+        .run();
+      db.insert(portForwards)
+        .values({
+          firewallIp,
+          internalIp: forward.internalIp,
+          port: forward.port,
+          protocol: forward.protocol,
+          ingressIp: forward.ingressIp ?? null,
+          description: forward.description,
+        })
+        .run();
+    },
+
+    remove(
+      firewallIp: string,
+      internalIp: string,
+      port: number,
+      protocol: Protocol,
+      ingressIp?: string,
+    ): void {
+      db.delete(portForwards)
+        .where(
+          and(
+            eq(portForwards.firewallIp, firewallIp),
+            eq(portForwards.internalIp, internalIp),
+            eq(portForwards.port, port),
+            eq(portForwards.protocol, protocol),
+            ingressMatch(ingressIp),
+          ),
+        )
+        .run();
+    },
+  };
+}

package/src/services/proxmox-reconcile.ts

@@ -1,5 +1,5 @@
 /**
- * Proxmox reconciliation (v2/CELILO_BASE.md D5).
+ * Proxmox reconciliation (openspec/specs/base-module-aspects/spec.md D5).
  *
  * When a base-module aspect declares `proxmox_reconcile.tfvars` and
  * its fan-out plan touches Proxmox-provisioned LXCs, the running
@@ -151,6 +151,6 @@ export function executeProxmoxReconcile(plan: ProxmoxReconcilePlan): void {
     );
   }
   log.warn(
-    'To persist these values, re-deploy the owning modules after the underlying celilo state catches up. Tracked as Phase 1c in v2/CELILO_BASE.md.',
+    'To persist these values, re-deploy the owning modules after the underlying celilo state catches up. Tracked as Phase 1c in openspec/specs/base-module-aspects/spec.md.',
   );
 }

package/src/services/public-web-republish.test.ts

@@ -0,0 +1,189 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DbClient } from '../db/client';
+import { modules, webRoutes } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import { providesPublicWeb, republishStaticWebConsumers } from './public-web-republish';
+
+const caddyManifest = {
+  provides: { capabilities: [{ name: 'public_web', version: '3.0.0' }] },
+} as unknown as ModuleManifest;
+
+const plainManifest = { provides: { capabilities: [] } } as unknown as ModuleManifest;
+
+describe('republishStaticWebConsumers', () => {
+  let dir: string;
+  let db: DbClient;
+
+  function seedModule(id: string) {
+    db.insert(modules)
+      .values({ id, name: id, version: '1.0.0', manifestData: {}, sourcePath: `/tmp/${id}` })
+      .run();
+  }
+
+  function seedRoute(
+    moduleId: string,
+    slug: string,
+    path: string,
+    type: 'static' | 'reverse_proxy',
+  ) {
+    db.insert(webRoutes)
+      .values({
+        slug,
+        moduleId,
+        type,
+        path,
+        hostname: 'iamtheinternet.org',
+        targetHost: type === 'reverse_proxy' ? '10.0.20.5' : null,
+        targetPort: type === 'reverse_proxy' ? 8080 : null,
+      })
+      .run();
+  }
+
+  function recordingHook(failFor: string[] = []) {
+    const calls: string[] = [];
+    const runHook = (async (moduleId: string) => {
+      calls.push(moduleId);
+      return failFor.includes(moduleId)
+        ? { success: false, outputs: {}, error: 'boom', duration: 1 }
+        : { success: true, outputs: {}, duration: 1 };
+    }) as unknown as Parameters<typeof republishStaticWebConsumers>[0]['runHook'];
+    return { runHook, calls };
+  }
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'pwr-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  test('providesPublicWeb reads the manifest', () => {
+    expect(providesPublicWeb(caddyManifest)).toBe(true);
+    expect(providesPublicWeb(plainManifest)).toBe(false);
+  });
+
+  test('does not run for a module that is not a public_web provider', async () => {
+    seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    const { runHook, calls } = recordingHook();
+
+    const result = await republishStaticWebConsumers({
+      providerModuleId: 'namecheap',
+      manifest: plainManifest,
+      db,
+      runHook,
+    });
+
+    expect(result).toEqual({ ran: false, republished: [], failures: [] });
+    expect(calls).toEqual([]);
+  });
+
+  test('re-runs on_install for every static consumer', async () => {
+    seedModule('caddy');
+    seedModule('hello-foo');
+    seedModule('hello-bar');
+    seedModule('celilo-website');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    seedRoute('hello-bar', 'bar', '/bar', 'static');
+    seedRoute('celilo-website', 'celilo-website', '/', 'static');
+    const { runHook, calls } = recordingHook();
+
+    const result = await republishStaticWebConsumers({
+      providerModuleId: 'caddy',
+      manifest: caddyManifest,
+      db,
+      runHook,
+    });
+
+    expect(result.ran).toBe(true);
+    expect(calls.sort()).toEqual(['celilo-website', 'hello-bar', 'hello-foo']);
+    expect(result.republished.sort()).toEqual(['celilo-website', 'hello-bar', 'hello-foo']);
+    expect(result.failures).toEqual([]);
+  });
+
+  test('skips reverse-proxy-only consumers (no assets on the provider host)', async () => {
+    seedModule('caddy');
+    seedModule('forgejo');
+    seedModule('hello-foo');
+    seedRoute('forgejo', 'git', '/git', 'reverse_proxy');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    const { runHook, calls } = recordingHook();
+
+    await republishStaticWebConsumers({
+      providerModuleId: 'caddy',
+      manifest: caddyManifest,
+      db,
+      runHook,
+    });
+
+    expect(calls).toEqual(['hello-foo']);
+  });
+
+  test('never re-runs the provider itself', async () => {
+    seedModule('caddy');
+    // A provider that also owns a static route (a landing page of its own).
+    seedRoute('caddy', 'caddy', '/', 'static');
+    const { runHook, calls } = recordingHook();
+
+    const result = await republishStaticWebConsumers({
+      providerModuleId: 'caddy',
+      manifest: caddyManifest,
+      db,
+      runHook,
+    });
+
+    expect(calls).toEqual([]);
+    expect(result.republished).toEqual([]);
+  });
+
+  test('de-duplicates a consumer that owns several static routes', async () => {
+    seedModule('caddy');
+    seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    seedRoute('hello-foo', 'foo-docs', '/foo/docs', 'static');
+    const { runHook, calls } = recordingHook();
+
+    await republishStaticWebConsumers({
+      providerModuleId: 'caddy',
+      manifest: caddyManifest,
+      db,
+      runHook,
+    });
+
+    expect(calls).toEqual(['hello-foo']);
+  });
+
+  test('one consumer failing does not stop the others', async () => {
+    seedModule('caddy');
+    seedModule('hello-foo');
+    seedModule('hello-bar');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    seedRoute('hello-bar', 'bar', '/bar', 'static');
+    const { runHook, calls } = recordingHook(['hello-foo']);
+
+    const result = await republishStaticWebConsumers({
+      providerModuleId: 'caddy',
+      manifest: caddyManifest,
+      db,
+      runHook,
+    });
+
+    expect(calls.sort()).toEqual(['hello-bar', 'hello-foo']);
+    expect(result.republished).toEqual(['hello-bar']);
+    expect(result.failures).toEqual([{ moduleId: 'hello-foo', error: 'boom' }]);
+  });
+});

package/src/services/public-web-republish.ts

@@ -0,0 +1,84 @@
+/**
+ * Restore consumer content after a public_web provider redeploy.
+ *
+ * A provider redeploy destroys the provider's host and rebuilds it, taking
+ * `/srv/www` with it. The consumers' `web_routes` rows survive — they belong to
+ * the consumer modules, not the provider — so the rebuilt provider renders
+ * `root * /srv/www/<slug>` for directories that no longer exist and every
+ * static site 404s. Nothing re-uploaded the assets; an operator had to redeploy
+ * each consumer by hand and there was no signal telling them to.
+ *
+ * The fix is to do exactly what that operator did: after a provider deploy
+ * lands, re-run `on_install` for every module holding a STATIC route.
+ * `publishStaticSite` is idempotent (it upserts the route and re-tars the
+ * source dir), so the re-run re-registers the route and refills the fresh
+ * `/srv/www/<slug>`.
+ *
+ * Reverse-proxy consumers are skipped: they keep no state on the provider host,
+ * so re-running them buys nothing and only widens the blast radius.
+ *
+ * ponytail: re-runs every static consumer unconditionally rather than probing
+ * which /srv/www/<slug> directories are actually missing — O(static consumers)
+ * per provider deploy, currently a handful. If a large fleet makes provider
+ * deploys slow, probe the provider host first and re-run only the misses.
+ */
+
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { webRoutes } from '../db/schema';
+import { createConsoleLogger } from '../hooks/logger';
+import { runNamedHook } from '../hooks/run-named-hook';
+import type { ModuleManifest } from '../manifest/schema';
+
+export interface RepublishFailure {
+  moduleId: string;
+  error: string;
+}
+
+export interface RepublishResult {
+  /** False when the deployed module isn't a public_web provider. */
+  ran: boolean;
+  /** Consumer modules whose on_install was re-run successfully. */
+  republished: string[];
+  failures: RepublishFailure[];
+}
+
+export function providesPublicWeb(manifest: ModuleManifest): boolean {
+  return (manifest.provides?.capabilities ?? []).some((c) => c.name === 'public_web');
+}
+
+export interface RepublishDeps {
+  providerModuleId: string;
+  manifest: ModuleManifest;
+  db: DbClient;
+  /** Injectable for tests. */
+  runHook?: typeof runNamedHook;
+}
+
+export async function republishStaticWebConsumers(deps: RepublishDeps): Promise<RepublishResult> {
+  const { providerModuleId, manifest, db, runHook = runNamedHook } = deps;
+
+  if (!providesPublicWeb(manifest)) {
+    return { ran: false, republished: [], failures: [] };
+  }
+
+  const staticRoutes = db.select().from(webRoutes).where(eq(webRoutes.type, 'static')).all();
+  const consumerIds = [
+    ...new Set(staticRoutes.map((r) => r.moduleId).filter((id) => id !== providerModuleId)),
+  ];
+
+  const republished: string[] = [];
+  const failures: RepublishFailure[] = [];
+
+  for (const moduleId of consumerIds) {
+    const logger = createConsoleLogger(moduleId, 'on_install');
+    const result = await runHook(moduleId, 'on_install', db, logger, {});
+    if (result.success) {
+      republished.push(moduleId);
+    } else {
+      failures.push({ moduleId, error: result.error ?? 'unknown error' });
+    }
+  }
+
+  return { ran: true, republished, failures };
+}

package/src/services/restore-from-file.ts

@@ -1,6 +1,6 @@
 /**
  * File-direct restore + system-file swap. Powers the `celilo restore`
- * top-level convenience command (Phase 4 of v2/SYSTEM_BACKUP_TERRAFORM_STATE.md).
+ * top-level convenience command (Phase 4 of openspec/specs/management-server-backup/spec.md).
  *
  * The generic `restoreModuleBackup` reads from a configured storage
  * destination — which on a fresh-bootstrap box doesn't exist yet.

package/src/services/restore-preflight.ts

@@ -13,7 +13,7 @@
  * Either signal is sufficient. Both are checked so a half-deployed
  * target (DB cleared but disk artifacts left over) still gets caught.
  *
- * Phase 4 of v2/SYSTEM_BACKUP_TERRAFORM_STATE.md (Design Decision 9).
+ * Phase 4 of openspec/specs/management-server-backup/spec.md (Design Decision 9).
  */
 
 import { existsSync, readdirSync, statSync } from 'node:fs';

package/src/services/system-identity.ts

@@ -16,7 +16,7 @@ export interface SystemIdentity {
 
 /**
  * Get a module's hostname and IP from its recorded deployed systems
- * (v2/MODULE_SYSTEMS_ADDRESSING.md). Returns null for a module with no host
+ * (openspec/specs/module-systems-addressing/spec.md). Returns null for a module with no host
  * (config-only providers like namecheap). Single-system modules have one; this
  * returns the first (Phase C iterates all systems for per-system events).
  */

package/src/services/system-init.test.ts

@@ -29,7 +29,7 @@ describe('System Init Service', () => {
       // a zone address).
       expect(defaults['network.bridge']).toBe('vmbr0');
       // Network/DNS addressing is no longer defaulted
-      // (v2/NETWORK_CONFIG_TO_FIREWALL.md): internal is discovered at
+      // (openspec/specs/progressive-zone-disclosure/spec.md): internal is discovered at
       // celilo-mgmt install; dmz/app/secure appear only when a firewall
       // module provides them.
       expect(defaults['network.dmz.subnet']).toBeUndefined();
@@ -184,7 +184,7 @@ describe('System Init Service', () => {
     test('keys off the sentinel, not config rows', async () => {
       // Config rows present but no sentinel → NOT initialized. Initialization
       // is tracked by the explicit `system.initialized` marker
-      // (v2/NETWORK_CONFIG_TO_FIREWALL.md), not by any network row.
+      // (openspec/specs/progressive-zone-disclosure/spec.md), not by any network row.
       db.insert(systemConfig).values({ key: 'network.dmz.subnet', value: '10.0.0.0/24' }).run();
       db.insert(systemConfig).values({ key: 'ssh.public_key', value: 'ssh-rsa AAAA...' }).run();
       expect(isSystemInitialized(db)).toBe(false);

package/src/services/system-init.ts

@@ -199,7 +199,7 @@ export function initializeSystem(
   }
 
   // Explicit init marker. Network/DNS addressing is no longer seeded with
-  // defaults (v2/NETWORK_CONFIG_TO_FIREWALL.md), so we can't key
+  // defaults (openspec/specs/progressive-zone-disclosure/spec.md), so we can't key
   // "is this system initialized?" off a network row anymore. Write a
   // dedicated sentinel instead. additionalProperties:true in the schema
   // permits this non-schema key.
@@ -238,7 +238,7 @@ export function loadExistingConfiguration(db: DbClient): Record<string, string>
  * Keys off the explicit `system.initialized` sentinel written by
  * initializeSystem(). Earlier this checked for a seeded network zone
  * subnet, but network/DNS addressing is no longer defaulted
- * (v2/NETWORK_CONFIG_TO_FIREWALL.md), so a fresh init leaves those
+ * (openspec/specs/progressive-zone-disclosure/spec.md), so a fresh init leaves those
  * rows absent.
  *
  * @param db - Database instance