@celilo/cli 0.11.0-alpha.0 → 0.12.0

package/src/services/web-route-cleanup.test.ts

@@ -0,0 +1,250 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { RunResult, Runner } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { capabilities, moduleConfigs, modules, webRoutes } from '../db/schema';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import type { RouteReconcileWaitResult } from './celilo-events';
+import { cleanupWebRoutesForModule } from './web-route-cleanup';
+
+const CADDY_IP = '10.0.10.10';
+
+const CLEAN_RECONCILE: RouteReconcileWaitResult = {
+  events: 1,
+  succeeded: 1,
+  failed: 0,
+  timedOut: false,
+  noDispatcher: false,
+};
+
+function createRecordingRunner(result: Partial<RunResult> = {}): Runner & { calls: string[] } {
+  const calls: string[] = [];
+  const runner = ((cmd: string) => {
+    calls.push(cmd);
+    return { ok: true, stdout: '', stderr: '', ...result } as RunResult;
+  }) as Runner & { calls: string[] };
+  runner.calls = calls;
+  return runner;
+}
+
+describe('cleanupWebRoutesForModule', () => {
+  let dir: string;
+  let db: DbClient;
+
+  async function seedModule(id: string): Promise<void> {
+    db.insert(modules)
+      .values({
+        id,
+        name: id,
+        version: '1.0.0',
+        manifestData: {},
+        sourcePath: `/tmp/${id}`,
+      })
+      .run();
+  }
+
+  async function seedCaddyProvider(): Promise<void> {
+    await seedModule('caddy');
+    db.insert(capabilities)
+      .values({ moduleId: 'caddy', capabilityName: 'public_web', version: '3.0.0', data: {} })
+      .run();
+    db.insert(moduleConfigs)
+      .values({
+        moduleId: 'caddy',
+        key: 'target_ip',
+        value: `${CADDY_IP}/24`,
+        valueJson: JSON.stringify(`${CADDY_IP}/24`),
+      })
+      .run();
+  }
+
+  function seedRoute(
+    moduleId: string,
+    slug: string,
+    path: string,
+    type: 'static' | 'reverse_proxy',
+  ) {
+    db.insert(webRoutes)
+      .values({
+        slug,
+        moduleId,
+        type,
+        path,
+        hostname: 'iamtheinternet.org',
+        targetHost: type === 'reverse_proxy' ? '10.0.20.5' : null,
+        targetPort: type === 'reverse_proxy' ? 8080 : null,
+      })
+      .run();
+  }
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'wrc-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+  });
+
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  test('deletes routes, reclaims the asset root, and emits the reconcile', async () => {
+    await seedCaddyProvider();
+    await seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+
+    const run = createRecordingRunner();
+    let emittedFor: string | undefined;
+
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'hello-foo',
+      db,
+      run,
+      emitAndWait: async (triggeredBy) => {
+        emittedFor = triggeredBy;
+        return CLEAN_RECONCILE;
+      },
+    });
+
+    expect(result.routesRemoved).toBe(1);
+    expect(result.assetDirsRemoved).toEqual(['/srv/www/foo']);
+    expect(result.warnings).toEqual([]);
+    expect(emittedFor).toBe('hello-foo');
+
+    expect(run.calls).toHaveLength(1);
+    expect(run.calls[0]).toContain(`root@${CADDY_IP}`);
+    expect(run.calls[0]).toContain('rm -rf');
+    expect(run.calls[0]).toContain('/srv/www/foo');
+
+    const remaining = db.select().from(webRoutes).where(eq(webRoutes.moduleId, 'hello-foo')).all();
+    expect(remaining).toHaveLength(0);
+  });
+
+  test('leaves other modules routes alone', async () => {
+    await seedCaddyProvider();
+    await seedModule('hello-foo');
+    await seedModule('hello-bar');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+    seedRoute('hello-bar', 'bar', '/bar', 'static');
+
+    const run = createRecordingRunner();
+    await cleanupWebRoutesForModule({
+      moduleId: 'hello-foo',
+      db,
+      run,
+      emitAndWait: async () => CLEAN_RECONCILE,
+    });
+
+    const survivors = db.select().from(webRoutes).all();
+    expect(survivors.map((r) => r.moduleId)).toEqual(['hello-bar']);
+    // Only /srv/www/foo is touched — the sibling's asset root must survive.
+    expect(run.calls.join('\n')).not.toContain('/srv/www/bar');
+  });
+
+  test('no-ops when the module owns no routes (its on_uninstall already ran)', async () => {
+    await seedCaddyProvider();
+    await seedModule('celilo-website');
+
+    const run = createRecordingRunner();
+    let emitted = false;
+
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'celilo-website',
+      db,
+      run,
+      emitAndWait: async () => {
+        emitted = true;
+        return CLEAN_RECONCILE;
+      },
+    });
+
+    expect(result).toEqual({ routesRemoved: 0, assetDirsRemoved: [], warnings: [] });
+    expect(run.calls).toHaveLength(0);
+    // No redundant reconcile on every module removal.
+    expect(emitted).toBe(false);
+  });
+
+  test('skips the remote rm for reverse-proxy routes (no assets on the host)', async () => {
+    await seedCaddyProvider();
+    await seedModule('forgejo');
+    seedRoute('forgejo', 'forgejo', '/git', 'reverse_proxy');
+
+    const run = createRecordingRunner();
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'forgejo',
+      db,
+      run,
+      emitAndWait: async () => CLEAN_RECONCILE,
+    });
+
+    expect(result.routesRemoved).toBe(1);
+    expect(run.calls).toHaveLength(0);
+  });
+
+  test('still deletes routes when the asset rm fails', async () => {
+    await seedCaddyProvider();
+    await seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+
+    const run = createRecordingRunner({ ok: false, stderr: 'host unreachable' });
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'hello-foo',
+      db,
+      run,
+      emitAndWait: async () => CLEAN_RECONCILE,
+    });
+
+    expect(result.routesRemoved).toBe(1);
+    expect(result.assetDirsRemoved).toEqual([]);
+    expect(result.warnings.join(' ')).toMatch(/Failed to remove \/srv\/www\/foo/);
+    expect(db.select().from(webRoutes).all()).toHaveLength(0);
+  });
+
+  test('warns when no provider is deployed to clean assets from', async () => {
+    await seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+
+    const run = createRecordingRunner();
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'hello-foo',
+      db,
+      run,
+      emitAndWait: async () => CLEAN_RECONCILE,
+    });
+
+    expect(result.routesRemoved).toBe(1);
+    expect(run.calls).toHaveLength(0);
+    expect(result.warnings.join(' ')).toMatch(/No public_web provider host found/);
+  });
+
+  test('warns — but does not throw — when the provider never reconciles', async () => {
+    await seedCaddyProvider();
+    await seedModule('hello-foo');
+    seedRoute('hello-foo', 'foo', '/foo', 'static');
+
+    const result = await cleanupWebRoutesForModule({
+      moduleId: 'hello-foo',
+      db,
+      run: createRecordingRunner(),
+      emitAndWait: async () => ({
+        events: 1,
+        succeeded: 0,
+        failed: 0,
+        timedOut: false,
+        noDispatcher: true,
+      }),
+    });
+
+    expect(result.routesRemoved).toBe(1);
+    expect(result.warnings.join(' ')).toMatch(/no event dispatcher is running/);
+  });
+});

package/src/services/web-route-cleanup.ts

@@ -0,0 +1,144 @@
+/**
+ * Route cleanup on module removal — the consumer half of ISS-0035.
+ *
+ * `register_route` / `unregister_routes` emit `public_web.routes_changed`, and
+ * the provider's `reconcile_routes` subscription re-renders its config from
+ * `web_routes`. But `unregister_routes` is only reachable from a module's own
+ * `on_uninstall` teardown script, and most consumers don't define one —
+ * celilo-website, celilo-registry, celilo-apt-repo and celilo-mgmt all publish
+ * a site and have no teardown. For those, `module remove` deleted the route
+ * rows through the FK cascade on `modules.id` and emitted NOTHING, so the
+ * provider never learned the route was gone: the rendered Caddyfile kept a site
+ * block for a module that no longer exists, and `/srv/www/<slug>` was orphaned.
+ *
+ * The fix lives here rather than in each module's teardown script for the usual
+ * reason: one call site in `performModuleRemove` covers every consumer and
+ * can't be forgotten by the next one. Modules that DO define an `on_uninstall`
+ * calling `unregister_routes` still win — their hook runs first and this
+ * becomes a no-op (zero owned routes).
+ *
+ * Everything here is best-effort. A failed asset `rm` or an undelivered
+ * reconcile is reported as a warning, never a thrown error: a stuck provider is
+ * no reason to strand a module in a half-removed state.
+ */
+
+import { execRunner, runAppCommand, shellEscape } from '@celilo/capabilities';
+import type { Runner } from '@celilo/capabilities';
+import { eq } from 'drizzle-orm';
+import type { DbClient } from '../db/client';
+import { capabilities, webRoutes } from '../db/schema';
+import { loadHookConfigMap } from '../hooks/load-hook-config';
+import { emitWebRoutesChangedAndWait } from './celilo-events';
+
+export interface WebRouteCleanupResult {
+  routesRemoved: number;
+  /** Asset roots actually deleted from the provider host. */
+  assetDirsRemoved: string[];
+  /** Non-fatal problems, for the caller to log. Never thrown. */
+  warnings: string[];
+}
+
+export interface WebRouteCleanupDeps {
+  moduleId: string;
+  db: DbClient;
+  /** Injectable remote runner — tests pass a mock. */
+  run?: Runner;
+  /** Injectable event emit + wait — tests pass a stub. */
+  emitAndWait?: typeof emitWebRoutesChangedAndWait;
+}
+
+/**
+ * The provider host that owns `/srv/www` — the same `config.target_ip` the
+ * public_web capability uses for its asset uploads.
+ */
+async function resolvePublicWebHostIp(db: DbClient): Promise<string | undefined> {
+  const providers = db
+    .select()
+    .from(capabilities)
+    .where(eq(capabilities.capabilityName, 'public_web'))
+    .all();
+  const provider = providers[0];
+  if (!provider) return undefined;
+
+  const config = await loadHookConfigMap(provider.moduleId, db);
+  // target_ip may carry a CIDR suffix; the capability strips it the same way.
+  return String(config.target_ip ?? '').split('/')[0] || undefined;
+}
+
+/**
+ * Delete a module's web routes, reclaim its static asset roots, and tell the
+ * provider to reconcile. Returns counts + warnings; never throws.
+ */
+export async function cleanupWebRoutesForModule(
+  deps: WebRouteCleanupDeps,
+): Promise<WebRouteCleanupResult> {
+  const { moduleId, db, run = execRunner, emitAndWait = emitWebRoutesChangedAndWait } = deps;
+
+  const owned = db.select().from(webRoutes).where(eq(webRoutes.moduleId, moduleId)).all();
+  if (owned.length === 0) {
+    // Either the module never registered a route, or its own on_uninstall
+    // already called unregister_routes (which emitted and reconciled). Nothing
+    // to do — and importantly, no event, so we don't trigger a redundant
+    // reconcile on every module removal.
+    return { routesRemoved: 0, assetDirsRemoved: [], warnings: [] };
+  }
+
+  const warnings: string[] = [];
+  const assetDirsRemoved: string[] = [];
+
+  const staticSlugs = owned.filter((r) => r.type === 'static').map((r) => r.slug);
+  if (staticSlugs.length > 0) {
+    const providerIp = await resolvePublicWebHostIp(db);
+    if (!providerIp) {
+      warnings.push(
+        `No public_web provider host found — left ${staticSlugs.length} asset dir(s) in /srv/www on disk`,
+      );
+    } else {
+      for (const slug of staticSlugs) {
+        const dir = `/srv/www/${slug}`;
+        // escape-hatch: reclaiming a consumer's static web root on the provider
+        // host. No capability owns asset-root teardown, and it is neither
+        // desired-state nor a service action.
+        const result = runAppCommand(
+          { ipv4_address: providerIp },
+          `rm -rf ${shellEscape(dir)}`,
+          run,
+          { timeoutMs: 30_000 },
+        );
+        if (result.ok) {
+          assetDirsRemoved.push(dir);
+        } else {
+          warnings.push(
+            `Failed to remove ${dir} on ${providerIp} (continuing): ${result.stderr || result.stdout || 'unknown'}`,
+          );
+        }
+      }
+    }
+  }
+
+  db.delete(webRoutes).where(eq(webRoutes.moduleId, moduleId)).run();
+
+  // Same signal a consumer's own unregister_routes emits. Unlike the deploy
+  // path (ISS-0081), an undelivered reconcile here is a warning, not a
+  // failure — removal must complete regardless.
+  const reconcile = await emitAndWait(moduleId);
+  if (reconcile.noDispatcher) {
+    warnings.push(
+      `Routes for ${moduleId} were deleted but no event dispatcher is running, so the public_web provider has not reconciled — it is still serving the removed route(s) until its next reconcile.`,
+    );
+  } else if (reconcile.timedOut) {
+    warnings.push(
+      `public_web provider did not confirm the route removal within the deadline (${reconcile.succeeded} ok, ${reconcile.failed} failed of ${reconcile.events} event(s)).`,
+    );
+  } else if (reconcile.failed > 0) {
+    warnings.push(
+      `public_web provider failed to apply the route removal (${reconcile.failed} delivery failure(s)) — it may still be serving the removed route(s).`,
+    );
+  } else if (reconcile.events > 0 && reconcile.succeeded === 0) {
+    warnings.push(
+      `Routes for ${moduleId} were deleted but NO provider reconciled — the provider has no reconcile_routes subscription, so the removed route(s) may still be served.`,
+    );
+  }
+
+  return { routesRemoved: owned.length, assetDirsRemoved, warnings };
+}

package/src/templates/generator.ts

@@ -158,7 +158,7 @@ export function getOutputFilename(templateFilename: string): string {
  *     a no-op and a real change an in-place UPDATE — never the destructive
  *     REPLACE the terraform-safety guard (create-only, see terraform-safety.ts)
  *     rejects. terraform = birth DNS, the `dns-client-config` aspect = ongoing
- *     DNS. The per-type lists are in the callback. See v2/LXC_INTERNAL_DNS.md.
+ *     DNS. The per-type lists are in the callback. See openspec/specs/lxc-dns-at-birth/spec.md.
  *
  * Anchored on the resource's opening line — it never brace-matches the nested
  * disk/network/features blocks, so it's robust to attribute order/formatting.
@@ -736,7 +736,7 @@ export async function generateTemplates(options: GenerateOptions): Promise<Gener
 
   // Dedicated DNS-ingress IP (ISS-0156). A dns_internal provider now deploys into
   // a PROTECTED zone (dmz) so it can see protected-zone query sources for
-  // split-horizon views (v2/INTERNAL_DNS_ZONE_VIEWS.md). `internal` devices have
+  // split-horizon views (openspec/specs/internal-dns-zone-views/spec.md). `internal` devices have
   // no route into the 10-net, so they reach the resolver through a firewall DNAT
   // on a dedicated `internal`-subnet address. A module opts in by declaring a
   // `dns_ingress_ip` infrastructure variable; we allocate a free IP from the

package/src/types/infrastructure.ts

@@ -96,7 +96,7 @@ export interface Machine {
   /**
    * Appliance / API-only machines (e.g., greenwave) where celilo has
    * no shell access. Base-module aspects skip these — see
-   * v2/CELILO_BASE.md D8. Optional on the type because most call
+   * openspec/specs/base-module-aspects/spec.md D8. Optional on the type because most call
    * sites don't care; the DB column defaults to false so the
    * effective value is well-defined regardless.
    */

package/src/variables/computed/computed-integration.test.ts

@@ -159,7 +159,7 @@ describe('computed capability fields — DB integration', () => {
     if (!result.success) expect(result.error).toMatch(/could not be resolved|Failed to evaluate/);
   });
 
-  // Gap B (v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md step 3): buildResolutionContext
+  // Gap B (openspec/specs/internal-dns-split-horizon/spec.md step 3): buildResolutionContext
   // must EAGERLY evaluate computed markers into the capabilities map, so the
   // `variables.imports` path (which reads context.capabilities directly, not via
   // resolveVariable) sees the real array.

package/src/variables/computed/evaluate.ts

@@ -8,7 +8,7 @@
  * integration (a later step) decide how refs map onto live data.
  *
  * The function allow-list is the v1 set from
- * v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md (D1.1):
+ * openspec/specs/internal-dns-split-horizon/spec.md (D1.1):
  *   keys, values, map, concat, unique, format
  *
  * Secret-projection rule: `values(secret.*)` is rejected — it would leak

package/src/variables/computed/parse.ts

@@ -1,7 +1,7 @@
 /**
  * Parser for the computed-variable DSL.
  *
- * Computed variables (v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md, D1) are
+ * Computed variables (openspec/specs/internal-dns-split-horizon/spec.md, D1) are
  * declared in a manifest with `type: computed` and a `value:` expression
  * derived on access from other values, e.g.:
  *

package/src/variables/context.test.ts

@@ -21,7 +21,7 @@ const TEST_DB_PATH = './test-context.db';
 /**
  * Select a proxmox container_service for a module + give it a hostname, so the
  * deployed-system recording in buildResolutionContext fires (IPAM allocates and
- * records into module_systems). v2/MODULE_SYSTEMS_ADDRESSING.md.
+ * records into module_systems). openspec/specs/module-systems-addressing/spec.md.
  */
 function setupProxmoxInfra(db: DbClient, moduleId: string, zone: string): void {
   const serviceId = `svc-${moduleId}`;

package/src/variables/context.ts

@@ -148,7 +148,7 @@ function autoDeriveInventoryVariables(
  * its deployed systems. `$self:` supports the optional `[N]` array-index suffix
  * (e.g. `$self:domains[0]`). `$infra:` references a provider's deployed system
  * by name (the replacement for the old `$self:target_ip` — see
- * v2/MODULE_SYSTEMS_ADDRESSING.md). Non-string values and strings that don't
+ * openspec/specs/module-systems-addressing/spec.md). Non-string values and strings that don't
  * start with one of those prefixes are returned unchanged.
  */
 function resolveSelfRefsInObject(
@@ -323,7 +323,7 @@ export async function buildResolutionContext(
     }
   }
 
-  // Deployed-system recording (v2/MODULE_SYSTEMS_ADDRESSING.md).
+  // Deployed-system recording (openspec/specs/module-systems-addressing/spec.md).
   // Record this module's system(s) into module_systems at generate time, so
   // `$infra:<name>.…` resolves in templates. The address comes from:
   //   - machine pool      → the machine's own IP; no vmid (no container).
@@ -468,7 +468,7 @@ export async function buildResolutionContext(
       providerSystems,
     );
 
-    // Evaluate computed-field markers (v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md
+    // Evaluate computed-field markers (openspec/specs/internal-dns-split-horizon/spec.md
     // D1) in the PROVIDER's context, so consumers — both `$capability:` template
     // refs AND `variables.imports` — see the real value (e.g. domain_list as an
     // array) rather than the raw marker. Only build the provider lookup when a
@@ -498,7 +498,7 @@ export async function buildResolutionContext(
     systemConfigMap[row.key] = row.value;
   }
 
-  // LXC nameserver list (v2/LXC_INTERNAL_DNS.md). Proxmox's `nameserver` is a
+  // LXC nameserver list (openspec/specs/lxc-dns-at-birth/spec.md). Proxmox's `nameserver` is a
   // space-separated primary/secondary list. Compose it so every celilo-built
   // LXC boots with a working resolver — before Ansible's apt update, and
   // independent of which Proxmox node it lands on (the nodes' DNS defaults
@@ -566,7 +566,7 @@ export async function buildResolutionContext(
   // Build the `$infra:<name>.<field>` lookup from the systems recorded above
   // (proxmox/machine at generate; DigitalOcean gets refreshed at deploy from
   // terraform outputs). This is what lets `$infra:` resolve in templates.
-  // v2/MODULE_SYSTEMS_ADDRESSING.md.
+  // openspec/specs/module-systems-addressing/spec.md.
   let infraSystemsMap: Record<string, import('./types').InfraSystemFields> = {};
   if (module?.manifestData) {
     const { buildInfraSystemsMap } = await import('../services/deployed-systems');

package/src/variables/lxc-nameserver.test.ts

@@ -9,7 +9,7 @@ const TEST_DB_PATH = './test-lxc-nameserver.db';
 
 /**
  * Coverage for the generate-time `lxc_nameserver` composition
- * (v2/LXC_INTERNAL_DNS.md): proxmox_lxc templates read `$self:lxc_nameserver`,
+ * (openspec/specs/lxc-dns-at-birth/spec.md): proxmox_lxc templates read `$self:lxc_nameserver`,
  * a space-separated primary/secondary list = internal dns_internal resolver
  * first (CIDR stripped), then the public dns.primary/dns.fallback resolvers;
  * public-only when no dns_internal provider is registered (bootstrap).

package/src/variables/resolver.ts

@@ -75,7 +75,7 @@ export async function resolveVariable(
 
     case 'infra': {
       // Format: $infra:<system-name>.<field> — references a deployed system by
-      // its stable handle (requires.systems[].name). v2/MODULE_SYSTEMS_ADDRESSING.md.
+      // its stable handle (requires.systems[].name). openspec/specs/module-systems-addressing/spec.md.
       const dot = variable.path.indexOf('.');
       if (dot === -1) {
         return {
@@ -208,7 +208,7 @@ export async function resolveVariable(
         };
       }
 
-      // Computed field (v2/INTERNAL_DNS_DHCP_AND_SPLIT_HORIZON.md D1):
+      // Computed field (openspec/specs/internal-dns-split-horizon/spec.md D1):
       // evaluate the DSL expression in the PROVIDER's context. Like the
       // lazy $self: block below, we look up the provider module, but here
       // we build a typed lookup over its config/secrets/system data.

package/src/variables/types.ts

@@ -24,7 +24,7 @@ export interface ResolutionContext {
   capabilities: Record<string, Record<string, unknown>>;
   /**
    * The module's deployed systems, keyed by name, for `$infra:<name>.<field>`
-   * resolution at generate time (v2/MODULE_SYSTEMS_ADDRESSING.md). Each value
+   * resolution at generate time (openspec/specs/module-systems-addressing/spec.md). Each value
    * exposes the snake_cased fields a template can reference: `ipv4_address`,
    * `hostname`, `zone`, `vmid`, `cidr`. Optional only so the many test-stub
    * `ResolutionContext` literals need no churn — production (buildResolutionContext)