@celilo/cli 3.1.0 → 4.1.0

package/src/services/deploy-validation.test.ts

@@ -19,11 +19,15 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { type DbClient, getDb } from '../db/client';
-import { modules, secrets } from '../db/schema';
+import { capabilities, modules, secrets } from '../db/schema';
 import type { ModuleManifest } from '../manifest/schema';
 import { resetTestDbPath } from '../test-utils/db-path';
 import { findMissingSecrets } from './config-interview';
-import { findMissingBuildArtifacts, findMissingRequiredVariables } from './deploy-validation';
+import {
+  findMissingBuildArtifacts,
+  findMissingRequiredVariables,
+  validateAndPrepareDeployment,
+} from './deploy-validation';
 
 function manifestWithStringMapSecret(): ModuleManifest {
   return {
@@ -366,3 +370,78 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
     expect(findMissingBuildArtifacts(manifest, dir)).toEqual(['dist/index.html', 'dist/db.sqlite']);
   });
 });
+
+/**
+ * celilo#1384. An OPTIONAL capability is optional in its PRESENCE, not in its
+ * contract: with a provider installed the module's hooks call it, so a
+ * provider on another major fails at runtime — and `on_install` failing is
+ * only a warning, so it fails silently. Measured: generic-cpanel 0.4.1+2
+ * (external_web 2.0.0) against tango-nexus 0.8.0 (optional external_web
+ * 1.0.0); nothing published, the site kept serving, nothing said so.
+ */
+describe('validateAndPrepareDeployment (optional capability version gate)', () => {
+  let tempDir: string;
+  let db: DbClient;
+
+  function seed(optionalVersion: string, providerVersion: string | null) {
+    db.insert(modules)
+      .values({
+        id: 'consumer',
+        name: 'Consumer',
+        sourcePath: tempDir,
+        version: '1.0.0',
+        manifestData: {
+          celilo_contract: '1.0',
+          id: 'consumer',
+          name: 'Consumer',
+          version: '1.0.0',
+          description: 'fixture',
+          optional: { capabilities: [{ name: 'external_web', version: optionalVersion }] },
+        },
+      })
+      .run();
+    if (providerVersion === null) return;
+    db.insert(modules)
+      .values({
+        id: 'provider',
+        name: 'Provider',
+        sourcePath: tempDir,
+        version: '1.0.0',
+        manifestData: {},
+      })
+      .run();
+    db.insert(capabilities)
+      .values({
+        moduleId: 'provider',
+        capabilityName: 'external_web',
+        version: providerVersion,
+        data: {},
+      })
+      .run();
+  }
+
+  beforeEach(() => {
+    tempDir = mkdtempSync(join(tmpdir(), 'celilo-optional-cap-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+    db = getDb();
+  });
+
+  afterEach(() => {
+    rmSync(tempDir, { recursive: true, force: true });
+    resetTestDbPath();
+  });
+
+  test('refuses the deploy when the installed provider is another major', async () => {
+    seed('1.0.0', '2.0.0');
+    const result = await validateAndPrepareDeployment('consumer', db);
+    expect(result.success).toBe(false);
+    expect(result.error).toContain('external_web');
+    expect(result.error).toContain('provider');
+  });
+
+  test('does not refuse when nobody provides the optional capability', async () => {
+    seed('1.0.0', null);
+    const result = await validateAndPrepareDeployment('consumer', db);
+    expect(result.error).toBeUndefined();
+  });
+});

package/src/services/deploy-validation.ts

@@ -113,6 +113,25 @@ export async function validateAndPrepareDeployment(
     }
   }
 
+  // An OPTIONAL capability is optional in its PRESENCE, not in its contract.
+  // When a provider is installed the module's hooks will call it, so a provider
+  // on another major breaks at runtime exactly as a required one would — except
+  // an `on_install` failure is only a warning, so it breaks silently
+  // (celilo#1384). No provider installed is still fine: the version check below
+  // skips a capability nobody provides.
+  if (manifest.optional?.capabilities) {
+    const optionalMismatches = await findCapabilityVersionMismatches(
+      manifest.optional.capabilities,
+      db,
+    );
+    if (optionalMismatches.length > 0) {
+      return {
+        success: false,
+        error: formatVersionMismatchError(moduleId, optionalMismatches),
+      };
+    }
+  }
+
   // Check for missing required variables BEFORE generating — if any are missing
   // we return them for the caller to interview, then the caller re-invokes deploy
   // after the user has answered. This prevents generation from failing mid-way

package/src/services/fleet-checks.ts

@@ -19,7 +19,7 @@
 
 import { readFileSync } from 'node:fs';
 import { join } from 'node:path';
-import { type Bus, describeError } from '@celilo/event-bus';
+import { type Bus, type BusHealth, describeError } from '@celilo/event-bus';
 import { desc, eq, inArray } from 'drizzle-orm';
 import { parse as parseYaml } from 'yaml';
 import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
@@ -203,6 +203,12 @@ export interface DispatcherCheckOptions {
    * Injected so the crash-loop aspect is testable without launchd.
    */
   launchdProbe?: (scope: SupervisorScope) => LaunchdUnitStatus | null;
+  /**
+   * A verdict already taken, normally `await bus.healthConfirmed()`. Callers
+   * that can await pass it so a load-starved dispatcher reads `lagging` rather
+   * than the `wedged` a single sample cannot distinguish it from (celilo#1399).
+   */
+  health?: BusHealth;
 }
 
 /**
@@ -214,7 +220,7 @@ export interface DispatcherCheckOptions {
  */
 export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): FleetFinding {
   const now = opts.now ?? Date.now();
-  const health = bus.health();
+  const health = opts.health ?? bus.health();
   const hb = bus.db
     .query<HeartbeatRow, []>(
       'SELECT dispatcher_id, last_heartbeat, started_at, pid, version FROM dispatcher_heartbeat ORDER BY last_heartbeat DESC LIMIT 1',
@@ -227,14 +233,25 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
 
   // (1) running — a fresh heartbeat. health() already classifies a
   // stale/absent heartbeat as no_dispatcher.
-  if (health.status === 'no_dispatcher' || !hb) {
+  if (health.status === 'no_dispatcher' || health.status === 'wedged' || !hb) {
     statuses.push('fail');
-    detail.push(
-      'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
-    );
-    remediations.push(
-      'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
-    );
+    if (health.status === 'wedged') {
+      // Registered, process alive, heartbeat stale, nothing running: it is present
+      // but dead, so "start the dispatcher" is the wrong instruction (celilo#1398).
+      detail.push(
+        `dispatcher WEDGED — pid ${hb?.pid ?? '?'} is alive and registered but its heartbeat is ${Math.round((health.lastHeartbeatAgeMs ?? 0) / 1000)}s old with nothing running (events are queueing, not delivered)`,
+      );
+      remediations.push(
+        health.remedy ?? 'restart the dispatcher: `celilo events restart-daemon --system`',
+      );
+    } else {
+      detail.push(
+        'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
+      );
+      remediations.push(
+        'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
+      );
+    }
     // A unit can be INSTALLED and still dead: the supervisor respawns a
     // program that cannot start in the unit's environment and it dies again,
     // forever. On macOS the classic cause is a PATH-less
@@ -274,6 +291,15 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
   detail.push(
     `running (pid ${hb.pid}, heartbeat ${Math.round(ageMs / 1000)}s ago, code v${hb.version})`,
   );
+  if (health.status === 'lagging') {
+    // Confirmed alive by a second beat: slow, not dead. Worth saying — an operator
+    // reading a 60s heartbeat wants to know it was checked — but not a failure,
+    // and explicitly not a restart (celilo#1399).
+    statuses.push('warn');
+    detail.push(
+      'heartbeat is stale but still advancing — the dispatcher is alive and starved for CPU, not wedged; it recovers when load drops',
+    );
+  }
   if (health.status === 'stuck') {
     statuses.push('warn');
     detail.push(
@@ -1395,7 +1421,11 @@ export async function runFleetChecks(
   const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
   return [
     checkSchemaDrift(db),
-    checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
+    checkDispatcher(bus, {
+      now: opts.now,
+      installedCodeMtimeMs: opts.installedCodeMtimeMs,
+      health: await bus.healthConfirmed(),
+    }),
     checkBuildBusPublishing(bus, db, { now: opts.now }),
     checkSubscribers(bus, db),
     checkCapabilityProviders(db),

package/src/services/machine-interface-zones.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, test } from 'bun:test';
+import type { NetworkZone } from '../db/schema';
+import {
+  applyReclassification,
+  diffInterfaceZones,
+  explainMissingZoneInterface,
+} from './machine-interface-zones';
+
+/** The zone model as it stands today: 10.0.120.0/24 is secure-mgmt. */
+const detect = async (ip: string): Promise<NetworkZone | 'unknown'> => {
+  if (ip.startsWith('10.0.120.')) return 'secure-mgmt';
+  if (ip.startsWith('10.0.10.')) return 'dmz';
+  return 'unknown';
+};
+
+/**
+ * celilo-mgr as celilo#1382 found it: added before `secure-mgmt` existed,
+ * while the detector labelled unmatched subnets `external`.
+ */
+const celiloMgr = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' }];
+
+describe('diffInterfaceZones', () => {
+  test('reports a label the current zone model disagrees with', async () => {
+    expect(await diffInterfaceZones(celiloMgr, detect)).toEqual([
+      { name: 'eth0', ipAddress: '10.0.120.10', from: 'external', to: 'secure-mgmt' },
+    ]);
+  });
+
+  test('reports nothing when the stored labels already agree', async () => {
+    const current = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' }];
+    expect(await diffInterfaceZones(current, detect)).toEqual([]);
+  });
+
+  test('reports only the interfaces that changed', async () => {
+    const mixed = [
+      { name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' },
+      { name: 'eth1', ipAddress: '10.0.10.5', zone: 'dmz' },
+    ];
+    const changes = await diffInterfaceZones(mixed, detect);
+    expect(changes.map((c) => c.name)).toEqual(['eth0']);
+  });
+});
+
+describe('applyReclassification', () => {
+  test('writes the new label and leaves the input untouched', async () => {
+    const changes = await diffInterfaceZones(celiloMgr, detect);
+    expect(applyReclassification(celiloMgr, changes)).toEqual([
+      { name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' },
+    ]);
+    expect(celiloMgr[0].zone).toBe('external');
+  });
+});
+
+describe('explainMissingZoneInterface', () => {
+  test('says MISLABELLED, and names the interface, when one would match', async () => {
+    const changes = await diffInterfaceZones(celiloMgr, detect);
+    const message = explainMissingZoneInterface(celiloMgr, 'secure-mgmt', changes);
+    expect(message).toContain('MISLABELLED');
+    expect(message).toContain('eth0');
+    expect(message).toContain('10.0.120.10');
+    expect(message).toContain('machine reclassify');
+  });
+
+  test('says the interface is absent when no relabel would produce one', async () => {
+    const dmzOnly = [{ name: 'eth0', ipAddress: '10.0.10.5', zone: 'dmz' }];
+    const changes = await diffInterfaceZones(dmzOnly, detect);
+    const message = explainMissingZoneInterface(dmzOnly, 'secure-mgmt', changes);
+    expect(message).toContain("has no interface in zone 'secure-mgmt'");
+    expect(message).not.toContain('MISLABELLED');
+    // Names what it DOES hold, so the operator can see why.
+    expect(message).toContain('10.0.10.5');
+  });
+});

package/src/services/machine-interface-zones.ts

@@ -0,0 +1,101 @@
+/**
+ * Re-classify a machine's STORED interface zone labels.
+ *
+ * Interface zones are decided once, at `machine add`, and never revisited
+ * (`machine-add.ts` / `machine-detector.ts` are the only writers). That is fine
+ * until the zone model itself changes — a new zone is added, or a subnet is
+ * declared for one that had none. Every machine added before that change keeps
+ * the label the old model produced, and nothing anywhere says so.
+ *
+ * Measured (celilo#1382): celilo-mgr was added before `secure-mgmt` existed,
+ * while the detector labelled unmatched subnets `external`. Its eth0
+ * (10.0.120.10) was stored as zone `external`. `resolveMachineIdentityAddress`
+ * matches on the stored label, so celilo-mgmt's upgrade failed with "no
+ * non-loopback interface in zone 'secure-mgmt'" — the interface existed, it was
+ * mislabelled. The fix was a hand-edit of the live DB, which is the shape of
+ * problem this module exists to remove.
+ *
+ * The diff is pure over an injected detector so it is testable without a DB.
+ */
+
+import type { NetworkZone } from '../db/schema';
+
+/**
+ * A stored interface row. Deliberately looser than `NetworkInterface`: the
+ * `machines.interfaces` column types `zone` as a plain string, and a stale
+ * label is exactly the case where it may not be a current `NetworkZone`.
+ */
+export interface StoredInterface {
+  name: string;
+  ipAddress: string;
+  zone: string;
+}
+
+/** One interface whose stored label disagrees with the current zone model. */
+export interface ZoneReclassification {
+  name: string;
+  ipAddress: string;
+  /** The label on disk. */
+  from: string;
+  /** What the current zone model says. */
+  to: NetworkZone | 'unknown';
+}
+
+export type ZoneDetector = (ip: string) => Promise<NetworkZone | 'unknown'>;
+
+/**
+ * Which of `interfaces` the current zone model labels differently.
+ *
+ * Returns only the CHANGES: an empty array means the stored labels already
+ * agree with the model, which is what an operator needs to hear.
+ */
+export async function diffInterfaceZones(
+  interfaces: readonly StoredInterface[],
+  detect: ZoneDetector,
+): Promise<ZoneReclassification[]> {
+  const changes: ZoneReclassification[] = [];
+  for (const iface of interfaces) {
+    const to = await detect(iface.ipAddress);
+    if (to !== iface.zone) {
+      changes.push({ name: iface.name, ipAddress: iface.ipAddress, from: iface.zone, to });
+    }
+  }
+  return changes;
+}
+
+/** `interfaces` with the changes applied. Does not mutate the input. */
+export function applyReclassification(
+  interfaces: readonly StoredInterface[],
+  changes: readonly ZoneReclassification[],
+): StoredInterface[] {
+  const byName = new Map(changes.map((c) => [c.name, c]));
+  return interfaces.map((iface) => {
+    const change = byName.get(iface.name);
+    return change ? { ...iface, zone: change.to } : { ...iface };
+  });
+}
+
+/**
+ * Why `resolveMachineIdentityAddress` found nothing for `zone`.
+ *
+ * Distinguishes the two cases that previously read identically. "No interface
+ * in zone X" is true of a machine that genuinely has none AND of one whose
+ * interface is sitting right there under a stale label — and the second is the
+ * common case after a zone-model change, where the remediation is a
+ * re-classification rather than new hardware.
+ */
+export function explainMissingZoneInterface(
+  interfaces: readonly StoredInterface[],
+  zone: NetworkZone,
+  changes: readonly ZoneReclassification[],
+): string {
+  const wouldMatch = changes.filter((c) => c.to === zone);
+  if (wouldMatch.length === 0) {
+    const held = interfaces.map((i) => `${i.name} (${i.ipAddress}, zone ${i.zone})`).join(', ');
+    return `has no interface in zone '${zone}'${held ? ` — it holds ${held}` : ' (no interfaces recorded)'}`;
+  }
+  const named = wouldMatch
+    .map((c) => `${c.name} (${c.ipAddress}, stored as '${c.from}')`)
+    .join(', ');
+  return `has an interface in zone '${zone}' that is MISLABELLED: ${named}. The stored label predates the current zone model. Fix it with: celilo machine reclassify <hostname> --apply`;
+}

package/src/services/module-deploy.stale-provider-guard.test.ts

@@ -0,0 +1,98 @@
+import { describe, expect, it } from 'bun:test';
+import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { ModuleManifest } from '../manifest/schema';
+import { staleProviderProjectRefusal } from './module-deploy';
+import { providerConfigVarsPath } from './provider-converge';
+
+// The converge copy of this guard has had a refusal test since celilo#1387
+// (provider-converge.test.ts). The module-deploy copy had none, so nobody had
+// watched it fail — and this guard has already failed in both directions:
+// celilo#1248 refused every provider, celilo#1387 read its own output and
+// refused nothing. Both silent. celilo#1408.
+
+function manifest(caps: string[]): ModuleManifest {
+  return {
+    provides: { capabilities: caps.map((name) => ({ name })) },
+  } as unknown as ModuleManifest;
+}
+
+async function withGenerated(run: (generatedPath: string) => Promise<void>) {
+  const dir = mkdtempSync(join(tmpdir(), 'celilo-stale-guard-'));
+  try {
+    await run(join(dir, 'generated'));
+  } finally {
+    rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+function plantStaleTree(generatedPath: string) {
+  const roles = join(generatedPath, 'ansible', 'roles');
+  mkdirSync(roles, { recursive: true });
+  writeFileSync(join(roles, 'main.yml'), '- name: does not read the new vars\n', 'utf-8');
+}
+
+describe('staleProviderProjectRefusal — the deploy-side guard, watched failing', () => {
+  it('refuses a public_web provider whose generated role predates the converge', async () => {
+    await withGenerated(async (generatedPath) => {
+      plantStaleTree(generatedPath);
+
+      const error = await staleProviderProjectRefusal(
+        'caddy',
+        manifest(['public_web']),
+        generatedPath,
+      );
+
+      expect(error).toContain('predates the provider converge');
+      expect(error).toContain('celilo module upgrade caddy');
+    });
+  });
+
+  it('does not reach its own output: nothing is written, so the scan cannot self-satisfy', async () => {
+    await withGenerated(async (generatedPath) => {
+      plantStaleTree(generatedPath);
+
+      await staleProviderProjectRefusal('caddy', manifest(['public_web']), generatedPath);
+
+      // celilo#1387's bug was write-order: the vars file lands inside the
+      // scanned tree and carries the needle, so a guard that runs after the
+      // write always passes. This guard writes nothing at all.
+      expect(existsSync(providerConfigVarsPath(generatedPath))).toBe(false);
+    });
+  });
+
+  it('passes a tree whose role reads provider_config_files', async () => {
+    await withGenerated(async (generatedPath) => {
+      const roles = join(generatedPath, 'ansible', 'roles');
+      mkdirSync(roles, { recursive: true });
+      writeFileSync(
+        join(roles, 'main.yml'),
+        '- name: place\n  loop: "{{ provider_config_files }}"\n',
+        'utf-8',
+      );
+
+      expect(
+        await staleProviderProjectRefusal('caddy', manifest(['public_web']), generatedPath),
+      ).toBe(undefined);
+    });
+  });
+
+  it('leaves a module that provides no public_web alone, stale tree or not', async () => {
+    await withGenerated(async (generatedPath) => {
+      plantStaleTree(generatedPath);
+
+      expect(
+        await staleProviderProjectRefusal('homebridge', manifest(['dns_internal']), generatedPath),
+      ).toBe(undefined);
+    });
+  });
+
+  it('passes when there is no generated ansible tree to judge', async () => {
+    await withGenerated(async (generatedPath) => {
+      expect(
+        await staleProviderProjectRefusal('caddy', manifest(['public_web']), generatedPath),
+      ).toBe(undefined);
+    });
+  });
+});

package/src/services/module-deploy.ts

@@ -628,6 +628,34 @@ export async function repointDnsPrimaryForProvider(
   );
 }
 
+/**
+ * The refusal above, as its own function so it can be watched failing.
+ *
+ * Returns the operator-facing error when `moduleId` is a public_web provider
+ * whose GENERATED project predates the converge role tasks, and undefined
+ * otherwise. Extracted for celilo#1408: the deploy path that calls it is long
+ * enough that no test had ever reached it, and this guard has already failed in
+ * both directions elsewhere (celilo#1248 refused everything, celilo#1387 read
+ * its own output and refused nothing). Both are silent. Rule 7.6.
+ *
+ * It only READS the tree — nothing is written before or by it, which is what
+ * keeps it from finding its own output the way #1387's copy did.
+ */
+export async function staleProviderProjectRefusal(
+  moduleId: string,
+  manifest: ModuleManifest,
+  generatedPath: string,
+): Promise<string | undefined> {
+  if (!manifest.provides?.capabilities?.some((cap) => cap.name === 'public_web')) return undefined;
+
+  const { ansibleTreeMentions } = await import('./provider-converge');
+  const ansiblePath = join(generatedPath, 'ansible');
+  if (!existsSync(ansiblePath)) return undefined;
+  if (ansibleTreeMentions(ansiblePath, 'provider_config_files')) return undefined;
+
+  return `The installed version of '${moduleId}' predates the provider converge: its role never reads provider_config_files, so celilo would deploy it and then have no way to place the config or the sites it renders. Upgrade it first (\`celilo module upgrade ${moduleId}\`), then deploy.`;
+}
+
 async function deployModuleImpl(
   moduleId: string,
   db: DbClient,
@@ -1474,19 +1502,26 @@ async function deployModuleImpl(
       }
     }
 
-    // The provider's own deploy runs the static-content converge alongside
-    // everything else (design D10, task 4.5): the release set is written into
-    // the generated inventory, the playbook's static_content-tagged tasks
-    // converge /srv/www, and a rebuilt host recovers with no consumer
-    // involvement. Same plan + writer the publish-time converge uses — one
-    // code path, two callers.
-    if (manifest.provides?.capabilities?.some((cap) => cap.name === 'public_web')) {
-      const { writeStaticContentVars } = await import('./static-content-converge');
-      const staticVars = await writeStaticContentVars(db, moduleId, generatedPath);
-      if (!staticVars.success) {
-        return { success: false, error: staticVars.error, phases };
-      }
-      log.success('Static-content release set written to the inventory');
+    // A public_web provider's deploy no longer plans the fleet's static content
+    // here. Core used to walk every declared route row, resolve every module's
+    // built site, and throw for the whole deploy when any one of them was
+    // missing — celilo#1383, where two out-of-tree modules made the public
+    // ingress undeployable. The provider now renders its own desired state in
+    // its hook and hands it to the converge, which runs the same role tasks
+    // (providers-converge-declared-state, design D4/D6). on_install runs
+    // immediately after this play, so /srv/www still converges on a rebuilt
+    // host — through one path instead of two.
+    //
+    // A provider whose role predates that converge would deploy cleanly and
+    // then never converge its static content again: core no longer writes the
+    // release set, and the old role has no task that reads what the provider
+    // rendered. Nothing would report it — the deploy succeeds, the publish
+    // succeeds, and /srv/www silently stops tracking what is declared. So the
+    // version skew is refused here, by the same tree scan the converge itself
+    // uses, and the message names the upgrade that fixes it.
+    const staleRefusal = await staleProviderProjectRefusal(moduleId, manifest, generatedPath);
+    if (staleRefusal) {
+      return { success: false, error: staleRefusal, phases };
     }
 
     try {