@celilo/cli 0.13.3 → 0.14.1

package/src/services/health-runner.ts

@@ -8,12 +8,12 @@
 import { eq } from 'drizzle-orm';
 import { FuelGauge } from '../cli/fuel-gauge';
 import type { DbClient } from '../db/client';
-import { modules } from '../db/schema';
+import { type ModuleState, modules } from '../db/schema';
 import { loadCapabilityFunctions } from '../hooks/capability-loader';
 import { invokeHook } from '../hooks/executor';
 import { loadHookConfigMap } from '../hooks/load-hook-config';
 import { createCapturingLogger, createConsoleLogger, createGaugeLogger } from '../hooks/logger';
-import type { HookResult } from '../hooks/types';
+import type { HookLogger, HookResult } from '../hooks/types';
 import type { ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
@@ -33,21 +33,108 @@ export interface HealthCheckResult {
   error?: string;
 }
 
+export interface HealthCheckOptions {
+  debug?: boolean;
+  noInteractive?: boolean;
+  onProgress?: (msg: string) => void;
+  /**
+   * Scheduled/monitor-driven run: no FuelGauge, no stdout, and — the part
+   * that matters beyond cosmetics — the module's lifecycle state is NOT
+   * touched. See `createHealthCheckPresentation` and the state-transition
+   * guard below.
+   */
+  unattended?: boolean;
+}
+
+/**
+ * How a health-check run reports itself while it runs.
+ *
+ * The four modes differ ONLY in which logger they build and whether a
+ * gauge needs stopping afterwards, so they are resolved here rather than
+ * branching around four copies of the `invokeHook` call (Rule 10.1 —
+ * presentation is not execution).
+ */
+interface HealthCheckPresentation {
+  logger: HookLogger;
+  /** Called once the hook returns; stops the gauge when there is one. */
+  finish(success: boolean): void;
+}
+
+const NO_OP_FINISH = (): void => {};
+
+function createHealthCheckPresentation(
+  moduleId: string,
+  options: HealthCheckOptions,
+): HealthCheckPresentation {
+  if (options.debug) {
+    return { logger: createConsoleLogger(moduleId, 'health_check'), finish: NO_OP_FINISH };
+  }
+
+  // Scheduled run: capture hook output rather than letting it reach a
+  // terminal nobody is watching.
+  if (options.unattended) {
+    return { logger: createCapturingLogger().logger, finish: NO_OP_FINISH };
+  }
+
+  // TUI mode: emit a progress message instead of drawing FuelGauge
+  // (which writes to stdout and would corrupt the alt-screen render).
+  // Use a capturing logger to absorb hook-level log lines that would
+  // otherwise leak to stdout via createConsoleLogger.
+  if (options.onProgress) {
+    options.onProgress(`Checking ${moduleId}`);
+    return { logger: createCapturingLogger().logger, finish: NO_OP_FINISH };
+  }
+
+  const gauge = new FuelGauge('Testing app', { skipAnimation: options.noInteractive });
+  gauge.start();
+  return {
+    logger: createGaugeLogger(gauge, moduleId, 'health_check'),
+    finish: (success) => gauge.stop(success),
+  };
+}
+
+/**
+ * Decide whether a health-check result should move a module's lifecycle
+ * state, returning the new state or `null` to leave it alone.
+ *
+ * A SCHEDULED run observes only. `VERIFIED` asserts that a check was
+ * deliberately run and passed; letting a recurring unattended check drive it
+ * would silently change what the state means — from "someone verified this"
+ * to "it was up N minutes ago" — and would churn it on every transient
+ * failure, damping that the alert lifecycle already provides and module state
+ * does not. Continuously observed health is reported through alerts and the
+ * `celilo module list` health column instead.
+ *
+ * Pure on purpose: this rule is the one thing in the scheduling work that
+ * regresses invisibly (a wrong boolean here just makes state flicker), so it
+ * is testable without a database or a hook subprocess.
+ *
+ * See openspec/changes/add-alerting/design.md D15.
+ */
+export function nextModuleState(
+  current: ModuleState,
+  status: 'healthy' | 'degraded' | 'unhealthy',
+  unattended: boolean,
+): ModuleState | null {
+  if (unattended) return null;
+  if (status === 'healthy' || status === 'degraded') return 'VERIFIED';
+  return current === 'VERIFIED' ? 'INSTALLED' : null;
+}
+
 /**
  * Run health checks for a single module
  *
  * `onProgress` (when supplied) replaces the FuelGauge animation. Used
  * by the audit TUI to stream progress messages into its own UI rather
  * than letting them leak to stdout before the alt-screen takes over.
+ *
+ * `unattended` marks a monitor-driven run: silent, and non-mutating with
+ * respect to module state.
  */
 export async function runModuleHealthCheck(
   moduleId: string,
   db: DbClient,
-  options: {
-    debug?: boolean;
-    noInteractive?: boolean;
-    onProgress?: (msg: string) => void;
-  } = {},
+  options: HealthCheckOptions = {},
 ): Promise<HealthCheckResult> {
   const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
   if (!module) {
@@ -83,75 +170,25 @@ export async function runModuleHealthCheck(
   // Run the hook. Logger is constructed BEFORE loadCapabilityFunctions
   // so the auto-logging wrapper (HOOK_API_V2 D6) can capture it for
   // every capability call.
-  let hookResult: HookResult;
-  if (options.debug) {
-    const logger = createConsoleLogger(moduleId, 'health_check');
-    const capabilityFunctions = await loadCapabilityFunctions(moduleId, db, logger);
-    hookResult = await invokeHook(
-      module.sourcePath,
-      'health_check',
-      manifest.celilo_contract,
-      hookDef,
-      {},
-      configMap,
-      secretMap,
-      logger,
-      {
-        debug: true,
-        capabilities: capabilityFunctions,
-        requiredCapabilities,
-        systems: getModuleSystems(moduleId, db),
-      },
-    );
-  } else if (options.onProgress) {
-    // TUI mode: emit a progress message instead of drawing FuelGauge
-    // (which writes to stdout and would corrupt the alt-screen render).
-    // Use a capturing logger to absorb hook-level log lines that would
-    // otherwise leak to stdout via createConsoleLogger.
-    options.onProgress(`Checking ${moduleId}`);
-    const { logger } = createCapturingLogger();
-    const capabilityFunctions = await loadCapabilityFunctions(moduleId, db, logger);
-    hookResult = await invokeHook(
-      module.sourcePath,
-      'health_check',
-      manifest.celilo_contract,
-      hookDef,
-      {},
-      configMap,
-      secretMap,
-      logger,
-      {
-        debug: false,
-        capabilities: capabilityFunctions,
-        requiredCapabilities,
-        systems: getModuleSystems(moduleId, db),
-      },
-    );
-  } else {
-    const gauge = new FuelGauge('Testing app', {
-      skipAnimation: options.noInteractive,
-    });
-    gauge.start();
-    const logger = createGaugeLogger(gauge, moduleId, 'health_check');
-    const capabilityFunctions = await loadCapabilityFunctions(moduleId, db, logger);
-    hookResult = await invokeHook(
-      module.sourcePath,
-      'health_check',
-      manifest.celilo_contract,
-      hookDef,
-      {},
-      configMap,
-      secretMap,
-      logger,
-      {
-        debug: false,
-        capabilities: capabilityFunctions,
-        requiredCapabilities,
-        systems: getModuleSystems(moduleId, db),
-      },
-    );
-    gauge.stop(hookResult.success);
-  }
+  const presentation = createHealthCheckPresentation(moduleId, options);
+  const capabilityFunctions = await loadCapabilityFunctions(moduleId, db, presentation.logger);
+  const hookResult: HookResult = await invokeHook(
+    module.sourcePath,
+    'health_check',
+    manifest.celilo_contract,
+    hookDef,
+    {},
+    configMap,
+    secretMap,
+    presentation.logger,
+    {
+      debug: options.debug ?? false,
+      capabilities: capabilityFunctions,
+      requiredCapabilities,
+      systems: getModuleSystems(moduleId, db),
+    },
+  );
+  presentation.finish(hookResult.success);
 
   if (!hookResult.success) {
     return {
@@ -178,12 +215,9 @@ export async function runModuleHealthCheck(
       ? 'degraded'
       : 'healthy';
 
-  // Update module state to VERIFIED if healthy or degraded
-  if (status === 'healthy' || status === 'degraded') {
-    db.update(modules).set({ state: 'VERIFIED' }).where(eq(modules.id, moduleId)).run();
-  } else if (module.state === 'VERIFIED') {
-    // Revert to INSTALLED if previously verified but now unhealthy
-    db.update(modules).set({ state: 'INSTALLED' }).where(eq(modules.id, moduleId)).run();
+  const nextState = nextModuleState(module.state, status, options.unattended ?? false);
+  if (nextState) {
+    db.update(modules).set({ state: nextState }).where(eq(modules.id, moduleId)).run();
   }
 
   return { moduleId, status, checks };

package/src/services/machine-pool.ts

@@ -2,6 +2,7 @@ import { randomUUID } from 'node:crypto';
 import { and, eq, inArray } from 'drizzle-orm';
 import { getDb } from '../db/client';
 import {
+  type AllocatableZone,
   type NetworkZone,
   containerServices,
   ipAllocations,
@@ -355,7 +356,7 @@ export async function getContainerSystemsByZone(
         eq(moduleInfrastructure.infrastructureType, 'container_service'),
         // ip_allocations.zone is narrower than NetworkZone (no 'external');
         // safe to cast — any 'external' input would just match zero rows.
-        inArray(ipAllocations.zone, zones as Array<'dmz' | 'app' | 'secure' | 'internal'>),
+        inArray(ipAllocations.zone, zones as AllocatableZone[]),
       ),
     );
 

package/src/services/module-deploy.ts

@@ -268,14 +268,25 @@ export async function deployModule(
     await import('./celilo-events');
   const { startOperation, completeOperation, failOperation } = await import('./module-operations');
 
+  const { closeDeployWindows, ensureMonitorOnDeploy, openDeployWindow } = await import(
+    './alerting/deploy-hooks'
+  );
+
   const opId = startOperation(moduleId, 'deploy');
   emitDeployStarted({ module: moduleId, startedAt });
 
+  // A deploy restarts services, which fails their own health checks. Without
+  // this the operator is paged about the deploy they are personally running.
+  openDeployWindow(db, moduleId, new Date());
+
   let result: DeployResult;
   try {
     result = await deployModuleImpl(moduleId, db, options);
   } catch (err) {
     const error = err instanceof Error ? err.message : String(err);
+    // Close on the failure path too: a window left open would silence this
+    // module forever, which is far worse than a few noisy alerts.
+    closeDeployWindows(db, moduleId, new Date());
     failOperation(opId, err);
     emitDeployFailed({
       module: moduleId,
@@ -287,7 +298,13 @@ export async function deployModule(
   }
 
   const durationMs = Math.max(0, Date.now() - startedAt);
+  closeDeployWindows(db, moduleId, new Date());
+
   if (result.success) {
+    // A module that declares a suggested interval starts being watched on its
+    // first successful deploy — watching is the default, not a thing to
+    // remember.
+    ensureMonitorOnDeploy(db, moduleId);
     completeOperation(opId);
     emitDeployCompleted({ module: moduleId, startedAt, durationMs });
   } else {

package/src/services/system-config-validator.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, it } from 'bun:test';
-import { readFileSync } from 'node:fs';
+import { existsSync, readFileSync, readdirSync } from 'node:fs';
 import { join } from 'node:path';
 import type { SystemConfigProperty, SystemConfigSchema } from './system-config-schema-types';
 import { validateKey, validateValue } from './system-config-validator';
@@ -182,4 +182,34 @@ describe('shipped system_config.json covers every placement zone', () => {
       expect(validateKey(`network.${zone}.gateway`, shipped).valid).toBe(true);
     });
   }
+
+  // The same defect one step further out: a module can declare
+  // `derive_from: "$system:<key>"` for a key the shipped schema never accepts.
+  // technitium's `vpn_subnet` derived from `$system:network.vpn.subnet` for
+  // months while `system config set network.vpn.subnet` rejected the key — the
+  // derive had no source and DNS silently lost its VPN split-horizon view.
+  // Enumerating the manifests keeps this from needing a hand-maintained list.
+  const MODULES_DIR = join(import.meta.dir, '../../../../modules');
+  const derivedSystemKeys = new Set<string>();
+  for (const entry of readdirSync(MODULES_DIR, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const manifest = join(MODULES_DIR, entry.name, 'manifest.yml');
+    if (!existsSync(manifest)) continue;
+    // `derive_from:` only — a bare `$system:` also appears in prose descriptions.
+    for (const m of readFileSync(manifest, 'utf-8').matchAll(
+      /derive_from:\s*["']?\$system:([\w.-]+)/g,
+    )) {
+      derivedSystemKeys.add(m[1]);
+    }
+  }
+
+  it('found system-derived keys to check (the scan itself must not silently pass)', () => {
+    expect(derivedSystemKeys.size).toBeGreaterThan(0);
+  });
+
+  for (const key of [...derivedSystemKeys].sort()) {
+    it(`accepts ${key}, which a module derives from`, () => {
+      expect(validateKey(key, shipped).valid).toBe(true);
+    });
+  }
 });

package/src/services/trusted-sources.test.ts

@@ -0,0 +1,221 @@
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { DbClient } from '../db/client';
+import { systemConfig } from '../db/schema';
+import { loadTrustedSubnets } from '../hooks/capability-loader';
+import { setupTestDatabase } from '../test-utils/setup-test-db';
+import {
+  buildTrustedSourceStore,
+  composeTrustedSubnets,
+  listAllTrustedSources,
+  listTrustedSourcesFor,
+  parseOperatorTrustedSubnets,
+} from './trusted-sources';
+
+const FW = '192.168.0.254';
+const VPN = '10.255.255.0/24';
+const CONTROL_PLANE = '192.168.0.0/24';
+
+describe('trusted-source store', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'ts-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+  });
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('stamps registeredBy from the binding, not the caller', () => {
+    const store = buildTrustedSourceStore(db, 'wireguard');
+    store.add(FW, { subnet: VPN, description: 'admin VPN clients' });
+
+    expect(store.list(FW)).toEqual([
+      { subnet: VPN, description: 'admin VPN clients', registeredBy: 'wireguard' },
+    ]);
+  });
+
+  it('upserts by subnet — registering twice yields one row', () => {
+    const store = buildTrustedSourceStore(db, 'wireguard');
+    store.add(FW, { subnet: VPN, description: 'first' });
+    store.add(FW, { subnet: VPN, description: 'second' });
+
+    expect(store.list(FW)).toHaveLength(1);
+    expect(store.list(FW)[0].description).toBe('second');
+  });
+
+  it('removes by subnet and leaves other firewalls alone', () => {
+    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
+    buildTrustedSourceStore(db, 'other').add('10.0.0.1', { subnet: VPN, description: 'elsewhere' });
+
+    buildTrustedSourceStore(db, 'wireguard').remove(FW, VPN);
+
+    expect(buildTrustedSourceStore(db, 'wireguard').list(FW)).toEqual([]);
+    expect(buildTrustedSourceStore(db, 'other').list('10.0.0.1')).toHaveLength(1);
+  });
+
+  it('reads one firewall’s registrations without a module binding', () => {
+    // Trust registered against one firewall is not trust granted by another —
+    // the read is scoped, and the reader needs no identity to look.
+    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
+    buildTrustedSourceStore(db, 'other').add('10.0.0.1', {
+      subnet: '172.16.9.0/24',
+      description: 'elsewhere',
+    });
+
+    expect(listTrustedSourcesFor(db, FW).map((s) => s.subnet)).toEqual([VPN]);
+    expect(listTrustedSourcesFor(db, '10.0.0.1').map((s) => s.subnet)).toEqual(['172.16.9.0/24']);
+  });
+
+  it('reports every registration across firewalls, with who registered it', () => {
+    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'vpn' });
+
+    expect(listAllTrustedSources(db)).toEqual([
+      { firewallIp: FW, subnet: VPN, description: 'vpn', registeredBy: 'wireguard' },
+    ]);
+  });
+});
+
+// Caught by e2e, and it is the whole reason the render input and the reporting
+// view are different calls. `on_uninstall` withdrew the VPN's trusted source,
+// logged success — and the reach rules were re-rendered anyway, because the
+// snapshot the loader had injected into the provider's config still listed the
+// subnet, and the provider unions that snapshot with the live registry. The set
+// could grow but never shrink. One value, two sources of truth.
+describe('the render input excludes registrations; the reporting view includes them', () => {
+  let dir: string;
+  let db: DbClient;
+
+  beforeEach(async () => {
+    dir = mkdtempSync(join(tmpdir(), 'ts-render-'));
+    const dbPath = join(dir, 'celilo.db');
+    process.env.CELILO_DB_PATH = dbPath;
+    db = await setupTestDatabase(dbPath);
+    db.insert(systemConfig).values({ key: 'network.internal.subnet', value: CONTROL_PLANE }).run();
+    buildTrustedSourceStore(db, 'wireguard').add(FW, { subnet: VPN, description: 'admin VPN' });
+  });
+  afterEach(() => {
+    db.$client.close();
+    process.env.CELILO_DB_PATH = undefined;
+    try {
+      rmSync(dir, { recursive: true, force: true });
+    } catch {
+      /* ignore */
+    }
+  });
+
+  it('omits registrations when no firewall is named (the render input)', () => {
+    // The provider unions the LIVE registry itself; folding a snapshot in here
+    // too is what stopped a withdrawal from ever taking effect.
+    expect(loadTrustedSubnets(db).map((e) => e.subnet)).toEqual([CONTROL_PLANE]);
+  });
+
+  it('includes them when a firewall IS named (the reporting view)', () => {
+    const composed = loadTrustedSubnets(db, FW);
+    expect(composed.map((e) => e.subnet)).toEqual([CONTROL_PLANE, VPN]);
+    expect(composed[1].registeredBy).toBe('wireguard');
+  });
+});
+
+describe('composing the trusted-subnet set', () => {
+  // The property that makes composition safe to ship AHEAD of any module that
+  // registers: a fleet with no contributors renders exactly what it rendered
+  // when the set was derived from the control plane alone.
+  it('with no registrations and no override, yields the derived subnet alone', () => {
+    expect(
+      composeTrustedSubnets({
+        controlPlaneSubnet: CONTROL_PLANE,
+        registered: [],
+        operatorOverride: [],
+      }),
+    ).toEqual([{ subnet: CONTROL_PLANE, origin: 'derived-control-plane' }]);
+  });
+
+  it('with no control plane and no contributors, yields nothing', () => {
+    expect(
+      composeTrustedSubnets({
+        controlPlaneSubnet: undefined,
+        registered: [],
+        operatorOverride: [],
+      }),
+    ).toEqual([]);
+  });
+
+  it('merges all three origins', () => {
+    const composed = composeTrustedSubnets({
+      controlPlaneSubnet: CONTROL_PLANE,
+      registered: [{ subnet: VPN, description: 'admin VPN', registeredBy: 'wireguard' }],
+      operatorOverride: ['172.16.5.0/24'],
+    });
+
+    expect(composed.map((e) => e.subnet)).toEqual([CONTROL_PLANE, VPN, '172.16.5.0/24']);
+    expect(composed.map((e) => e.origin)).toEqual([
+      'derived-control-plane',
+      'registered',
+      'operator-override',
+    ]);
+  });
+
+  it('names WHO holds zone-wide reach, so it is never anonymous', () => {
+    const composed = composeTrustedSubnets({
+      controlPlaneSubnet: CONTROL_PLANE,
+      registered: [{ subnet: VPN, description: 'admin VPN', registeredBy: 'wireguard' }],
+      operatorOverride: [],
+    });
+
+    expect(composed[1]).toEqual({
+      subnet: VPN,
+      origin: 'registered',
+      registeredBy: 'wireguard',
+      description: 'admin VPN',
+    });
+  });
+
+  it('deduplicates an overlapping subnet, keeping the strongest claim', () => {
+    const composed = composeTrustedSubnets({
+      controlPlaneSubnet: CONTROL_PLANE,
+      registered: [{ subnet: CONTROL_PLANE, description: 'dup', registeredBy: 'wireguard' }],
+      operatorOverride: [CONTROL_PLANE],
+    });
+
+    expect(composed).toEqual([{ subnet: CONTROL_PLANE, origin: 'derived-control-plane' }]);
+  });
+
+  it('an operator override adds to the derived set rather than replacing it', () => {
+    // Replacing would let an operator strand celilo's own control plane, which
+    // blocks the SSH every hook and converge runs over.
+    const composed = composeTrustedSubnets({
+      controlPlaneSubnet: CONTROL_PLANE,
+      registered: [],
+      operatorOverride: ['172.16.5.0/24'],
+    });
+
+    expect(composed.map((e) => e.subnet)).toContain(CONTROL_PLANE);
+  });
+});
+
+describe('parsing the operator override', () => {
+  it('splits, trims, and drops empties', () => {
+    expect(parseOperatorTrustedSubnets(' 10.1.0.0/24 , 10.2.0.0/24 ,, ')).toEqual([
+      '10.1.0.0/24',
+      '10.2.0.0/24',
+    ]);
+  });
+
+  it('an unset key contributes nothing', () => {
+    expect(parseOperatorTrustedSubnets(undefined)).toEqual([]);
+    expect(parseOperatorTrustedSubnets('')).toEqual([]);
+  });
+});