@celilo/cli 1.12.0 → 1.14.0

package/src/console/projection.ts

@@ -0,0 +1,364 @@
+/**
+ * The narrow reads the web console polls.
+ *
+ * Separate from the CLI's own reads for a measured reason: `celilo module list
+ * --json` returns 156 KB for 23 modules because it embeds every module's
+ * `manifestData` blob. That is the right payload for a human debugging one
+ * module and the wrong one for a loop that runs every few seconds, so the
+ * console gets a projection with the manifest left out and the two derived
+ * facts it actually renders (observed health, backup freshness) already
+ * computed.
+ *
+ * Reads only. Nothing here writes, and the console's API principal is granted
+ * only read ops, so a write added here would fail at the boundary rather than
+ * succeed quietly.
+ */
+
+import { FIREWALL_CAPABILITY_NAME, orderFirewallChain } from '@celilo/capabilities';
+import { desc, eq } from 'drizzle-orm';
+import { z } from 'zod';
+import type { DbClient } from '../db/client';
+import {
+  NETWORK_ZONES,
+  backups,
+  capabilities,
+  moduleSystems,
+  modules,
+  monitors,
+} from '../db/schema';
+import { loadObservedHealthDetail } from '../services/alerting/observed-health';
+import { loadBackupAuditInfo } from '../services/audit/backup-source';
+import { backupStaleThresholdMs, moduleHasBackupHook } from '../services/audit/backups';
+import { effectiveBackupSchedule } from '../services/backup-schedule';
+import { listCapabilityBindings } from '../services/capability-bindings';
+import type { ConsumedCapabilities } from '../services/consumer-cleanup';
+import type { DelegationChain } from './closure';
+
+/**
+ * Just the capability lists, validated at the boundary.
+ *
+ * `modules.manifestData` is opaque JSON in the database, which is a trust
+ * boundary even though celilo wrote it: rows outlive the code that wrote them,
+ * and an older celilo's row is exactly the case a clean-database test can never
+ * produce. `.catch` rather than `.parse` because one unreadable manifest must
+ * end that branch of the walk, not blank the whole topology.
+ */
+const CONSUMED_CAPABILITIES = z
+  .object({
+    requires: z
+      .object({ capabilities: z.array(z.object({ name: z.string() })).optional() })
+      .optional(),
+    optional: z
+      .object({ capabilities: z.array(z.object({ name: z.string() })).optional() })
+      .optional(),
+  })
+  .catch({});
+
+export interface ConsoleSystem {
+  hostname: string;
+  address: string;
+  zone: string;
+}
+
+export interface ConsoleModule {
+  id: string;
+  version: string;
+  state: string;
+  health: { cell: string; monitored: boolean; firingCount: number; suppressed: boolean };
+  systems: ConsoleSystem[];
+  /**
+   * Epoch MILLISECONDS of the newest COMPLETED backup, or null if never.
+   *
+   * Milliseconds because every other instant that crosses this boundary is in
+   * milliseconds, and a single field in seconds is a factor of a thousand that
+   * no type catches. The console renders ages, so the failure mode is a
+   * plausible wrong number rather than a crash: the alerts route computed one
+   * age in the wrong unit and rendered every alert as `0m`, a fleet where
+   * nothing had been wrong for over a minute.
+   */
+  lastBackupAt: number | null;
+  lastBackupFailed: boolean;
+  /**
+   * Nothing is checking this module.
+   *
+   * No enabled monitor, or one whose interval is manual, or one that has never
+   * run. Distinct from healthy: an unwatched module's silence is not evidence
+   * that it is fine, and `ok` and `unwatched` look identical on a dashboard
+   * that only tracks health.
+   */
+  unwatched: boolean;
+  /**
+   * It IS checked, on a schedule, and its monitor has no escalation policy.
+   *
+   * The alert is raised and reaches nobody. Worse than unwatched in one
+   * respect, because it looks monitored, so it is reported separately rather
+   * than folded into the health cell.
+   */
+  pagesNobody: boolean;
+  /**
+   * The newest successful backup is older than this module's cadence allows.
+   *
+   * Resolved HERE, through the same accessor the backup sweep and the drift
+   * audit use, so the three cannot disagree about what a module's cadence is.
+   * False for a module with no `on_backup` hook: there is nothing to run, so it
+   * is not overdue.
+   */
+  backupStale: boolean;
+}
+
+export interface ConsoleStatus {
+  /**
+   * The canonical zone order, most exposed first. Served rather than compiled
+   * into the console so a zone added to `NETWORK_ZONES` gets a band without a
+   * console release.
+   */
+  zones: string[];
+  modules: ConsoleModule[];
+}
+
+/**
+ * The dashboard's single poll: the zone order, and every module with the two
+ * derived columns the roster shows.
+ *
+ * One call rather than three because a poll that changes nothing should cost
+ * one round trip, and because the topology, the roster and the backup column
+ * all read the same rows.
+ */
+export function consoleStatus(db: DbClient): ConsoleStatus {
+  const health = loadObservedHealthDetail(db);
+  const systemsByModule = new Map<string, ConsoleSystem[]>();
+
+  for (const row of db.select().from(moduleSystems).all()) {
+    const list = systemsByModule.get(row.moduleId) ?? [];
+    list.push({ hostname: row.hostname, address: row.ipv4Address, zone: row.zone });
+    systemsByModule.set(row.moduleId, list);
+  }
+
+  const backupByModule = latestBackupByModule(db);
+  const watch = watchFactsByModule(db);
+  const overdue = staleBackupModules(db);
+
+  const rows = db
+    .select({ id: modules.id, version: modules.version, state: modules.state })
+    .from(modules)
+    .all();
+
+  return {
+    zones: [...NETWORK_ZONES],
+    modules: rows
+      .map((row): ConsoleModule => {
+        const backup = backupByModule.get(row.id);
+        return {
+          id: row.id,
+          version: row.version,
+          state: row.state,
+          health: health.get(row.id) ?? {
+            // A module that is not deployed has nothing to observe. Saying
+            // "not observed" here would report undeployed modules as a finding.
+            cell: 'not deployed',
+            monitored: false,
+            firingCount: 0,
+            suppressed: false,
+          },
+          systems: systemsByModule.get(row.id) ?? [],
+          lastBackupAt: backup?.lastSuccessAt ?? null,
+          lastBackupFailed: backup?.lastFailed ?? false,
+          // Absent from the map means no enabled monitor at all, which is the
+          // loudest form of unwatched rather than the quietest.
+          unwatched: watch.get(row.id)?.unwatched ?? true,
+          pagesNobody: watch.get(row.id)?.pagesNobody ?? false,
+          backupStale: overdue.has(row.id),
+        };
+      })
+      .sort((a, b) => a.id.localeCompare(b.id)),
+  };
+}
+
+/**
+ * What celilo is doing to watch each module, if anything.
+ *
+ * Two facts, kept apart because an operator acts differently on each. A module
+ * nothing checks is invisible. A module that IS checked on a schedule and whose
+ * monitor routes to no escalation policy is worse in one respect: it looks
+ * monitored, the alert gets raised on time, and it reaches nobody.
+ *
+ * `interval manual` and `never run` both count as unwatched. A monitor that
+ * exists and has not executed produces exactly as much evidence as one that
+ * does not exist, and the console must not report the first as coverage.
+ */
+interface WatchFacts {
+  unwatched: boolean;
+  pagesNobody: boolean;
+}
+
+function watchFactsByModule(db: DbClient): Map<string, WatchFacts> {
+  const result = new Map<string, WatchFacts>();
+  for (const monitor of db.select().from(monitors).where(eq(monitors.enabled, true)).all()) {
+    const running = monitor.intervalMinutes > 0 && monitor.lastRunAt !== null;
+    const existing = result.get(monitor.target);
+    // A module can carry more than one monitor. It is watched if ANY of them
+    // runs, and pages nobody only if every running one lacks a policy.
+    result.set(monitor.target, {
+      unwatched: (existing?.unwatched ?? true) && !running,
+      pagesNobody:
+        (existing?.pagesNobody ?? false) || (running && monitor.escalationPolicyId === null),
+    });
+  }
+  return result;
+}
+
+/**
+ * Modules whose newest successful backup is older than their cadence allows.
+ *
+ * Resolved through `effectiveBackupSchedule` and `backupStaleThresholdMs`, the
+ * same two functions the drift audit uses. The cadence in force comes from the
+ * operator's override and then the manifest and defaults to daily when neither
+ * says anything, so a console that worked it out again would be a second
+ * definition of "overdue" and would disagree the first time an override was
+ * set.
+ *
+ * A module with no `on_backup` hook is never stale. There is nothing to run.
+ */
+function staleBackupModules(db: DbClient): Set<string> {
+  const stale = new Set<string>();
+  const now = Date.now();
+  for (const info of loadBackupAuditInfo(db)) {
+    if (!moduleHasBackupHook(info.manifest)) continue;
+    const cadence = effectiveBackupSchedule(info.manifest, info.scheduleOverride);
+    const threshold = backupStaleThresholdMs(cadence);
+    // `manual` has no threshold. Opting out is a decision, not a fault.
+    if (threshold === null) continue;
+    const last = info.lastSuccessfulBackupAt;
+    if (last === null || now - last > threshold) stale.add(info.id);
+  }
+  return stale;
+}
+
+interface BackupFacts {
+  lastSuccessAt: number | null;
+  lastFailed: boolean;
+}
+
+/**
+ * Newest completed backup per module, and whether the most recent ATTEMPT
+ * failed.
+ *
+ * Both, because they answer different questions and the console shows both. A
+ * module can hold a fresh successful backup and still have failed last night,
+ * and a roster that showed only the success date would call that healthy.
+ */
+function latestBackupByModule(db: DbClient): Map<string, BackupFacts> {
+  const result = new Map<string, BackupFacts>();
+  const rows = db
+    .select({
+      moduleId: backups.moduleId,
+      status: backups.status,
+      startedAt: backups.startedAt,
+      completedAt: backups.completedAt,
+    })
+    .from(backups)
+    .orderBy(desc(backups.startedAt))
+    .all();
+
+  for (const row of rows) {
+    if (!row.moduleId) continue; // a system-state backup belongs to no module
+    const existing = result.get(row.moduleId);
+    if (!existing) {
+      result.set(row.moduleId, {
+        lastSuccessAt:
+          row.status === 'completed' && row.completedAt ? epochMillis(row.completedAt) : null,
+        lastFailed: row.status === 'failed',
+      });
+      continue;
+    }
+    if (existing.lastSuccessAt === null && row.status === 'completed' && row.completedAt) {
+      existing.lastSuccessAt = epochMillis(row.completedAt);
+    }
+  }
+  return result;
+}
+
+/**
+ * Milliseconds, deliberately, and named so nobody has to guess.
+ *
+ * This was `epochSeconds`, and the protocol on the other side of the boundary
+ * documents the same field as milliseconds. The two never met because the
+ * console server does not exist yet, so the factor of a thousand sat there
+ * looking like working code. It is the exact shape of the bug that rendered
+ * every alert as `0m`.
+ */
+function epochMillis(value: Date): number {
+  return value.getTime();
+}
+
+/**
+ * Which providers a module has actually called into, by capability.
+ *
+ * The difference between a manifest and a fleet. A manifest says what a module
+ * CAN consume; this says what it resolved to and reached for. `tango-nexus`
+ * declares four optional capabilities and is bound to one, and until
+ * celilo#1072 landed there was no way to tell those apart.
+ *
+ * The console previously inferred this: a REQUIRED capability of a deployed
+ * module must have bound, because the deploy would have failed otherwise. That
+ * was the only sound inference available and it is now an approximation, so it
+ * is gone rather than kept as a fallback (Rule 3.9). A module with no rows here
+ * has called into nothing, which is a real answer and not a missing one.
+ */
+export function loadBindings(db: DbClient, moduleId: string): Map<string, string> {
+  return new Map(
+    listCapabilityBindings(db, moduleId).map((b) => [b.capabilityName, b.providerModuleId]),
+  );
+}
+
+/**
+ * Everything the closure walk needs, read in ONE place.
+ *
+ * The chains are in here rather than beside it deliberately. A separate loader
+ * a caller has to remember is a caller that forgets, and the answer when it
+ * forgets is an empty chain list that reads exactly like a fleet with no
+ * delegation. There is one call site, and it cannot get a partial set.
+ */
+export function loadClosureInputs(db: DbClient): {
+  manifests: Map<string, ConsumedCapabilities>;
+  providerStates: { moduleId: string; state: string }[];
+  chains: DelegationChain[];
+} {
+  return {
+    manifests: new Map(
+      db
+        .select({ id: modules.id, manifestData: modules.manifestData })
+        .from(modules)
+        .all()
+        .map((row) => [row.id, CONSUMED_CAPABILITIES.parse(row.manifestData)] as const),
+    ),
+    providerStates: db.select({ moduleId: modules.id, state: modules.state }).from(modules).all(),
+    chains: loadFirewallChain(db),
+  };
+}
+
+/**
+ * The `firewall` providers in delegation order, as at most one chain.
+ *
+ * At most one because `firewall` is the only capability celilo chains, and
+ * `orderFirewallChain` — the SAME function the hook loader wires the live
+ * providers with — is what decides the order. A fleet with one firewall, or
+ * with none that reaches the internet, yields no chain at all.
+ */
+export function loadFirewallChain(db: DbClient): DelegationChain[] {
+  const providers = db
+    .select({ moduleId: capabilities.moduleId, data: capabilities.data })
+    .from(capabilities)
+    .where(eq(capabilities.capabilityName, FIREWALL_CAPABILITY_NAME))
+    .all();
+
+  const ordered = orderFirewallChain(providers).map((provider) => provider.moduleId);
+  return ordered.length === 0 ? [] : [{ capability: FIREWALL_CAPABILITY_NAME, moduleIds: ordered }];
+}
+
+/** A module's row, or undefined. Used to reject a closure request for a module that is gone. */
+export function moduleExists(db: DbClient, moduleId: string): boolean {
+  return (
+    db.select({ id: modules.id }).from(modules).where(eq(modules.id, moduleId)).all().length > 0
+  );
+}

package/src/db/schema.ts

@@ -1,3 +1,4 @@
+import { INSTANCE_STATES, type InstanceState } from '@celilo/capabilities';
 import { sql } from 'drizzle-orm';
 import {
   index,
@@ -300,22 +301,14 @@ export const moduleIntegrity = sqliteTable('module_integrity', {
 });
 
 /**
- * Lifecycle states an instance moves through, reported by the deploy worker's
- * `list` so a parent can compare desired against observed (design D6).
+ * Lifecycle states an instance moves through.
  *
- * `failed` and `pending` must stay distinguishable: a parent that cannot tell
- * "has not started" from "will never work" either retries a bug forever or
- * gives up on work that was merely queued.
+ * Re-exported from `@celilo/capabilities` rather than redefined here. `list`
+ * PROMISES these to a caller's reconcile loop, so the vocabulary belongs to the
+ * contract; a second copy in the schema is how a column and an interface come
+ * to disagree about what `failed` means.
  */
-export const INSTANCE_STATES = [
-  'pending',
-  'provisioning',
-  'ready',
-  'failed',
-  'destroying',
-] as const;
-
-export type InstanceState = (typeof INSTANCE_STATES)[number];
+export { INSTANCE_STATES, type InstanceState };
 
 /**
  * Instances of a submodule (openspec/changes/submodules, D2 and D4).
@@ -492,6 +485,18 @@ export const moduleBuilds = sqliteTable('module_builds', {
  *   that is someone else's cloud, this is a network the fleet's own firewall
  *   holds a leg on and must translate for.
  */
+/**
+ * ⚠️ ADDING A ZONE ALSO MEANS TOUCHING THE WEB CONSOLE.
+ *
+ * A zone comes into being here and nowhere else. How it is DRAWN — its accent
+ * colour and the one-line description a person reads under its name — lives in
+ * `apps/console/src/zones.ts`, because that is presentation and this is not.
+ *
+ * A zone added here and missed there draws grey with "no description yet",
+ * which is legible and wrong. `apps/console/tests/zones.test.ts` fails until
+ * the entry exists, so the pull request that adds the zone is the one that
+ * finds out, rather than the release that ships it.
+ */
 export const NETWORK_ZONES = [
   'isp-transit',
   'internal',

package/src/hooks/broker.test.ts

@@ -58,12 +58,10 @@ async function runCapabilityHook(): Promise<Record<string, unknown>> {
       stateDir: dir,
       capabilities: demoCapabilities(),
     };
-    return await executeHookScript(
-      join(FIXTURES, 'capability-calling-hook.ts'),
-      context,
-      30_000,
-      30_000,
-    );
+    return await executeHookScript(join(FIXTURES, 'capability-calling-hook.ts'), context, {
+      timeoutMs: 30_000,
+      idleTimeoutMs: 30_000,
+    });
   } finally {
     rmSync(dir, { recursive: true, force: true });
   }

package/src/hooks/capability-loader-control-plane-api.test.ts

@@ -0,0 +1,124 @@
+/**
+ * `control_plane_api` reaches a consuming module, and only a consuming module.
+ *
+ * The rest of `loadCapabilityFunctions` hands over every capability it can
+ * build, declared or not — its own comment says "not just required ones", and
+ * `capability-loader-bindings.test.ts` asserts that shape deliberately. That is
+ * fine for a capability whose worst outcome is an unused OIDC client.
+ *
+ * This one mints an SSH principal into celilo's own control plane, so it is
+ * gated on the `requires` line instead. These pin both halves, because the
+ * failure that matters is silent in both directions: a console that declared it
+ * and got nothing fails at install with "no provider is loaded" for a capability
+ * no module ever provides, and a module that declared nothing and got it anyway
+ * holds control-plane read that no manifest review would have shown.
+ */
+
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import type { HookLogger } from '@celilo/capabilities';
+import type { DbClient } from '../db/client';
+import { cleanupTestDatabase, setupTestDatabase } from '../test-utils/database';
+import { loadCapabilityFunctions } from './capability-loader';
+
+const noopLogger: HookLogger = {
+  info() {},
+  warn() {},
+  error() {},
+  success() {},
+};
+
+function installModule(db: DbClient, moduleId: string, manifest: Record<string, unknown>): void {
+  db.$client.run(
+    `INSERT INTO modules (id, name, version, source_path, manifest_data) VALUES (?, ?, '1.0.0', '/tmp/${moduleId}', ?)`,
+    [moduleId, moduleId, JSON.stringify(manifest)],
+  );
+}
+
+const DECLARES = {
+  requires: { capabilities: [{ name: 'control_plane_api', version: '1.0.0' }] },
+};
+
+describe('control_plane_api injection', () => {
+  let db: DbClient;
+
+  beforeEach(async () => {
+    db = await setupTestDatabase();
+  });
+
+  afterEach(async () => {
+    await cleanupTestDatabase(db);
+  });
+
+  test('a module that requires it gets it, with no provider module deployed', async () => {
+    // The point of the whole decision: nothing provides this capability, and it
+    // is still there. A test that first deployed a provider would be measuring
+    // the ordinary path.
+    installModule(db, 'celilo-web-console', DECLARES);
+
+    const capabilities = await loadCapabilityFunctions('celilo-web-console', db, noopLogger);
+
+    expect(capabilities.control_plane_api).toBeTruthy();
+    const api = capabilities.control_plane_api as Record<string, unknown>;
+    expect(typeof api.enrol_principal).toBe('function');
+    expect(typeof api.revoke_principal).toBe('function');
+  });
+
+  test('a module that declares it as optional gets it too', async () => {
+    // `optional` is a declaration a reviewer reads on the same line of the same
+    // file. Treating it as no declaration would fail the module at runtime for a
+    // capability its manifest names.
+    installModule(db, 'someday-console', {
+      optional: { capabilities: [{ name: 'control_plane_api', version: '1.0.0' }] },
+    });
+
+    const capabilities = await loadCapabilityFunctions('someday-console', db, noopLogger);
+
+    expect(capabilities.control_plane_api).toBeTruthy();
+  });
+
+  test('a module that declares nothing does NOT get it', async () => {
+    installModule(db, 'hello-foo', { requires: { capabilities: [] } });
+
+    const capabilities = await loadCapabilityFunctions('hello-foo', db, noopLogger);
+
+    expect(capabilities.control_plane_api).toBeUndefined();
+  });
+
+  test('a module that requires something ELSE does NOT get it', async () => {
+    // The near miss: a manifest with a populated `requires` block is the shape a
+    // too-loose check ("does this module require anything?") would wave through.
+    installModule(db, 'hello-bar', {
+      requires: { capabilities: [{ name: 'idp', version: '1.0.0' }] },
+    });
+
+    const capabilities = await loadCapabilityFunctions('hello-bar', db, noopLogger);
+
+    expect(capabilities.control_plane_api).toBeUndefined();
+  });
+
+  test('a module celilo has no record of does NOT get it', async () => {
+    // A hook can run for a module id with no row (a removal path, a typo). The
+    // manifest lookup returns nothing, and "no manifest" must read as "declared
+    // nothing" rather than throwing or defaulting open.
+    const capabilities = await loadCapabilityFunctions('never-imported', db, noopLogger);
+
+    expect(capabilities.control_plane_api).toBeUndefined();
+  });
+
+  test('the injected object is scoped to the module it was built for', async () => {
+    // Scoping is what stops a module rotating a neighbour's key. It is asserted
+    // through the loader rather than only on `buildControlPlaneApi`, because the
+    // loader is where the wrong argument (a provider id, a hook name) would be
+    // passed.
+    installModule(db, 'hello-foo', DECLARES);
+
+    const capabilities = await loadCapabilityFunctions('hello-foo', db, noopLogger);
+    const api = capabilities.control_plane_api as {
+      revoke_principal(request: { name: string }): Promise<unknown>;
+    };
+
+    await expect(api.revoke_principal({ name: 'celilo-web-console' })).rejects.toThrow(
+      /only revoke its own principal/,
+    );
+  });
+});