@celilo/cli 0.17.0 → 0.18.0

package/src/cli/tui/audit-state.ts

@@ -77,12 +77,14 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
   'module_configs',
   'health',
   'backups',
+  'abandoned_operations',
   'undeployed_modules',
   'unconfigured_modules',
   'services_credentials',
   'secrets_decryptable',
   'services_reachable',
   'machines_reachable',
+  'disk_space',
   'transport_reads',
   'trusted_sources',
 ];
@@ -96,12 +98,14 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
   module_configs: 'Module configs',
   health: 'Module health',
   backups: 'Backups',
+  abandoned_operations: 'Abandoned operations',
   undeployed_modules: 'Undeployed modules',
   unconfigured_modules: 'Unconfigured modules',
   services_credentials: 'Service credentials',
   secrets_decryptable: 'Secrets',
   services_reachable: 'Service reachability',
   machines_reachable: 'Machine reachability',
+  disk_space: 'Disk space',
   transport_reads: 'Transport readability',
   trusted_sources: 'Trusted networks',
 };

package/src/cli/tui/audit-tui.test.tsx

@@ -35,7 +35,6 @@ const FIXTURE: SystemAuditReport = {
 };
 
 function strip(s: string): string {
-  // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping CSI escape sequences
   return s.replace(/\[[0-9;]*m/g, '');
 }
 

package/src/db/schema.ts

@@ -249,14 +249,31 @@ export const moduleBuilds = sqliteTable('module_builds', {
 });
 
 /**
- * Network zones type
- * - internal: Home network (192.168.0.0/24)
- * - dmz: Public-facing services in home lab (10.0.10.0/24)
- * - app: Internal application services (10.0.20.0/24)
- * - secure: Authentication and database services (10.0.30.0/24)
+ * Network zones — the DEFINED vocabulary, not the active set.
+ *
+ * A zone being listed here means celilo knows the name and will accept
+ * `network.<zone>.*` config for it. It does NOT mean the zone exists on a given
+ * fleet: per openspec/specs/progressive-zone-disclosure/spec.md, "deployable
+ * zones are those with a configured subnet", so a zone becomes REAL only when
+ * something declares `network.<zone>.subnet` — normally the module that supplies
+ * the network. Defining a zone here costs nothing and activates nothing.
+ *
+ * - internal: the semi-trusted LAN the management server sits on
+ * - dmz: public-facing services
+ * - app: internal application services
+ * - secure: authentication and database services
  * - secure-mgmt: celilo's own control plane (management server + management-plane
  *   modules). Outside the data-plane tier chain; reaches every tier by trust.
- * - external: Internet-hosted services (outside home network)
+ * - external: internet-hosted services — a cloud/VPS provider's network, NOT a
+ *   VPN. Addressed by the provider, which is why it is excluded from IPAM below.
+ * - control-plane-vpn: the ADMINISTRATIVE remote-access client subnet, activated
+ *   by the module that terminates the tunnel (`wireguard` writes
+ *   `network.control-plane-vpn.subnet`). Named for its purpose rather than its
+ *   technology because a fleet may run more than one VPN — a site-to-site link
+ *   or a user VPN is a different network with different trust, and `vpn` would
+ *   have been the wrong name to have to share. Distinct from `external` too:
+ *   that is someone else's cloud, this is a network the fleet's own firewall
+ *   holds a leg on and must translate for.
  */
 export const NETWORK_ZONES = [
   'internal',
@@ -265,6 +282,7 @@ export const NETWORK_ZONES = [
   'secure',
   'secure-mgmt',
   'external',
+  'control-plane-vpn',
 ] as const;
 
 /**
@@ -276,15 +294,36 @@ export const NETWORK_ZONES = [
 export type NetworkZone = (typeof NETWORK_ZONES)[number];
 
 /**
- * Zones an IP allocation or reservation can name: every NetworkZone except
- * `external`, whose systems are addressed by the provider, not by our IPAM.
+ * Zones an IP allocation or reservation can name: every NetworkZone whose
+ * addresses celilo hands out.
+ *
+ * Two are excluded, for the same underlying reason — somebody else is the
+ * address authority:
+ *   - `external`, whose systems are addressed by the cloud/VPS provider;
+ *   - `vpn`, whose client addresses are assigned by the module terminating the
+ *     tunnel. celilo allocating into that subnet would collide with the VPN
+ *     server's own assignments.
  *
  * Derived rather than hand-written for the same reason as NetworkZone above —
  * the previous hand-written union was copied into two column definitions and a
  * cast in machine-pool.ts, and the cast had already drifted (it was missing
  * `secure-mgmt`, and its comment claimed the only difference was `external`).
  */
-export type AllocatableZone = Exclude<NetworkZone, 'external'>;
+export type AllocatableZone = Exclude<NetworkZone, 'external' | 'control-plane-vpn'>;
+
+/** The zones whose addresses celilo hands out, as a runtime list. */
+export const ALLOCATABLE_ZONES: AllocatableZone[] = NETWORK_ZONES.filter(
+  (zone): zone is AllocatableZone => zone !== 'external' && zone !== 'control-plane-vpn',
+);
+
+/**
+ * Is this a zone celilo allocates addresses in? Use this rather than testing
+ * `zone !== 'external'` by hand — that check predates `vpn` and read as "the one
+ * externally-addressed zone" when there are now two.
+ */
+export function isAllocatableZone(zone: NetworkZone): zone is AllocatableZone {
+  return (ALLOCATABLE_ZONES as string[]).includes(zone);
+}
 
 /**
  * Container services table
@@ -679,6 +718,11 @@ export const backups = sqliteTable('backups', {
   status: text('status').$type<BackupStatus>().notNull().default('in_progress'),
   errorMessage: text('error_message'),
   name: text('name'), // optional human-readable name/annotation
+  // The process assembling this backup's staging directory. Lets the staging
+  // reaper (services/backup-staging.ts) tell a live backup from one whose
+  // process was killed before its `finally` could clean up. Nullable: rows
+  // written before this column existed have no pid and age out via the TTL.
+  pid: integer('pid'),
   startedAt: integer('started_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
   completedAt: integer('completed_at', { mode: 'timestamp' }),
 });

package/src/hooks/capability-loader.ts

@@ -28,7 +28,14 @@ import type {
 } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities, modules, secrets, systemConfig, webRoutes } from '../db/schema';
+import {
+  NETWORK_ZONES,
+  capabilities,
+  modules,
+  secrets,
+  systemConfig,
+  webRoutes,
+} from '../db/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
@@ -593,6 +600,7 @@ function buildCapabilityInterface(
           zoneTiers: zones?.zoneTiers ?? [],
           trustedSubnets: zones?.trustedSubnets ?? [],
           controlPlaneSubnet: zones?.controlPlaneSubnet,
+          frontedSubnets: zones?.frontedSubnets ?? [],
         },
         store,
         undefined, // no upstream — the chain path handles that
@@ -699,6 +707,22 @@ interface FirewallZones {
   trustedSubnets: string[];
   /** celilo's control-plane network, as a DESTINATION for trusted sources. */
   controlPlaneSubnet?: string;
+  /**
+   * Every declared zone subnet — read from
+   * `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
+   * data-plane tiers.
+   *
+   * A downstream firewall must translate for every network behind it, and the
+   * tiers are only the ones that happen to form the dmz→app→secure chain.
+   * `secure-mgmt` and the control-plane VPN are equally behind it and equally
+   * unroutable untranslated. Reading the canonical zone list rather than the tier
+   * list means declaring a zone's subnet is sufficient to get it translated — no
+   * module registration required.
+   *
+   * The egress network is NOT excluded here: the firewall derives that from its
+   * own routing table, so nothing on this side has to guess which zone it is.
+   */
+  frontedSubnets: string[];
 }
 
 /** The module that IS celilo's control plane; its network is what we trust. */
@@ -802,6 +826,10 @@ function loadFirewallZones(db: DbClient): FirewallZones {
     zoneTiers,
     trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
     controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
+    // Every zone celilo knows about, not the tier list — see the field doc.
+    frontedSubnets: NETWORK_ZONES.map((zone) => readZoneSubnet(db, zone)).filter(
+      (subnet): subnet is string => !!subnet,
+    ),
   };
 }
 
@@ -953,6 +981,7 @@ async function buildFirewallChain(
         zoneTiers: zones.zoneTiers,
         trustedSubnets: zones.trustedSubnets,
         controlPlaneSubnet: zones.controlPlaneSubnet,
+        frontedSubnets: zones.frontedSubnets,
       },
       store,
       currentUpstream,

package/src/ipam/allocator.ts

@@ -9,16 +9,26 @@ import type { BunSQLiteDatabase } from 'drizzle-orm/bun-sqlite';
 import type { DbClient } from '../db/client';
 import { ipAllocations, ipReservations, systemConfig, vmidReservations } from '../db/schema';
 import type { NewIpAllocation, NewIpReservation, NewVmidReservation } from '../db/schema';
+import { ALLOCATABLE_ZONES, type AllocatableZone } from '../db/schema';
 import type * as schema from '../db/schema';
 import { generateIPsInSubnet, isIPInRange, isInSubnet, stripCIDR } from './subnet-parser';
 
 // Type that accepts both database client and transaction
 type DbOrTransaction = BunSQLiteDatabase<typeof schema> | DbClient;
 
-/** Zones that support IPAM auto-allocation of VMID and container IP */
-export type IpamZone = 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal';
+/**
+ * Zones that support IPAM auto-allocation of VMID and container IP.
+ *
+ * ALIASED to `AllocatableZone`, not hand-written. This was its own union
+ * (`'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'`) that happened to
+ * agree with `AllocatableZone` — a fourth hand-maintained copy of the zone list,
+ * in a repo where three earlier copies had already drifted and left comments
+ * saying so. Deriving means a zone celilo stops allocating for (as `vpn` is,
+ * since the tunnel module assigns client addresses) cannot be missed here.
+ */
+export type IpamZone = AllocatableZone;
 
-const IPAM_ZONES: IpamZone[] = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
+const IPAM_ZONES: IpamZone[] = ALLOCATABLE_ZONES;
 
 /**
  * Infer which zone an IP address belongs to by checking configured zone subnets.

package/src/services/alerting/builtin-monitors.test.ts

@@ -27,6 +27,12 @@ describe('targetKindForCategory', () => {
     expect(targetKindForCategory('backups')).toBe('module');
   });
 
+  // Machine-scoped so suppression inherits the machine topology: an unreachable
+  // host suppresses its own disk alert instead of paging twice for one dead box.
+  test('disk_space is machine-scoped', () => {
+    expect(targetKindForCategory('disk_space')).toBe('machine');
+  });
+
   // Whole-system categories have no narrower subject to suppress against.
   test('unmapped category falls back to system', () => {
     expect(targetKindForCategory('cli_version')).toBe('system');
@@ -46,6 +52,42 @@ describe('severityForDriftSeverity', () => {
 });
 
 describe('failingKeysFromFindings', () => {
+  // Asserts the WHOLE key, not a prefix. A prefix-only assertion is exactly why
+  // #596 survived — machines_reachable emits a UUID where suppression expects a
+  // hostname, and `toContain('builtin:machines_reachable/machine:')` passes for
+  // both. This check must be pinned to the identifier itself.
+  test('disk_space keys on the hostname, in full', () => {
+    const keys = failingKeysFromFindings(
+      'disk_space',
+      [
+        finding({
+          category: 'disk_space',
+          code: 'disk_critical',
+          severity: 'blocked',
+          subject: 'celilo-mgr',
+          message: 'celilo-mgr: root filesystem 96% full',
+        }),
+      ],
+      'critical',
+    );
+
+    expect(keys).toHaveLength(1);
+    expect(keys[0]?.key).toBe('builtin:disk_space/machine:celilo-mgr');
+    expect(keys[0]?.severity).toBe('critical');
+  });
+
+  // An unmeasurable host records without paging.
+  test('an unmeasured disk downgrades to warning', () => {
+    const keys = failingKeysFromFindings(
+      'disk_space',
+      [finding({ category: 'disk_space', severity: 'todo', subject: 'iot' })],
+      'critical',
+    );
+
+    expect(keys[0]?.key).toBe('builtin:disk_space/machine:iot');
+    expect(keys[0]?.severity).toBe('warning');
+  });
+
   test('projects findings into builtin keys', () => {
     const keys = failingKeysFromFindings('machines_reachable', [finding()], 'critical');
     expect(keys).toEqual([

package/src/services/alerting/builtin-monitors.ts

@@ -24,10 +24,12 @@ import { type FailingKey, builtinAlertKey } from './keys';
  */
 const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
   machines_reachable: 'machine',
+  disk_space: 'machine',
   transport_reads: 'module',
   services_reachable: 'service',
   services_credentials: 'service',
   backups: 'module',
+  abandoned_operations: 'module',
   health: 'module',
   module_versions: 'module',
   module_configs: 'module',

package/src/services/alerting/builtin-source.ts

@@ -13,16 +13,21 @@
  */
 
 import type { DbClient } from '../../db/client';
+import { auditAbandonedOperations, loadAbandonedOperations } from '../audit/abandoned-operations';
 import { loadBackupAuditInfo } from '../audit/backup-source';
 import { auditBackups } from '../audit/backups';
+import { auditDiskSpace } from '../audit/disk-space';
 import { auditMachinesReachable } from '../audit/machines-reachable';
 import type { DriftCategory, DriftFinding } from '../audit/types';
+import { probeDiskUsage } from '../disk-probe';
 import { probeMachines } from '../machine-probe';
 
 /** Categories a monitor can currently schedule. */
 export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
   'machines_reachable',
   'backups',
+  'disk_space',
+  'abandoned_operations',
 ];
 
 export function isSchedulableBuiltin(category: string): category is DriftCategory {
@@ -37,12 +42,22 @@ export async function runBuiltinCheckForMonitor(
     return auditMachinesReachable({ results: await probeMachines() });
   }
 
+  // Same shape as machines_reachable — one SSH round trip per machine, bounded
+  // — but it MEASURES the local box rather than exempting it. See disk-probe.ts.
+  if (category === 'disk_space') {
+    return auditDiskSpace({ results: await probeDiskUsage() });
+  }
+
   // Local DB reads only — cheap enough to run on every sweep, which is
   // the whole reason this category is schedulable and most are not.
   if (category === 'backups') {
     return auditBackups({ modules: loadBackupAuditInfo(db) });
   }
 
+  if (category === 'abandoned_operations') {
+    return auditAbandonedOperations({ records: loadAbandonedOperations(db) });
+  }
+
   throw new Error(
     `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
   );

package/src/services/audit/abandoned-operations.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'bun:test';
+import {
+  ABANDONED_THRESHOLD,
+  ABANDONED_WINDOW_MS,
+  type AbandonedOperationRecord,
+  auditAbandonedOperations,
+} from './abandoned-operations';
+
+const NOW = Date.parse('2026-08-05T12:00:00Z');
+const now = () => NOW;
+
+function records(
+  moduleId: string,
+  operation: 'deploy' | 'backup',
+  count: number,
+  ageMs = 60_000,
+): AbandonedOperationRecord[] {
+  return Array.from({ length: count }, () => ({
+    moduleId,
+    operation,
+    startedAt: NOW - ageMs,
+  }));
+}
+
+describe('auditAbandonedOperations', () => {
+  it('says nothing about a single abandonment — that is an operator hitting Ctrl-C', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', ABANDONED_THRESHOLD - 1),
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  it('flags a module whose same operation keeps dying', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', 63),
+      now,
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.subject).toBe('forgejo');
+    expect(findings[0]?.severity).toBe('drift');
+    expect(findings[0]?.message).toContain('63 backup operations abandoned');
+  });
+
+  // The point of grouping: a module whose backups are being killed and whose
+  // deploys are fine should say exactly that.
+  it('reports each operation kind separately', () => {
+    const findings = auditAbandonedOperations({
+      records: [...records('forgejo', 'backup', 5), ...records('technitium', 'deploy', 5)],
+      now,
+    });
+
+    expect(findings.map((f) => f.subject)).toEqual(['forgejo', 'technitium']);
+    expect(findings[1]?.message).toContain('deploy operations');
+  });
+
+  it('ignores abandonments older than the window, so a fixed module stops being flagged', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', 60, ABANDONED_WINDOW_MS + 60_000),
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  it('does not merge two modules into one unactionable count', () => {
+    const findings = auditAbandonedOperations({
+      records: [...records('forgejo', 'backup', 2), ...records('lunacycle', 'backup', 2)],
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+});

package/src/services/audit/disk-space.test.ts

@@ -0,0 +1,111 @@
+import { describe, expect, test } from 'bun:test';
+import {
+  DISK_BLOCKED_PERCENT,
+  DISK_DRIFT_PERCENT,
+  type DiskUsageResult,
+  auditDiskSpace,
+} from './disk-space';
+
+function usage(over: Partial<DiskUsageResult> = {}): DiskUsageResult {
+  return {
+    hostname: 'celilo-mgr',
+    ipAddress: '10.0.120.10',
+    usedPercent: 10,
+    availableBytes: 102 * 1024 ** 3,
+    ...over,
+  };
+}
+
+describe('auditDiskSpace', () => {
+  test('a healthy filesystem produces no finding', () => {
+    expect(auditDiskSpace({ results: [usage({ usedPercent: 10 })] })).toEqual([]);
+  });
+
+  test('just below the threshold stays quiet', () => {
+    const findings = auditDiskSpace({ results: [usage({ usedPercent: DISK_DRIFT_PERCENT - 1 })] });
+    expect(findings).toEqual([]);
+  });
+
+  test('at the drift threshold reports drift, naming host and usage', () => {
+    const findings = auditDiskSpace({ results: [usage({ usedPercent: DISK_DRIFT_PERCENT })] });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.severity).toBe('drift');
+    expect(findings[0]?.code).toBe('disk_low');
+    expect(findings[0]?.message).toContain('celilo-mgr');
+    expect(findings[0]?.message).toContain('85%');
+  });
+
+  test('at the blocked threshold escalates to blocked', () => {
+    const findings = auditDiskSpace({ results: [usage({ usedPercent: DISK_BLOCKED_PERCENT })] });
+
+    expect(findings[0]?.severity).toBe('blocked');
+    expect(findings[0]?.code).toBe('disk_critical');
+  });
+
+  // The alert has to name the host to act on, and suppression resolves a
+  // machine's ancestor key from its HOSTNAME (alerting/suppression.ts). A UUID
+  // subject produces a key suppression can never match — that is #596, filed
+  // against machines_reachable. This check must not repeat it.
+  test('subjects the finding on the hostname, never a UUID', () => {
+    const findings = auditDiskSpace({
+      results: [usage({ hostname: 'iot', usedPercent: 99 })],
+    });
+
+    expect(findings[0]?.subject).toBe('iot');
+  });
+
+  test('only the offending host is reported in a mixed fleet', () => {
+    const findings = auditDiskSpace({
+      results: [
+        usage({ hostname: 'celilo-mgr', usedPercent: 96 }),
+        usage({ hostname: 'iot', usedPercent: 12 }),
+        usage({ hostname: 'dns-ext', usedPercent: 40 }),
+      ],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.subject).toBe('celilo-mgr');
+  });
+
+  test('reports free space alongside the percentage', () => {
+    const findings = auditDiskSpace({
+      results: [usage({ usedPercent: 96, availableBytes: 6 * 1024 ** 3 })],
+    });
+
+    expect(findings[0]?.message).toContain('6.0 GB free');
+  });
+
+  // Unmeasurable must not read as healthy — but it must not page either, since
+  // machines_reachable is already alerting for the same dead host.
+  test('an unmeasurable host is recorded as todo, not as healthy', () => {
+    const findings = auditDiskSpace({
+      results: [usage({ hostname: 'iot', usedPercent: null, message: 'ssh: connect timed out' })],
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.severity).toBe('todo');
+    expect(findings[0]?.code).toBe('disk_unmeasured');
+    expect(findings[0]?.details).toContain('timed out');
+  });
+
+  test('an unreachable host does not hide a filling one', () => {
+    const findings = auditDiskSpace({
+      results: [
+        usage({ hostname: 'iot', usedPercent: null, message: 'unreachable' }),
+        usage({ hostname: 'celilo-mgr', usedPercent: 97 }),
+      ],
+    });
+
+    expect(findings.map((f) => f.subject).sort()).toEqual(['celilo-mgr', 'iot']);
+    expect(findings.find((f) => f.subject === 'celilo-mgr')?.severity).toBe('blocked');
+  });
+
+  test('every finding carries the disk_space category', () => {
+    const findings = auditDiskSpace({
+      results: [usage({ usedPercent: 99 }), usage({ hostname: 'iot', usedPercent: null })],
+    });
+
+    expect(findings.every((f) => f.category === 'disk_space')).toBe(true);
+  });
+});

package/src/services/audit/disk-space.ts

@@ -0,0 +1,114 @@
+/**
+ * Disk-space check.
+ *
+ * Catches the failure that had no detector at all: a filesystem filling up.
+ * celilo-mgr reached 34% and climbing at ~4.8 GB/hour from leaked backup
+ * staging, and the only reason anyone noticed was an operator running `df` by
+ * eye. Every one of the eighteen configured monitors watched module health or
+ * machine reachability; not one looked at disk.
+ *
+ * Reports EARLY, not at exhaustion. A check that fires once a filesystem is
+ * full reports an outage instead of preventing one, so the thresholds leave
+ * room to act: `drift` at 85% is "you have time", `blocked` at 95% is "you do
+ * not". On a 117 GB root that is ~17 GB and ~6 GB of headroom respectively —
+ * hours at the leak rate that motivated this, days at any normal one.
+ *
+ * The audit consumes pre-computed measurements so it stays unit-testable
+ * without a live filesystem or SSH client, exactly like `machines-reachable`.
+ */
+
+import type { DriftFinding } from './types';
+
+/** Usage at or above this is divergence worth an operator's attention. */
+export const DISK_DRIFT_PERCENT = 85;
+
+/** Usage at or above this is close enough to exhaustion to gate on. */
+export const DISK_BLOCKED_PERCENT = 95;
+
+export interface DiskUsageResult {
+  /**
+   * User-facing hostname, and the identifier the finding is keyed by.
+   *
+   * NOT the machine's UUID. Suppression resolves a machine's ancestor key from
+   * its hostname (`machineAlertKey` in alerting/suppression.ts), so a finding
+   * subjected on the UUID produces an alert key suppression can never match —
+   * which is exactly the bug filed as #596 against `machines_reachable`. Using
+   * the hostname here also satisfies CLAUDE.md: users never see UUIDs.
+   */
+  hostname: string;
+  ipAddress: string;
+  /** Percent of the root filesystem in use, or null when it could not be measured. */
+  usedPercent: number | null;
+  /** Bytes still available. Omitted when unmeasured. */
+  availableBytes?: number;
+  /** Why the measurement failed, when `usedPercent` is null. */
+  message?: string;
+}
+
+export interface DiskSpaceAuditDeps {
+  results: DiskUsageResult[];
+}
+
+function humanBytes(bytes: number): string {
+  const units = ['B', 'KB', 'MB', 'GB', 'TB'];
+  let value = bytes;
+  let unit = 0;
+  while (value >= 1024 && unit < units.length - 1) {
+    value /= 1024;
+    unit++;
+  }
+  return `${value.toFixed(value < 10 && unit > 0 ? 1 : 0)} ${units[unit]}`;
+}
+
+export function auditDiskSpace(deps: DiskSpaceAuditDeps): DriftFinding[] {
+  const findings: DriftFinding[] = [];
+
+  for (const result of deps.results) {
+    // Unmeasurable is NOT healthy — but it does not page either. The host is
+    // already unreachable, `machines_reachable` is already alerting on it, and
+    // a second page for one dead host is noise. `todo` records without
+    // notifying, which is the existing severity for exactly that.
+    if (result.usedPercent === null) {
+      findings.push({
+        category: 'disk_space',
+        severity: 'todo',
+        code: 'disk_unmeasured',
+        message: `${result.hostname}: disk usage could not be measured`,
+        details: result.message,
+        remediation:
+          'The host is unreachable or `df` failed on it. `machines_reachable` covers reachability; this finding only records that disk is currently unknown, not that it is healthy.',
+        actionable: false,
+        subject: result.hostname,
+      });
+      continue;
+    }
+
+    if (result.usedPercent < DISK_DRIFT_PERCENT) continue;
+
+    const critical = result.usedPercent >= DISK_BLOCKED_PERCENT;
+    const free =
+      result.availableBytes === undefined ? '' : `, ${humanBytes(result.availableBytes)} free`;
+
+    findings.push({
+      category: 'disk_space',
+      severity: critical ? 'blocked' : 'drift',
+      code: critical ? 'disk_critical' : 'disk_low',
+      message: `${result.hostname}: root filesystem ${result.usedPercent}% full${free}`,
+      details: critical
+        ? `At or above ${DISK_BLOCKED_PERCENT}% the host is close enough to exhaustion that writes can begin failing. On the management server a full root filesystem takes the event bus and dispatcher with it, including whatever scheduled work would otherwise clean up.`
+        : `At or above ${DISK_DRIFT_PERCENT}% there is still room to act. Find the growth before it becomes an outage rather than after.`,
+      remediation: [
+        `Find what is growing on ${result.hostname}:`,
+        '  df -h /',
+        '  sudo du -xh --max-depth=2 / | sort -h | tail -30',
+        'Then fix the source — a retention policy, a size cap, or a prune',
+        'schedule. A one-off delete leaves the same thing growing.',
+      ].join('\n'),
+      // Multi-step diagnosis, not a one-shot celilo command.
+      actionable: false,
+      subject: result.hostname,
+    });
+  }
+
+  return findings;
+}

package/src/services/audit/index.test.ts

@@ -26,6 +26,7 @@ const emptyDeps = {
   moduleConfigs: { modules: [] },
   health: { results: [] },
   backups: { modules: [] },
+  abandonedOperations: { records: [] },
   undeployedModules: { modules: [] },
   unconfiguredModules: { modules: [] },
   servicesCredentials: { results: [] },