@celilo/cli 0.17.0 → 0.19.0

package/src/cli/index.ts

@@ -31,6 +31,7 @@ import {
   handleEventsRepair,
   handleEventsReply,
   handleEventsRespond,
+  handleEventsRestartDaemon,
   handleEventsResyncSubscriptions,
   handleEventsRun,
   handleEventsRunHook,
@@ -303,6 +304,7 @@ Subcommands:
   respond                      Run the terminal responder; answer deploy prompts from another shell
   install-daemon [--system]    Write a systemd/launchd unit for the dispatcher (--system: management-plane scope)
   uninstall-daemon [--system]  Remove the installed supervisor unit
+  restart-daemon [--system]    Restart the dispatcher and verify the new process is on current code
   show-daemon [--system]       Print the currently installed unit file
 
 Description:
@@ -1439,6 +1441,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
         return handleEventsInstallDaemon(parsed.args, parsed.flags);
       case 'uninstall-daemon':
         return handleEventsUninstallDaemon(parsed.args, parsed.flags);
+      case 'restart-daemon':
+        return handleEventsRestartDaemon(parsed.args, parsed.flags);
       case 'show-daemon':
         return handleEventsShowDaemon(parsed.args, parsed.flags);
       default:
@@ -2133,7 +2137,7 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
     }
 
     if (parsed.subcommand === 'migrate') {
-      return handleSystemMigrate();
+      return handleSystemMigrate(parsed.args, parsed.flags);
     }
 
     return {

package/src/cli/tui/audit-state.ts

@@ -77,12 +77,14 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
   'module_configs',
   'health',
   'backups',
+  'abandoned_operations',
   'undeployed_modules',
   'unconfigured_modules',
   'services_credentials',
   'secrets_decryptable',
   'services_reachable',
   'machines_reachable',
+  'disk_space',
   'transport_reads',
   'trusted_sources',
 ];
@@ -96,12 +98,14 @@ export const CATEGORY_LABELS: Record<DriftCategory, string> = {
   module_configs: 'Module configs',
   health: 'Module health',
   backups: 'Backups',
+  abandoned_operations: 'Abandoned operations',
   undeployed_modules: 'Undeployed modules',
   unconfigured_modules: 'Unconfigured modules',
   services_credentials: 'Service credentials',
   secrets_decryptable: 'Secrets',
   services_reachable: 'Service reachability',
   machines_reachable: 'Machine reachability',
+  disk_space: 'Disk space',
   transport_reads: 'Transport readability',
   trusted_sources: 'Trusted networks',
 };

package/src/cli/tui/audit-tui.test.tsx

@@ -35,7 +35,6 @@ const FIXTURE: SystemAuditReport = {
 };
 
 function strip(s: string): string {
-  // biome-ignore lint/suspicious/noControlCharactersInRegex: stripping CSI escape sequences
   return s.replace(/\[[0-9;]*m/g, '');
 }
 

package/src/db/migration-status.test.ts

@@ -0,0 +1,114 @@
+/**
+ * celilo#604: a table COUNT reported "up to date" for a DB missing a migrated
+ * COLUMN, so the rollout's own stop-condition was uncheckable by the tools the
+ * runbook named. These assert the column case specifically.
+ */
+
+import { Database } from 'bun:sqlite';
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { getMigrationStatus, readMigrationJournal } from './migration-status';
+
+/** A journal + a `__drizzle_migrations` table, wired the way drizzle wires them. */
+function seed(dir: string, entries: Array<{ when: number; tag: string }>, appliedWhens: number[]) {
+  mkdirSync(join(dir, 'meta'), { recursive: true });
+  writeFileSync(
+    join(dir, 'meta', '_journal.json'),
+    JSON.stringify({ version: '7', dialect: 'sqlite', entries }),
+  );
+  const db = new Database(':memory:');
+  db.run(
+    'CREATE TABLE `__drizzle_migrations` (id INTEGER PRIMARY KEY, hash TEXT, created_at NUMERIC)',
+  );
+  for (const when of appliedWhens) {
+    db.run('INSERT INTO `__drizzle_migrations` (hash, created_at) VALUES (?, ?)', [
+      `h${when}`,
+      when,
+    ]);
+  }
+  return db;
+}
+
+describe('getMigrationStatus', () => {
+  let dir: string;
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-migstatus-'));
+  });
+  afterEach(() => rmSync(dir, { recursive: true, force: true }));
+
+  it('names the latest applied migration and every pending one', () => {
+    const db = seed(
+      dir,
+      [
+        { when: 100, tag: '0018_drop_alert_policy_snapshot' },
+        { when: 200, tag: '0019_backup_pid' },
+        { when: 300, tag: '0020_future' },
+      ],
+      [100, 200],
+    );
+
+    const status = getMigrationStatus(db, dir);
+
+    expect(status.appliedCount).toBe(2);
+    expect(status.latestApplied).toBe('0019_backup_pid');
+    expect(status.pending).toEqual(['0020_future']);
+  });
+
+  it('reports nothing pending once every journal entry is applied', () => {
+    const db = seed(dir, [{ when: 100, tag: '0000_init' }], [100]);
+    const status = getMigrationStatus(db, dir);
+    expect(status.pending).toEqual([]);
+    expect(status.latestApplied).toBe('0000_init');
+  });
+
+  it('reports a DB that has never been migrated as all-pending', () => {
+    mkdirSync(join(dir, 'meta'), { recursive: true });
+    writeFileSync(
+      join(dir, 'meta', '_journal.json'),
+      JSON.stringify({ entries: [{ when: 1, tag: '0000_init' }] }),
+    );
+    const status = getMigrationStatus(new Database(':memory:'), dir);
+    expect(status.appliedCount).toBe(0);
+    expect(status.latestApplied).toBeNull();
+    expect(status.pending).toEqual(['0000_init']);
+  });
+
+  it('counts columns, not just tables — the signal a column migration needs', () => {
+    const db = seed(dir, [], []);
+    const status = getMigrationStatus(db, dir);
+    // Every schema table is missing here, so the counts are the code's totals.
+    expect(status.tableCount).toBeGreaterThan(0);
+    expect(status.columnCount).toBeGreaterThan(status.tableCount);
+  });
+
+  it('reads journal entries oldest-first regardless of file order', () => {
+    mkdirSync(join(dir, 'meta'), { recursive: true });
+    writeFileSync(
+      join(dir, 'meta', '_journal.json'),
+      JSON.stringify({
+        entries: [
+          { when: 300, tag: 'c' },
+          { when: 100, tag: 'a' },
+        ],
+      }),
+    );
+    expect(readMigrationJournal(dir).map((e) => e.tag)).toEqual(['a', 'c']);
+  });
+
+  it('returns an empty journal when the folder has none', () => {
+    expect(readMigrationJournal(join(dir, 'nope'))).toEqual([]);
+  });
+});
+
+describe('getMigrationStatus against the real migrations folder', () => {
+  it('sees backups.pid as missing when the column migration did not land', () => {
+    // The exact celilo#604 shape: every table present, one migrated column not.
+    const db = new Database(':memory:');
+    db.run('CREATE TABLE backups (id TEXT PRIMARY KEY)');
+    const status = getMigrationStatus(db, join(import.meta.dir, '../../drizzle'));
+    expect(status.missingColumns).toContain('backups.pid');
+    expect(status.pending).toContain('0019_backup_pid');
+  });
+});

package/src/db/migration-status.ts

@@ -0,0 +1,78 @@
+/**
+ * Read-only migration interrogation (celilo#604).
+ *
+ * `system migrate` used to report "Schema current: 35 tables" — a table COUNT,
+ * which cannot distinguish "the column migration applied" from "nothing
+ * happened". 0019_backup_pid adds a column; a rollout runbook asserting
+ * "applied 19 → 20, backups.pid present" had no product surface to check it
+ * against and had to reach for `sqlite3` over SSH.
+ *
+ * This names migrations. Drizzle's `__drizzle_migrations.created_at` is the
+ * journal entry's `when`, so the join back to a human tag is exact.
+ */
+
+import type { Database } from 'bun:sqlite';
+import { existsSync, readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { findSchemaDrift } from './schema-introspection';
+
+export interface MigrationStatus {
+  /** Rows in `__drizzle_migrations` — what the runbook calls "applied count". */
+  appliedCount: number;
+  /** Tag of the newest applied migration, e.g. `0019_backup_pid`. */
+  latestApplied: string | null;
+  /** Tags present on disk that this DB has not applied, oldest first. */
+  pending: string[];
+  /** Tables the running code declares that the DB lacks. */
+  missingTables: string[];
+  /** `table.column` the running code declares that the DB lacks. */
+  missingColumns: string[];
+  tableCount: number;
+  columnCount: number;
+}
+
+interface JournalEntry {
+  when: number;
+  tag: string;
+}
+
+/** Journal entries (oldest first), or [] when the folder has no journal. */
+export function readMigrationJournal(migrationsFolder: string): JournalEntry[] {
+  const path = join(migrationsFolder, 'meta', '_journal.json');
+  if (!existsSync(path)) return [];
+  const parsed = JSON.parse(readFileSync(path, 'utf-8')) as { entries?: JournalEntry[] };
+  return [...(parsed.entries ?? [])].sort((a, b) => a.when - b.when);
+}
+
+/** `created_at` timestamps recorded in `__drizzle_migrations` (empty if untracked). */
+function appliedTimestamps(sqlite: Database): number[] {
+  try {
+    return sqlite
+      .query<{ created_at: number }, []>(
+        'SELECT created_at FROM `__drizzle_migrations` ORDER BY created_at',
+      )
+      .all()
+      .map((r) => r.created_at);
+  } catch {
+    // No migrations table — a DB that has never been through the migrator.
+    return [];
+  }
+}
+
+export function getMigrationStatus(sqlite: Database, migrationsFolder: string): MigrationStatus {
+  const journal = readMigrationJournal(migrationsFolder);
+  const applied = new Set(appliedTimestamps(sqlite));
+  const appliedTags = journal.filter((e) => applied.has(e.when)).map((e) => e.tag);
+  const pending = journal.filter((e) => !applied.has(e.when)).map((e) => e.tag);
+  const drift = findSchemaDrift(sqlite);
+
+  return {
+    appliedCount: applied.size,
+    latestApplied: appliedTags.at(-1) ?? null,
+    pending,
+    missingTables: drift.missingTables,
+    missingColumns: drift.missingColumns,
+    tableCount: drift.tableCount,
+    columnCount: drift.columnCount,
+  };
+}

package/src/db/schema-introspection.ts

@@ -20,6 +20,12 @@ export interface SchemaDrift {
   missingColumns: string[];
   /** Total number of tables the code's schema declares. */
   tableCount: number;
+  /**
+   * Total number of columns the code's schema declares. Reported alongside
+   * tableCount so the doctor can say what it actually checked — a table count
+   * alone reads as "columns unverified" even when they were (celilo#604).
+   */
+  columnCount: number;
 }
 
 /** Every table name + column names the drizzle schema declares. */
@@ -84,5 +90,6 @@ export function findSchemaDrift(sqlite: Database): SchemaDrift {
       if (!cols.has(c)) missingColumns.push(`${t.name}.${c}`);
     }
   }
-  return { missingTables, missingColumns, tableCount: tables.length };
+  const columnCount = tables.reduce((n, t) => n + t.columns.length, 0);
+  return { missingTables, missingColumns, tableCount: tables.length, columnCount };
 }

package/src/db/schema.ts

@@ -249,14 +249,31 @@ export const moduleBuilds = sqliteTable('module_builds', {
 });
 
 /**
- * Network zones type
- * - internal: Home network (192.168.0.0/24)
- * - dmz: Public-facing services in home lab (10.0.10.0/24)
- * - app: Internal application services (10.0.20.0/24)
- * - secure: Authentication and database services (10.0.30.0/24)
+ * Network zones — the DEFINED vocabulary, not the active set.
+ *
+ * A zone being listed here means celilo knows the name and will accept
+ * `network.<zone>.*` config for it. It does NOT mean the zone exists on a given
+ * fleet: per openspec/specs/progressive-zone-disclosure/spec.md, "deployable
+ * zones are those with a configured subnet", so a zone becomes REAL only when
+ * something declares `network.<zone>.subnet` — normally the module that supplies
+ * the network. Defining a zone here costs nothing and activates nothing.
+ *
+ * - internal: the semi-trusted LAN the management server sits on
+ * - dmz: public-facing services
+ * - app: internal application services
+ * - secure: authentication and database services
  * - secure-mgmt: celilo's own control plane (management server + management-plane
  *   modules). Outside the data-plane tier chain; reaches every tier by trust.
- * - external: Internet-hosted services (outside home network)
+ * - external: internet-hosted services — a cloud/VPS provider's network, NOT a
+ *   VPN. Addressed by the provider, which is why it is excluded from IPAM below.
+ * - control-plane-vpn: the ADMINISTRATIVE remote-access client subnet, activated
+ *   by the module that terminates the tunnel (`wireguard` writes
+ *   `network.control-plane-vpn.subnet`). Named for its purpose rather than its
+ *   technology because a fleet may run more than one VPN — a site-to-site link
+ *   or a user VPN is a different network with different trust, and `vpn` would
+ *   have been the wrong name to have to share. Distinct from `external` too:
+ *   that is someone else's cloud, this is a network the fleet's own firewall
+ *   holds a leg on and must translate for.
  */
 export const NETWORK_ZONES = [
   'internal',
@@ -265,6 +282,7 @@ export const NETWORK_ZONES = [
   'secure',
   'secure-mgmt',
   'external',
+  'control-plane-vpn',
 ] as const;
 
 /**
@@ -276,15 +294,36 @@ export const NETWORK_ZONES = [
 export type NetworkZone = (typeof NETWORK_ZONES)[number];
 
 /**
- * Zones an IP allocation or reservation can name: every NetworkZone except
- * `external`, whose systems are addressed by the provider, not by our IPAM.
+ * Zones an IP allocation or reservation can name: every NetworkZone whose
+ * addresses celilo hands out.
+ *
+ * Two are excluded, for the same underlying reason — somebody else is the
+ * address authority:
+ *   - `external`, whose systems are addressed by the cloud/VPS provider;
+ *   - `vpn`, whose client addresses are assigned by the module terminating the
+ *     tunnel. celilo allocating into that subnet would collide with the VPN
+ *     server's own assignments.
  *
  * Derived rather than hand-written for the same reason as NetworkZone above —
  * the previous hand-written union was copied into two column definitions and a
  * cast in machine-pool.ts, and the cast had already drifted (it was missing
  * `secure-mgmt`, and its comment claimed the only difference was `external`).
  */
-export type AllocatableZone = Exclude<NetworkZone, 'external'>;
+export type AllocatableZone = Exclude<NetworkZone, 'external' | 'control-plane-vpn'>;
+
+/** The zones whose addresses celilo hands out, as a runtime list. */
+export const ALLOCATABLE_ZONES: AllocatableZone[] = NETWORK_ZONES.filter(
+  (zone): zone is AllocatableZone => zone !== 'external' && zone !== 'control-plane-vpn',
+);
+
+/**
+ * Is this a zone celilo allocates addresses in? Use this rather than testing
+ * `zone !== 'external'` by hand — that check predates `vpn` and read as "the one
+ * externally-addressed zone" when there are now two.
+ */
+export function isAllocatableZone(zone: NetworkZone): zone is AllocatableZone {
+  return (ALLOCATABLE_ZONES as string[]).includes(zone);
+}
 
 /**
  * Container services table
@@ -679,6 +718,11 @@ export const backups = sqliteTable('backups', {
   status: text('status').$type<BackupStatus>().notNull().default('in_progress'),
   errorMessage: text('error_message'),
   name: text('name'), // optional human-readable name/annotation
+  // The process assembling this backup's staging directory. Lets the staging
+  // reaper (services/backup-staging.ts) tell a live backup from one whose
+  // process was killed before its `finally` could clean up. Nullable: rows
+  // written before this column existed have no pid and age out via the TTL.
+  pid: integer('pid'),
   startedAt: integer('started_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
   completedAt: integer('completed_at', { mode: 'timestamp' }),
 });

package/src/hooks/capability-loader.ts

@@ -28,7 +28,14 @@ import type {
 } from '@celilo/capabilities';
 import { and, eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
-import { capabilities, modules, secrets, systemConfig, webRoutes } from '../db/schema';
+import {
+  NETWORK_ZONES,
+  capabilities,
+  modules,
+  secrets,
+  systemConfig,
+  webRoutes,
+} from '../db/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { emitWebRoutesChangedAndWait } from '../services/celilo-events';
@@ -593,6 +600,7 @@ function buildCapabilityInterface(
           zoneTiers: zones?.zoneTiers ?? [],
           trustedSubnets: zones?.trustedSubnets ?? [],
           controlPlaneSubnet: zones?.controlPlaneSubnet,
+          frontedSubnets: zones?.frontedSubnets ?? [],
         },
         store,
         undefined, // no upstream — the chain path handles that
@@ -699,6 +707,22 @@ interface FirewallZones {
   trustedSubnets: string[];
   /** celilo's control-plane network, as a DESTINATION for trusted sources. */
   controlPlaneSubnet?: string;
+  /**
+   * Every declared zone subnet — read from
+   * `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
+   * data-plane tiers.
+   *
+   * A downstream firewall must translate for every network behind it, and the
+   * tiers are only the ones that happen to form the dmz→app→secure chain.
+   * `secure-mgmt` and the control-plane VPN are equally behind it and equally
+   * unroutable untranslated. Reading the canonical zone list rather than the tier
+   * list means declaring a zone's subnet is sufficient to get it translated — no
+   * module registration required.
+   *
+   * The egress network is NOT excluded here: the firewall derives that from its
+   * own routing table, so nothing on this side has to guess which zone it is.
+   */
+  frontedSubnets: string[];
 }
 
 /** The module that IS celilo's control plane; its network is what we trust. */
@@ -802,6 +826,10 @@ function loadFirewallZones(db: DbClient): FirewallZones {
     zoneTiers,
     trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
     controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
+    // Every zone celilo knows about, not the tier list — see the field doc.
+    frontedSubnets: NETWORK_ZONES.map((zone) => readZoneSubnet(db, zone)).filter(
+      (subnet): subnet is string => !!subnet,
+    ),
   };
 }
 
@@ -953,6 +981,7 @@ async function buildFirewallChain(
         zoneTiers: zones.zoneTiers,
         trustedSubnets: zones.trustedSubnets,
         controlPlaneSubnet: zones.controlPlaneSubnet,
+        frontedSubnets: zones.frontedSubnets,
       },
       store,
       currentUpstream,

package/src/ipam/allocator.ts

@@ -9,16 +9,26 @@ import type { BunSQLiteDatabase } from 'drizzle-orm/bun-sqlite';
 import type { DbClient } from '../db/client';
 import { ipAllocations, ipReservations, systemConfig, vmidReservations } from '../db/schema';
 import type { NewIpAllocation, NewIpReservation, NewVmidReservation } from '../db/schema';
+import { ALLOCATABLE_ZONES, type AllocatableZone } from '../db/schema';
 import type * as schema from '../db/schema';
 import { generateIPsInSubnet, isIPInRange, isInSubnet, stripCIDR } from './subnet-parser';
 
 // Type that accepts both database client and transaction
 type DbOrTransaction = BunSQLiteDatabase<typeof schema> | DbClient;
 
-/** Zones that support IPAM auto-allocation of VMID and container IP */
-export type IpamZone = 'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal';
+/**
+ * Zones that support IPAM auto-allocation of VMID and container IP.
+ *
+ * ALIASED to `AllocatableZone`, not hand-written. This was its own union
+ * (`'dmz' | 'app' | 'secure' | 'secure-mgmt' | 'internal'`) that happened to
+ * agree with `AllocatableZone` — a fourth hand-maintained copy of the zone list,
+ * in a repo where three earlier copies had already drifted and left comments
+ * saying so. Deriving means a zone celilo stops allocating for (as `vpn` is,
+ * since the tunnel module assigns client addresses) cannot be missed here.
+ */
+export type IpamZone = AllocatableZone;
 
-const IPAM_ZONES: IpamZone[] = ['internal', 'dmz', 'app', 'secure', 'secure-mgmt'];
+const IPAM_ZONES: IpamZone[] = ALLOCATABLE_ZONES;
 
 /**
  * Infer which zone an IP address belongs to by checking configured zone subnets.

package/src/services/alerting/builtin-monitors.test.ts

@@ -27,6 +27,12 @@ describe('targetKindForCategory', () => {
     expect(targetKindForCategory('backups')).toBe('module');
   });
 
+  // Machine-scoped so suppression inherits the machine topology: an unreachable
+  // host suppresses its own disk alert instead of paging twice for one dead box.
+  test('disk_space is machine-scoped', () => {
+    expect(targetKindForCategory('disk_space')).toBe('machine');
+  });
+
   // Whole-system categories have no narrower subject to suppress against.
   test('unmapped category falls back to system', () => {
     expect(targetKindForCategory('cli_version')).toBe('system');
@@ -46,6 +52,42 @@ describe('severityForDriftSeverity', () => {
 });
 
 describe('failingKeysFromFindings', () => {
+  // Asserts the WHOLE key, not a prefix. A prefix-only assertion is exactly why
+  // #596 survived — machines_reachable emits a UUID where suppression expects a
+  // hostname, and `toContain('builtin:machines_reachable/machine:')` passes for
+  // both. This check must be pinned to the identifier itself.
+  test('disk_space keys on the hostname, in full', () => {
+    const keys = failingKeysFromFindings(
+      'disk_space',
+      [
+        finding({
+          category: 'disk_space',
+          code: 'disk_critical',
+          severity: 'blocked',
+          subject: 'celilo-mgr',
+          message: 'celilo-mgr: root filesystem 96% full',
+        }),
+      ],
+      'critical',
+    );
+
+    expect(keys).toHaveLength(1);
+    expect(keys[0]?.key).toBe('builtin:disk_space/machine:celilo-mgr');
+    expect(keys[0]?.severity).toBe('critical');
+  });
+
+  // An unmeasurable host records without paging.
+  test('an unmeasured disk downgrades to warning', () => {
+    const keys = failingKeysFromFindings(
+      'disk_space',
+      [finding({ category: 'disk_space', severity: 'todo', subject: 'iot' })],
+      'critical',
+    );
+
+    expect(keys[0]?.key).toBe('builtin:disk_space/machine:iot');
+    expect(keys[0]?.severity).toBe('warning');
+  });
+
   test('projects findings into builtin keys', () => {
     const keys = failingKeysFromFindings('machines_reachable', [finding()], 'critical');
     expect(keys).toEqual([

package/src/services/alerting/builtin-monitors.ts

@@ -24,10 +24,12 @@ import { type FailingKey, builtinAlertKey } from './keys';
  */
 const TARGET_KIND_BY_CATEGORY: Partial<Record<DriftCategory, string>> = {
   machines_reachable: 'machine',
+  disk_space: 'machine',
   transport_reads: 'module',
   services_reachable: 'service',
   services_credentials: 'service',
   backups: 'module',
+  abandoned_operations: 'module',
   health: 'module',
   module_versions: 'module',
   module_configs: 'module',

package/src/services/alerting/builtin-source.ts

@@ -13,16 +13,21 @@
  */
 
 import type { DbClient } from '../../db/client';
+import { auditAbandonedOperations, loadAbandonedOperations } from '../audit/abandoned-operations';
 import { loadBackupAuditInfo } from '../audit/backup-source';
 import { auditBackups } from '../audit/backups';
+import { auditDiskSpace } from '../audit/disk-space';
 import { auditMachinesReachable } from '../audit/machines-reachable';
 import type { DriftCategory, DriftFinding } from '../audit/types';
+import { probeDiskUsage } from '../disk-probe';
 import { probeMachines } from '../machine-probe';
 
 /** Categories a monitor can currently schedule. */
 export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
   'machines_reachable',
   'backups',
+  'disk_space',
+  'abandoned_operations',
 ];
 
 export function isSchedulableBuiltin(category: string): category is DriftCategory {
@@ -37,12 +42,22 @@ export async function runBuiltinCheckForMonitor(
     return auditMachinesReachable({ results: await probeMachines() });
   }
 
+  // Same shape as machines_reachable — one SSH round trip per machine, bounded
+  // — but it MEASURES the local box rather than exempting it. See disk-probe.ts.
+  if (category === 'disk_space') {
+    return auditDiskSpace({ results: await probeDiskUsage() });
+  }
+
   // Local DB reads only — cheap enough to run on every sweep, which is
   // the whole reason this category is schedulable and most are not.
   if (category === 'backups') {
     return auditBackups({ modules: loadBackupAuditInfo(db) });
   }
 
+  if (category === 'abandoned_operations') {
+    return auditAbandonedOperations({ records: loadAbandonedOperations(db) });
+  }
+
   throw new Error(
     `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
   );

package/src/services/audit/abandoned-operations.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from 'bun:test';
+import {
+  ABANDONED_THRESHOLD,
+  ABANDONED_WINDOW_MS,
+  type AbandonedOperationRecord,
+  auditAbandonedOperations,
+} from './abandoned-operations';
+
+const NOW = Date.parse('2026-08-05T12:00:00Z');
+const now = () => NOW;
+
+function records(
+  moduleId: string,
+  operation: 'deploy' | 'backup',
+  count: number,
+  ageMs = 60_000,
+): AbandonedOperationRecord[] {
+  return Array.from({ length: count }, () => ({
+    moduleId,
+    operation,
+    startedAt: NOW - ageMs,
+  }));
+}
+
+describe('auditAbandonedOperations', () => {
+  it('says nothing about a single abandonment — that is an operator hitting Ctrl-C', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', ABANDONED_THRESHOLD - 1),
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  it('flags a module whose same operation keeps dying', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', 63),
+      now,
+    });
+
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.subject).toBe('forgejo');
+    expect(findings[0]?.severity).toBe('drift');
+    expect(findings[0]?.message).toContain('63 backup operations abandoned');
+  });
+
+  // The point of grouping: a module whose backups are being killed and whose
+  // deploys are fine should say exactly that.
+  it('reports each operation kind separately', () => {
+    const findings = auditAbandonedOperations({
+      records: [...records('forgejo', 'backup', 5), ...records('technitium', 'deploy', 5)],
+      now,
+    });
+
+    expect(findings.map((f) => f.subject)).toEqual(['forgejo', 'technitium']);
+    expect(findings[1]?.message).toContain('deploy operations');
+  });
+
+  it('ignores abandonments older than the window, so a fixed module stops being flagged', () => {
+    const findings = auditAbandonedOperations({
+      records: records('forgejo', 'backup', 60, ABANDONED_WINDOW_MS + 60_000),
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+
+  it('does not merge two modules into one unactionable count', () => {
+    const findings = auditAbandonedOperations({
+      records: [...records('forgejo', 'backup', 2), ...records('lunacycle', 'backup', 2)],
+      now,
+    });
+    expect(findings).toEqual([]);
+  });
+});