@celilo/cli 0.16.2 → 0.18.0

package/src/services/disk-probe.ts

@@ -0,0 +1,145 @@
+/**
+ * Disk-usage probe for the machine pool.
+ *
+ * Structurally a sibling of `machine-probe.ts` — same pool, same SSH bounding —
+ * with one deliberate difference that is the whole reason this file has its own
+ * comment.
+ *
+ * ⚠️ THE LOCAL BOX IS MEASURED, NOT EXEMPTED.
+ *
+ * `probeMachines()` reports the management server reachable WITHOUT probing it,
+ * and that is correct there: celilo runs as a user with no SSH key for itself,
+ * so `ssh root@127.0.0.1` fails on a perfectly healthy host — and the question
+ * is meaningless anyway, since if this box were unreachable this code would not
+ * be running.
+ *
+ * None of that transfers to disk. The management server stages backups, caches
+ * modules, holds the celilo DB and writes the logs; it is the likeliest host in
+ * the fleet to fill, and it is the host that DID fill. A disk check that copied
+ * the probe's structure and inherited its local shortcut would skip the only
+ * machine the check exists to protect — reporting "all clear" about a
+ * filesystem it never looked at.
+ *
+ * So the local box reads `statfs` directly. There is no reachability question
+ * to answer about the machine running the code, only a usage one.
+ */
+
+import { execFile } from 'node:child_process';
+import { statfs } from 'node:fs/promises';
+import { promisify } from 'node:util';
+import type { DiskUsageResult } from './audit/disk-space';
+import { listMachines } from './machine-pool';
+import { LOCAL_MACHINE_IP } from './ssh-key-manager';
+
+const execFileAsync = promisify(execFile);
+
+/**
+ * Percent-used the way `df` reports it.
+ *
+ * Deliberately not `1 - bavail/blocks`. A filesystem reserves blocks for root,
+ * so free-to-root and free-to-everyone-else differ; `df` computes capacity
+ * against the space an ordinary process can actually use, and an operator
+ * comparing this alert to their own `df` output must see the same number.
+ */
+export function percentUsed(totalBlocks: number, freeBlocks: number, availBlocks: number): number {
+  const used = totalBlocks - freeBlocks;
+  const usable = used + availBlocks;
+  if (usable <= 0) return 0;
+  return Math.round((used / usable) * 100);
+}
+
+/** Parse the data row of `df -P /`. Returns null when the output is unusable. */
+export function parseDfOutput(
+  stdout: string,
+): { usedPercent: number; availableBytes: number } | null {
+  // -P guarantees one line per filesystem, so the row we want is the second.
+  const line = stdout.trim().split('\n')[1];
+  if (!line) return null;
+
+  // Filesystem 1024-blocks Used Available Capacity Mounted-on
+  const fields = line.trim().split(/\s+/);
+  if (fields.length < 5) return null;
+
+  const available = Number(fields[3]);
+  const percent = Number(fields[4]?.replace('%', ''));
+  if (!Number.isFinite(available) || !Number.isFinite(percent)) return null;
+
+  return { usedPercent: percent, availableBytes: available * 1024 };
+}
+
+async function probeLocal(hostname: string, ipAddress: string): Promise<DiskUsageResult> {
+  try {
+    const stats = await statfs('/');
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: percentUsed(stats.blocks, stats.bfree, stats.bavail),
+      availableBytes: stats.bavail * stats.bsize,
+    };
+  } catch (err) {
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: null,
+      message: err instanceof Error ? err.message : String(err),
+    };
+  }
+}
+
+async function probeRemote(
+  hostname: string,
+  ipAddress: string,
+  sshUser: string,
+): Promise<DiskUsageResult> {
+  try {
+    // Same bounding as machine-probe: BatchMode so a password prompt can never
+    // hang the probe, ConnectTimeout so an unresponsive host — the condition
+    // this check exists to notice — cannot wedge it.
+    const { stdout } = await execFileAsync(
+      'ssh',
+      [
+        '-o',
+        'BatchMode=yes',
+        '-o',
+        'ConnectTimeout=5',
+        '-o',
+        'StrictHostKeyChecking=no',
+        '-o',
+        'UserKnownHostsFile=/dev/null',
+        `${sshUser}@${ipAddress}`,
+        'df -P /',
+      ],
+      { timeout: 8000 },
+    );
+
+    const parsed = parseDfOutput(stdout);
+    if (!parsed) {
+      return {
+        hostname,
+        ipAddress,
+        usedPercent: null,
+        message: `unparseable df output: ${stdout}`,
+      };
+    }
+    return { hostname, ipAddress, ...parsed };
+  } catch (err) {
+    const e = err as { stderr?: string; message?: string };
+    return {
+      hostname,
+      ipAddress,
+      usedPercent: null,
+      message: e.stderr?.trim() || e.message || 'df probe failed',
+    };
+  }
+}
+
+export async function probeDiskUsage(): Promise<DiskUsageResult[]> {
+  const machines = await listMachines();
+  return Promise.all(
+    machines.map((m) =>
+      m.ipAddress === LOCAL_MACHINE_IP
+        ? probeLocal(m.hostname, m.ipAddress)
+        : probeRemote(m.hostname, m.ipAddress, m.sshUser),
+    ),
+  );
+}

package/src/services/fleet-checks.ts

@@ -208,6 +208,21 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     remediations.push('`celilo events repair` to sweep stuck deliveries');
   }
 
+  // (1b) sole — a duplicate dispatcher can no longer START (it refuses), but one
+  // stranded before that shipped keeps running, and it makes every other check
+  // here ambiguous: `hb` is whichever of them wrote last. Fail rather than warn —
+  // celilo-mgr ran two for 40 days precisely because nothing reported it (#580).
+  if (health.dispatcherCount > 1) {
+    statuses.push('fail');
+    const pids = health.dispatchers.map((d) => d.pid).join(', ');
+    detail.push(
+      `${health.dispatcherCount} dispatchers are live on this bus (pids ${pids}) — only one may run`,
+    );
+    remediations.push(
+      'stop the unsupervised one: compare `systemctl show celilo-events.service -p MainPID` against those pids and kill the pid systemd does not own',
+    );
+  }
+
   // (2) supervised — a unit file exists (user or system scope). A
   // running dispatcher with NO unit is the orphan case: works now, gone
   // after reboot.

package/src/services/module-operations.test.ts

@@ -7,9 +7,12 @@ import { closeDb } from '../db/client';
 import { runMigrations } from '../db/migrate';
 import {
   InFlightError,
+  OPERATIONS_SWEEP_PATTERN,
+  OPERATIONS_SWEEP_SUBSCRIBER,
   OPERATION_TTL_MS,
   checkInFlight,
   completeOperation,
+  ensureOperationsSweepSubscriber,
   failOperation,
   isPidRunnable,
   refuseIfInFlight,
@@ -213,3 +216,22 @@ describe('module-operations', () => {
     });
   });
 });
+
+describe('ensureOperationsSweepSubscriber', () => {
+  it('registers the hourly reclaim against the existing clear command', () => {
+    const calls: Array<{ name: string; pattern: string; handler: string; registeredBy?: string }> =
+      [];
+    ensureOperationsSweepSubscriber({ subscribe: (options) => calls.push(options) });
+
+    expect(calls).toEqual([
+      {
+        name: OPERATIONS_SWEEP_SUBSCRIBER,
+        pattern: OPERATIONS_SWEEP_PATTERN,
+        handler: 'celilo module operations clear',
+        registeredBy: 'celilo-module-operations',
+      },
+    ]);
+    // Finer than the TTL, so a wedged row never survives long.
+    expect(OPERATIONS_SWEEP_PATTERN).toBe('timer.tick.1h');
+  });
+});

package/src/services/module-operations.ts

@@ -26,7 +26,8 @@
  * lock for 20 days and blocked every backup on the fleet.
  *
  * Stale rows are ignored rather than deleted; `celilo module operations`
- * lists them and `... clear` sweeps them.
+ * lists them and `... clear` sweeps them — on an hourly bus tick, not
+ * only when a human remembers (see `ensureOperationsSweepSubscriber`).
  */
 
 import { spawnSync } from 'node:child_process';
@@ -119,6 +120,52 @@ export function isPidRunnable(pid: number): boolean {
   return state !== 'T' && state !== 'Z';
 }
 
+/**
+ * The `errorMessage` written when a row is released as abandoned.
+ *
+ * Load-bearing, not cosmetic: releasing marks rows `failed` rather than
+ * deleting them, and this exact string is what later distinguishes "the
+ * operation reported a failure" from "the operation never reported
+ * anything and the sweep reclaimed it". The abandoned-operations audit
+ * counts rows by it (`services/audit/abandoned-operations.ts`).
+ */
+export const ABANDONED_RELEASE_MESSAGE = 'abandoned — released by "celilo module operations clear"';
+
+/**
+ * Reclaim abandoned rows on a schedule instead of when a human remembers.
+ *
+ * Registered as an ordinary bus subscriber whose handler is the existing
+ * `celilo module operations clear`, exactly like the backup sweep
+ * (`services/backup-sweep.ts`) — no new command and no new scheduler.
+ * Hourly is far finer than the two-hour TTL, so a wedge never survives
+ * long, and clearing is idempotent: a pass with nothing abandoned is a
+ * single read.
+ *
+ * `clear` without `--all` only touches rows that `checkInFlight` already
+ * ignores, so the sweep can never release a lock a live operation holds.
+ */
+export const OPERATIONS_SWEEP_SUBSCRIBER = 'celilo-operations-sweep';
+export const OPERATIONS_SWEEP_PATTERN = 'timer.tick.1h';
+
+export interface SubscriberRegistrar {
+  subscribe(options: {
+    name: string;
+    pattern: string;
+    handler: string;
+    registeredBy?: string;
+  }): unknown;
+}
+
+/** Idempotent: `bus.subscribe` upserts by name. */
+export function ensureOperationsSweepSubscriber(bus: SubscriberRegistrar): void {
+  bus.subscribe({
+    name: OPERATIONS_SWEEP_SUBSCRIBER,
+    pattern: OPERATIONS_SWEEP_PATTERN,
+    handler: 'celilo module operations clear',
+    registeredBy: 'celilo-module-operations',
+  });
+}
+
 export interface InFlightConflict {
   operation: ModuleOperation;
   /** A short, operator-readable description: "deploy of homebridge (pid 12345)". */

package/src/services/module-subscriptions.test.ts

@@ -173,7 +173,7 @@ describe('register / unregister roundtrip', () => {
     try {
       const row = bus.db
         .query<{ name: string; pattern: string; handler: string }, []>(
-          'SELECT name, pattern, handler FROM subscribers',
+          "SELECT name, pattern, handler FROM subscribers WHERE name = 'celilo-backup-sweep'",
         )
         .get();
       expect(row).toEqual({
@@ -186,6 +186,28 @@ describe('register / unregister roundtrip', () => {
     }
   });
 
+  // Any module can hold the operation lock, so registering ANY module — even
+  // one with no subscriptions and no backup hook — is enough to arm the sweep
+  // that reclaims abandoned rows (#581).
+  it('arms the abandoned-operations sweep for any module', () => {
+    registerModuleSubscriptions(baseManifest({}), '/p');
+
+    const bus = openBus({ dbPath, events: defineEvents({}) });
+    try {
+      const row = bus.db
+        .query<{ pattern: string; handler: string }, []>(
+          "SELECT pattern, handler FROM subscribers WHERE name = 'celilo-operations-sweep'",
+        )
+        .get();
+      expect(row).toEqual({
+        pattern: 'timer.tick.1h',
+        handler: 'celilo module operations clear',
+      });
+    } finally {
+      bus.close();
+    }
+  });
+
   it('registers each subscription as a row, names scoped to module id', () => {
     const result = registerModuleSubscriptions(
       baseManifest({
@@ -210,7 +232,7 @@ describe('register / unregister roundtrip', () => {
     try {
       const rows = bus.db
         .query<{ name: string; pattern: string; handler: string }, []>(
-          'SELECT name, pattern, handler FROM subscribers ORDER BY name',
+          "SELECT name, pattern, handler FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
         )
         .all();
       expect(rows).toEqual([
@@ -243,10 +265,14 @@ describe('register / unregister roundtrip', () => {
     const bus = openBus({ dbPath, events: defineEvents({}) });
     try {
       const rows = bus.db
-        .query<{ count: number }, []>('SELECT COUNT(*) AS count FROM subscribers')
+        .query<{ count: number }, []>(
+          "SELECT COUNT(*) AS count FROM subscribers WHERE name LIKE '%.%'",
+        )
         .get();
       expect(rows?.count).toBe(1);
-      const row = bus.db.query<{ handler: string }, []>('SELECT handler FROM subscribers').get();
+      const row = bus.db
+        .query<{ handler: string }, []>("SELECT handler FROM subscribers WHERE name LIKE '%.%'")
+        .get();
       expect(row?.handler).toBe('echo second');
     } finally {
       bus.close();
@@ -279,7 +305,9 @@ describe('register / unregister roundtrip', () => {
     const bus = openBus({ dbPath, events: defineEvents({}) });
     try {
       const rows = bus.db
-        .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
+        .query<{ name: string }, []>(
+          "SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
+        )
         .all();
       expect(rows).toEqual([{ name: 'authentik.a' }]);
     } finally {
@@ -337,8 +365,13 @@ describe('resyncAllSubscriptions (ISS-0088)', () => {
   function subscriberNames(): string[] {
     const bus = openBus({ dbPath: busPath, events: defineEvents({}) });
     try {
+      // Module subscriptions are dot-scoped (`<module-id>.<sub-name>`); celilo's
+      // own housekeeping subscribers (the backup and operations sweeps) are not,
+      // and are not what these tests are about.
       return bus.db
-        .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
+        .query<{ name: string }, []>(
+          "SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
+        )
         .all()
         .map((r) => r.name);
     } finally {

package/src/services/module-subscriptions.ts

@@ -22,6 +22,7 @@ import { getDb } from '../db/client';
 import { modules } from '../db/schema';
 import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
 import { ensureBackupSweepSubscriber } from './backup-sweep';
+import { ensureOperationsSweepSubscriber } from './module-operations';
 
 /**
  * The bus is opened by the celilo CLI without an event registry — the
@@ -82,9 +83,6 @@ function resolveHandler(sub: ModuleSubscription, moduleId: string, modulePath: s
 /**
  * Register all of a module's subscriptions on the bus. Idempotent —
  * re-running with the same manifest updates existing rows in place.
- *
- * If the module's manifest declares no subscriptions, this is a
- * cheap no-op (the bus DB isn't even touched).
  */
 export function registerModuleSubscriptions(
   manifest: ModuleManifest,
@@ -92,10 +90,14 @@ export function registerModuleSubscriptions(
 ): { registered: number } {
   const subs = manifest.subscriptions ?? [];
   const backupSweep = Boolean(manifest.hooks?.on_backup);
-  if (subs.length === 0 && !backupSweep) return { registered: 0 };
 
   const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
   try {
+    // Unconditional: every module can hold the operation lock (a deploy at
+    // minimum), so the first module on a fleet is what arms the sweep that
+    // reclaims abandoned rows. Idempotent.
+    ensureOperationsSweepSubscriber(bus);
+
     // A module that can be backed up is also what switches the scheduled
     // backup sweep on. Registering here rather than at system init means the
     // sweep appears the moment the fleet has something to back up, and — since

package/src/services/module-types-generator.test.ts

@@ -63,17 +63,20 @@ describe('variableTypeToTs', () => {
 });
 
 describe('generateModuleTypes', () => {
-  test('emits a file header and empty interface for a manifest with no variables', () => {
+  test('emits a file header and a no-keys alias for a manifest with no variables', () => {
     const out = generateModuleTypes(baseManifest({ id: 'empty', name: 'Empty Module' }));
     expect(out).toContain('// Generated from manifest.yml');
     expect(out).toContain('Do not edit by hand');
-    expect(out).toContain('export type EmptyConfig = {');
+    // `= {}` means "any non-nullish value", which is the opposite of an empty
+    // config surface — and biome bans it (lint/complexity/noBannedTypes).
+    expect(out).toContain('export type EmptyConfig = Record<string, never>;');
+    expect(out).not.toContain('export type EmptyConfig = {');
     expect(out).toContain('(No variables declared — module has no typed config surface)');
   });
 
   test('produces the right type-alias name from a kebab-case module ID', () => {
     const out = generateModuleTypes(baseManifest({ id: 'dns-external', name: 'DNS External' }));
-    expect(out).toContain('export type DnsExternalConfig = {');
+    expect(out).toContain('export type DnsExternalConfig =');
   });
 
   test('renders required fields as non-optional', () => {

package/src/services/module-types-generator.ts

@@ -145,8 +145,6 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
   lines.push(' * gives type aliases an implicit index signature but withholds one from');
   lines.push(' * interfaces (which can be declaration-merged). See v2/issues.');
   lines.push(' */');
-  lines.push(`export type ${typeName} = {`);
-
   const ownsFields: string[] = [];
   const importsFields: string[] = [];
 
@@ -172,6 +170,17 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
     importsFields.push(...rendered);
   }
 
+  if (ownsFields.length === 0 && importsFields.length === 0) {
+    // `= {}` is the banned "any non-nullish value" type, not "no keys" — and an
+    // empty config surface means exactly no keys.
+    lines.push('// (No variables declared — module has no typed config surface)');
+    lines.push(`export type ${typeName} = Record<string, never>;`);
+    lines.push('');
+    return lines.map((line) => line.trimEnd()).join('\n');
+  }
+
+  lines.push(`export type ${typeName} = {`);
+
   if (ownsFields.length > 0) {
     lines.push('  // Module-owned variables (from variables.owns)');
     lines.push(...ownsFields);
@@ -183,12 +192,8 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
     lines.push(...importsFields);
   }
 
-  if (ownsFields.length === 0 && importsFields.length === 0) {
-    lines.push('  // (No variables declared — module has no typed config surface)');
-  }
-
   lines.push('};');
   lines.push('');
 
-  return lines.join('\n');
+  return lines.map((line) => line.trimEnd()).join('\n');
 }

package/src/services/storage-providers/local.ts

@@ -14,7 +14,8 @@ import {
 import { dirname, join, relative } from 'node:path';
 import type { StorageProvider, StorageVerifyResult } from './types';
 
-const BACKUP_PREFIX = 'celilo-backups';
+/** Subdirectory of the configured path that actually holds archives. */
+export const BACKUP_PREFIX = 'celilo-backups';
 
 export interface LocalStorageConfig {
   path: string;

package/src/services/update/orchestrator.test.ts

@@ -76,12 +76,14 @@ const cleanAudit: AuditDeps = {
   moduleConfigs: { modules: [] },
   health: { results: [] },
   backups: { modules: [] },
+  abandonedOperations: { records: [] },
   undeployedModules: { modules: [] },
   unconfiguredModules: { modules: [] },
   servicesCredentials: { results: [] },
   secretsDecryptable: { results: [] },
   servicesReachable: { results: [] },
   machinesReachable: { results: [] },
+  transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
   trustedSources: { firewalls: [] },
 };
 

package/src/variables/context.ts

@@ -5,6 +5,7 @@ import type { DbClient } from '../db/client';
 import {
   capabilities,
   containerServices,
+  isAllocatableZone,
   machines,
   moduleConfigs,
   moduleInfrastructure,
@@ -391,7 +392,11 @@ export async function buildResolutionContext(
             .get();
           isProxmox = svc?.providerName === 'proxmox';
         }
-        if (isProxmox && zone !== 'external') {
+        // `isAllocatableZone`, not `zone !== 'external'`: that check was written
+        // when `external` was the only zone celilo does not address, and it
+        // silently became wrong the moment `vpn` joined it — a VPN client subnet
+        // is assigned by the tunnel module, so allocating into it would collide.
+        if (isProxmox && isAllocatableZone(zone)) {
           await db.transaction(async (tx) => {
             const existing = await getAllocation(moduleId, tx);
             const allocation = existing ?? (await allocateResources(moduleId, zone, tx));