@celilo/cli 0.14.4 → 0.16.1

package/src/cli/commands/system-audit.ts

@@ -20,12 +20,11 @@ import { existsSync, readFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 import { promisify } from 'node:util';
-import { eq, isNotNull } from 'drizzle-orm';
+import { isNotNull } from 'drizzle-orm';
 import { testDigitalOceanConnection } from '../../api-clients/digitalocean';
 import { testProxmoxConnection } from '../../api-clients/proxmox';
 import { getDb } from '../../db/client';
 import {
-  backups,
   capabilitySecrets,
   moduleConfigs as moduleConfigsTbl,
   modules,
@@ -38,6 +37,7 @@ import { decryptSecret } from '../../secrets/encryption';
 import { getOrCreateMasterKey } from '../../secrets/master-key';
 import { runAudit } from '../../services/audit';
 import type { DriftFinding, SystemAuditReport } from '../../services/audit';
+import { loadBackupAuditInfo } from '../../services/audit/backup-source';
 import {
   type LatestCliVersionFetcher,
   fetchLatestCliVersion,
@@ -154,28 +154,6 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
 
   const registryClient = new RegistryClient();
 
-  // Most recent successful backup per module.
-  // The `backups` table is added via inline ALTER statements in db/client.ts
-  // for upgraded databases; a freshly-initialized DB created from the
-  // drizzle journal alone may not have it yet. Treat absence as "no
-  // backups recorded" rather than crashing the audit.
-  const latestBackupByModule = new Map<string, number>();
-  try {
-    const successfulBackups = db
-      .select()
-      .from(backups)
-      .where(eq(backups.status, 'completed'))
-      .all();
-    for (const b of successfulBackups) {
-      if (!b.moduleId || !b.completedAt) continue;
-      const ts = b.completedAt.getTime();
-      const prev = latestBackupByModule.get(b.moduleId);
-      if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts);
-    }
-  } catch {
-    // backups table missing — leave map empty
-  }
-
   // Build per-module config map for module-configs check.
   const allConfigs = db
     .select()
@@ -196,12 +174,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
     configs: configsByModule.get(m.id) ?? {},
   }));
 
-  const installedBackupInfo = deployedModules.map((m) => ({
-    id: m.id,
-    state: m.state,
-    manifest: m.manifestData as ModuleManifest,
-    lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
-  }));
+  const installedBackupInfo = loadBackupAuditInfo(db);
 
   // Compose per-module TF_VAR_* env vars in parallel — each call hits
   // the secret store / DB so they're not free, but they're all

package/src/cli/completion.ts

@@ -181,10 +181,12 @@ export async function getCompletions(words: string[], current: number): Promise<
       'destroy',
       'health',
       'logs',
+      'journal',
       'run-hook',
       'secret',
       'status',
       'where',
+      'operations',
       'terraform-unlock',
       'types',
       'validate',
@@ -198,6 +200,12 @@ export async function getCompletions(words: string[], current: number): Promise<
     return filterSuggestions(subcommands, args[2] || '');
   }
 
+  // Module operations subcommands (celilo module operations list/clear)
+  if (command === 'module' && args[1] === 'operations' && currentIndex === 2) {
+    const subcommands = ['list', 'clear'];
+    return filterSuggestions(subcommands, args[2] || '');
+  }
+
   // Module types subcommands (celilo module types generate/check)
   if (command === 'module' && args[1] === 'types' && currentIndex === 2) {
     const subcommands = ['generate', 'check'];
@@ -416,6 +424,7 @@ export async function getCompletions(words: string[], current: number): Promise<
       'deploy',
       'destroy',
       'logs',
+      'journal',
       'remove',
       'build',
       'backup',
@@ -500,7 +509,17 @@ export async function getCompletions(words: string[], current: number): Promise<
 
   // Backup subcommands
   if (command === 'backup' && currentIndex === 1) {
-    const subcommands = ['create', 'list', 'restore', 'delete', 'prune', 'name', 'import', 'pull'];
+    const subcommands = [
+      'create',
+      'sweep',
+      'list',
+      'restore',
+      'delete',
+      'prune',
+      'name',
+      'import',
+      'pull',
+    ];
     return filterSuggestions(subcommands, args[1] || '');
   }
 

package/src/cli/generate-zsh-completion.ts

@@ -324,6 +324,10 @@ _celilo_config_keys() {
         fi
         config_keys+=("$key:$desc_text")
       done < <(echo "$manifest_json" | jq -r '.variables.owns[]? | "\\(.name)|\\(.description // "No description")|\\(.required // false)"' 2>/dev/null)
+      # Celilo-managed keys apply to EVERY module regardless of its manifest
+      # (#515), so offer them even when the manifest declares none.
+      config_keys+=("auto_upgrade:Let the registry poll upgrade this module unattended (true|false)")
+      config_keys+=("upgrade_policy:Deploy posture floor for upgrades (by-semver|always-safe|always-fast)")
       if [[ \${#config_keys[@]} -gt 0 ]]; then
         _describe 'config key' config_keys
         return 0

package/src/cli/index.ts

@@ -64,8 +64,10 @@ import { handleModuleDeploy } from './commands/module-deploy';
 import { handleModuleGenerate } from './commands/module-generate';
 import { handleModuleHealth } from './commands/module-health';
 import { handleModuleImport } from './commands/module-import';
+import { handleModuleJournal } from './commands/module-journal';
 import { handleModuleList } from './commands/module-list';
 import { handleModuleLogs } from './commands/module-logs';
+import { handleModuleOperations } from './commands/module-operations';
 import { handleModulePublish } from './commands/module-publish';
 import { handleModuleRemove } from './commands/module-remove';
 import { handleModuleSearch } from './commands/module-search';
@@ -747,6 +749,9 @@ Subcommands:
       --storage <id>       Use specific storage destination
       --no-interactive     Non-interactive mode (for cron)
 
+  sweep                    Back up every module whose declared schedule is due
+                           (run by the event bus on timer.tick.1h, not by hand)
+
   list [module-id]         List available backups
     Options:
       --limit <n>          Number of backups to show (default: 20)
@@ -1464,8 +1469,12 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
         return handleModuleWhere(parsed.args, parsed.flags);
       case 'logs':
         return handleModuleLogs(parsed.args, parsed.flags);
+      case 'journal':
+        return handleModuleJournal(parsed.args, parsed.flags);
       case 'health':
         return handleModuleHealth(parsed.args, parsed.flags);
+      case 'operations':
+        return handleModuleOperations(parsed.args, parsed.flags);
       case 'remove':
         return handleModuleRemove(parsed.args, parsed.flags);
       case 'update':
@@ -1864,6 +1873,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
       return handleBackupCreate(parsed.args, parsed.flags);
     }
 
+    if (parsed.subcommand === 'sweep') {
+      const { handleBackupSweep } = await import('./commands/backup-sweep');
+      return handleBackupSweep();
+    }
+
     if (parsed.subcommand === 'list') {
       const { handleBackupList } = await import('./commands/backup-list');
       return handleBackupList(parsed.args, parsed.flags);

package/src/db/schema.ts

@@ -1048,9 +1048,11 @@ export const alerts = sqliteTable(
     deferredRouteId: text('deferred_route_id').references(() => routes.id, {
       onDelete: 'set null',
     }),
-    escalationPolicyId: text('escalation_policy_id').references(() => escalationPolicies.id, {
-      onDelete: 'set null',
-    }),
+    // No escalation policy is stored here on purpose. It lives on the MONITOR
+    // and is resolved fresh on every sweep, so assigning a policy takes effect
+    // on alerts that are already firing. Snapshotting it at alert creation made
+    // `escalation-policy assign` a no-op for exactly the alert an operator was
+    // trying to route — the one already paging nobody (#481).
     message: text('message').notNull(),
     details: text('details'),
     resolvedAt: integer('resolved_at', { mode: 'timestamp' }),

package/src/manifest/schema.ts

@@ -694,7 +694,10 @@ export const ModuleManifestSchema = z
 
     backup: z
       .object({
-        schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('manual'),
+        // Absent means `daily`. Opting out of backups entirely is a real
+        // decision and takes an explicit `manual` — see
+        // [[services/backup-schedule.ts]].
+        schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
         retention: z
           .object({
             count: z.number().int().positive().default(7),

package/src/module/packaging/build.ts

@@ -59,6 +59,10 @@ const EXCLUDE_PATTERNS = [
   '.DS_Store',
   '*.netapp',
   '*.test.ts',
+  // Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
+  // check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
+  // make every module's packaged content change whenever the shared base moves.
+  'tsconfig.json',
   'checksums.json',
   'signature.sig',
 ];

package/src/services/alerting/builtin-source.ts

@@ -12,22 +12,37 @@
  * as "nothing is wrong".
  */
 
+import type { DbClient } from '../../db/client';
+import { loadBackupAuditInfo } from '../audit/backup-source';
+import { auditBackups } from '../audit/backups';
 import { auditMachinesReachable } from '../audit/machines-reachable';
 import type { DriftCategory, DriftFinding } from '../audit/types';
 import { probeMachines } from '../machine-probe';
 
 /** Categories a monitor can currently schedule. */
-export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = ['machines_reachable'];
+export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
+  'machines_reachable',
+  'backups',
+];
 
 export function isSchedulableBuiltin(category: string): category is DriftCategory {
   return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
 }
 
-export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
+export async function runBuiltinCheckForMonitor(
+  category: DriftCategory,
+  db: DbClient,
+): Promise<DriftFinding[]> {
   if (category === 'machines_reachable') {
     return auditMachinesReachable({ results: await probeMachines() });
   }
 
+  // Local DB reads only — cheap enough to run on every sweep, which is
+  // the whole reason this category is schedulable and most are not.
+  if (category === 'backups') {
+    return auditBackups({ modules: loadBackupAuditInfo(db) });
+  }
+
   throw new Error(
     `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
   );

package/src/services/alerting/delivery-loop.test.ts

@@ -68,6 +68,11 @@ beforeAll(async () => {
       ...process.env,
       SIGNAL_RPC_PORT: String(PORT),
       SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
+      // The flag the module's systemd unit must pass. The simulator defaults
+      // to signal-cli's real default (`on-start`), where the daemon drains
+      // replies into its own SSE stream and REFUSES `receive` — so a loop test
+      // may only read replies from a daemon started the way celilo deploys it.
+      SIGNAL_RECEIVE_MODE: 'manual',
     },
     stdout: 'pipe',
     stderr: 'pipe',
@@ -169,7 +174,6 @@ describe('delivery loop against the signal-cli simulator', () => {
       silencedUntil: null,
       escalationStep: 0,
       nextEscalationAt: null,
-      escalationPolicyId: null,
       message: '/var 94% used',
       details: null,
       resolvedAt: null,

package/src/services/alerting/format.test.ts

@@ -34,7 +34,6 @@ function alert(over: Partial<Alert> = {}): Alert {
     silencedUntil: null,
     escalationStep: 0,
     nextEscalationAt: null,
-    escalationPolicyId: null,
     message: '/var 94% used',
     details: null,
     resolvedAt: null,

package/src/services/alerting/inbound-poller.test.ts

@@ -27,7 +27,7 @@ describe('pollInbound', () => {
 
   function deps(messages: InboundMessage[], over: Partial<InboundPollDeps> = {}): InboundPollDeps {
     return {
-      receiveFrom: async () => ({ messages, cursor: 'cursor-1' }),
+      receiveFrom: async () => ({ status: 'received', messages, cursor: 'cursor-1' }),
       readCursor: (t) => cursors.get(t) ?? null,
       writeCursor: (t, c) => {
         cursors.set(t, c);
@@ -118,6 +118,7 @@ describe('pollInbound', () => {
     });
 
   const alertState = () => db.select().from(alerts).where(eq(alerts.id, 'alert-1')).get()?.state;
+  const ackedBy = () => db.select().from(alerts).where(eq(alerts.id, 'alert-1')).get()?.ackedBy;
 
   test('only transports with routes are polled', () => {
     expect(transportsWithRoutes(db)).toEqual(['signal']);
@@ -190,7 +191,7 @@ describe('pollInbound', () => {
     const delivery = page(peterRoute.id);
     const report = await pollInbound(db, deps([inbound(WIFE, delivery.token)]));
 
-    expect(report.rejected).toBe(1);
+    expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
     expect(report.acked).toBe(0);
     expect(alertState()).toBe('firing');
   });
@@ -200,7 +201,7 @@ describe('pollInbound', () => {
     const delivery = page(peterRoute.id);
     const report = await pollInbound(db, deps([inbound(STRANGER, delivery.token)]));
 
-    expect(report.ignored).toBe(1);
+    expect(report.unheard).toEqual([{ senderAddress: STRANGER, reason: 'unknown_sender' }]);
     expect(alertState()).toBe('firing');
   });
 
@@ -228,7 +229,7 @@ describe('pollInbound', () => {
     expect(first.acked).toBe(1);
     // The token is spent, so the replay is refused rather than acking again.
     expect(second.acked).toBe(0);
-    expect(second.rejected).toBe(1);
+    expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
     expect(alertState()).toBe('acked');
   });
 
@@ -262,37 +263,263 @@ describe('pollInbound', () => {
     });
 
     const report = await pollInbound(db, deps([inbound(PETER, 'ack')]));
-    expect(report.rejected).toBe(1);
+    expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'ambiguous' }]);
     expect(alertState()).toBe('firing');
   });
 
   // One dead transport must not stop the others being read.
   test('an unreachable transport is skipped without throwing', async () => {
-    const report = await pollInbound(db, deps([], { receiveFrom: async () => null }));
+    const report = await pollInbound(
+      db,
+      deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
+    );
     expect(report.transportsPolled).toBe(1);
     expect(report.messagesRead).toBe(0);
   });
 
+  /**
+   * The regression that cost a week. signal-cli's daemon refused celilo's read
+   * call outright ("Receive command cannot be used if messages are already
+   * being received") and the poller reported the same "0 message(s)" it
+   * reports when nobody has replied. The reason must reach the report, or the
+   * two are indistinguishable from outside.
+   */
+  test('a transport that REFUSES the read says so instead of looking quiet', async () => {
+    const refusal = 'Receive command cannot be used if messages are already being received.';
+    const report = await pollInbound(
+      db,
+      deps([], { receiveFrom: async () => ({ status: 'failed', error: refusal }) }),
+    );
+
+    expect(report.failures).toEqual([{ transportModuleId: 'signal', error: refusal }]);
+    // ...and a genuinely quiet transport must NOT look like a failure.
+    const quiet = await pollInbound(db, deps([]));
+    expect(quiet.failures).toEqual([]);
+    expect(quiet.messagesRead).toBe(0);
+  });
+
+  test('a unidirectional transport is not reported as a failure', async () => {
+    const report = await pollInbound(
+      db,
+      deps([], { receiveFrom: async () => ({ status: 'unidirectional' }) }),
+    );
+    expect(report.failures).toEqual([]);
+    expect(report.transportsPolled).toBe(1);
+  });
+
   test('the cursor is persisted after a successful poll', async () => {
     await pollInbound(db, deps([]));
     expect(cursors.get('signal')).toBe('cursor-1');
   });
 
   test('the cursor is NOT advanced when the transport could not be reached', async () => {
-    await pollInbound(db, deps([], { receiveFrom: async () => null }));
+    await pollInbound(
+      db,
+      deps([], { receiveFrom: async () => ({ status: 'failed', error: 'connection refused' }) }),
+    );
     expect(cursors.get('signal')).toBeUndefined();
   });
 
   test('gibberish is rejected without touching the alert', async () => {
     page(peterRoute.id);
     const report = await pollInbound(db, deps([inbound(PETER, 'what is going on')]));
-    expect(report.rejected).toBe(1);
+    expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
     expect(alertState()).toBe('firing');
   });
 
+  // The seam that was untested. `acknowledgeAlert` returning null was covered
+  // (ack.test.ts), and pollInbound was covered — but not what the REPORT says
+  // when the two meet. Observed live: a reply produced `1 acked` while every
+  // alert still read `ackedBy: null`, because the counter incremented next to
+  // the call instead of observing its result.
+  describe('a token whose alert no longer exists', () => {
+    test('is NOT counted as acked', async () => {
+      const delivery = page(peterRoute.id);
+      db.delete(alerts).run(); // the alert this token names is gone
+
+      const report = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+      expect(report.acked).toBe(0);
+    });
+
+    test('is reported as stale_target rather than silently', async () => {
+      const delivery = page(peterRoute.id);
+      db.delete(alerts).run();
+
+      const report = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+      expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'stale_target' }]);
+    });
+
+    // The token is still single-use: a dead target must not leave it replayable.
+    test('still consumes the token', async () => {
+      const delivery = page(peterRoute.id);
+      db.delete(alerts).run();
+
+      await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+      const second = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+      expect(second.acked).toBe(0);
+      expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unknown_token' }]);
+    });
+
+    // Nobody should be told "someone took it" when nobody did.
+    test('broadcasts nothing', async () => {
+      const delivery = page(peterRoute.id);
+      db.delete(alerts).run();
+
+      const report = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+      expect(report.broadcast).toBe(0);
+    });
+  });
+
+  // The counter must track the database, not the attempt. If these two ever
+  // disagree again, the operator sees a success that did not happen.
+  test('report.acked agrees with what the alert row actually says', async () => {
+    const delivery = page(peterRoute.id);
+
+    const report = await pollInbound(db, deps([inbound(PETER, delivery.token)]));
+
+    expect(report.acked).toBe(1);
+    expect(alertState()).toBe('acked');
+    expect(ackedBy()).toBe(peterRoute.personId);
+  });
+
   test('a route pointing at a transport nobody uses is not polled', () => {
     db.delete(alerts).run();
     expect(transportsWithRoutes(db)).toEqual(['signal']);
     expect(wifeRoute.transportModuleId).toBe('signal');
   });
+
+  /**
+   * Answering a deploy question, at the SEAM.
+   *
+   * This is the coverage #533 was missing. `parseInterviewAnswer` was unit
+   * tested and correct; `pollInbound` was unit tested and correct; and an
+   * answer sent in the shape the page asks for was rejected as `unrecognised`
+   * for months, because nothing exercised an interview delivery THROUGH the
+   * poller. Two green units, wrong wiring — the same shape as the ack counter
+   * above.
+   */
+  describe('an answer to a deploy question', () => {
+    const ask = (routeId: string, eventId = '42') =>
+      mintDelivery(db, {
+        kind: 'interview',
+        targetId: eventId,
+        routeId,
+        now: NOW,
+        ttlMs: 60_000,
+      });
+
+    test('is published against the waiting question and counted as answered', async () => {
+      const question = ask(peterRoute.id, '42');
+      const answered: { eventId: string; value: string }[] = [];
+
+      const report = await pollInbound(
+        db,
+        deps([inbound(PETER, `${question.token} admin@example.org`)], {
+          answerInterview: (eventId, value) => answered.push({ eventId, value }),
+        }),
+      );
+
+      expect(answered).toEqual([{ eventId: '42', value: 'admin@example.org' }]);
+      expect(report.answered).toBe(1);
+      // An answer is not an acknowledgement; nothing about an alert moved.
+      expect(report.acked).toBe(0);
+      expect(report.unheard).toEqual([]);
+    });
+
+    test('a value containing spaces survives verbatim', async () => {
+      const question = ask(peterRoute.id);
+      const answered: string[] = [];
+
+      await pollInbound(
+        db,
+        deps([inbound(PETER, `${question.token} my value: with, punctuation`)], {
+          answerInterview: (_id, value) => answered.push(value),
+        }),
+      );
+
+      expect(answered).toEqual(['my value: with, punctuation']);
+    });
+
+    test('the token is consumed, so an answer cannot be replayed', async () => {
+      const question = ask(peterRoute.id);
+      const answered: string[] = [];
+      const withResponder = (body: string) =>
+        pollInbound(
+          db,
+          deps([inbound(PETER, body)], { answerInterview: (_id, v) => answered.push(v) }),
+        );
+
+      await withResponder(`${question.token} first`);
+      const second = await withResponder(`${question.token} second`);
+
+      expect(answered).toEqual(['first']);
+      expect(second.answered).toBe(0);
+      // `unrecognised`, not `unknown_token`: once the token no longer resolves,
+      // `<word> <text>` is indistinguishable from an ordinary sentence, so
+      // celilo declines to assert the operator mistyped a token.
+      expect(second.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
+    });
+
+    // Naming the question but supplying nothing is its own answer, and the
+    // token must survive so the operator's next attempt can work.
+    test('a token with no value reports needs_value and leaves the token live', async () => {
+      const question = ask(peterRoute.id);
+
+      const first = await pollInbound(
+        db,
+        deps([inbound(PETER, question.token)], { answerInterview: () => {} }),
+      );
+      expect(first.answered).toBe(0);
+      expect(first.unheard).toEqual([{ senderAddress: PETER, reason: 'needs_value' }]);
+
+      const answered: string[] = [];
+      const second = await pollInbound(
+        db,
+        deps([inbound(PETER, `${question.token} admin@example.org`)], {
+          answerInterview: (_id, v) => answered.push(v),
+        }),
+      );
+      expect(second.answered).toBe(1);
+      expect(answered).toEqual(['admin@example.org']);
+    });
+
+    // With no responder attached there is nothing to publish against, so the
+    // question stays unanswered AND the token stays usable.
+    test('with no responder attached the token is not burned', async () => {
+      const question = ask(peterRoute.id);
+
+      const report = await pollInbound(db, deps([inbound(PETER, `${question.token} value`)]));
+      expect(report.answered).toBe(0);
+      expect(report.unheard).toEqual([{ senderAddress: PETER, reason: 'unrecognised' }]);
+
+      const answered: string[] = [];
+      const retry = await pollInbound(
+        db,
+        deps([inbound(PETER, `${question.token} value`)], {
+          answerInterview: (_id, v) => answered.push(v),
+        }),
+      );
+      expect(retry.answered).toBe(1);
+      expect(answered).toEqual(['value']);
+    });
+
+    test('an answer from a number the question was not sent to is refused', async () => {
+      const question = ask(peterRoute.id);
+      const answered: string[] = [];
+
+      const report = await pollInbound(
+        db,
+        deps([inbound(WIFE, `${question.token} admin@example.org`)], {
+          answerInterview: (_id, v) => answered.push(v),
+        }),
+      );
+
+      expect(answered).toEqual([]);
+      expect(report.unheard).toEqual([{ senderAddress: WIFE, reason: 'wrong_sender' }]);
+    });
+  });
 });