@celilo/cli 0.14.4 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +18 -2
  3. package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +3 -3
  6. package/src/cli/commands/alerts-list.ts +10 -0
  7. package/src/cli/commands/alerts-poll.ts +12 -6
  8. package/src/cli/commands/alerts-sweep.ts +22 -81
  9. package/src/cli/commands/backup-sweep.ts +65 -0
  10. package/src/cli/commands/module-operations.test.ts +93 -0
  11. package/src/cli/commands/module-operations.ts +134 -0
  12. package/src/cli/commands/module-upgrade.test.ts +32 -20
  13. package/src/cli/commands/module-upgrade.ts +37 -32
  14. package/src/cli/commands/monitor.ts +26 -6
  15. package/src/cli/commands/system-audit.ts +3 -30
  16. package/src/cli/completion.ts +18 -1
  17. package/src/cli/index.ts +11 -0
  18. package/src/db/schema.ts +5 -3
  19. package/src/manifest/schema.ts +4 -1
  20. package/src/module/packaging/build.ts +4 -0
  21. package/src/services/alerting/builtin-source.ts +17 -2
  22. package/src/services/alerting/delivery-loop.test.ts +5 -1
  23. package/src/services/alerting/format.test.ts +0 -1
  24. package/src/services/alerting/inbound-poller.test.ts +44 -8
  25. package/src/services/alerting/inbound-poller.ts +65 -28
  26. package/src/services/alerting/notify-deps.ts +113 -0
  27. package/src/services/alerting/run-monitor.ts +0 -1
  28. package/src/services/alerting/store.test.ts +1 -1
  29. package/src/services/alerting/store.ts +0 -2
  30. package/src/services/alerting/sweep-runner.test.ts +11 -2
  31. package/src/services/alerting/sweep-runner.ts +14 -7
  32. package/src/services/audit/backup-source.ts +54 -0
  33. package/src/services/audit/backups.test.ts +7 -2
  34. package/src/services/audit/backups.ts +10 -18
  35. package/src/services/backup-cipher.test.ts +188 -0
  36. package/src/services/backup-cipher.ts +178 -0
  37. package/src/services/backup-create.ts +20 -30
  38. package/src/services/backup-envelope-roundtrip.test.ts +6 -26
  39. package/src/services/backup-restore.ts +10 -16
  40. package/src/services/backup-schedule.ts +35 -0
  41. package/src/services/backup-sweep.test.ts +148 -0
  42. package/src/services/backup-sweep.ts +124 -0
  43. package/src/services/deploy-posture.ts +15 -2
  44. package/src/services/module-operations.test.ts +67 -6
  45. package/src/services/module-operations.ts +69 -19
  46. package/src/services/module-subscriptions.test.ts +33 -2
  47. package/src/services/module-subscriptions.ts +10 -1
  48. package/src/services/module-validator/typescript-build.test.ts +20 -1
  49. package/src/services/module-validator/typescript-build.ts +9 -5
  50. package/src/services/restore-from-file.ts +6 -21
  51. package/src/templates/generator.test.ts +88 -0
  52. package/src/templates/generator.ts +119 -16
@@ -0,0 +1,134 @@
1
+ /**
2
+ * `celilo module operations` — see and release the module-operation lock.
3
+ *
4
+ * Backup and restore refuse to run while another operation is in flight.
5
+ * When that refusal is wrong, the operator previously had no way to see
6
+ * the lock at all, let alone clear it: the error said "wait for it to
7
+ * complete", which for a suspended process is advice that can never come
8
+ * true. A `module deploy` Ctrl-Z'd on a lost terminal blocked every
9
+ * backup on the fleet for 20 days on exactly that advice.
10
+ *
11
+ * `clear` marks rows failed rather than deleting them — the history of
12
+ * what was abandoned, and when, is worth more than a tidy table.
13
+ */
14
+
15
+ import { and, eq } from 'drizzle-orm';
16
+ import { getDb } from '../../db/client';
17
+ import { type ModuleOperation, moduleOperations } from '../../db/schema';
18
+ import { OPERATION_TTL_MS, isPidRunnable } from '../../services/module-operations';
19
+ import type { CommandResult } from '../types';
20
+
21
+ /** Why a row is not holding the lock, or null when it still is. */
22
+ function abandonedReason(row: ModuleOperation, now: number): string | null {
23
+ if (now - row.startedAt.getTime() > OPERATION_TTL_MS) return 'expired';
24
+ if (!isPidRunnable(row.pid)) return 'not running';
25
+ return null;
26
+ }
27
+
28
+ function formatAge(ms: number): string {
29
+ const minutes = Math.floor(ms / 60_000);
30
+ if (minutes < 60) return `${minutes}m`;
31
+ const hours = Math.floor(minutes / 60);
32
+ if (hours < 24) return `${hours}h`;
33
+ return `${Math.floor(hours / 24)}d`;
34
+ }
35
+
36
+ function inProgressRows(): ModuleOperation[] {
37
+ return getDb()
38
+ .select()
39
+ .from(moduleOperations)
40
+ .where(eq(moduleOperations.status, 'in_progress'))
41
+ .all();
42
+ }
43
+
44
+ function handleList(): CommandResult {
45
+ const now = Date.now();
46
+ const rows = inProgressRows();
47
+
48
+ if (rows.length === 0) {
49
+ console.log('\nNo module operations in progress.\n');
50
+ return { success: true, message: 'no operations in progress' };
51
+ }
52
+
53
+ console.log('\nModule operations in progress:\n');
54
+ let holding = 0;
55
+ for (const row of rows) {
56
+ const reason = abandonedReason(row, now);
57
+ if (!reason) holding++;
58
+ const age = formatAge(now - row.startedAt.getTime());
59
+ const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
60
+ console.log(
61
+ ` ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago ${status}`,
62
+ );
63
+ }
64
+
65
+ const abandoned = rows.length - holding;
66
+ console.log('');
67
+ if (abandoned > 0) {
68
+ console.log(`${abandoned} abandoned row(s) — "celilo module operations clear" sweeps them.\n`);
69
+ }
70
+
71
+ return {
72
+ success: true,
73
+ message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
74
+ };
75
+ }
76
+
77
+ /**
78
+ * Release abandoned rows. `--all` also releases rows whose process still
79
+ * looks alive.
80
+ *
81
+ * The permissive form exists because the pathological case is precisely
82
+ * the one where our liveness detection was wrong — a pid that has been
83
+ * recycled by an unrelated process reads as perfectly healthy. Refusing
84
+ * to clear it would recreate the outage this command exists to end. It
85
+ * is opt-in and names what it is overriding.
86
+ */
87
+ function handleClear(flags: Record<string, boolean | string>): CommandResult {
88
+ const db = getDb();
89
+ const now = Date.now();
90
+ const force = flags.all === true;
91
+ const rows = inProgressRows();
92
+
93
+ const targets = force ? rows : rows.filter((row) => abandonedReason(row, now) !== null);
94
+
95
+ if (targets.length === 0) {
96
+ const held = rows.length;
97
+ if (held > 0) {
98
+ return {
99
+ success: true,
100
+ message: `Nothing to clear — ${held} operation(s) still look genuinely in flight. Use --all to release them anyway.`,
101
+ };
102
+ }
103
+ return { success: true, message: 'Nothing to clear — no operations in progress.' };
104
+ }
105
+
106
+ for (const row of targets) {
107
+ db.update(moduleOperations)
108
+ .set({
109
+ status: 'failed',
110
+ completedAt: new Date(),
111
+ errorMessage: 'abandoned — released by "celilo module operations clear"',
112
+ })
113
+ .where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
114
+ .run();
115
+ console.log(` released ${row.operation} of ${row.moduleId} (pid ${row.pid})`);
116
+ }
117
+
118
+ return { success: true, message: `Released ${targets.length} operation(s)` };
119
+ }
120
+
121
+ export function handleModuleOperations(
122
+ args: string[],
123
+ flags: Record<string, boolean | string>,
124
+ ): CommandResult {
125
+ const action = args[0];
126
+
127
+ if (!action || action === 'list') return handleList();
128
+ if (action === 'clear') return handleClear(flags);
129
+
130
+ return {
131
+ success: false,
132
+ error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
133
+ };
134
+ }
@@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test';
2
2
  import type { ModuleManifest } from '../../manifest/schema';
3
3
  import {
4
4
  type PollCandidate,
5
+ isPollInvocation,
5
6
  needsPreUpgradeBackup,
6
7
  pickAutoUpgrade,
7
8
  pickUpgradePolicy,
@@ -15,39 +16,50 @@ function manifest(hooks?: ModuleManifest['hooks']): ModuleManifest {
15
16
  const withBackupHook = manifest({ on_backup: { script: './scripts/backup.ts', timeout: 300000 } });
16
17
  const noBackupHook = manifest({ on_install: { script: './scripts/setup.ts', timeout: 180000 } });
17
18
 
18
- describe('pickUpgradePolicy (ISS-0138 — config override > manifest default > by-semver)', () => {
19
- test('operator config wins over the manifest default', () => {
20
- expect(pickUpgradePolicy('always-safe', 'always-fast')).toBe('always-safe');
19
+ describe('pickUpgradePolicy (ISS-0138 — operator config, else by-semver)', () => {
20
+ test('operator config decides', () => {
21
+ expect(pickUpgradePolicy('always-safe')).toBe('always-safe');
22
+ expect(pickUpgradePolicy('always-fast')).toBe('always-fast');
21
23
  });
22
24
 
23
- test('manifest default applies when there is no config override', () => {
24
- expect(pickUpgradePolicy(undefined, 'always-fast')).toBe('always-fast');
25
+ test('defaults to by-semver when unset', () => {
26
+ expect(pickUpgradePolicy(undefined)).toBe('by-semver');
25
27
  });
26
28
 
27
- test('defaults to by-semver when neither is set', () => {
28
- expect(pickUpgradePolicy(undefined, undefined)).toBe('by-semver');
29
+ test('an unknown value falls back to by-semver (no crash on bad input)', () => {
30
+ expect(pickUpgradePolicy('bogus')).toBe('by-semver');
29
31
  });
32
+ });
30
33
 
31
- test('an unknown value falls back to by-semver (no crash on bad input)', () => {
32
- expect(pickUpgradePolicy('bogus', undefined)).toBe('by-semver');
33
- expect(pickUpgradePolicy(undefined, 'nonsense')).toBe('by-semver');
34
+ describe('pickAutoUpgrade (ISS-0139 opt-in via operator config, default off)', () => {
35
+ test('config value (string or boolean) decides', () => {
36
+ expect(pickAutoUpgrade('true')).toBe(true);
37
+ expect(pickAutoUpgrade('false')).toBe(false);
38
+ expect(pickAutoUpgrade(true)).toBe(true);
39
+ expect(pickAutoUpgrade(false)).toBe(false);
40
+ });
41
+
42
+ test('defaults to OFF (opt-in) when unset', () => {
43
+ expect(pickAutoUpgrade(undefined)).toBe(false);
34
44
  });
35
45
  });
36
46
 
37
- describe('pickAutoUpgrade (ISS-0139 opt-in: config override > manifest default > off)', () => {
38
- test('config override (string or boolean) wins', () => {
39
- expect(pickAutoUpgrade('true', false)).toBe(true);
40
- expect(pickAutoUpgrade('false', true)).toBe(false);
41
- expect(pickAutoUpgrade(true, false)).toBe(true);
47
+ // The dispatcher spawns a subprocess handler as `<handler> <event_id>`
48
+ // (openspec/specs/event-bus/spec.md), so the registry-poll subscription's
49
+ // handler arrives as `celilo module upgrade --poll 5517`. Without --poll the
50
+ // event id landed in the module-name slot and every 15m tick died with
51
+ // "Module not found: 5517" — the poll never ran on celilo-mgr for weeks.
52
+ describe('isPollInvocation (the CD poll must survive the appended event id)', () => {
53
+ test('--poll wins over a positional (the dispatcher-appended event id)', () => {
54
+ expect(isPollInvocation(['5517'], { poll: true })).toBe(true);
42
55
  });
43
56
 
44
- test('manifest default applies with no override', () => {
45
- expect(pickAutoUpgrade(undefined, true)).toBe(true);
46
- expect(pickAutoUpgrade(undefined, false)).toBe(false);
57
+ test('bare `module upgrade` is still the poll', () => {
58
+ expect(isPollInvocation([], {})).toBe(true);
47
59
  });
48
60
 
49
- test('defaults to OFF (opt-in) when neither is set', () => {
50
- expect(pickAutoUpgrade(undefined, undefined)).toBe(false);
61
+ test('a named module without --poll is a single-module upgrade', () => {
62
+ expect(isPollInvocation(['lunacycle'], {})).toBe(false);
51
63
  });
52
64
  });
53
65
 
@@ -37,33 +37,29 @@ import { classifyVersionChange, fetchAndUpdate } from './module-update';
37
37
  const VALID_POLICIES: readonly UpgradePolicy[] = ['by-semver', 'always-safe', 'always-fast'];
38
38
 
39
39
  /**
40
- * Pick the effective upgrade policy. Pure (Rule 10): operator config override
41
- * wins over the manifest default; an unknown/absent value falls back to
42
- * `by-semver`.
40
+ * Pick the effective upgrade policy. Pure (Rule 10): operator config decides;
41
+ * an unknown/absent value falls back to `by-semver`.
42
+ *
43
+ * Config-only by construction — the manifest schema is `.strict()` and declares
44
+ * no `upgrade_policy`, so a manifest that set one could never pass validation.
45
+ * (The old manifest-default branch read a key no valid manifest can carry —
46
+ * Rule 3.9 dead code, deleted rather than left as a corpse.)
43
47
  */
44
- export function pickUpgradePolicy(
45
- fromConfig: string | undefined,
46
- fromManifest: string | undefined,
47
- ): UpgradePolicy {
48
- const candidate = fromConfig ?? fromManifest;
49
- return VALID_POLICIES.includes(candidate as UpgradePolicy)
50
- ? (candidate as UpgradePolicy)
48
+ export function pickUpgradePolicy(fromConfig: string | undefined): UpgradePolicy {
49
+ return VALID_POLICIES.includes(fromConfig as UpgradePolicy)
50
+ ? (fromConfig as UpgradePolicy)
51
51
  : 'by-semver';
52
52
  }
53
53
 
54
54
  /**
55
55
  * Resolve whether a module opts into auto-upgrade by the poll. Pure (Rule 10):
56
- * operator config override wins over the manifest default; the default is OFF
57
- * (opt-in — a production module isn't auto-upgraded unless chosen).
56
+ * operator config only (same `.strict()` reason as pickUpgradePolicy), default
57
+ * OFF — a production module isn't auto-upgraded unless the operator chose it:
58
+ * `celilo module config set <module> auto_upgrade true`.
58
59
  */
59
- export function pickAutoUpgrade(
60
- fromConfig: string | boolean | undefined,
61
- fromManifest: boolean | undefined,
62
- ): boolean {
60
+ export function pickAutoUpgrade(fromConfig: string | boolean | undefined): boolean {
63
61
  if (typeof fromConfig === 'boolean') return fromConfig;
64
- if (fromConfig === 'true') return true;
65
- if (fromConfig === 'false') return false;
66
- return fromManifest ?? false;
62
+ return fromConfig === 'true';
67
63
  }
68
64
 
69
65
  /**
@@ -113,14 +109,11 @@ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] {
113
109
  return targets;
114
110
  }
115
111
 
116
- /** Resolve a module's auto_upgrade flag from config (override) + manifest (default). */
117
- function resolveAutoUpgrade(moduleId: string, manifest: ModuleManifest): boolean {
112
+ /** Resolve a module's auto_upgrade opt-in from operator config. */
113
+ function resolveAutoUpgrade(moduleId: string): boolean {
118
114
  const cfg = getModuleConfigValue(moduleId, 'auto_upgrade');
119
- const fromConfig =
120
- typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined;
121
115
  return pickAutoUpgrade(
122
- fromConfig,
123
- (manifest as ModuleManifest & { auto_upgrade?: boolean }).auto_upgrade,
116
+ typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined,
124
117
  );
125
118
  }
126
119
 
@@ -158,13 +151,11 @@ async function upgradeOneModule(
158
151
  (updatedRow?.manifestData as ModuleManifest | undefined) ??
159
152
  (mod.manifestData as ModuleManifest);
160
153
 
161
- // Posture. Version delta is installed→target; the policy comes from the
162
- // TARGET manifest (the version being installed declares its upgrade risk),
163
- // with an operator config override still winning.
154
+ // Posture. Version delta is installed→target; the policy comes from operator
155
+ // config (`celilo module config set <m> upgrade_policy …`), else by-semver.
164
156
  const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy');
165
157
  const modulePolicy = pickUpgradePolicy(
166
158
  typeof configPolicy?.value === 'string' ? configPolicy.value : undefined,
167
- (targetManifest as ModuleManifest & { upgrade_policy?: string }).upgrade_policy,
168
159
  );
169
160
  // Per-release deploy_posture override lives in the .netapp release metadata;
170
161
  // reading it requires fetching the package first. Deferred — the classifier
@@ -244,7 +235,7 @@ async function runRegistryPoll(
244
235
  moduleId: mod.id,
245
236
  installed: mod.version,
246
237
  latest: await latestRegistryVersion(client, mod.id),
247
- autoUpgrade: resolveAutoUpgrade(mod.id, mod.manifestData as ModuleManifest),
238
+ autoUpgrade: resolveAutoUpgrade(mod.id),
248
239
  });
249
240
  }
250
241
 
@@ -278,6 +269,20 @@ async function runRegistryPoll(
278
269
  };
279
270
  }
280
271
 
272
+ /**
273
+ * Pure (Rule 10.1): is this the CD poll rather than a single-module upgrade?
274
+ *
275
+ * `--poll` is what a bus subscription MUST use. The dispatcher spawns a
276
+ * subprocess handler as `<handler> <event_id>` (openspec/specs/event-bus/spec.md),
277
+ * so a bare `celilo module upgrade` handler arrives as `celilo module upgrade
278
+ * 5517` — the event id lands in the optional module-name slot and the poll dies
279
+ * with "Module not found: 5517" every tick. An explicit flag makes the poll
280
+ * invocation immune to the appended id instead of relying on argv position.
281
+ */
282
+ export function isPollInvocation(args: string[], flags: Record<string, string | boolean>): boolean {
283
+ return Boolean(flags.poll) || !getArg(args, 0);
284
+ }
285
+
281
286
  export async function handleModuleUpgrade(
282
287
  args: string[],
283
288
  flags: Record<string, string | boolean> = {},
@@ -285,8 +290,8 @@ export async function handleModuleUpgrade(
285
290
  const db = getDb();
286
291
  const moduleId = getArg(args, 0);
287
292
 
288
- // No name → the CD poll over all auto_upgrade modules.
289
- if (!moduleId) {
293
+ // `--poll` or no name → the CD poll over all auto_upgrade modules.
294
+ if (isPollInvocation(args, flags) || !moduleId) {
290
295
  return runRegistryPoll(db, flags);
291
296
  }
292
297
 
@@ -11,7 +11,10 @@ import { getEventBusPath } from '../../config/paths';
11
11
  import { getDb } from '../../db/client';
12
12
  import type { MonitorKind } from '../../db/schema';
13
13
  import { parseIntervalMinutes } from '../../manifest/schema';
14
- import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
14
+ import {
15
+ isSchedulableBuiltin,
16
+ runBuiltinCheckForMonitor,
17
+ } from '../../services/alerting/builtin-source';
15
18
  import { loadModuleCoverage } from '../../services/alerting/coverage-source';
16
19
  import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
17
20
  import {
@@ -21,6 +24,7 @@ import {
21
24
  listMonitors,
22
25
  setMonitorEnabled,
23
26
  } from '../../services/alerting/monitors';
27
+ import { listPolicies } from '../../services/alerting/people';
24
28
  import { runOneMonitor } from '../../services/alerting/run-monitor';
25
29
  import { promoteReadyAlerts } from '../../services/alerting/store';
26
30
  import type { DriftCategory } from '../../services/audit/types';
@@ -37,7 +41,7 @@ function buildDeps() {
37
41
  return {
38
42
  runModuleCheck: (moduleId: string) =>
39
43
  runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
40
- runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category),
44
+ runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db),
41
45
  loadModuleCoverage: () => loadModuleCoverage(db),
42
46
  now: () => new Date(),
43
47
  graceMs: DEFAULT_GRACE_MS,
@@ -53,15 +57,26 @@ function handleList(): CommandResult {
53
57
  return { success: true, message: 'No monitors configured' };
54
58
  }
55
59
 
60
+ // The POLICY column is the answer to "why did nothing page me". A monitor
61
+ // with no policy is one whose alerts reach nobody, and until this column
62
+ // existed there was no way to see that from the CLI at all — `assign`
63
+ // reported success and nothing anywhere reflected the result (#481).
64
+ const policies = new Map(listPolicies(getDb()).map((p) => [p.id, p.name]));
65
+ const policyOf = (id: string | null) =>
66
+ id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
67
+
56
68
  const width = Math.max(6, ...rows.map((r) => r.target.length));
69
+ const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
57
70
  console.log('');
58
- console.log(`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(6)} STATE`);
71
+ console.log(
72
+ `${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(6)} ${'POLICY'.padEnd(policyWidth)} STATE`,
73
+ );
59
74
  for (const monitor of rows) {
60
75
  const state = monitor.enabled ? 'enabled' : 'disabled';
61
76
  const suffix = monitor.lastRunAt ? '' : ' (never run)';
62
77
  const every = `${monitor.intervalMinutes}m`;
63
78
  console.log(
64
- `${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${state}${suffix}`,
79
+ `${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)} ${state}${suffix}`,
65
80
  );
66
81
  }
67
82
  console.log('');
@@ -89,9 +104,14 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
89
104
  }
90
105
 
91
106
  // A target naming an audit category is a built-in check; anything else is a
92
- // module's health_check hook.
107
+ // module's health_check hook. `isSchedulableBuiltin` is checked explicitly
108
+ // because not every category is snake_case — `backups` is one word, and the
109
+ // underscore heuristic alone would file it as a module hook against a module
110
+ // that does not exist.
93
111
  const kind: MonitorKind =
94
- target === HEALTH_COVERAGE_CHECK || target.includes('_') ? 'builtin_check' : 'module_hook';
112
+ target === HEALTH_COVERAGE_CHECK || isSchedulableBuiltin(target) || target.includes('_')
113
+ ? 'builtin_check'
114
+ : 'module_hook';
95
115
 
96
116
  createMonitor(db, { kind, target, intervalMinutes });
97
117
 
@@ -20,12 +20,11 @@ import { existsSync, readFileSync } from 'node:fs';
20
20
  import { dirname, join } from 'node:path';
21
21
  import { fileURLToPath } from 'node:url';
22
22
  import { promisify } from 'node:util';
23
- import { eq, isNotNull } from 'drizzle-orm';
23
+ import { isNotNull } from 'drizzle-orm';
24
24
  import { testDigitalOceanConnection } from '../../api-clients/digitalocean';
25
25
  import { testProxmoxConnection } from '../../api-clients/proxmox';
26
26
  import { getDb } from '../../db/client';
27
27
  import {
28
- backups,
29
28
  capabilitySecrets,
30
29
  moduleConfigs as moduleConfigsTbl,
31
30
  modules,
@@ -38,6 +37,7 @@ import { decryptSecret } from '../../secrets/encryption';
38
37
  import { getOrCreateMasterKey } from '../../secrets/master-key';
39
38
  import { runAudit } from '../../services/audit';
40
39
  import type { DriftFinding, SystemAuditReport } from '../../services/audit';
40
+ import { loadBackupAuditInfo } from '../../services/audit/backup-source';
41
41
  import {
42
42
  type LatestCliVersionFetcher,
43
43
  fetchLatestCliVersion,
@@ -154,28 +154,6 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
154
154
 
155
155
  const registryClient = new RegistryClient();
156
156
 
157
- // Most recent successful backup per module.
158
- // The `backups` table is added via inline ALTER statements in db/client.ts
159
- // for upgraded databases; a freshly-initialized DB created from the
160
- // drizzle journal alone may not have it yet. Treat absence as "no
161
- // backups recorded" rather than crashing the audit.
162
- const latestBackupByModule = new Map<string, number>();
163
- try {
164
- const successfulBackups = db
165
- .select()
166
- .from(backups)
167
- .where(eq(backups.status, 'completed'))
168
- .all();
169
- for (const b of successfulBackups) {
170
- if (!b.moduleId || !b.completedAt) continue;
171
- const ts = b.completedAt.getTime();
172
- const prev = latestBackupByModule.get(b.moduleId);
173
- if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts);
174
- }
175
- } catch {
176
- // backups table missing — leave map empty
177
- }
178
-
179
157
  // Build per-module config map for module-configs check.
180
158
  const allConfigs = db
181
159
  .select()
@@ -196,12 +174,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
196
174
  configs: configsByModule.get(m.id) ?? {},
197
175
  }));
198
176
 
199
- const installedBackupInfo = deployedModules.map((m) => ({
200
- id: m.id,
201
- state: m.state,
202
- manifest: m.manifestData as ModuleManifest,
203
- lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
204
- }));
177
+ const installedBackupInfo = loadBackupAuditInfo(db);
205
178
 
206
179
  // Compose per-module TF_VAR_* env vars in parallel — each call hits
207
180
  // the secret store / DB so they're not free, but they're all
@@ -185,6 +185,7 @@ export async function getCompletions(words: string[], current: number): Promise<
185
185
  'secret',
186
186
  'status',
187
187
  'where',
188
+ 'operations',
188
189
  'terraform-unlock',
189
190
  'types',
190
191
  'validate',
@@ -198,6 +199,12 @@ export async function getCompletions(words: string[], current: number): Promise<
198
199
  return filterSuggestions(subcommands, args[2] || '');
199
200
  }
200
201
 
202
+ // Module operations subcommands (celilo module operations list/clear)
203
+ if (command === 'module' && args[1] === 'operations' && currentIndex === 2) {
204
+ const subcommands = ['list', 'clear'];
205
+ return filterSuggestions(subcommands, args[2] || '');
206
+ }
207
+
201
208
  // Module types subcommands (celilo module types generate/check)
202
209
  if (command === 'module' && args[1] === 'types' && currentIndex === 2) {
203
210
  const subcommands = ['generate', 'check'];
@@ -500,7 +507,17 @@ export async function getCompletions(words: string[], current: number): Promise<
500
507
 
501
508
  // Backup subcommands
502
509
  if (command === 'backup' && currentIndex === 1) {
503
- const subcommands = ['create', 'list', 'restore', 'delete', 'prune', 'name', 'import', 'pull'];
510
+ const subcommands = [
511
+ 'create',
512
+ 'sweep',
513
+ 'list',
514
+ 'restore',
515
+ 'delete',
516
+ 'prune',
517
+ 'name',
518
+ 'import',
519
+ 'pull',
520
+ ];
504
521
  return filterSuggestions(subcommands, args[1] || '');
505
522
  }
506
523
 
package/src/cli/index.ts CHANGED
@@ -66,6 +66,7 @@ import { handleModuleHealth } from './commands/module-health';
66
66
  import { handleModuleImport } from './commands/module-import';
67
67
  import { handleModuleList } from './commands/module-list';
68
68
  import { handleModuleLogs } from './commands/module-logs';
69
+ import { handleModuleOperations } from './commands/module-operations';
69
70
  import { handleModulePublish } from './commands/module-publish';
70
71
  import { handleModuleRemove } from './commands/module-remove';
71
72
  import { handleModuleSearch } from './commands/module-search';
@@ -747,6 +748,9 @@ Subcommands:
747
748
  --storage <id> Use specific storage destination
748
749
  --no-interactive Non-interactive mode (for cron)
749
750
 
751
+ sweep Back up every module whose declared schedule is due
752
+ (run by the event bus on timer.tick.1h, not by hand)
753
+
750
754
  list [module-id] List available backups
751
755
  Options:
752
756
  --limit <n> Number of backups to show (default: 20)
@@ -1466,6 +1470,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1466
1470
  return handleModuleLogs(parsed.args, parsed.flags);
1467
1471
  case 'health':
1468
1472
  return handleModuleHealth(parsed.args, parsed.flags);
1473
+ case 'operations':
1474
+ return handleModuleOperations(parsed.args, parsed.flags);
1469
1475
  case 'remove':
1470
1476
  return handleModuleRemove(parsed.args, parsed.flags);
1471
1477
  case 'update':
@@ -1864,6 +1870,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
1864
1870
  return handleBackupCreate(parsed.args, parsed.flags);
1865
1871
  }
1866
1872
 
1873
+ if (parsed.subcommand === 'sweep') {
1874
+ const { handleBackupSweep } = await import('./commands/backup-sweep');
1875
+ return handleBackupSweep();
1876
+ }
1877
+
1867
1878
  if (parsed.subcommand === 'list') {
1868
1879
  const { handleBackupList } = await import('./commands/backup-list');
1869
1880
  return handleBackupList(parsed.args, parsed.flags);
package/src/db/schema.ts CHANGED
@@ -1048,9 +1048,11 @@ export const alerts = sqliteTable(
1048
1048
  deferredRouteId: text('deferred_route_id').references(() => routes.id, {
1049
1049
  onDelete: 'set null',
1050
1050
  }),
1051
- escalationPolicyId: text('escalation_policy_id').references(() => escalationPolicies.id, {
1052
- onDelete: 'set null',
1053
- }),
1051
+ // No escalation policy is stored here on purpose. It lives on the MONITOR
1052
+ // and is resolved fresh on every sweep, so assigning a policy takes effect
1053
+ // on alerts that are already firing. Snapshotting it at alert creation made
1054
+ // `escalation-policy assign` a no-op for exactly the alert an operator was
1055
+ // trying to route — the one already paging nobody (#481).
1054
1056
  message: text('message').notNull(),
1055
1057
  details: text('details'),
1056
1058
  resolvedAt: integer('resolved_at', { mode: 'timestamp' }),
@@ -694,7 +694,10 @@ export const ModuleManifestSchema = z
694
694
 
695
695
  backup: z
696
696
  .object({
697
- schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('manual'),
697
+ // Absent means `daily`. Opting out of backups entirely is a real
698
+ // decision and takes an explicit `manual` — see
699
+ // [[services/backup-schedule.ts]].
700
+ schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
698
701
  retention: z
699
702
  .object({
700
703
  count: z.number().int().positive().default(7),
@@ -59,6 +59,10 @@ const EXCLUDE_PATTERNS = [
59
59
  '.DS_Store',
60
60
  '*.netapp',
61
61
  '*.test.ts',
62
+ // Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
63
+ // check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
64
+ // make every module's packaged content change whenever the shared base moves.
65
+ 'tsconfig.json',
62
66
  'checksums.json',
63
67
  'signature.sig',
64
68
  ];
@@ -12,22 +12,37 @@
12
12
  * as "nothing is wrong".
13
13
  */
14
14
 
15
+ import type { DbClient } from '../../db/client';
16
+ import { loadBackupAuditInfo } from '../audit/backup-source';
17
+ import { auditBackups } from '../audit/backups';
15
18
  import { auditMachinesReachable } from '../audit/machines-reachable';
16
19
  import type { DriftCategory, DriftFinding } from '../audit/types';
17
20
  import { probeMachines } from '../machine-probe';
18
21
 
19
22
  /** Categories a monitor can currently schedule. */
20
- export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = ['machines_reachable'];
23
+ export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
24
+ 'machines_reachable',
25
+ 'backups',
26
+ ];
21
27
 
22
28
  export function isSchedulableBuiltin(category: string): category is DriftCategory {
23
29
  return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
24
30
  }
25
31
 
26
- export async function runBuiltinCheckForMonitor(category: DriftCategory): Promise<DriftFinding[]> {
32
+ export async function runBuiltinCheckForMonitor(
33
+ category: DriftCategory,
34
+ db: DbClient,
35
+ ): Promise<DriftFinding[]> {
27
36
  if (category === 'machines_reachable') {
28
37
  return auditMachinesReachable({ results: await probeMachines() });
29
38
  }
30
39
 
40
+ // Local DB reads only — cheap enough to run on every sweep, which is
41
+ // the whole reason this category is schedulable and most are not.
42
+ if (category === 'backups') {
43
+ return auditBackups({ modules: loadBackupAuditInfo(db) });
44
+ }
45
+
31
46
  throw new Error(
32
47
  `Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
33
48
  );
@@ -68,6 +68,11 @@ beforeAll(async () => {
68
68
  ...process.env,
69
69
  SIGNAL_RPC_PORT: String(PORT),
70
70
  SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
71
+ // The flag the module's systemd unit must pass. The simulator defaults
72
+ // to signal-cli's real default (`on-start`), where the daemon drains
73
+ // replies into its own SSE stream and REFUSES `receive` — so a loop test
74
+ // may only read replies from a daemon started the way celilo deploys it.
75
+ SIGNAL_RECEIVE_MODE: 'manual',
71
76
  },
72
77
  stdout: 'pipe',
73
78
  stderr: 'pipe',
@@ -169,7 +174,6 @@ describe('delivery loop against the signal-cli simulator', () => {
169
174
  silencedUntil: null,
170
175
  escalationStep: 0,
171
176
  nextEscalationAt: null,
172
- escalationPolicyId: null,
173
177
  message: '/var 94% used',
174
178
  details: null,
175
179
  resolvedAt: null,