@celilo/cli 0.17.0 → 0.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. package/CELILO_CORE_MODULES.md +1 -1
  2. package/CELILO_SUBSYSTEMS.md +38 -9
  3. package/drizzle/0019_backup_pid.sql +18 -0
  4. package/drizzle/meta/_journal.json +7 -0
  5. package/package.json +5 -5
  6. package/schemas/system_config.json +1 -1
  7. package/src/api/remote-client.test.ts +62 -0
  8. package/src/api/serve.ts +14 -6
  9. package/src/cli/command-tree-parser.ts +0 -1
  10. package/src/cli/commands/apt-upgrade.test.ts +20 -1
  11. package/src/cli/commands/apt-upgrade.ts +12 -2
  12. package/src/cli/commands/backup-sweep.ts +62 -0
  13. package/src/cli/commands/events.ts +90 -0
  14. package/src/cli/commands/module-operations.test.ts +45 -1
  15. package/src/cli/commands/module-operations.ts +35 -12
  16. package/src/cli/commands/module-show.ts +1 -0
  17. package/src/cli/commands/module-update.test.ts +72 -1
  18. package/src/cli/commands/module-update.ts +45 -22
  19. package/src/cli/commands/system-audit.ts +2 -0
  20. package/src/cli/commands/system-migrate.test.ts +56 -0
  21. package/src/cli/commands/system-migrate.ts +92 -4
  22. package/src/cli/commands/system-update.ts +5 -0
  23. package/src/cli/completion.ts +19 -0
  24. package/src/cli/fuel-gauge.ts +0 -1
  25. package/src/cli/generate-zsh-completion.ts +1 -1
  26. package/src/cli/index.ts +5 -1
  27. package/src/cli/tui/audit-state.ts +4 -0
  28. package/src/cli/tui/audit-tui.test.tsx +0 -1
  29. package/src/db/migration-status.test.ts +114 -0
  30. package/src/db/migration-status.ts +78 -0
  31. package/src/db/schema-introspection.ts +8 -1
  32. package/src/db/schema.ts +53 -9
  33. package/src/hooks/capability-loader.ts +30 -1
  34. package/src/ipam/allocator.ts +13 -3
  35. package/src/services/alerting/builtin-monitors.test.ts +42 -0
  36. package/src/services/alerting/builtin-monitors.ts +2 -0
  37. package/src/services/alerting/builtin-source.ts +15 -0
  38. package/src/services/audit/abandoned-operations.test.ts +73 -0
  39. package/src/services/audit/abandoned-operations.ts +0 -0
  40. package/src/services/audit/disk-space.test.ts +111 -0
  41. package/src/services/audit/disk-space.ts +114 -0
  42. package/src/services/audit/index.test.ts +1 -0
  43. package/src/services/audit/index.ts +9 -0
  44. package/src/services/audit/types.ts +2 -0
  45. package/src/services/backup-create.ts +4 -4
  46. package/src/services/backup-in-flight-refusal.test.ts +2 -0
  47. package/src/services/backup-metadata.ts +4 -0
  48. package/src/services/backup-staging.test.ts +134 -0
  49. package/src/services/backup-staging.ts +192 -0
  50. package/src/services/backup-sweep.test.ts +68 -0
  51. package/src/services/backup-sweep.ts +62 -0
  52. package/src/services/bus-interview.ts +11 -5
  53. package/src/services/config-interview.ts +1 -1
  54. package/src/services/deploy-ansible.ts +0 -1
  55. package/src/services/disk-probe.test.ts +74 -0
  56. package/src/services/disk-probe.ts +145 -0
  57. package/src/services/events-daemon.test.ts +244 -0
  58. package/src/services/events-daemon.ts +295 -8
  59. package/src/services/fleet-checks.test.ts +75 -4
  60. package/src/services/fleet-checks.ts +97 -12
  61. package/src/services/interview-errors.ts +20 -0
  62. package/src/services/module-operations.test.ts +22 -0
  63. package/src/services/module-operations.ts +48 -1
  64. package/src/services/module-subscriptions.test.ts +39 -6
  65. package/src/services/module-subscriptions.ts +6 -4
  66. package/src/services/module-types-generator.test.ts +6 -3
  67. package/src/services/module-types-generator.ts +12 -7
  68. package/src/services/remote-responder.test.ts +70 -0
  69. package/src/services/remote-responder.ts +27 -10
  70. package/src/services/responder-probe.ts +3 -1
  71. package/src/services/update/orchestrator.test.ts +1 -0
  72. package/src/variables/context.ts +6 -1
@@ -39,9 +39,9 @@ function seedHeartbeat(
39
39
  );
40
40
  }
41
41
 
42
- /** Write a supervisor unit file so readInstalledUnit('user') sees it. */
43
- function installFakeUnit(home: string): void {
44
- const path = getDaemonUnitPath('linux', home, 'user');
42
+ /** Write a supervisor unit file so readInstalledUnit(scope) sees it. */
43
+ function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', systemRoot = '/'): void {
44
+ const path = getDaemonUnitPath('linux', home, scope, systemRoot);
45
45
  mkdirSync(dirname(path), { recursive: true });
46
46
  writeFileSync(path, '[Unit]\nDescription=fake\n');
47
47
  }
@@ -114,6 +114,51 @@ describe('checkDispatcher', () => {
114
114
  expect(f.detail.join(' ')).toContain('not under a supervisor');
115
115
  });
116
116
 
117
+ // #610 — celilo-mgr had a dead system unit and a live user-scope unit of the
118
+ // SAME name. The old file-exists test called that "supervised".
119
+ it('fails when the running dispatcher is not the pid any installed unit supervises', () => {
120
+ seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 588704 });
121
+ installFakeUnit(home);
122
+ const f = checkDispatcher(bus, {
123
+ now: now,
124
+ home,
125
+ platform: 'linux',
126
+ unitMainPid: () => 3639051, // systemd supervises a different process
127
+ });
128
+ expect(f.status).toBe('fail');
129
+ expect(f.detail.join(' ')).toContain('not the process any installed unit supervises');
130
+ });
131
+
132
+ it('fails when both a user-scope and a system-scope unit are installed', () => {
133
+ seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
134
+ installFakeUnit(home);
135
+ installFakeUnit(home, 'system', dir);
136
+ const f = checkDispatcher(bus, {
137
+ now: now,
138
+ home,
139
+ systemRoot: dir,
140
+ platform: 'linux',
141
+ unitMainPid: () => 4242,
142
+ });
143
+ expect(f.status).toBe('fail');
144
+ expect(f.detail.join(' ')).toContain('same unit name, different services');
145
+ });
146
+
147
+ // Null is ignorance, not evidence: a systemd probe that fails must not
148
+ // manufacture an orphan report.
149
+ it('makes no supervision claim when the unit pid cannot be determined', () => {
150
+ seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000, pid: 4242 });
151
+ installFakeUnit(home);
152
+ const f = checkDispatcher(bus, {
153
+ now: now,
154
+ home,
155
+ platform: 'linux',
156
+ installedCodeMtimeMs: now - 2 * MINUTE,
157
+ unitMainPid: () => null,
158
+ });
159
+ expect(f.status).toBe('ok');
160
+ });
161
+
117
162
  it('warns when the dispatcher started before the installed code (stale)', () => {
118
163
  seedHeartbeat(bus, { startedAt: now - 10 * MINUTE, lastHeartbeat: now - 1000 });
119
164
  installFakeUnit(home);
@@ -425,7 +470,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
425
470
  it('is ok when every schema table is present (fresh migrated DB)', () => {
426
471
  const f = checkSchemaDrift(db);
427
472
  expect(f.status).toBe('ok');
428
- expect(f.summary).toContain('schema tables present');
473
+ expect(f.summary).toContain('schema tables');
429
474
  });
430
475
 
431
476
  it('fails and names a table the running CLI expects but the DB lacks', () => {
@@ -435,6 +480,32 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
435
480
  expect(f.detail.join(' ')).toContain('dns_internal_records');
436
481
  expect(f.remediation).toContain('migrations');
437
482
  });
483
+
484
+ // celilo#604: this is the state the rollout could not check. Every table
485
+ // is present, one MIGRATED COLUMN is not, and the doctor must not call
486
+ // that "migrations applied".
487
+ it('fails and names a migrated COLUMN the DB lacks, with every table present', () => {
488
+ db.$client.run('ALTER TABLE backups DROP COLUMN pid');
489
+ const f = checkSchemaDrift(db);
490
+ expect(f.status).toBe('fail');
491
+ expect(f.detail.join(' ')).toContain('backups.pid');
492
+ expect(f.summary).not.toContain('present');
493
+ });
494
+
495
+ it('says it checked columns, not only tables', () => {
496
+ const f = checkSchemaDrift(db);
497
+ expect(f.status).toBe('ok');
498
+ expect(f.summary).toContain('columns present');
499
+ });
500
+
501
+ it('fails when a journal migration has not been applied on this box', () => {
502
+ db.$client.run(
503
+ 'DELETE FROM `__drizzle_migrations` WHERE created_at = (SELECT MAX(created_at) FROM `__drizzle_migrations`)',
504
+ );
505
+ const f = checkSchemaDrift(db);
506
+ expect(f.status).toBe('fail');
507
+ expect(f.detail.join(' ')).toContain('unapplied migration');
508
+ });
438
509
  });
439
510
  });
440
511
 
@@ -20,7 +20,8 @@
20
20
  import type { Bus } from '@celilo/event-bus';
21
21
  import { inArray } from 'drizzle-orm';
22
22
  import { getModuleStoragePath } from '../config/paths';
23
- import type { DbClient } from '../db/client';
23
+ import { type DbClient, findMigrationsFolder } from '../db/client';
24
+ import { getMigrationStatus } from '../db/migration-status';
24
25
  import { capabilities as capabilitiesTable, modules } from '../db/schema';
25
26
  import { findSchemaDrift } from '../db/schema-introspection';
26
27
  import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
@@ -30,7 +31,13 @@ import type { ModuleManifest } from '../manifest/schema';
30
31
  const CONTROL_PLANE_MODULE = 'celilo-mgmt';
31
32
  import { getModuleSystems } from './deployed-systems';
32
33
  import { listDnsInternalRecords } from './dns-internal-records';
33
- import { type SupervisorPlatform, readInstalledUnit } from './events-daemon';
34
+ import {
35
+ SUPERVISOR_SCOPES,
36
+ type SupervisorPlatform,
37
+ type SupervisorScope,
38
+ readInstalledUnit,
39
+ unitMainPid,
40
+ } from './events-daemon';
34
41
  import { resolveSubscription } from './module-subscriptions';
35
42
 
36
43
  /**
@@ -107,26 +114,43 @@ function worst(statuses: FleetFindingStatus[]): FleetFindingStatus {
107
114
  * its schema is current, so presence is the honest signal.
108
115
  */
109
116
  export function checkSchemaDrift(db: DbClient): FleetFinding {
110
- const { missingTables, missingColumns, tableCount } = findSchemaDrift(db.$client);
117
+ const { missingTables, missingColumns, tableCount, columnCount } = findSchemaDrift(db.$client);
111
118
 
112
119
  const detail: string[] = [];
113
120
  if (missingTables.length > 0) detail.push(`missing table(s): ${missingTables.join(', ')}`);
114
121
  if (missingColumns.length > 0) detail.push(`missing column(s): ${missingColumns.join(', ')}`);
122
+
123
+ // Also name unapplied migrations. Presence is the honest signal for schema
124
+ // objects, but a migration can carry an index or a data fix that presence
125
+ // can't see — and an operator reading "migrations applied" deserves to know
126
+ // when some aren't (celilo#604). Best-effort: an install layout where the
127
+ // journal can't be found must not fail the check.
128
+ let pending: string[] = [];
129
+ try {
130
+ pending = getMigrationStatus(db.$client, findMigrationsFolder()).pending;
131
+ } catch {
132
+ // No journal reachable — the presence check above still stands.
133
+ }
134
+ if (pending.length > 0) detail.push(`unapplied migration(s): ${pending.join(', ')}`);
135
+
115
136
  const status: FleetFindingStatus = detail.length > 0 ? 'fail' : 'ok';
116
137
 
117
138
  return {
118
139
  id: 'schema',
119
140
  title: 'database schema matches the running CLI (migrations applied)',
120
141
  status,
142
+ // Say tables AND columns: "all 35 tables present" reads as though columns
143
+ // went unchecked, which is what sent a rollout to sqlite3 over SSH to
144
+ // confirm a column migration the doctor had in fact already verified.
121
145
  summary:
122
146
  status === 'ok'
123
- ? `all ${tableCount} schema tables present`
147
+ ? `all ${tableCount} schema tables and ${columnCount} columns present, no unapplied migrations`
124
148
  : 'database schema is behind the running CLI — migrations not applied',
125
149
  detail,
126
150
  remediation:
127
151
  status === 'ok'
128
152
  ? null
129
- : 'run `celilo system migrate` to apply pending migrations on this box — see ISS-0100',
153
+ : 'run `celilo system migrate` to apply pending migrations on this box (`celilo system migrate --status` names them) — see ISS-0100',
130
154
  autoFixable: false,
131
155
  };
132
156
  }
@@ -152,6 +176,14 @@ export interface DispatcherCheckOptions {
152
176
  /** Override for readInstalledUnit — tests point this at a temp home. */
153
177
  home?: string;
154
178
  platform?: SupervisorPlatform;
179
+ /** Prefix for system-scope unit paths. Test seam — see getDaemonUnitPath. */
180
+ systemRoot?: string;
181
+ /**
182
+ * Which pid each scope's unit supervises. Injected so the check is testable
183
+ * without systemd. Returning null means "can't tell" — the check then makes
184
+ * no supervision claim rather than guessing.
185
+ */
186
+ unitMainPid?: (scope: SupervisorScope) => number | null;
155
187
  }
156
188
 
157
189
  /**
@@ -208,16 +240,69 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
208
240
  remediations.push('`celilo events repair` to sweep stuck deliveries');
209
241
  }
210
242
 
211
- // (2) supervised — a unit file exists (user or system scope). A
212
- // running dispatcher with NO unit is the orphan case: works now, gone
213
- // after reboot.
214
- const supervised =
215
- readInstalledUnit({ scope: 'user', home: opts.home, platform: opts.platform }).exists ||
216
- readInstalledUnit({ scope: 'system', home: opts.home, platform: opts.platform }).exists;
217
- if (!supervised) {
243
+ // (1b) sole — a duplicate dispatcher can no longer START (it refuses), but one
244
+ // stranded before that shipped keeps running, and it makes every other check
245
+ // here ambiguous: `hb` is whichever of them wrote last. Fail rather than warn —
246
+ // celilo-mgr ran two for 40 days precisely because nothing reported it (#580).
247
+ if (health.dispatcherCount > 1) {
248
+ statuses.push('fail');
249
+ const pids = health.dispatchers.map((d) => d.pid).join(', ');
250
+ detail.push(
251
+ `${health.dispatcherCount} dispatchers are live on this bus (pids ${pids}) — only one may run`,
252
+ );
253
+ remediations.push(
254
+ 'stop the unsupervised one: compare `systemctl show celilo-events.service -p MainPID` against those pids and kill the pid systemd does not own',
255
+ );
256
+ }
257
+
258
+ // (2) supervised — not just "a unit file exists on disk", but "the process
259
+ // that is actually running IS the one an installed unit supervises".
260
+ //
261
+ // The file-exists test this replaces reported green on celilo-mgr while the
262
+ // system unit was dead and a user-scope unit of the SAME NAME served
263
+ // production (#610). A check whose entire job is catching an unsupervised
264
+ // dispatcher cannot be satisfied by a file nobody is running.
265
+ const installedScopes = SUPERVISOR_SCOPES.filter(
266
+ (scope) =>
267
+ readInstalledUnit({
268
+ scope,
269
+ home: opts.home,
270
+ platform: opts.platform,
271
+ systemRoot: opts.systemRoot,
272
+ }).exists,
273
+ );
274
+ if (installedScopes.length === 0) {
218
275
  statuses.push('warn');
219
276
  detail.push('not under a supervisor unit — will not survive a reboot (orphan process)');
220
277
  remediations.push('`celilo events install-daemon` then enable the unit so it is supervised');
278
+ } else {
279
+ if (installedScopes.length > 1) {
280
+ statuses.push('fail');
281
+ detail.push(
282
+ 'both a user-scope AND a system-scope unit are installed — same unit name, different services; ' +
283
+ 'one will lose the race on every boot and retry forever',
284
+ );
285
+ remediations.push(
286
+ 'keep exactly one: `celilo events uninstall-daemon` (user) or `celilo events uninstall-daemon --system`, and disable it in systemd',
287
+ );
288
+ }
289
+ // Only accuse when systemd actually answered. A null probe is ignorance,
290
+ // not evidence of an orphan.
291
+ const probe =
292
+ opts.unitMainPid ?? ((scope: SupervisorScope) => unitMainPid(scope, opts.platform));
293
+ const supervisedPids = installedScopes
294
+ .map((scope) => ({ scope, pid: probe(scope) }))
295
+ .filter((entry): entry is { scope: SupervisorScope; pid: number } => entry.pid !== null);
296
+ if (supervisedPids.length > 0 && !supervisedPids.some((entry) => entry.pid === hb.pid)) {
297
+ statuses.push('fail');
298
+ detail.push(
299
+ `the running dispatcher (pid ${hb.pid}) is not the process any installed unit supervises ` +
300
+ `(${supervisedPids.map((e) => `${e.scope}=${e.pid}`).join(', ')}) — restarting the unit will not restart it`,
301
+ );
302
+ remediations.push(
303
+ 'stop the unsupervised process and let the unit own the dispatcher, or reinstall the unit for the scope that is actually running it',
304
+ );
305
+ }
221
306
  }
222
307
 
223
308
  // (3) current — started before the installed code was last written ⇒
@@ -0,0 +1,20 @@
1
+ /**
2
+ * The one error type that means "this interview question could not be
3
+ * answered". Its own module so `responder-probe` (no responder listening) and
4
+ * `bus-interview` (a responder replied that it couldn't decide) can both throw
5
+ * it without importing each other.
6
+ *
7
+ * Callers catch this to distinguish an *unanswered* question from an answered
8
+ * one — the distinction `module update` conflated when it reported a breaking
9
+ * update as "operator declined" that no operator had ever seen.
10
+ */
11
+ export class InterviewUnansweredError extends Error {
12
+ constructor(
13
+ /** The interview event type that went unanswered, e.g. `interview.required.<scope>.<key>`. */
14
+ readonly queryType: string,
15
+ message: string,
16
+ ) {
17
+ super(message);
18
+ this.name = 'InterviewUnansweredError';
19
+ }
20
+ }
@@ -7,9 +7,12 @@ import { closeDb } from '../db/client';
7
7
  import { runMigrations } from '../db/migrate';
8
8
  import {
9
9
  InFlightError,
10
+ OPERATIONS_SWEEP_PATTERN,
11
+ OPERATIONS_SWEEP_SUBSCRIBER,
10
12
  OPERATION_TTL_MS,
11
13
  checkInFlight,
12
14
  completeOperation,
15
+ ensureOperationsSweepSubscriber,
13
16
  failOperation,
14
17
  isPidRunnable,
15
18
  refuseIfInFlight,
@@ -213,3 +216,22 @@ describe('module-operations', () => {
213
216
  });
214
217
  });
215
218
  });
219
+
220
+ describe('ensureOperationsSweepSubscriber', () => {
221
+ it('registers the hourly reclaim against the existing clear command', () => {
222
+ const calls: Array<{ name: string; pattern: string; handler: string; registeredBy?: string }> =
223
+ [];
224
+ ensureOperationsSweepSubscriber({ subscribe: (options) => calls.push(options) });
225
+
226
+ expect(calls).toEqual([
227
+ {
228
+ name: OPERATIONS_SWEEP_SUBSCRIBER,
229
+ pattern: OPERATIONS_SWEEP_PATTERN,
230
+ handler: 'celilo module operations clear',
231
+ registeredBy: 'celilo-module-operations',
232
+ },
233
+ ]);
234
+ // Finer than the TTL, so a wedged row never survives long.
235
+ expect(OPERATIONS_SWEEP_PATTERN).toBe('timer.tick.1h');
236
+ });
237
+ });
@@ -26,7 +26,8 @@
26
26
  * lock for 20 days and blocked every backup on the fleet.
27
27
  *
28
28
  * Stale rows are ignored rather than deleted; `celilo module operations`
29
- * lists them and `... clear` sweeps them.
29
+ * lists them and `... clear` sweeps them — on an hourly bus tick, not
30
+ * only when a human remembers (see `ensureOperationsSweepSubscriber`).
30
31
  */
31
32
 
32
33
  import { spawnSync } from 'node:child_process';
@@ -119,6 +120,52 @@ export function isPidRunnable(pid: number): boolean {
119
120
  return state !== 'T' && state !== 'Z';
120
121
  }
121
122
 
123
+ /**
124
+ * The `errorMessage` written when a row is released as abandoned.
125
+ *
126
+ * Load-bearing, not cosmetic: releasing marks rows `failed` rather than
127
+ * deleting them, and this exact string is what later distinguishes "the
128
+ * operation reported a failure" from "the operation never reported
129
+ * anything and the sweep reclaimed it". The abandoned-operations audit
130
+ * counts rows by it (`services/audit/abandoned-operations.ts`).
131
+ */
132
+ export const ABANDONED_RELEASE_MESSAGE = 'abandoned — released by "celilo module operations clear"';
133
+
134
+ /**
135
+ * Reclaim abandoned rows on a schedule instead of when a human remembers.
136
+ *
137
+ * Registered as an ordinary bus subscriber whose handler is the existing
138
+ * `celilo module operations clear`, exactly like the backup sweep
139
+ * (`services/backup-sweep.ts`) — no new command and no new scheduler.
140
+ * Hourly is far finer than the two-hour TTL, so a wedge never survives
141
+ * long, and clearing is idempotent: a pass with nothing abandoned is a
142
+ * single read.
143
+ *
144
+ * `clear` without `--all` only touches rows that `checkInFlight` already
145
+ * ignores, so the sweep can never release a lock a live operation holds.
146
+ */
147
+ export const OPERATIONS_SWEEP_SUBSCRIBER = 'celilo-operations-sweep';
148
+ export const OPERATIONS_SWEEP_PATTERN = 'timer.tick.1h';
149
+
150
+ export interface SubscriberRegistrar {
151
+ subscribe(options: {
152
+ name: string;
153
+ pattern: string;
154
+ handler: string;
155
+ registeredBy?: string;
156
+ }): unknown;
157
+ }
158
+
159
+ /** Idempotent: `bus.subscribe` upserts by name. */
160
+ export function ensureOperationsSweepSubscriber(bus: SubscriberRegistrar): void {
161
+ bus.subscribe({
162
+ name: OPERATIONS_SWEEP_SUBSCRIBER,
163
+ pattern: OPERATIONS_SWEEP_PATTERN,
164
+ handler: 'celilo module operations clear',
165
+ registeredBy: 'celilo-module-operations',
166
+ });
167
+ }
168
+
122
169
  export interface InFlightConflict {
123
170
  operation: ModuleOperation;
124
171
  /** A short, operator-readable description: "deploy of homebridge (pid 12345)". */
@@ -173,7 +173,7 @@ describe('register / unregister roundtrip', () => {
173
173
  try {
174
174
  const row = bus.db
175
175
  .query<{ name: string; pattern: string; handler: string }, []>(
176
- 'SELECT name, pattern, handler FROM subscribers',
176
+ "SELECT name, pattern, handler FROM subscribers WHERE name = 'celilo-backup-sweep'",
177
177
  )
178
178
  .get();
179
179
  expect(row).toEqual({
@@ -186,6 +186,28 @@ describe('register / unregister roundtrip', () => {
186
186
  }
187
187
  });
188
188
 
189
+ // Any module can hold the operation lock, so registering ANY module — even
190
+ // one with no subscriptions and no backup hook — is enough to arm the sweep
191
+ // that reclaims abandoned rows (#581).
192
+ it('arms the abandoned-operations sweep for any module', () => {
193
+ registerModuleSubscriptions(baseManifest({}), '/p');
194
+
195
+ const bus = openBus({ dbPath, events: defineEvents({}) });
196
+ try {
197
+ const row = bus.db
198
+ .query<{ pattern: string; handler: string }, []>(
199
+ "SELECT pattern, handler FROM subscribers WHERE name = 'celilo-operations-sweep'",
200
+ )
201
+ .get();
202
+ expect(row).toEqual({
203
+ pattern: 'timer.tick.1h',
204
+ handler: 'celilo module operations clear',
205
+ });
206
+ } finally {
207
+ bus.close();
208
+ }
209
+ });
210
+
189
211
  it('registers each subscription as a row, names scoped to module id', () => {
190
212
  const result = registerModuleSubscriptions(
191
213
  baseManifest({
@@ -210,7 +232,7 @@ describe('register / unregister roundtrip', () => {
210
232
  try {
211
233
  const rows = bus.db
212
234
  .query<{ name: string; pattern: string; handler: string }, []>(
213
- 'SELECT name, pattern, handler FROM subscribers ORDER BY name',
235
+ "SELECT name, pattern, handler FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
214
236
  )
215
237
  .all();
216
238
  expect(rows).toEqual([
@@ -243,10 +265,14 @@ describe('register / unregister roundtrip', () => {
243
265
  const bus = openBus({ dbPath, events: defineEvents({}) });
244
266
  try {
245
267
  const rows = bus.db
246
- .query<{ count: number }, []>('SELECT COUNT(*) AS count FROM subscribers')
268
+ .query<{ count: number }, []>(
269
+ "SELECT COUNT(*) AS count FROM subscribers WHERE name LIKE '%.%'",
270
+ )
247
271
  .get();
248
272
  expect(rows?.count).toBe(1);
249
- const row = bus.db.query<{ handler: string }, []>('SELECT handler FROM subscribers').get();
273
+ const row = bus.db
274
+ .query<{ handler: string }, []>("SELECT handler FROM subscribers WHERE name LIKE '%.%'")
275
+ .get();
250
276
  expect(row?.handler).toBe('echo second');
251
277
  } finally {
252
278
  bus.close();
@@ -279,7 +305,9 @@ describe('register / unregister roundtrip', () => {
279
305
  const bus = openBus({ dbPath, events: defineEvents({}) });
280
306
  try {
281
307
  const rows = bus.db
282
- .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
308
+ .query<{ name: string }, []>(
309
+ "SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
310
+ )
283
311
  .all();
284
312
  expect(rows).toEqual([{ name: 'authentik.a' }]);
285
313
  } finally {
@@ -337,8 +365,13 @@ describe('resyncAllSubscriptions (ISS-0088)', () => {
337
365
  function subscriberNames(): string[] {
338
366
  const bus = openBus({ dbPath: busPath, events: defineEvents({}) });
339
367
  try {
368
+ // Module subscriptions are dot-scoped (`<module-id>.<sub-name>`); celilo's
369
+ // own housekeeping subscribers (the backup and operations sweeps) are not,
370
+ // and are not what these tests are about.
340
371
  return bus.db
341
- .query<{ name: string }, []>('SELECT name FROM subscribers ORDER BY name')
372
+ .query<{ name: string }, []>(
373
+ "SELECT name FROM subscribers WHERE name LIKE '%.%' ORDER BY name",
374
+ )
342
375
  .all()
343
376
  .map((r) => r.name);
344
377
  } finally {
@@ -22,6 +22,7 @@ import { getDb } from '../db/client';
22
22
  import { modules } from '../db/schema';
23
23
  import type { ModuleManifest, ModuleSubscription } from '../manifest/schema';
24
24
  import { ensureBackupSweepSubscriber } from './backup-sweep';
25
+ import { ensureOperationsSweepSubscriber } from './module-operations';
25
26
 
26
27
  /**
27
28
  * The bus is opened by the celilo CLI without an event registry — the
@@ -82,9 +83,6 @@ function resolveHandler(sub: ModuleSubscription, moduleId: string, modulePath: s
82
83
  /**
83
84
  * Register all of a module's subscriptions on the bus. Idempotent —
84
85
  * re-running with the same manifest updates existing rows in place.
85
- *
86
- * If the module's manifest declares no subscriptions, this is a
87
- * cheap no-op (the bus DB isn't even touched).
88
86
  */
89
87
  export function registerModuleSubscriptions(
90
88
  manifest: ModuleManifest,
@@ -92,10 +90,14 @@ export function registerModuleSubscriptions(
92
90
  ): { registered: number } {
93
91
  const subs = manifest.subscriptions ?? [];
94
92
  const backupSweep = Boolean(manifest.hooks?.on_backup);
95
- if (subs.length === 0 && !backupSweep) return { registered: 0 };
96
93
 
97
94
  const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
98
95
  try {
96
+ // Unconditional: every module can hold the operation lock (a deploy at
97
+ // minimum), so the first module on a fleet is what arms the sweep that
98
+ // reclaims abandoned rows. Idempotent.
99
+ ensureOperationsSweepSubscriber(bus);
100
+
99
101
  // A module that can be backed up is also what switches the scheduled
100
102
  // backup sweep on. Registering here rather than at system init means the
101
103
  // sweep appears the moment the fleet has something to back up, and — since
@@ -63,17 +63,20 @@ describe('variableTypeToTs', () => {
63
63
  });
64
64
 
65
65
  describe('generateModuleTypes', () => {
66
- test('emits a file header and empty interface for a manifest with no variables', () => {
66
+ test('emits a file header and a no-keys alias for a manifest with no variables', () => {
67
67
  const out = generateModuleTypes(baseManifest({ id: 'empty', name: 'Empty Module' }));
68
68
  expect(out).toContain('// Generated from manifest.yml');
69
69
  expect(out).toContain('Do not edit by hand');
70
- expect(out).toContain('export type EmptyConfig = {');
70
+ // `= {}` means "any non-nullish value", which is the opposite of an empty
71
+ // config surface — and biome bans it (lint/complexity/noBannedTypes).
72
+ expect(out).toContain('export type EmptyConfig = Record<string, never>;');
73
+ expect(out).not.toContain('export type EmptyConfig = {');
71
74
  expect(out).toContain('(No variables declared — module has no typed config surface)');
72
75
  });
73
76
 
74
77
  test('produces the right type-alias name from a kebab-case module ID', () => {
75
78
  const out = generateModuleTypes(baseManifest({ id: 'dns-external', name: 'DNS External' }));
76
- expect(out).toContain('export type DnsExternalConfig = {');
79
+ expect(out).toContain('export type DnsExternalConfig =');
77
80
  });
78
81
 
79
82
  test('renders required fields as non-optional', () => {
@@ -145,8 +145,6 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
145
145
  lines.push(' * gives type aliases an implicit index signature but withholds one from');
146
146
  lines.push(' * interfaces (which can be declaration-merged). See v2/issues.');
147
147
  lines.push(' */');
148
- lines.push(`export type ${typeName} = {`);
149
-
150
148
  const ownsFields: string[] = [];
151
149
  const importsFields: string[] = [];
152
150
 
@@ -172,6 +170,17 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
172
170
  importsFields.push(...rendered);
173
171
  }
174
172
 
173
+ if (ownsFields.length === 0 && importsFields.length === 0) {
174
+ // `= {}` is the banned "any non-nullish value" type, not "no keys" — and an
175
+ // empty config surface means exactly no keys.
176
+ lines.push('// (No variables declared — module has no typed config surface)');
177
+ lines.push(`export type ${typeName} = Record<string, never>;`);
178
+ lines.push('');
179
+ return lines.map((line) => line.trimEnd()).join('\n');
180
+ }
181
+
182
+ lines.push(`export type ${typeName} = {`);
183
+
175
184
  if (ownsFields.length > 0) {
176
185
  lines.push(' // Module-owned variables (from variables.owns)');
177
186
  lines.push(...ownsFields);
@@ -183,12 +192,8 @@ export function generateModuleTypes(manifest: ModuleManifest): string {
183
192
  lines.push(...importsFields);
184
193
  }
185
194
 
186
- if (ownsFields.length === 0 && importsFields.length === 0) {
187
- lines.push(' // (No variables declared — module has no typed config surface)');
188
- }
189
-
190
195
  lines.push('};');
191
196
  lines.push('');
192
197
 
193
- return lines.join('\n');
198
+ return lines.map((line) => line.trimEnd()).join('\n');
194
199
  }
@@ -76,3 +76,73 @@ test('answers responder.probe with kind daemon', async () => {
76
76
  responder.close();
77
77
  }
78
78
  }, 15_000);
79
+
80
+ /**
81
+ * When the client can't reach a decider it says so. The responder must relay
82
+ * that as an error reply — the waiting command then fails loudly instead of the
83
+ * question quietly resolving to `defaultValue`, which is how a breaking update
84
+ * came to be recorded as "operator declined".
85
+ */
86
+ test('an ask that cannot be answered replies with an error, not the default', async () => {
87
+ const responder = startRemoteResponder({
88
+ busDbPath,
89
+ ask: async () => {
90
+ throw new Error("stdin isn't a terminal and no answer was pre-staged");
91
+ },
92
+ });
93
+
94
+ const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
95
+ try {
96
+ const replies = (await bus.query(
97
+ 'interview.required.module-upgrade:iptables.apply_breaking' as never,
98
+ {
99
+ scope: 'module-upgrade:iptables',
100
+ key: 'apply_breaking',
101
+ kind: 'confirm',
102
+ message: 'Apply breaking update for iptables?',
103
+ required: true,
104
+ defaultValue: 'false',
105
+ } as never,
106
+ { timeoutMs: 8000, pollIntervalMs: 100, expect: 'first' } as never,
107
+ )) as BusEvent[];
108
+
109
+ expect(replies).toHaveLength(1);
110
+ const payload = replies[0].payload as { value?: unknown; error?: string };
111
+ expect(payload.value).toBeUndefined();
112
+ expect(payload.error).toContain('terminal');
113
+ } finally {
114
+ bus.close();
115
+ responder.close();
116
+ }
117
+ }, 15_000);
118
+
119
+ test('forwards the question scope/key so a client can pre-stage an answer', async () => {
120
+ const asked: WireInterview[] = [];
121
+ const responder = startRemoteResponder({
122
+ busDbPath,
123
+ ask: async (iv) => {
124
+ asked.push(iv);
125
+ return true;
126
+ },
127
+ });
128
+
129
+ const bus = openBus({ dbPath: busDbPath, events: NO_SCHEMAS });
130
+ try {
131
+ await bus.query(
132
+ 'interview.required.module-upgrade:iptables.apply_breaking' as never,
133
+ {
134
+ scope: 'module-upgrade:iptables',
135
+ key: 'apply_breaking',
136
+ kind: 'confirm',
137
+ message: 'Apply breaking update for iptables?',
138
+ required: true,
139
+ } as never,
140
+ { timeoutMs: 8000, pollIntervalMs: 100, expect: 'first' } as never,
141
+ );
142
+ expect(asked[0].scope).toBe('module-upgrade:iptables');
143
+ expect(asked[0].key).toBe('apply_breaking');
144
+ } finally {
145
+ bus.close();
146
+ responder.close();
147
+ }
148
+ }, 15_000);