@celilo/cli 0.23.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (61) hide show
  1. package/CELILO_CORE_MODULES.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +27 -7
  3. package/package.json +6 -5
  4. package/src/cli/commands/alerts-act.ts +1 -1
  5. package/src/cli/commands/backup-create.ts +26 -11
  6. package/src/cli/commands/backup-list.test.ts +83 -0
  7. package/src/cli/commands/backup-list.ts +67 -3
  8. package/src/cli/commands/backup-prune.ts +17 -17
  9. package/src/cli/commands/backup-sweep.ts +20 -8
  10. package/src/cli/commands/firewall-interface-list.test.ts +85 -0
  11. package/src/cli/commands/firewall-interface-list.ts +123 -0
  12. package/src/cli/commands/machine-add.ts +30 -2
  13. package/src/cli/commands/module-config.test.ts +64 -2
  14. package/src/cli/commands/module-config.ts +159 -8
  15. package/src/cli/commands/module-status.ts +124 -0
  16. package/src/cli/commands/monitor.ts +116 -19
  17. package/src/cli/commands/system-migrate.ts +14 -0
  18. package/src/cli/commands/system-update.ts +4 -1
  19. package/src/cli/completion.ts +35 -9
  20. package/src/cli/index.ts +59 -2
  21. package/src/cli/tui/audit-state.ts +2 -0
  22. package/src/hooks/capability-loader.ts +130 -4
  23. package/src/hooks/types.ts +2 -1
  24. package/src/manifest/contracts/v1.ts +16 -0
  25. package/src/manifest/schema.ts +40 -65
  26. package/src/services/alerting/builtin-monitors.test.ts +18 -10
  27. package/src/services/alerting/cadence-migration.test.ts +155 -0
  28. package/src/services/alerting/cadence-migration.ts +90 -0
  29. package/src/services/alerting/coverage-source.ts +8 -11
  30. package/src/services/alerting/deploy-hooks.test.ts +16 -7
  31. package/src/services/alerting/deploy-hooks.ts +11 -5
  32. package/src/services/alerting/health-cadence.test.ts +58 -0
  33. package/src/services/alerting/health-cadence.ts +128 -0
  34. package/src/services/alerting/health-coverage.ts +18 -8
  35. package/src/services/alerting/monitors.ts +50 -15
  36. package/src/services/alerting/sweep-runner.test.ts +51 -3
  37. package/src/services/alerting/sweep-runner.ts +30 -7
  38. package/src/services/audit/backup-source.ts +24 -1
  39. package/src/services/audit/backups.test.ts +95 -10
  40. package/src/services/audit/backups.ts +40 -37
  41. package/src/services/audit/interface-classification.test.ts +220 -0
  42. package/src/services/audit/interface-classification.ts +167 -0
  43. package/src/services/audit/types.ts +2 -1
  44. package/src/services/backup-age-agreement.test.ts +118 -0
  45. package/src/services/backup-create.ts +36 -30
  46. package/src/services/backup-metadata.ts +52 -1
  47. package/src/services/backup-retention.test.ts +123 -0
  48. package/src/services/backup-retention.ts +66 -5
  49. package/src/services/backup-schedule.test.ts +166 -0
  50. package/src/services/backup-schedule.ts +105 -15
  51. package/src/services/backup-staging.ts +14 -1
  52. package/src/services/backup-sweep.test.ts +22 -3
  53. package/src/services/backup-sweep.ts +15 -5
  54. package/src/services/cadence.test.ts +97 -0
  55. package/src/services/cadence.ts +165 -0
  56. package/src/services/machine-detector.ts +23 -1
  57. package/src/services/module-config.ts +33 -0
  58. package/src/services/storage-providers/s3.test.ts +96 -13
  59. package/src/services/storage-providers/s3.ts +48 -15
  60. package/src/services/zone-detector.test.ts +34 -3
  61. package/src/services/zone-detector.ts +33 -13
@@ -154,6 +154,32 @@ export async function resolveFirewallNatIp(db: DbClient): Promise<string | undef
154
154
  return undefined;
155
155
  }
156
156
 
157
+ /**
158
+ * The addresses of every machine a `firewall` capability provider manages.
159
+ *
160
+ * This is the authoritative answer to "which machines are firewalls", and it has
161
+ * to be: a machine's stored `role` is decided by `machine add`, from the zones
162
+ * declared AT THAT MOMENT. The normal order is `machine add` and THEN deploy
163
+ * iptables, whose `on_install` writes the zone subnets — so the firewall is
164
+ * recorded as a plain host and stays that way. A firewall provider naming an IP
165
+ * is a statement, not an inference from a snapshot.
166
+ */
167
+ export async function listFirewallIps(db: DbClient): Promise<string[]> {
168
+ const providers = db
169
+ .select()
170
+ .from(capabilities)
171
+ .where(eq(capabilities.capabilityName, 'firewall'))
172
+ .all();
173
+ const ips: string[] = [];
174
+ for (const provider of providers) {
175
+ const config = await loadModuleConfig(provider.moduleId, db);
176
+ if (typeof config.firewall_ip === 'string' && config.firewall_ip) {
177
+ ips.push(config.firewall_ip);
178
+ }
179
+ }
180
+ return ips;
181
+ }
182
+
157
183
  /**
158
184
  * Caddy's zone-routable IP — its own DMZ ingress address (`target_ip`, the same
159
185
  * value public_web exposes as `dmz_ip`). This is the in-zone split-horizon
@@ -606,6 +632,9 @@ function buildCapabilityInterface(
606
632
  undefined, // no upstream — the chain path handles that
607
633
  undefined, // logger is applied by wrapWithLogging at the loader site
608
634
  trustedSourceStore,
635
+ // LIVE, for the same reason as the chain path: a declaration written
636
+ // during this hook run must be visible to the converge that follows it.
637
+ zones ? { list: zones.declaredNetworks } : undefined,
609
638
  );
610
639
  return stampProvider(iface, providerModuleId);
611
640
  }
@@ -707,6 +736,18 @@ interface FirewallZones {
707
736
  trustedSubnets: string[];
708
737
  /** celilo's control-plane network, as a DESTINATION for trusted sources. */
709
738
  controlPlaneSubnet?: string;
739
+ /**
740
+ * Every zone with a declared subnet, WITH its name — the input to interface
741
+ * classification. Excludes `external`, which is a residual rather than a
742
+ * subnet (design D1, amended).
743
+ */
744
+ /**
745
+ * A LIVE read, not a snapshot. Everything else here is captured when the
746
+ * capability is built — before any hook runs — and for declarations that is
747
+ * wrong: `wireguard` declares its VPN subnet and brings `wg0` up inside one
748
+ * hook run, and the converge that follows has to see the declaration.
749
+ */
750
+ declaredNetworks: () => Array<{ zone: string; subnet: string }>;
710
751
  /**
711
752
  * Every declared zone subnet — read from
712
753
  * `network.<zone>.subnet` across all of `NETWORK_ZONES`, not just the three
@@ -806,6 +847,35 @@ export function loadTrustedSubnets(db: DbClient, firewallIp?: string): TrustedSu
806
847
  * of them. Zones with no configured subnet are omitted — their traffic stays
807
848
  * denied (fail-closed).
808
849
  */
850
+ /**
851
+ * Every network celilo can attribute an interface to: one entry per
852
+ * `network.<name>.subnet` in system config.
853
+ *
854
+ * Read from the config rather than from `NETWORK_ZONES`, because celilo declares
855
+ * networks that are not placement zones. `wireguard`'s `on_install` writes
856
+ * `network.control-plane-vpn.subnet` before it brings `wg0` up, exactly as the
857
+ * design requires (D3), and the firewall is supposed to classify `wg0` "by the
858
+ * same subnet containment it uses for every other leg". Reading only
859
+ * `NETWORK_ZONES` left that declaration invisible: the converge could not
860
+ * attribute `wg0` and refused — and on a firewall with a baseline it would have
861
+ * ISOLATED it, shutting down the admin VPN.
862
+ *
863
+ * `external` is deliberately absent even when something has set
864
+ * `network.external.subnet`: it is the RESIDUAL, decided by `isPubliclyRoutable`
865
+ * and never by containment (design D1, amended). Matching an interface to
866
+ * `external` by subnet would reintroduce the overload this change removes.
867
+ */
868
+ export function readDeclaredNetworks(db: DbClient): Array<{ zone: string; subnet: string }> {
869
+ return db
870
+ .select()
871
+ .from(systemConfig)
872
+ .all()
873
+ .flatMap((row) => {
874
+ const zone = /^network\.(.+)\.subnet$/.exec(row.key)?.[1];
875
+ return zone && zone !== 'external' && row.value ? [{ zone, subnet: row.value }] : [];
876
+ });
877
+ }
878
+
809
879
  function loadFirewallZones(db: DbClient): FirewallZones {
810
880
  const zoneTiers: Array<{ name: string; subnet: string }> = [];
811
881
  for (const zone of ZONE_TIER_ORDER) {
@@ -822,17 +892,65 @@ function loadFirewallZones(db: DbClient): FirewallZones {
822
892
  // snapshot put the subnet straight back, so the set could grow but never
823
893
  // shrink. That is exactly what happened: `on_uninstall` withdrew the VPN's
824
894
  // trusted source, logged success, and the reach rules were re-rendered anyway.
895
+ // Every zone that HAS a declared subnet, keeping the zone NAME. Interface
896
+ // classification needs the name — `external` and `internal` are the only two
897
+ // legs a default route may leave through (design D11), which is not a question
898
+ // a bare list of subnets can answer.
899
+ //
900
+ // `external` is deliberately absent from this list even when something has set
901
+ // `network.external.subnet`: `external` is the RESIDUAL, decided by
902
+ // `isPubliclyRoutable`, never by containment (design D1, amended). Matching an
903
+ // interface to `external` by subnet would reintroduce the overload this change
904
+ // removes.
905
+ //
906
+ // EVERY declared network, not just the six `NETWORK_ZONES`. celilo declares
907
+ // networks that are not placement zones — `network.control-plane-vpn.subnet`,
908
+ // written by `wireguard`'s `on_install` — and the design is explicit that the
909
+ // firewall "classifies `wg0` by the same subnet containment it uses for every
910
+ // other leg". Reading only `NETWORK_ZONES` left that declaration invisible:
911
+ // the module declared the subnet before bringing the interface up, exactly as
912
+ // designed, and the converge still could not attribute `wg0` and refused. On a
913
+ // firewall that already had a baseline it would have gone further and ISOLATED
914
+ // the interface — celilo shutting down the admin VPN, which for a remote
915
+ // operator is the way back in.
916
+ //
917
+ // So the source is the config itself. A hardcoded list here could only ever
918
+ // describe the networks celilo knew about when this line was written.
919
+
825
920
  return {
826
921
  zoneTiers,
827
922
  trustedSubnets: loadTrustedSubnets(db).map((e) => e.subnet),
828
923
  controlPlaneSubnet: loadControlPlaneSubnet(db) ?? readZoneSubnet(db, 'internal'),
829
- // Every zone celilo knows about, not the tier list — see the field doc.
830
- frontedSubnets: NETWORK_ZONES.map((zone) => readZoneSubnet(db, zone)).filter(
831
- (subnet): subnet is string => !!subnet,
832
- ),
924
+ declaredNetworks: () => readDeclaredNetworks(db),
925
+ // Placement ZONES only — deliberately NOT `declaredNetworks`, which is wider.
926
+ // Fronting a subnet renders translation for it; the control-plane VPN
927
+ // reaches the fleet as a trusted source instead, and giving it a second
928
+ // mechanism would change the rendered ruleset for every existing fleet.
929
+ frontedSubnets: NETWORK_ZONES.filter((zone) => zone !== 'external').flatMap((zone) => {
930
+ const subnet = readZoneSubnet(db, zone);
931
+ return subnet ? [subnet] : [];
932
+ }),
833
933
  };
834
934
  }
835
935
 
936
+ /**
937
+ * The interface baseline as stored in module config: a comma-separated list of
938
+ * interface names, or absent.
939
+ *
940
+ * Absent and empty are the SAME thing here and both mean "no baseline" — a
941
+ * firewall with zero interfaces does not exist, so an empty string can only be
942
+ * a cleared or never-written value. Treating it as a baseline of nothing would
943
+ * mean every interface is "new", which is the opposite of what it says.
944
+ */
945
+ function parseInterfaceBaseline(raw: unknown): string[] | undefined {
946
+ if (typeof raw !== 'string') return undefined;
947
+ const names = raw
948
+ .split(',')
949
+ .map((n) => n.trim())
950
+ .filter((n) => n.length > 0);
951
+ return names.length > 0 ? names : undefined;
952
+ }
953
+
836
954
  async function buildFirewallChain(
837
955
  allProviders: Array<{
838
956
  id: number;
@@ -982,11 +1100,19 @@ async function buildFirewallChain(
982
1100
  trustedSubnets: zones.trustedSubnets,
983
1101
  controlPlaneSubnet: zones.controlPlaneSubnet,
984
1102
  frontedSubnets: zones.frontedSubnets,
1103
+ // The recorded baseline (D12), from this firewall's own module config.
1104
+ // Absent means the box has never converged cleanly, so an interface
1105
+ // celilo cannot attribute refuses rather than being disabled.
1106
+ interfaceBaseline: parseInterfaceBaseline(provConfig.interface_baseline),
985
1107
  },
986
1108
  store,
987
1109
  currentUpstream,
988
1110
  logger,
989
1111
  trustedSourceStore,
1112
+ // LIVE, not a snapshot: a declaration written during this hook run —
1113
+ // `wireguard` declaring its VPN subnet before creating `wg0` — must be
1114
+ // visible to the converge that follows it in the same run.
1115
+ { list: zones.declaredNetworks },
990
1116
  );
991
1117
 
992
1118
  debugLog(`firewall chain: wired ${provider.moduleId} → ${hasExternal.moduleId}`);
@@ -120,7 +120,8 @@ export type HookName =
120
120
  | 'on_backup_analyze'
121
121
  | 'on_restore'
122
122
  | 'on_system_event'
123
- | 'refresh_registrations';
123
+ | 'refresh_registrations'
124
+ | 'reassert_dhcp_dns';
124
125
 
125
126
  /**
126
127
  * Hook manifest section - maps hook names to definitions
@@ -194,6 +194,22 @@ export const V1_HOOKS: ContractHooks = {
194
194
  },
195
195
  outputs: {},
196
196
  },
197
+ /**
198
+ * Periodic re-assertion of the resolver a dns_internal provider hands
199
+ * out over DHCP (celilo#739). Some routers regenerate that value from
200
+ * their own upstream list on a timer and silently undo what on_install
201
+ * set — the write succeeds and reads back correct, so only a client
202
+ * renewing later ever sees the wrong resolver. No framework-injected
203
+ * inputs: the hook derives the address it wants from its own config and
204
+ * systems, and reads the device before writing. Typically driven by a
205
+ * `timer.tick.1m` subscription, because the tick interval is the
206
+ * worst-case window in which a renewing client can be handed the wrong
207
+ * resolver for a full lease.
208
+ */
209
+ reassert_dhcp_dns: {
210
+ inputs: {},
211
+ outputs: {},
212
+ },
197
213
  /**
198
214
  * Build-bus upstream publish hook. The executor passes the
199
215
  * PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,
@@ -1,5 +1,10 @@
1
1
  import { z } from 'zod';
2
2
  import { NETWORK_ZONES } from '../db/schema';
3
+ import {
4
+ BACKUP_CADENCE_FLOOR_MINUTES,
5
+ MONITOR_INTERVAL_FLOOR_MINUTES,
6
+ cadenceSchema,
7
+ } from '../services/cadence';
3
8
  import { SUPPORTED_CONTRACT_VERSIONS } from './contracts';
4
9
 
5
10
  /**
@@ -273,73 +278,26 @@ export const LifecycleHookSchema = z.object({
273
278
  timeout: z.number().positive().optional(),
274
279
  });
275
280
 
276
- /**
277
- * Floor for a module's suggested `health_check` interval.
278
- *
279
- * The monitor sweep rides the event bus's fixed `timer.tick.5m`
280
- * (packages/event-bus/src/timer.ts), so an interval below five minutes
281
- * cannot be honoured — it would silently round up. Rejecting it at manifest
282
- * validation is better than accepting a promise celilo can't keep.
283
- */
284
- export const MONITOR_INTERVAL_FLOOR_MINUTES = 5;
285
-
286
- const DURATION_PATTERN = /^(\d+)(m|h|d)$/;
287
-
288
- /**
289
- * Parse a duration string (`15m`, `1h`, `1d`) to whole minutes.
290
- * Returns null when the string is not a well-formed duration.
291
- */
292
- export function parseIntervalMinutes(value: string): number | null {
293
- const match = DURATION_PATTERN.exec(value);
294
- if (!match) return null;
295
- const amount = Number.parseInt(match[1], 10);
296
- if (!Number.isFinite(amount) || amount <= 0) return null;
297
- const unit = match[2];
298
- if (unit === 'm') return amount;
299
- if (unit === 'h') return amount * 60;
300
- return amount * 60 * 24;
301
- }
302
-
303
281
  /**
304
282
  * `health_check` accepts everything a lifecycle hook does, plus an optional
305
283
  * `interval` — the module author's SUGGESTED monitoring cadence.
306
284
  *
307
- * It is only a suggestion: the operator's `monitors` row is the effective
308
- * schedule and survives module upgrades (openspec/changes/add-alerting
309
- * design D3). A module that omits it is simply not monitored until an
310
- * operator creates a monitor by hand.
285
+ * It is only a suggestion: the operator's `health_check_interval` override
286
+ * decides, and resolution happens at read time so a corrected suggestion
287
+ * reaches every install that has not overridden
288
+ * ([[services/alerting/health-cadence.ts]]). A module that omits it is not
289
+ * monitored until someone names a cadence.
290
+ *
291
+ * The same spellings as every other cadence in celilo — a named period, a
292
+ * duration, or `manual` — with the floor derived from the alerting sweep's
293
+ * tick.
311
294
  */
312
295
  export const HealthCheckHookSchema = LifecycleHookSchema.extend({
313
- // `.regex` duplicates the well-formedness half of the `superRefine` below on
314
- // purpose: only `.regex` survives the export to JSON Schema, and that export is
315
- // what validates `modules/*/manifest.yml` in the editor. The 5-minute floor
316
- // cannot be expressed in JSON Schema at all, so it stays a refinement which
317
- // is why both exist rather than one. Keeping them in sync is the point of
318
- // sharing DURATION_PATTERN.
319
- interval: z
320
- .string()
321
- .regex(DURATION_PATTERN)
322
- .optional()
323
- .describe(
324
- 'Suggested monitoring cadence, e.g. "15m", "1h", "1d". Must be 5m or longer — the monitor sweep runs on a 5-minute grid.',
325
- )
326
- .superRefine((value, ctx) => {
327
- if (value === undefined) return;
328
- const minutes = parseIntervalMinutes(value);
329
- if (minutes === null) {
330
- ctx.addIssue({
331
- code: z.ZodIssueCode.custom,
332
- message: `Invalid health_check interval "${value}". Use a duration like "15m", "1h", or "1d".`,
333
- });
334
- return;
335
- }
336
- if (minutes < MONITOR_INTERVAL_FLOOR_MINUTES) {
337
- ctx.addIssue({
338
- code: z.ZodIssueCode.custom,
339
- message: `health_check interval "${value}" is below the ${MONITOR_INTERVAL_FLOOR_MINUTES}m floor — the monitor sweep runs on a ${MONITOR_INTERVAL_FLOOR_MINUTES}-minute grid and cannot run it more often. Use "${MONITOR_INTERVAL_FLOOR_MINUTES}m" or longer.`,
340
- });
341
- }
342
- }),
296
+ interval: cadenceSchema({
297
+ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES,
298
+ description:
299
+ 'Suggested monitoring cadence: a duration like "15m", "1h" or "1d", a named period (hourly, daily, weekly, monthly), or "manual". The monitor sweep runs on a 5-minute grid, so nothing finer than 5m can be served.',
300
+ }).optional(),
343
301
  });
344
302
 
345
303
  /**
@@ -639,6 +597,16 @@ export const ModuleManifestSchema = z
639
597
  * designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B3).
640
598
  */
641
599
  refresh_registrations: LifecycleHookSchema.optional(),
600
+ /**
601
+ * Periodic re-assertion of the resolver a dns_internal provider
602
+ * hands out over DHCP. Some routers regenerate that value from
603
+ * their own upstream list on a timer, silently undoing what
604
+ * on_install set. The hook reads the device before writing, so a
605
+ * quiet minute costs one query. Subscribe it to `timer.tick.1m` —
606
+ * the tick interval IS the worst-case window in which a renewing
607
+ * client can be handed the wrong resolver. See celilo#739.
608
+ */
609
+ reassert_dhcp_dns: LifecycleHookSchema.optional(),
642
610
  /**
643
611
  * Build-bus upstream publish hooks. Array (a module can react
644
612
  * to multiple upstream packages with different actions). See
@@ -694,10 +662,17 @@ export const ModuleManifestSchema = z
694
662
 
695
663
  backup: z
696
664
  .object({
697
- // Absent means `daily`. Opting out of backups entirely is a real
698
- // decision and takes an explicit `manual` — see
699
- // [[services/backup-schedule.ts]].
700
- schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
665
+ // The author's SUGGESTED cadence. The operator's `backup_schedule`
666
+ // override wins and is resolved at read time — see
667
+ // [[services/backup-schedule.ts]]. Absent from both means `daily`;
668
+ // opting out of backups entirely is a real decision and takes an
669
+ // explicit `manual`.
670
+ //
671
+ schedule: cadenceSchema({
672
+ floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES,
673
+ description:
674
+ 'Suggested backup cadence: a named period (hourly, daily, weekly, monthly), a duration like "6h", or "manual" to opt out. The backup sweep runs hourly, so nothing finer than 1h can be served.',
675
+ }).default('daily'),
701
676
  retention: z
702
677
  .object({
703
678
  count: z.number().int().positive().default(7),
@@ -116,7 +116,7 @@ describe('failingKeysFromFindings', () => {
116
116
  });
117
117
 
118
118
  describe('healthCoverageFailingKeys', () => {
119
- const base = { hasHealthCheckHook: true, hasEnabledMonitor: true } as const;
119
+ const base = { hasHealthCheckHook: true, cadence: { minutes: 15 } } as const;
120
120
 
121
121
  test('a monitored module produces no finding', () => {
122
122
  expect(healthCoverageFailingKeys([{ id: 'caddy', state: 'VERIFIED', ...base }])).toEqual([]);
@@ -124,16 +124,26 @@ describe('healthCoverageFailingKeys', () => {
124
124
 
125
125
  test('a module with no health_check hook is surfaced', () => {
126
126
  const keys = healthCoverageFailingKeys([
127
- { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, hasEnabledMonitor: false },
127
+ { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
128
128
  ]);
129
129
  expect(keys).toHaveLength(1);
130
130
  expect(keys[0].key).toBe('builtin:health_coverage/module:signal');
131
131
  expect(keys[0].message).toContain('no health_check hook');
132
132
  });
133
133
 
134
- test('a module with a hook but no monitor is surfaced differently', () => {
134
+ test('an operator opt-out raises no finding it is a decision, not a gap', () => {
135
+ // The finding would ask for the action they just declined, so nothing they
136
+ // could do would ever clear it.
137
+ expect(
138
+ healthCoverageFailingKeys([
139
+ { id: 'lunacycle', state: 'VERIFIED', hasHealthCheckHook: true, cadence: 'manual' },
140
+ ]),
141
+ ).toEqual([]);
142
+ });
143
+
144
+ test('a module with a hook but no cadence is surfaced differently', () => {
135
145
  const keys = healthCoverageFailingKeys([
136
- { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true, hasEnabledMonitor: false },
146
+ { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true, cadence: null },
137
147
  ]);
138
148
  expect(keys).toHaveLength(1);
139
149
  expect(keys[0].message).toContain('nothing schedules it');
@@ -142,7 +152,7 @@ describe('healthCoverageFailingKeys', () => {
142
152
  // A coverage gap is real but is not an outage — it must never page.
143
153
  test('findings are warning severity', () => {
144
154
  const keys = healthCoverageFailingKeys([
145
- { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false, hasEnabledMonitor: false },
155
+ { id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false, cadence: null },
146
156
  ]);
147
157
  expect(keys[0].severity).toBe('warning');
148
158
  });
@@ -153,9 +163,7 @@ describe('healthCoverageFailingKeys', () => {
153
163
  'ignores a module in state %s',
154
164
  (state) => {
155
165
  expect(
156
- healthCoverageFailingKeys([
157
- { id: 'x', state, hasHealthCheckHook: false, hasEnabledMonitor: false },
158
- ]),
166
+ healthCoverageFailingKeys([{ id: 'x', state, hasHealthCheckHook: false, cadence: null }]),
159
167
  ).toEqual([]);
160
168
  },
161
169
  );
@@ -163,8 +171,8 @@ describe('healthCoverageFailingKeys', () => {
163
171
  test('reports only the modules that are actually uncovered', () => {
164
172
  const keys = healthCoverageFailingKeys([
165
173
  { id: 'caddy', state: 'VERIFIED', ...base },
166
- { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, hasEnabledMonitor: false },
167
- { id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true, hasEnabledMonitor: false },
174
+ { id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
175
+ { id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true, cadence: null },
168
176
  ]);
169
177
  expect(keys.map((k) => k.key)).toEqual([
170
178
  'builtin:health_coverage/module:signal',
@@ -0,0 +1,155 @@
1
+ /**
2
+ * The migrate step, and the invariant it protects.
3
+ *
4
+ * A `module_hook` monitor's stored `intervalMinutes` and `enabled` are no
5
+ * longer read: cadence resolves from `module_configs`. That is only safe on an
6
+ * existing fleet because `celilo system migrate` — which the `.deb` postinst
7
+ * runs on every apt upgrade — carries the diverging rows over first. Without
8
+ * it, the upgrade that shipped this change would silently revert every
9
+ * operator's hand-set cadence and resume watching modules they had disabled.
10
+ */
11
+
12
+ import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
13
+ import { mkdtempSync, rmSync } from 'node:fs';
14
+ import { tmpdir } from 'node:os';
15
+ import { join } from 'node:path';
16
+ import { closeDb, getDb } from '../../db/client';
17
+ import { runMigrations } from '../../db/migrate';
18
+ import { modules, monitors } from '../../db/schema';
19
+ import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
20
+ import { migrateMonitorCadences } from './cadence-migration';
21
+ import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, loadModuleHealthCadences } from './health-cadence';
22
+
23
+ function addModule(id: string, interval?: string): void {
24
+ getDb()
25
+ .insert(modules)
26
+ .values({
27
+ id,
28
+ name: id,
29
+ sourcePath: `/tmp/${id}`,
30
+ version: '1.0.0',
31
+ state: 'INSTALLED',
32
+ manifestData: {
33
+ id,
34
+ hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
35
+ },
36
+ })
37
+ .run();
38
+ }
39
+
40
+ function addMonitor(target: string, intervalMinutes: number, enabled = true): void {
41
+ getDb()
42
+ .insert(monitors)
43
+ .values({
44
+ id: `monitor-${target}`,
45
+ kind: 'module_hook',
46
+ target,
47
+ intervalMinutes,
48
+ enabled,
49
+ })
50
+ .run();
51
+ }
52
+
53
+ function override(moduleId: string): string | undefined {
54
+ const row = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, getDb());
55
+ return row === null ? undefined : String(row.value);
56
+ }
57
+
58
+ describe('migrateMonitorCadences', () => {
59
+ let dir: string;
60
+
61
+ beforeEach(async () => {
62
+ dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-migration-'));
63
+ process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
64
+ await runMigrations(process.env.CELILO_DB_PATH);
65
+ });
66
+
67
+ afterEach(() => {
68
+ closeDb();
69
+ process.env.CELILO_DB_PATH = undefined;
70
+ rmSync(dir, { recursive: true, force: true });
71
+ });
72
+
73
+ test("a cadence that diverges from the manifest is carried over, so the upgrade can't revert it", () => {
74
+ addModule('caddy', '15m');
75
+ addMonitor('caddy', 60); // an operator re-cadenced this to hourly
76
+
77
+ const report = migrateMonitorCadences(getDb());
78
+
79
+ expect(override('caddy')).toBe('hourly');
80
+ expect(report.written.get('caddy')).toBe('hourly');
81
+ });
82
+
83
+ test('a disabled monitor becomes manual, so a module nobody wanted watched stays unwatched', () => {
84
+ addModule('lunacycle', '15m');
85
+ addMonitor('lunacycle', 15, false);
86
+
87
+ migrateMonitorCadences(getDb());
88
+
89
+ expect(override('lunacycle')).toBe('manual');
90
+ });
91
+
92
+ test('a cadence matching the manifest writes nothing — it must stay free to follow corrections', () => {
93
+ addModule('forgejo', '15m');
94
+ addMonitor('forgejo', 15);
95
+
96
+ const report = migrateMonitorCadences(getDb());
97
+
98
+ expect(override('forgejo')).toBeUndefined();
99
+ expect(report.unchanged).toContain('forgejo');
100
+ });
101
+
102
+ test('an existing override is never overwritten', () => {
103
+ addModule('authentik', '15m');
104
+ addMonitor('authentik', 60);
105
+ upsertModuleConfig(getDb(), 'authentik', HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'daily');
106
+
107
+ migrateMonitorCadences(getDb());
108
+
109
+ expect(override('authentik')).toBe('daily');
110
+ });
111
+
112
+ test('a second run changes nothing', () => {
113
+ addModule('caddy', '15m');
114
+ addMonitor('caddy', 60);
115
+
116
+ migrateMonitorCadences(getDb());
117
+ const second = migrateMonitorCadences(getDb());
118
+
119
+ expect(second.written.size).toBe(0);
120
+ expect(override('caddy')).toBe('hourly');
121
+ });
122
+ });
123
+
124
+ describe("a module_hook row's stored cadence is not consulted", () => {
125
+ let dir: string;
126
+
127
+ beforeEach(async () => {
128
+ dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-row-'));
129
+ process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
130
+ await runMigrations(process.env.CELILO_DB_PATH);
131
+ });
132
+
133
+ afterEach(() => {
134
+ closeDb();
135
+ process.env.CELILO_DB_PATH = undefined;
136
+ rmSync(dir, { recursive: true, force: true });
137
+ });
138
+
139
+ // A future reader that reintroduces the dependency on the row fails here
140
+ // rather than silently regressing to write-time resolution (design.md D8).
141
+ test('the resolved cadence follows the manifest, not the row', () => {
142
+ addModule('caddy', '15m');
143
+ addMonitor('caddy', 9999);
144
+
145
+ expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 15 });
146
+ });
147
+
148
+ test('the resolved cadence follows the override, not the row', () => {
149
+ addModule('caddy', '15m');
150
+ addMonitor('caddy', 9999);
151
+ upsertModuleConfig(getDb(), 'caddy', HEALTH_CHECK_INTERVAL_CONFIG_KEY, '1h');
152
+
153
+ expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 60 });
154
+ });
155
+ });
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Carrying an existing fleet's health-check cadences into `module_configs`.
3
+ *
4
+ * Not bookkeeping. Before this change a `module_hook` monitor's cadence lived
5
+ * on its row, seeded from the manifest at first deploy and editable only by raw
6
+ * SQL; after it, the row is not read and the cadence comes from the operator's
7
+ * override or the manifest's suggestion. So on the FIRST apt upgrade that
8
+ * carries this code, every module whose row diverges from its manifest — which
9
+ * is every module anyone ever re-cadenced by hand — would silently revert to
10
+ * the author's suggestion, and every module an operator deliberately disabled
11
+ * would start being watched again.
12
+ *
13
+ * Two steps, both idempotent, both writing only where no override exists:
14
+ *
15
+ * 1. a row whose cadence differs from what its manifest suggests gets that
16
+ * cadence written as an override;
17
+ * 2. a disabled row gets `manual`.
18
+ *
19
+ * Runs from `celilo system migrate`, which the `.deb` postinst already invokes
20
+ * on every upgrade. Rolling back is safe in both directions: the rows written
21
+ * here are inert to older code, which reads the monitor row, and rolling
22
+ * forward again finds them already present.
23
+ */
24
+
25
+ import { eq } from 'drizzle-orm';
26
+ import type { DbClient } from '../../db/client';
27
+ import { modules, monitors } from '../../db/schema';
28
+ import type { ModuleManifest } from '../../manifest/schema';
29
+ import { formatCadence, parseCadence } from '../cadence';
30
+ import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
31
+ import { HEALTH_CHECK_INTERVAL_CONFIG_KEY } from './health-cadence';
32
+
33
+ export interface CadenceMigrationReport {
34
+ /** `moduleId → written value`, for the operator-visible summary. */
35
+ written: Map<string, string>;
36
+ /** Modules left alone: an override already existed, or nothing diverged. */
37
+ unchanged: string[];
38
+ }
39
+
40
+ export function migrateMonitorCadences(db: DbClient): CadenceMigrationReport {
41
+ const report: CadenceMigrationReport = { written: new Map(), unchanged: [] };
42
+
43
+ for (const monitor of db.select().from(monitors).where(eq(monitors.kind, 'module_hook')).all()) {
44
+ const moduleId = monitor.target;
45
+
46
+ // An operator who has already set one has said the last word; never
47
+ // overwrite it, which is also what makes a second run a no-op.
48
+ if (getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db)) {
49
+ report.unchanged.push(moduleId);
50
+ continue;
51
+ }
52
+
53
+ const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
54
+ if (!module) {
55
+ // A monitor whose module is gone has nothing to carry over.
56
+ report.unchanged.push(moduleId);
57
+ continue;
58
+ }
59
+
60
+ // Disabled wins over cadence: an operator who stopped watching a module
61
+ // meant that, whatever interval the row happens to carry.
62
+ if (!monitor.enabled) {
63
+ upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
64
+ report.written.set(moduleId, 'manual');
65
+ continue;
66
+ }
67
+
68
+ const manifest = module.manifestData as ModuleManifest;
69
+ const suggested = manifest.hooks?.health_check?.interval;
70
+ const suggestedMinutes = suggested ? parseCadence(suggested) : null;
71
+ const suggestedIsSame =
72
+ suggestedMinutes !== null &&
73
+ suggestedMinutes !== 'manual' &&
74
+ suggestedMinutes.minutes === monitor.intervalMinutes;
75
+
76
+ if (suggestedIsSame) {
77
+ // The row never diverged, so read-time resolution already produces it —
78
+ // and writing an override here would freeze this module against every
79
+ // future manifest correction, which is the failure this change removes.
80
+ report.unchanged.push(moduleId);
81
+ continue;
82
+ }
83
+
84
+ const carried = formatCadence({ minutes: monitor.intervalMinutes });
85
+ upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, carried);
86
+ report.written.set(moduleId, carried);
87
+ }
88
+
89
+ return report;
90
+ }