@celilo/cli 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +70 -3
- package/src/cli/commands/module-config.ts +262 -28
- package/src/cli/commands/module-status.ts +155 -12
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/load-hook-config.test.ts +169 -1
- package/src/hooks/load-hook-config.ts +118 -20
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/config-provenance.test.ts +155 -0
- package/src/services/config-provenance.ts +104 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
- package/src/variables/context.ts +69 -15
- package/src/variables/declarative-derivation.test.ts +53 -0
- package/src/variables/declarative-derivation.ts +13 -2
package/src/hooks/types.ts
CHANGED
|
@@ -120,7 +120,8 @@ export type HookName =
|
|
|
120
120
|
| 'on_backup_analyze'
|
|
121
121
|
| 'on_restore'
|
|
122
122
|
| 'on_system_event'
|
|
123
|
-
| 'refresh_registrations'
|
|
123
|
+
| 'refresh_registrations'
|
|
124
|
+
| 'reassert_dhcp_dns';
|
|
124
125
|
|
|
125
126
|
/**
|
|
126
127
|
* Hook manifest section - maps hook names to definitions
|
|
@@ -194,6 +194,22 @@ export const V1_HOOKS: ContractHooks = {
|
|
|
194
194
|
},
|
|
195
195
|
outputs: {},
|
|
196
196
|
},
|
|
197
|
+
/**
|
|
198
|
+
* Periodic re-assertion of the resolver a dns_internal provider hands
|
|
199
|
+
* out over DHCP (celilo#739). Some routers regenerate that value from
|
|
200
|
+
* their own upstream list on a timer and silently undo what on_install
|
|
201
|
+
* set — the write succeeds and reads back correct, so only a client
|
|
202
|
+
* renewing later ever sees the wrong resolver. No framework-injected
|
|
203
|
+
* inputs: the hook derives the address it wants from its own config and
|
|
204
|
+
* systems, and reads the device before writing. Typically driven by a
|
|
205
|
+
* `timer.tick.1m` subscription, because the tick interval is the
|
|
206
|
+
* worst-case window in which a renewing client can be handed the wrong
|
|
207
|
+
* resolver for a full lease.
|
|
208
|
+
*/
|
|
209
|
+
reassert_dhcp_dns: {
|
|
210
|
+
inputs: {},
|
|
211
|
+
outputs: {},
|
|
212
|
+
},
|
|
197
213
|
/**
|
|
198
214
|
* Build-bus upstream publish hook. The executor passes the
|
|
199
215
|
* PublishEvent fields as env vars (CELILO_EVENT_PAYLOAD,
|
package/src/manifest/schema.ts
CHANGED
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import { NETWORK_ZONES } from '../db/schema';
|
|
3
|
+
import {
|
|
4
|
+
BACKUP_CADENCE_FLOOR_MINUTES,
|
|
5
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
6
|
+
cadenceSchema,
|
|
7
|
+
} from '../services/cadence';
|
|
3
8
|
import { SUPPORTED_CONTRACT_VERSIONS } from './contracts';
|
|
4
9
|
|
|
5
10
|
/**
|
|
@@ -273,73 +278,26 @@ export const LifecycleHookSchema = z.object({
|
|
|
273
278
|
timeout: z.number().positive().optional(),
|
|
274
279
|
});
|
|
275
280
|
|
|
276
|
-
/**
|
|
277
|
-
* Floor for a module's suggested `health_check` interval.
|
|
278
|
-
*
|
|
279
|
-
* The monitor sweep rides the event bus's fixed `timer.tick.5m`
|
|
280
|
-
* (packages/event-bus/src/timer.ts), so an interval below five minutes
|
|
281
|
-
* cannot be honoured — it would silently round up. Rejecting it at manifest
|
|
282
|
-
* validation is better than accepting a promise celilo can't keep.
|
|
283
|
-
*/
|
|
284
|
-
export const MONITOR_INTERVAL_FLOOR_MINUTES = 5;
|
|
285
|
-
|
|
286
|
-
const DURATION_PATTERN = /^(\d+)(m|h|d)$/;
|
|
287
|
-
|
|
288
|
-
/**
|
|
289
|
-
* Parse a duration string (`15m`, `1h`, `1d`) to whole minutes.
|
|
290
|
-
* Returns null when the string is not a well-formed duration.
|
|
291
|
-
*/
|
|
292
|
-
export function parseIntervalMinutes(value: string): number | null {
|
|
293
|
-
const match = DURATION_PATTERN.exec(value);
|
|
294
|
-
if (!match) return null;
|
|
295
|
-
const amount = Number.parseInt(match[1], 10);
|
|
296
|
-
if (!Number.isFinite(amount) || amount <= 0) return null;
|
|
297
|
-
const unit = match[2];
|
|
298
|
-
if (unit === 'm') return amount;
|
|
299
|
-
if (unit === 'h') return amount * 60;
|
|
300
|
-
return amount * 60 * 24;
|
|
301
|
-
}
|
|
302
|
-
|
|
303
281
|
/**
|
|
304
282
|
* `health_check` accepts everything a lifecycle hook does, plus an optional
|
|
305
283
|
* `interval` — the module author's SUGGESTED monitoring cadence.
|
|
306
284
|
*
|
|
307
|
-
* It is only a suggestion: the operator's `
|
|
308
|
-
*
|
|
309
|
-
*
|
|
310
|
-
*
|
|
285
|
+
* It is only a suggestion: the operator's `health_check_interval` override
|
|
286
|
+
* decides, and resolution happens at read time so a corrected suggestion
|
|
287
|
+
* reaches every install that has not overridden
|
|
288
|
+
* ([[services/alerting/health-cadence.ts]]). A module that omits it is not
|
|
289
|
+
* monitored until someone names a cadence.
|
|
290
|
+
*
|
|
291
|
+
* The same spellings as every other cadence in celilo — a named period, a
|
|
292
|
+
* duration, or `manual` — with the floor derived from the alerting sweep's
|
|
293
|
+
* tick.
|
|
311
294
|
*/
|
|
312
295
|
export const HealthCheckHookSchema = LifecycleHookSchema.extend({
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
// sharing DURATION_PATTERN.
|
|
319
|
-
interval: z
|
|
320
|
-
.string()
|
|
321
|
-
.regex(DURATION_PATTERN)
|
|
322
|
-
.optional()
|
|
323
|
-
.describe(
|
|
324
|
-
'Suggested monitoring cadence, e.g. "15m", "1h", "1d". Must be 5m or longer — the monitor sweep runs on a 5-minute grid.',
|
|
325
|
-
)
|
|
326
|
-
.superRefine((value, ctx) => {
|
|
327
|
-
if (value === undefined) return;
|
|
328
|
-
const minutes = parseIntervalMinutes(value);
|
|
329
|
-
if (minutes === null) {
|
|
330
|
-
ctx.addIssue({
|
|
331
|
-
code: z.ZodIssueCode.custom,
|
|
332
|
-
message: `Invalid health_check interval "${value}". Use a duration like "15m", "1h", or "1d".`,
|
|
333
|
-
});
|
|
334
|
-
return;
|
|
335
|
-
}
|
|
336
|
-
if (minutes < MONITOR_INTERVAL_FLOOR_MINUTES) {
|
|
337
|
-
ctx.addIssue({
|
|
338
|
-
code: z.ZodIssueCode.custom,
|
|
339
|
-
message: `health_check interval "${value}" is below the ${MONITOR_INTERVAL_FLOOR_MINUTES}m floor — the monitor sweep runs on a ${MONITOR_INTERVAL_FLOOR_MINUTES}-minute grid and cannot run it more often. Use "${MONITOR_INTERVAL_FLOOR_MINUTES}m" or longer.`,
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
}),
|
|
296
|
+
interval: cadenceSchema({
|
|
297
|
+
floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
298
|
+
description:
|
|
299
|
+
'Suggested monitoring cadence: a duration like "15m", "1h" or "1d", a named period (hourly, daily, weekly, monthly), or "manual". The monitor sweep runs on a 5-minute grid, so nothing finer than 5m can be served.',
|
|
300
|
+
}).optional(),
|
|
343
301
|
});
|
|
344
302
|
|
|
345
303
|
/**
|
|
@@ -639,6 +597,16 @@ export const ModuleManifestSchema = z
|
|
|
639
597
|
* designs/DISPATCHER_DAEMON_AND_TIMER_EVENTS.md (B3).
|
|
640
598
|
*/
|
|
641
599
|
refresh_registrations: LifecycleHookSchema.optional(),
|
|
600
|
+
/**
|
|
601
|
+
* Periodic re-assertion of the resolver a dns_internal provider
|
|
602
|
+
* hands out over DHCP. Some routers regenerate that value from
|
|
603
|
+
* their own upstream list on a timer, silently undoing what
|
|
604
|
+
* on_install set. The hook reads the device before writing, so a
|
|
605
|
+
* quiet minute costs one query. Subscribe it to `timer.tick.1m` —
|
|
606
|
+
* the tick interval IS the worst-case window in which a renewing
|
|
607
|
+
* client can be handed the wrong resolver. See celilo#739.
|
|
608
|
+
*/
|
|
609
|
+
reassert_dhcp_dns: LifecycleHookSchema.optional(),
|
|
642
610
|
/**
|
|
643
611
|
* Build-bus upstream publish hooks. Array (a module can react
|
|
644
612
|
* to multiple upstream packages with different actions). See
|
|
@@ -694,10 +662,17 @@ export const ModuleManifestSchema = z
|
|
|
694
662
|
|
|
695
663
|
backup: z
|
|
696
664
|
.object({
|
|
697
|
-
//
|
|
698
|
-
//
|
|
699
|
-
// [[services/backup-schedule.ts]].
|
|
700
|
-
|
|
665
|
+
// The author's SUGGESTED cadence. The operator's `backup_schedule`
|
|
666
|
+
// override wins and is resolved at read time — see
|
|
667
|
+
// [[services/backup-schedule.ts]]. Absent from both means `daily`;
|
|
668
|
+
// opting out of backups entirely is a real decision and takes an
|
|
669
|
+
// explicit `manual`.
|
|
670
|
+
//
|
|
671
|
+
schedule: cadenceSchema({
|
|
672
|
+
floorMinutes: BACKUP_CADENCE_FLOOR_MINUTES,
|
|
673
|
+
description:
|
|
674
|
+
'Suggested backup cadence: a named period (hourly, daily, weekly, monthly), a duration like "6h", or "manual" to opt out. The backup sweep runs hourly, so nothing finer than 1h can be served.',
|
|
675
|
+
}).default('daily'),
|
|
701
676
|
retention: z
|
|
702
677
|
.object({
|
|
703
678
|
count: z.number().int().positive().default(7),
|
|
@@ -116,7 +116,7 @@ describe('failingKeysFromFindings', () => {
|
|
|
116
116
|
});
|
|
117
117
|
|
|
118
118
|
describe('healthCoverageFailingKeys', () => {
|
|
119
|
-
const base = { hasHealthCheckHook: true,
|
|
119
|
+
const base = { hasHealthCheckHook: true, cadence: { minutes: 15 } } as const;
|
|
120
120
|
|
|
121
121
|
test('a monitored module produces no finding', () => {
|
|
122
122
|
expect(healthCoverageFailingKeys([{ id: 'caddy', state: 'VERIFIED', ...base }])).toEqual([]);
|
|
@@ -124,16 +124,26 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
124
124
|
|
|
125
125
|
test('a module with no health_check hook is surfaced', () => {
|
|
126
126
|
const keys = healthCoverageFailingKeys([
|
|
127
|
-
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false,
|
|
127
|
+
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
|
|
128
128
|
]);
|
|
129
129
|
expect(keys).toHaveLength(1);
|
|
130
130
|
expect(keys[0].key).toBe('builtin:health_coverage/module:signal');
|
|
131
131
|
expect(keys[0].message).toContain('no health_check hook');
|
|
132
132
|
});
|
|
133
133
|
|
|
134
|
-
test('
|
|
134
|
+
test('an operator opt-out raises no finding — it is a decision, not a gap', () => {
|
|
135
|
+
// The finding would ask for the action they just declined, so nothing they
|
|
136
|
+
// could do would ever clear it.
|
|
137
|
+
expect(
|
|
138
|
+
healthCoverageFailingKeys([
|
|
139
|
+
{ id: 'lunacycle', state: 'VERIFIED', hasHealthCheckHook: true, cadence: 'manual' },
|
|
140
|
+
]),
|
|
141
|
+
).toEqual([]);
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('a module with a hook but no cadence is surfaced differently', () => {
|
|
135
145
|
const keys = healthCoverageFailingKeys([
|
|
136
|
-
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true,
|
|
146
|
+
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: true, cadence: null },
|
|
137
147
|
]);
|
|
138
148
|
expect(keys).toHaveLength(1);
|
|
139
149
|
expect(keys[0].message).toContain('nothing schedules it');
|
|
@@ -142,7 +152,7 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
142
152
|
// A coverage gap is real but is not an outage — it must never page.
|
|
143
153
|
test('findings are warning severity', () => {
|
|
144
154
|
const keys = healthCoverageFailingKeys([
|
|
145
|
-
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false,
|
|
155
|
+
{ id: 'caddy', state: 'VERIFIED', hasHealthCheckHook: false, cadence: null },
|
|
146
156
|
]);
|
|
147
157
|
expect(keys[0].severity).toBe('warning');
|
|
148
158
|
});
|
|
@@ -153,9 +163,7 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
153
163
|
'ignores a module in state %s',
|
|
154
164
|
(state) => {
|
|
155
165
|
expect(
|
|
156
|
-
healthCoverageFailingKeys([
|
|
157
|
-
{ id: 'x', state, hasHealthCheckHook: false, hasEnabledMonitor: false },
|
|
158
|
-
]),
|
|
166
|
+
healthCoverageFailingKeys([{ id: 'x', state, hasHealthCheckHook: false, cadence: null }]),
|
|
159
167
|
).toEqual([]);
|
|
160
168
|
},
|
|
161
169
|
);
|
|
@@ -163,8 +171,8 @@ describe('healthCoverageFailingKeys', () => {
|
|
|
163
171
|
test('reports only the modules that are actually uncovered', () => {
|
|
164
172
|
const keys = healthCoverageFailingKeys([
|
|
165
173
|
{ id: 'caddy', state: 'VERIFIED', ...base },
|
|
166
|
-
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false,
|
|
167
|
-
{ id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true,
|
|
174
|
+
{ id: 'signal', state: 'INSTALLED', hasHealthCheckHook: false, cadence: null },
|
|
175
|
+
{ id: 'forgejo', state: 'INSTALLED', hasHealthCheckHook: true, cadence: null },
|
|
168
176
|
]);
|
|
169
177
|
expect(keys.map((k) => k.key)).toEqual([
|
|
170
178
|
'builtin:health_coverage/module:signal',
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The migrate step, and the invariant it protects.
|
|
3
|
+
*
|
|
4
|
+
* A `module_hook` monitor's stored `intervalMinutes` and `enabled` are no
|
|
5
|
+
* longer read: cadence resolves from `module_configs`. That is only safe on an
|
|
6
|
+
* existing fleet because `celilo system migrate` — which the `.deb` postinst
|
|
7
|
+
* runs on every apt upgrade — carries the diverging rows over first. Without
|
|
8
|
+
* it, the upgrade that shipped this change would silently revert every
|
|
9
|
+
* operator's hand-set cadence and resume watching modules they had disabled.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
13
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
14
|
+
import { tmpdir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { closeDb, getDb } from '../../db/client';
|
|
17
|
+
import { runMigrations } from '../../db/migrate';
|
|
18
|
+
import { modules, monitors } from '../../db/schema';
|
|
19
|
+
import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
|
|
20
|
+
import { migrateMonitorCadences } from './cadence-migration';
|
|
21
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, loadModuleHealthCadences } from './health-cadence';
|
|
22
|
+
|
|
23
|
+
function addModule(id: string, interval?: string): void {
|
|
24
|
+
getDb()
|
|
25
|
+
.insert(modules)
|
|
26
|
+
.values({
|
|
27
|
+
id,
|
|
28
|
+
name: id,
|
|
29
|
+
sourcePath: `/tmp/${id}`,
|
|
30
|
+
version: '1.0.0',
|
|
31
|
+
state: 'INSTALLED',
|
|
32
|
+
manifestData: {
|
|
33
|
+
id,
|
|
34
|
+
hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
|
|
35
|
+
},
|
|
36
|
+
})
|
|
37
|
+
.run();
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function addMonitor(target: string, intervalMinutes: number, enabled = true): void {
|
|
41
|
+
getDb()
|
|
42
|
+
.insert(monitors)
|
|
43
|
+
.values({
|
|
44
|
+
id: `monitor-${target}`,
|
|
45
|
+
kind: 'module_hook',
|
|
46
|
+
target,
|
|
47
|
+
intervalMinutes,
|
|
48
|
+
enabled,
|
|
49
|
+
})
|
|
50
|
+
.run();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function override(moduleId: string): string | undefined {
|
|
54
|
+
const row = getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, getDb());
|
|
55
|
+
return row === null ? undefined : String(row.value);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
describe('migrateMonitorCadences', () => {
|
|
59
|
+
let dir: string;
|
|
60
|
+
|
|
61
|
+
beforeEach(async () => {
|
|
62
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-migration-'));
|
|
63
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
64
|
+
await runMigrations(process.env.CELILO_DB_PATH);
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
afterEach(() => {
|
|
68
|
+
closeDb();
|
|
69
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
70
|
+
rmSync(dir, { recursive: true, force: true });
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
test("a cadence that diverges from the manifest is carried over, so the upgrade can't revert it", () => {
|
|
74
|
+
addModule('caddy', '15m');
|
|
75
|
+
addMonitor('caddy', 60); // an operator re-cadenced this to hourly
|
|
76
|
+
|
|
77
|
+
const report = migrateMonitorCadences(getDb());
|
|
78
|
+
|
|
79
|
+
expect(override('caddy')).toBe('hourly');
|
|
80
|
+
expect(report.written.get('caddy')).toBe('hourly');
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test('a disabled monitor becomes manual, so a module nobody wanted watched stays unwatched', () => {
|
|
84
|
+
addModule('lunacycle', '15m');
|
|
85
|
+
addMonitor('lunacycle', 15, false);
|
|
86
|
+
|
|
87
|
+
migrateMonitorCadences(getDb());
|
|
88
|
+
|
|
89
|
+
expect(override('lunacycle')).toBe('manual');
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test('a cadence matching the manifest writes nothing — it must stay free to follow corrections', () => {
|
|
93
|
+
addModule('forgejo', '15m');
|
|
94
|
+
addMonitor('forgejo', 15);
|
|
95
|
+
|
|
96
|
+
const report = migrateMonitorCadences(getDb());
|
|
97
|
+
|
|
98
|
+
expect(override('forgejo')).toBeUndefined();
|
|
99
|
+
expect(report.unchanged).toContain('forgejo');
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test('an existing override is never overwritten', () => {
|
|
103
|
+
addModule('authentik', '15m');
|
|
104
|
+
addMonitor('authentik', 60);
|
|
105
|
+
upsertModuleConfig(getDb(), 'authentik', HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'daily');
|
|
106
|
+
|
|
107
|
+
migrateMonitorCadences(getDb());
|
|
108
|
+
|
|
109
|
+
expect(override('authentik')).toBe('daily');
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test('a second run changes nothing', () => {
|
|
113
|
+
addModule('caddy', '15m');
|
|
114
|
+
addMonitor('caddy', 60);
|
|
115
|
+
|
|
116
|
+
migrateMonitorCadences(getDb());
|
|
117
|
+
const second = migrateMonitorCadences(getDb());
|
|
118
|
+
|
|
119
|
+
expect(second.written.size).toBe(0);
|
|
120
|
+
expect(override('caddy')).toBe('hourly');
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
describe("a module_hook row's stored cadence is not consulted", () => {
|
|
125
|
+
let dir: string;
|
|
126
|
+
|
|
127
|
+
beforeEach(async () => {
|
|
128
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-cadence-row-'));
|
|
129
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
130
|
+
await runMigrations(process.env.CELILO_DB_PATH);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
afterEach(() => {
|
|
134
|
+
closeDb();
|
|
135
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
136
|
+
rmSync(dir, { recursive: true, force: true });
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// A future reader that reintroduces the dependency on the row fails here
|
|
140
|
+
// rather than silently regressing to write-time resolution (design.md D8).
|
|
141
|
+
test('the resolved cadence follows the manifest, not the row', () => {
|
|
142
|
+
addModule('caddy', '15m');
|
|
143
|
+
addMonitor('caddy', 9999);
|
|
144
|
+
|
|
145
|
+
expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 15 });
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test('the resolved cadence follows the override, not the row', () => {
|
|
149
|
+
addModule('caddy', '15m');
|
|
150
|
+
addMonitor('caddy', 9999);
|
|
151
|
+
upsertModuleConfig(getDb(), 'caddy', HEALTH_CHECK_INTERVAL_CONFIG_KEY, '1h');
|
|
152
|
+
|
|
153
|
+
expect(loadModuleHealthCadences(getDb()).get('caddy')?.cadence).toEqual({ minutes: 60 });
|
|
154
|
+
});
|
|
155
|
+
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Carrying an existing fleet's health-check cadences into `module_configs`.
|
|
3
|
+
*
|
|
4
|
+
* Not bookkeeping. Before this change a `module_hook` monitor's cadence lived
|
|
5
|
+
* on its row, seeded from the manifest at first deploy and editable only by raw
|
|
6
|
+
* SQL; after it, the row is not read and the cadence comes from the operator's
|
|
7
|
+
* override or the manifest's suggestion. So on the FIRST apt upgrade that
|
|
8
|
+
* carries this code, every module whose row diverges from its manifest — which
|
|
9
|
+
* is every module anyone ever re-cadenced by hand — would silently revert to
|
|
10
|
+
* the author's suggestion, and every module an operator deliberately disabled
|
|
11
|
+
* would start being watched again.
|
|
12
|
+
*
|
|
13
|
+
* Two steps, both idempotent, both writing only where no override exists:
|
|
14
|
+
*
|
|
15
|
+
* 1. a row whose cadence differs from what its manifest suggests gets that
|
|
16
|
+
* cadence written as an override;
|
|
17
|
+
* 2. a disabled row gets `manual`.
|
|
18
|
+
*
|
|
19
|
+
* Runs from `celilo system migrate`, which the `.deb` postinst already invokes
|
|
20
|
+
* on every upgrade. Rolling back is safe in both directions: the rows written
|
|
21
|
+
* here are inert to older code, which reads the monitor row, and rolling
|
|
22
|
+
* forward again finds them already present.
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
import { eq } from 'drizzle-orm';
|
|
26
|
+
import type { DbClient } from '../../db/client';
|
|
27
|
+
import { modules, monitors } from '../../db/schema';
|
|
28
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
29
|
+
import { formatCadence, parseCadence } from '../cadence';
|
|
30
|
+
import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
|
|
31
|
+
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY } from './health-cadence';
|
|
32
|
+
|
|
33
|
+
export interface CadenceMigrationReport {
|
|
34
|
+
/** `moduleId → written value`, for the operator-visible summary. */
|
|
35
|
+
written: Map<string, string>;
|
|
36
|
+
/** Modules left alone: an override already existed, or nothing diverged. */
|
|
37
|
+
unchanged: string[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function migrateMonitorCadences(db: DbClient): CadenceMigrationReport {
|
|
41
|
+
const report: CadenceMigrationReport = { written: new Map(), unchanged: [] };
|
|
42
|
+
|
|
43
|
+
for (const monitor of db.select().from(monitors).where(eq(monitors.kind, 'module_hook')).all()) {
|
|
44
|
+
const moduleId = monitor.target;
|
|
45
|
+
|
|
46
|
+
// An operator who has already set one has said the last word; never
|
|
47
|
+
// overwrite it, which is also what makes a second run a no-op.
|
|
48
|
+
if (getModuleConfigValue(moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, db)) {
|
|
49
|
+
report.unchanged.push(moduleId);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const module = db.select().from(modules).where(eq(modules.id, moduleId)).get();
|
|
54
|
+
if (!module) {
|
|
55
|
+
// A monitor whose module is gone has nothing to carry over.
|
|
56
|
+
report.unchanged.push(moduleId);
|
|
57
|
+
continue;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Disabled wins over cadence: an operator who stopped watching a module
|
|
61
|
+
// meant that, whatever interval the row happens to carry.
|
|
62
|
+
if (!monitor.enabled) {
|
|
63
|
+
upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, 'manual');
|
|
64
|
+
report.written.set(moduleId, 'manual');
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const manifest = module.manifestData as ModuleManifest;
|
|
69
|
+
const suggested = manifest.hooks?.health_check?.interval;
|
|
70
|
+
const suggestedMinutes = suggested ? parseCadence(suggested) : null;
|
|
71
|
+
const suggestedIsSame =
|
|
72
|
+
suggestedMinutes !== null &&
|
|
73
|
+
suggestedMinutes !== 'manual' &&
|
|
74
|
+
suggestedMinutes.minutes === monitor.intervalMinutes;
|
|
75
|
+
|
|
76
|
+
if (suggestedIsSame) {
|
|
77
|
+
// The row never diverged, so read-time resolution already produces it —
|
|
78
|
+
// and writing an override here would freeze this module against every
|
|
79
|
+
// future manifest correction, which is the failure this change removes.
|
|
80
|
+
report.unchanged.push(moduleId);
|
|
81
|
+
continue;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const carried = formatCadence({ minutes: monitor.intervalMinutes });
|
|
85
|
+
upsertModuleConfig(db, moduleId, HEALTH_CHECK_INTERVAL_CONFIG_KEY, carried);
|
|
86
|
+
report.written.set(moduleId, carried);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
return report;
|
|
90
|
+
}
|
|
@@ -6,21 +6,18 @@
|
|
|
6
6
|
* lives here. See Rule 2.3.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import { eq } from 'drizzle-orm';
|
|
10
9
|
import type { DbClient } from '../../db/client';
|
|
11
|
-
import { modules
|
|
10
|
+
import { modules } from '../../db/schema';
|
|
12
11
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
12
|
+
import { loadModuleHealthCadences } from './health-cadence';
|
|
13
13
|
import type { ModuleCoverageInput } from './health-coverage';
|
|
14
14
|
|
|
15
15
|
export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
.all()
|
|
22
|
-
.map((row) => row.target),
|
|
23
|
-
);
|
|
16
|
+
// Coverage now asks the same question the sweep asks — "what is this module's
|
|
17
|
+
// effective cadence" — rather than "does an enabled monitor row exist". The
|
|
18
|
+
// row's `enabled` column is `builtin_check`-only, so reading it here would
|
|
19
|
+
// have reported a module as watched that the sweep had stopped scheduling.
|
|
20
|
+
const cadences = loadModuleHealthCadences(db);
|
|
24
21
|
|
|
25
22
|
return db
|
|
26
23
|
.select({ id: modules.id, state: modules.state, manifestData: modules.manifestData })
|
|
@@ -32,7 +29,7 @@ export function loadModuleCoverage(db: DbClient): ModuleCoverageInput[] {
|
|
|
32
29
|
id: module.id,
|
|
33
30
|
state: module.state,
|
|
34
31
|
hasHealthCheckHook: Boolean(manifest.hooks?.health_check),
|
|
35
|
-
|
|
32
|
+
cadence: cadences.get(module.id)?.cadence ?? null,
|
|
36
33
|
};
|
|
37
34
|
});
|
|
38
35
|
}
|
|
@@ -101,21 +101,30 @@ describe('deploy hooks', () => {
|
|
|
101
101
|
expect(monitor.intervalMinutes).toBe(15);
|
|
102
102
|
});
|
|
103
103
|
|
|
104
|
-
|
|
104
|
+
// The row carries severity, escalation policy and lastRunAt; WHETHER it
|
|
105
|
+
// runs comes from the module's effective cadence. So a module with the hook
|
|
106
|
+
// and no suggested interval still gets a row — otherwise an operator who
|
|
107
|
+
// sets `health_check_interval` on it would have nothing to schedule against.
|
|
108
|
+
test('a module with the hook but no interval still gets a row to carry its state', () => {
|
|
105
109
|
addModule('caddy', { hooks: { health_check: { script: 'x' } } });
|
|
106
|
-
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(
|
|
110
|
+
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(true);
|
|
111
|
+
expect(db.select().from(monitors).all()).toHaveLength(1);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test('a module with no health_check hook at all gets nothing', () => {
|
|
115
|
+
addModule('signal', { hooks: {} });
|
|
116
|
+
expect(ensureMonitorOnDeploy(db, 'signal')).toBe(false);
|
|
107
117
|
expect(db.select().from(monitors).all()).toEqual([]);
|
|
108
118
|
});
|
|
109
119
|
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
test('
|
|
120
|
+
// Not because the row protects an operator's setting any more — that lives
|
|
121
|
+
// in module_configs now — but because a redeploy must not duplicate rows.
|
|
122
|
+
test('a second deploy does not create a second row', () => {
|
|
113
123
|
addModule('caddy', { hooks: { health_check: { script: 'x', interval: '15m' } } });
|
|
114
124
|
ensureMonitorOnDeploy(db, 'caddy');
|
|
115
|
-
db.update(monitors).set({ intervalMinutes: 60 }).run();
|
|
116
125
|
|
|
117
126
|
expect(ensureMonitorOnDeploy(db, 'caddy')).toBe(false);
|
|
118
|
-
expect(db.select().from(monitors).all()
|
|
127
|
+
expect(db.select().from(monitors).all()).toHaveLength(1);
|
|
119
128
|
});
|
|
120
129
|
|
|
121
130
|
test('an unknown module does not throw', () => {
|
|
@@ -92,10 +92,14 @@ export function modulesInDeployWindow(db: DbClient): Set<string> {
|
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
/**
|
|
95
|
-
* Create
|
|
95
|
+
* Create the monitor row for a freshly deployed module that declares a
|
|
96
|
+
* `health_check` hook.
|
|
96
97
|
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
98
|
+
* A row for any such module, not only one whose manifest suggests an interval:
|
|
99
|
+
* the row carries severity, escalation policy and `lastRunAt`, while whether it
|
|
100
|
+
* runs comes from the module's effective cadence. Without a row, an operator
|
|
101
|
+
* setting `health_check_interval` on a module whose author named none would
|
|
102
|
+
* have nothing to schedule against.
|
|
99
103
|
*/
|
|
100
104
|
export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
|
|
101
105
|
try {
|
|
@@ -103,8 +107,10 @@ export function ensureMonitorOnDeploy(db: DbClient, moduleId: string): boolean {
|
|
|
103
107
|
if (!module) return false;
|
|
104
108
|
|
|
105
109
|
const manifest = module.manifestData as ModuleManifest;
|
|
106
|
-
const
|
|
107
|
-
|
|
110
|
+
const healthCheck = manifest.hooks?.health_check;
|
|
111
|
+
if (!healthCheck) return false;
|
|
112
|
+
|
|
113
|
+
return ensureMonitorForModule(db, moduleId, healthCheck.interval) !== null;
|
|
108
114
|
} catch {
|
|
109
115
|
return false;
|
|
110
116
|
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
3
|
+
import { effectiveHealthCheckCadence, isScheduled } from './health-cadence';
|
|
4
|
+
|
|
5
|
+
function manifestWith(interval?: string): ModuleManifest {
|
|
6
|
+
return {
|
|
7
|
+
hooks: { health_check: { script: './health.ts', ...(interval ? { interval } : {}) } },
|
|
8
|
+
} as unknown as ModuleManifest;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
describe('effectiveHealthCheckCadence', () => {
|
|
12
|
+
test("the manifest's suggestion applies when nobody has overridden", () => {
|
|
13
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
test('the operator override wins', () => {
|
|
17
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
// The failure this whole change exists for: the cadence used to be seeded onto
|
|
21
|
+
// the monitor row at first deploy and never reconsulted, so an author who
|
|
22
|
+
// corrected a bad interval never reached an existing install.
|
|
23
|
+
test('a corrected suggestion reaches an un-overridden module', () => {
|
|
24
|
+
const before = effectiveHealthCheckCadence(manifestWith('1h'), undefined);
|
|
25
|
+
const afterUpgrade = effectiveHealthCheckCadence(manifestWith('15m'), undefined);
|
|
26
|
+
|
|
27
|
+
expect(before).toEqual({ minutes: 60 });
|
|
28
|
+
expect(afterUpgrade).toEqual({ minutes: 15 });
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
test('a corrected suggestion does NOT disturb an overridden module', () => {
|
|
32
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), '1h')).toEqual({ minutes: 60 });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test('`manual` stops it being scheduled, and is not the same as unset', () => {
|
|
36
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), 'manual')).toBe('manual');
|
|
37
|
+
expect(isScheduled('manual')).toBe(false);
|
|
38
|
+
expect(isScheduled(null)).toBe(false);
|
|
39
|
+
expect(isScheduled({ minutes: 15 })).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('unsetting the override resumes the manifest cadence', () => {
|
|
43
|
+
// `undefined` is what the accessor sees once the row is deleted.
|
|
44
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), undefined)).toEqual({ minutes: 15 });
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
test('nobody naming a cadence is null — a gap, not an opt-out', () => {
|
|
48
|
+
expect(effectiveHealthCheckCadence(manifestWith(), undefined)).toBeNull();
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
test('an operator can set a cadence a module never suggested', () => {
|
|
52
|
+
expect(effectiveHealthCheckCadence(manifestWith(), 'daily')).toEqual({ minutes: 1440 });
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
test('an unparseable override falls back to the manifest rather than going quiet', () => {
|
|
56
|
+
expect(effectiveHealthCheckCadence(manifestWith('15m'), 'fifteen')).toEqual({ minutes: 15 });
|
|
57
|
+
});
|
|
58
|
+
});
|