@celilo/cli 0.23.0 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +27 -7
- package/package.json +6 -5
- package/src/cli/commands/alerts-act.ts +1 -1
- package/src/cli/commands/backup-create.ts +26 -11
- package/src/cli/commands/backup-list.test.ts +83 -0
- package/src/cli/commands/backup-list.ts +67 -3
- package/src/cli/commands/backup-prune.ts +17 -17
- package/src/cli/commands/backup-sweep.ts +20 -8
- package/src/cli/commands/firewall-interface-list.test.ts +85 -0
- package/src/cli/commands/firewall-interface-list.ts +123 -0
- package/src/cli/commands/machine-add.ts +30 -2
- package/src/cli/commands/module-config.test.ts +70 -3
- package/src/cli/commands/module-config.ts +262 -28
- package/src/cli/commands/module-status.ts +155 -12
- package/src/cli/commands/monitor.ts +116 -19
- package/src/cli/commands/system-migrate.ts +14 -0
- package/src/cli/commands/system-update.ts +4 -1
- package/src/cli/completion.ts +35 -9
- package/src/cli/index.ts +59 -2
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/hooks/capability-loader.ts +130 -4
- package/src/hooks/load-hook-config.test.ts +169 -1
- package/src/hooks/load-hook-config.ts +118 -20
- package/src/hooks/types.ts +2 -1
- package/src/manifest/contracts/v1.ts +16 -0
- package/src/manifest/schema.ts +40 -65
- package/src/services/alerting/builtin-monitors.test.ts +18 -10
- package/src/services/alerting/cadence-migration.test.ts +155 -0
- package/src/services/alerting/cadence-migration.ts +90 -0
- package/src/services/alerting/coverage-source.ts +8 -11
- package/src/services/alerting/deploy-hooks.test.ts +16 -7
- package/src/services/alerting/deploy-hooks.ts +11 -5
- package/src/services/alerting/health-cadence.test.ts +58 -0
- package/src/services/alerting/health-cadence.ts +128 -0
- package/src/services/alerting/health-coverage.ts +18 -8
- package/src/services/alerting/monitors.ts +50 -15
- package/src/services/alerting/sweep-runner.test.ts +51 -3
- package/src/services/alerting/sweep-runner.ts +30 -7
- package/src/services/audit/backup-source.ts +24 -1
- package/src/services/audit/backups.test.ts +95 -10
- package/src/services/audit/backups.ts +40 -37
- package/src/services/audit/interface-classification.test.ts +220 -0
- package/src/services/audit/interface-classification.ts +167 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +118 -0
- package/src/services/backup-create.ts +36 -30
- package/src/services/backup-metadata.ts +52 -1
- package/src/services/backup-retention.test.ts +123 -0
- package/src/services/backup-retention.ts +66 -5
- package/src/services/backup-schedule.test.ts +166 -0
- package/src/services/backup-schedule.ts +105 -15
- package/src/services/backup-staging.ts +14 -1
- package/src/services/backup-sweep.test.ts +22 -3
- package/src/services/backup-sweep.ts +15 -5
- package/src/services/cadence.test.ts +97 -0
- package/src/services/cadence.ts +165 -0
- package/src/services/config-provenance.test.ts +155 -0
- package/src/services/config-provenance.ts +104 -0
- package/src/services/machine-detector.ts +23 -1
- package/src/services/module-config.ts +33 -0
- package/src/services/storage-providers/s3.test.ts +96 -13
- package/src/services/storage-providers/s3.ts +48 -15
- package/src/services/zone-detector.test.ts +34 -3
- package/src/services/zone-detector.ts +33 -13
- package/src/variables/context.ts +69 -15
- package/src/variables/declarative-derivation.test.ts +53 -0
- package/src/variables/declarative-derivation.ts +13 -2
|
@@ -5,11 +5,124 @@
|
|
|
5
5
|
import { eq } from 'drizzle-orm';
|
|
6
6
|
import { getDb } from '../../db/client';
|
|
7
7
|
import { capabilities, moduleConfigs, modules, secrets } from '../../db/schema';
|
|
8
|
+
import type { ModuleManifest } from '../../manifest/schema';
|
|
9
|
+
import {
|
|
10
|
+
HEALTH_CHECK_INTERVAL_CONFIG_KEY,
|
|
11
|
+
effectiveHealthCheckCadence,
|
|
12
|
+
} from '../../services/alerting/health-cadence';
|
|
13
|
+
import {
|
|
14
|
+
BACKUP_RETENTION_COUNT_CONFIG_KEY,
|
|
15
|
+
BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY,
|
|
16
|
+
effectiveBackupRetention,
|
|
17
|
+
prunesNothing,
|
|
18
|
+
} from '../../services/backup-retention';
|
|
19
|
+
import {
|
|
20
|
+
BACKUP_SCHEDULE_CONFIG_KEY,
|
|
21
|
+
effectiveBackupSchedule,
|
|
22
|
+
} from '../../services/backup-schedule';
|
|
23
|
+
import { formatCadence } from '../../services/cadence';
|
|
24
|
+
import { declaredVariables, isDerivedVariable } from '../../services/config-provenance';
|
|
8
25
|
import { getModuleSystems } from '../../services/deployed-systems';
|
|
26
|
+
import { configOverride, parseStoredConfigValue } from '../../services/module-config';
|
|
9
27
|
import { formatPlacementLine, reconcilePlacement } from '../../services/placement-reconcile';
|
|
10
28
|
import { getArg, validateRequiredArgs } from '../parser';
|
|
11
29
|
import type { CommandResult } from '../types';
|
|
12
30
|
|
|
31
|
+
/**
|
|
32
|
+
* PURE (Rule 10.1): the per-module policy block — what celilo will do to this
|
|
33
|
+
* module, and on whose authority.
|
|
34
|
+
*
|
|
35
|
+
* Both halves are shown deliberately. The stored override alone does not tell
|
|
36
|
+
* an operator what they changed it FROM, and the effective value alone does not
|
|
37
|
+
* tell them whether they set it or the module's author did.
|
|
38
|
+
*/
|
|
39
|
+
export function formatCadencePolicy(input: {
|
|
40
|
+
manifest: ModuleManifest;
|
|
41
|
+
configs: Record<string, unknown>;
|
|
42
|
+
}): string {
|
|
43
|
+
const { manifest, configs } = input;
|
|
44
|
+
const lines = ['Policy:'];
|
|
45
|
+
|
|
46
|
+
if (manifest.hooks?.on_backup) {
|
|
47
|
+
const override = configOverride(configs, BACKUP_SCHEDULE_CONFIG_KEY);
|
|
48
|
+
const effective = formatCadence(effectiveBackupSchedule(manifest, override));
|
|
49
|
+
const suggested = manifest.backup?.schedule;
|
|
50
|
+
if (override !== undefined) {
|
|
51
|
+
lines.push(
|
|
52
|
+
` backup cadence: ${effective} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
|
|
53
|
+
);
|
|
54
|
+
} else if (suggested !== undefined) {
|
|
55
|
+
lines.push(` backup cadence: ${effective} (from the manifest)`);
|
|
56
|
+
} else {
|
|
57
|
+
lines.push(` backup cadence: ${effective} (celilo default; nothing declared or set)`);
|
|
58
|
+
}
|
|
59
|
+
lines.push(` backup retention: ${describeRetention(manifest, configs)}`);
|
|
60
|
+
} else {
|
|
61
|
+
lines.push(' backup cadence: not backed up (module declares no on_backup hook)');
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (manifest.hooks?.health_check) {
|
|
65
|
+
const override = configOverride(configs, HEALTH_CHECK_INTERVAL_CONFIG_KEY);
|
|
66
|
+
const effective = effectiveHealthCheckCadence(manifest, override);
|
|
67
|
+
const suggested = manifest.hooks.health_check.interval;
|
|
68
|
+
const value = effective === null ? 'not watched (no cadence set)' : formatCadence(effective);
|
|
69
|
+
if (override !== undefined) {
|
|
70
|
+
lines.push(
|
|
71
|
+
` health check: ${value} (operator override; manifest suggests ${suggested ?? 'nothing'})`,
|
|
72
|
+
);
|
|
73
|
+
} else if (suggested !== undefined) {
|
|
74
|
+
lines.push(` health check: ${value} (from the manifest)`);
|
|
75
|
+
} else {
|
|
76
|
+
lines.push(` health check: ${value}`);
|
|
77
|
+
}
|
|
78
|
+
} else {
|
|
79
|
+
lines.push(' health check: not watched (module declares no health_check hook)');
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
return lines.join('\n');
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Retention in one line, per dimension, saying "unbounded" rather than a number
|
|
87
|
+
* wherever nothing bounds it. An operator reading a bound they never set would
|
|
88
|
+
* reasonably assume backups are being deleted — and the reverse assumption,
|
|
89
|
+
* that something is pruning when nothing is, is how a disk fills.
|
|
90
|
+
*/
|
|
91
|
+
function describeRetention(manifest: ModuleManifest, configs: Record<string, unknown>): string {
|
|
92
|
+
const policy = effectiveBackupRetention(manifest, configs);
|
|
93
|
+
if (prunesNothing(policy)) return 'none — every backup is kept';
|
|
94
|
+
|
|
95
|
+
const declared = manifest.backup?.retention;
|
|
96
|
+
const dimension = (
|
|
97
|
+
effective: number,
|
|
98
|
+
override: string | undefined,
|
|
99
|
+
suggested: number | undefined,
|
|
100
|
+
unit: string,
|
|
101
|
+
): string => {
|
|
102
|
+
if (effective === Number.POSITIVE_INFINITY) return `unbounded ${unit}`;
|
|
103
|
+
const source =
|
|
104
|
+
override !== undefined
|
|
105
|
+
? `operator override; manifest suggests ${suggested ?? 'nothing'}`
|
|
106
|
+
: 'from the manifest';
|
|
107
|
+
return `${effective} ${unit} (${source})`;
|
|
108
|
+
};
|
|
109
|
+
|
|
110
|
+
return [
|
|
111
|
+
dimension(
|
|
112
|
+
policy.count,
|
|
113
|
+
configOverride(configs, BACKUP_RETENTION_COUNT_CONFIG_KEY),
|
|
114
|
+
declared?.count,
|
|
115
|
+
'copies',
|
|
116
|
+
),
|
|
117
|
+
dimension(
|
|
118
|
+
policy.maxAgeDays,
|
|
119
|
+
configOverride(configs, BACKUP_RETENTION_MAX_AGE_DAYS_CONFIG_KEY),
|
|
120
|
+
declared?.max_age_days,
|
|
121
|
+
'days',
|
|
122
|
+
),
|
|
123
|
+
].join(', ');
|
|
124
|
+
}
|
|
125
|
+
|
|
13
126
|
/**
|
|
14
127
|
* Handle module status command
|
|
15
128
|
*
|
|
@@ -98,22 +211,52 @@ export async function handleModuleStatus(args: string[]): Promise<CommandResult>
|
|
|
98
211
|
sections.push(placementLines.join('\n'));
|
|
99
212
|
}
|
|
100
213
|
|
|
101
|
-
// Section 2: Configuration
|
|
102
|
-
if
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
214
|
+
// Section 2: Configuration, split by who owns each value. A flat list
|
|
215
|
+
// presented a value celilo computed as if the operator had chosen it, which
|
|
216
|
+
// is how a derived value gets "corrected" by hand and silently reverted.
|
|
217
|
+
// `source` is the authority for the split, never the presence of a
|
|
218
|
+
// `derive_from` — see services/config-provenance.ts.
|
|
219
|
+
const declared = declaredVariables(module.manifestData as ModuleManifest);
|
|
220
|
+
const isDerivedKey = (key: string) => {
|
|
221
|
+
const variable = declared.get(key);
|
|
222
|
+
return variable !== undefined && isDerivedVariable(variable);
|
|
223
|
+
};
|
|
224
|
+
// `value` is the human-readable display form (e.g. "test-host" for a string,
|
|
225
|
+
// "2222" for a number, JSON-stringified for complex types) — populated by
|
|
226
|
+
// upsertModuleConfig alongside the canonical valueJson. Using it here keeps
|
|
227
|
+
// the status output free of JSON-quote noise around primitives.
|
|
228
|
+
const userConfigs = configs.filter((c) => !isDerivedKey(c.key));
|
|
229
|
+
const derivedConfigs = configs.filter((c) => isDerivedKey(c.key));
|
|
230
|
+
|
|
231
|
+
if (userConfigs.length > 0) {
|
|
232
|
+
sections.push(
|
|
233
|
+
['Configuration:', ...userConfigs.map((c) => ` ${c.key}: ${c.value}`)].join('\n'),
|
|
234
|
+
);
|
|
113
235
|
} else {
|
|
114
236
|
sections.push('Configuration: (none)');
|
|
115
237
|
}
|
|
116
238
|
|
|
239
|
+
if (derivedConfigs.length > 0) {
|
|
240
|
+
sections.push(
|
|
241
|
+
[
|
|
242
|
+
'Derived by celilo (not settable):',
|
|
243
|
+
...derivedConfigs.map((c) => ` ${c.key}: ${c.value} [${declared.get(c.key)?.source}]`),
|
|
244
|
+
].join('\n'),
|
|
245
|
+
);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// Section 2b: Per-module policy — what celilo will DO to this module, and
|
|
249
|
+
// whether that came from the operator or from the module's author. A raw
|
|
250
|
+
// config key does not tell an operator what the manifest said, and an
|
|
251
|
+
// effective value alone does not tell them whether they are the one who set
|
|
252
|
+
// it. Both, always.
|
|
253
|
+
sections.push(
|
|
254
|
+
formatCadencePolicy({
|
|
255
|
+
manifest: module.manifestData as ModuleManifest,
|
|
256
|
+
configs: Object.fromEntries(configs.map((c) => [c.key, parseStoredConfigValue(c)])),
|
|
257
|
+
}),
|
|
258
|
+
);
|
|
259
|
+
|
|
117
260
|
// Section 3: Secrets
|
|
118
261
|
if (moduleSecrets.length > 0) {
|
|
119
262
|
const secretLines = ['Secrets:'];
|
|
@@ -10,12 +10,12 @@ import { defineEvents, openBus } from '@celilo/event-bus';
|
|
|
10
10
|
import { getEventBusPath } from '../../config/paths';
|
|
11
11
|
import { getDb } from '../../db/client';
|
|
12
12
|
import type { MonitorKind } from '../../db/schema';
|
|
13
|
-
import { parseIntervalMinutes } from '../../manifest/schema';
|
|
14
13
|
import {
|
|
15
14
|
isSchedulableBuiltin,
|
|
16
15
|
runBuiltinCheckForMonitor,
|
|
17
16
|
} from '../../services/alerting/builtin-source';
|
|
18
17
|
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
18
|
+
import { loadModuleHealthCadences } from '../../services/alerting/health-cadence';
|
|
19
19
|
import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
|
|
20
20
|
import {
|
|
21
21
|
createMonitor,
|
|
@@ -23,11 +23,18 @@ import {
|
|
|
23
23
|
findMonitorByTarget,
|
|
24
24
|
listMonitors,
|
|
25
25
|
setMonitorEnabled,
|
|
26
|
+
updateMonitorInterval,
|
|
26
27
|
} from '../../services/alerting/monitors';
|
|
27
28
|
import { listPolicies } from '../../services/alerting/people';
|
|
28
29
|
import { runOneMonitor } from '../../services/alerting/run-monitor';
|
|
29
30
|
import { promoteReadyAlerts } from '../../services/alerting/store';
|
|
30
31
|
import type { DriftCategory } from '../../services/audit/types';
|
|
32
|
+
import {
|
|
33
|
+
MONITOR_INTERVAL_FLOOR_MINUTES,
|
|
34
|
+
cadenceSchema,
|
|
35
|
+
formatCadence,
|
|
36
|
+
parseCadence,
|
|
37
|
+
} from '../../services/cadence';
|
|
31
38
|
import { runModuleHealthCheck } from '../../services/health-runner';
|
|
32
39
|
import type { CommandResult } from '../types';
|
|
33
40
|
|
|
@@ -65,18 +72,34 @@ function handleList(): CommandResult {
|
|
|
65
72
|
const policyOf = (id: string | null) =>
|
|
66
73
|
id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
|
|
67
74
|
|
|
75
|
+
// A `module_hook` row's stored interval is not what the sweep uses, so
|
|
76
|
+
// printing it would be a confident lie. Resolve the same way the sweep does.
|
|
77
|
+
const cadences = loadModuleHealthCadences(getDb());
|
|
78
|
+
const everyOf = (monitor: (typeof rows)[number]): string => {
|
|
79
|
+
if (monitor.kind !== 'module_hook') return `${monitor.intervalMinutes}m`;
|
|
80
|
+
const cadence = cadences.get(monitor.target)?.cadence ?? null;
|
|
81
|
+
return cadence === null ? '—' : formatCadence(cadence);
|
|
82
|
+
};
|
|
83
|
+
|
|
68
84
|
const width = Math.max(6, ...rows.map((r) => r.target.length));
|
|
69
85
|
const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
|
|
70
86
|
console.log('');
|
|
71
87
|
console.log(
|
|
72
|
-
`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(
|
|
88
|
+
`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(8)} ${'POLICY'.padEnd(policyWidth)} STATE`,
|
|
73
89
|
);
|
|
74
90
|
for (const monitor of rows) {
|
|
75
|
-
const
|
|
91
|
+
const every = everyOf(monitor);
|
|
92
|
+
const state =
|
|
93
|
+
monitor.kind === 'module_hook'
|
|
94
|
+
? every === 'manual' || every === '—'
|
|
95
|
+
? 'not watched'
|
|
96
|
+
: 'watched'
|
|
97
|
+
: monitor.enabled
|
|
98
|
+
? 'enabled'
|
|
99
|
+
: 'disabled';
|
|
76
100
|
const suffix = monitor.lastRunAt ? '' : ' (never run)';
|
|
77
|
-
const every = `${monitor.intervalMinutes}m`;
|
|
78
101
|
console.log(
|
|
79
|
-
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(
|
|
102
|
+
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(8)} ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)} ${state}${suffix}`,
|
|
80
103
|
);
|
|
81
104
|
}
|
|
82
105
|
console.log('');
|
|
@@ -90,18 +113,6 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
|
|
|
90
113
|
}
|
|
91
114
|
|
|
92
115
|
const db = getDb();
|
|
93
|
-
if (findMonitorByTarget(db, target)) {
|
|
94
|
-
return { success: false, error: `A monitor for "${target}" already exists.` };
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
const interval = typeof flags.interval === 'string' ? flags.interval : '15m';
|
|
98
|
-
const intervalMinutes = parseIntervalMinutes(interval);
|
|
99
|
-
if (intervalMinutes === null) {
|
|
100
|
-
return {
|
|
101
|
-
success: false,
|
|
102
|
-
error: `Invalid --interval "${interval}". Use a duration like "15m", "1h", or "1d".`,
|
|
103
|
-
};
|
|
104
|
-
}
|
|
105
116
|
|
|
106
117
|
// A target naming an audit category is a built-in check; anything else is a
|
|
107
118
|
// module's health_check hook. `isSchedulableBuiltin` is checked explicitly
|
|
@@ -113,6 +124,30 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
|
|
|
113
124
|
? 'builtin_check'
|
|
114
125
|
: 'module_hook';
|
|
115
126
|
|
|
127
|
+
// Checked BEFORE the already-exists check, so an operator reaching for the
|
|
128
|
+
// cadence knob is told where it moved rather than told the row exists. A
|
|
129
|
+
// module's cadence does not live on its row, so accepting `--interval` here
|
|
130
|
+
// would store a number nothing reads — refuse rather than ignore.
|
|
131
|
+
if (kind === 'module_hook' && typeof flags.interval === 'string') {
|
|
132
|
+
return { success: false, error: moduleCadenceRedirect(target, '--interval') };
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
if (findMonitorByTarget(db, target)) {
|
|
136
|
+
return {
|
|
137
|
+
success: false,
|
|
138
|
+
error:
|
|
139
|
+
kind === 'module_hook'
|
|
140
|
+
? `A monitor for "${target}" already exists — a deploy creates one for every module with a health_check hook.\n\nTo change how often it runs: celilo module config set ${target} health_check_interval <cadence>`
|
|
141
|
+
: `A monitor for "${target}" already exists.\n\nTo change how often it runs: celilo monitor set-interval ${target} <cadence>`,
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const interval = typeof flags.interval === 'string' ? flags.interval : '15m';
|
|
146
|
+
const invalid = validateMonitorCadence(interval);
|
|
147
|
+
if (invalid) return { success: false, error: invalid };
|
|
148
|
+
const cadence = parseCadence(interval);
|
|
149
|
+
const intervalMinutes = cadence !== null && cadence !== 'manual' ? cadence.minutes : 0;
|
|
150
|
+
|
|
116
151
|
createMonitor(db, { kind, target, intervalMinutes });
|
|
117
152
|
|
|
118
153
|
// Creating the first monitor is also what switches the sweep on. Registering
|
|
@@ -127,7 +162,10 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
|
|
|
127
162
|
|
|
128
163
|
return {
|
|
129
164
|
success: true,
|
|
130
|
-
message:
|
|
165
|
+
message:
|
|
166
|
+
kind === 'module_hook'
|
|
167
|
+
? `Watching ${target}. How often is per-module policy: celilo module config set ${target} health_check_interval <cadence>`
|
|
168
|
+
: `Monitoring ${target} every ${interval} (sweep runs on timer.tick.5m)`,
|
|
131
169
|
};
|
|
132
170
|
}
|
|
133
171
|
|
|
@@ -167,11 +205,68 @@ function handleToggle(args: string[], enabled: boolean): CommandResult {
|
|
|
167
205
|
const db = getDb();
|
|
168
206
|
const monitor = findMonitorByTarget(db, target);
|
|
169
207
|
if (!monitor) return { success: false, error: `No monitor for "${target}".` };
|
|
208
|
+
if (monitor.kind === 'module_hook') {
|
|
209
|
+
return { success: false, error: moduleCadenceRedirect(target, `monitor ${verb}`) };
|
|
210
|
+
}
|
|
170
211
|
|
|
171
212
|
setMonitorEnabled(db, monitor.id, enabled, new Date());
|
|
172
213
|
return { success: true, message: `Monitor for ${target} ${enabled ? 'enabled' : 'disabled'}` };
|
|
173
214
|
}
|
|
174
215
|
|
|
216
|
+
/**
|
|
217
|
+
* Two ways to change one module's cadence would disagree about what `module
|
|
218
|
+
* status` shows, so the fleet-level commands refuse a module target and name
|
|
219
|
+
* the per-module one. The targets are disjoint in practice — an operator has
|
|
220
|
+
* either a module or an audit check in hand — and an explicit error teaches
|
|
221
|
+
* better than silence.
|
|
222
|
+
*/
|
|
223
|
+
function moduleCadenceRedirect(target: string, what: string): string {
|
|
224
|
+
return (
|
|
225
|
+
`${what} does not apply to "${target}" — it is a module, and a module's cadence is per-module policy rather than a monitor setting. Its monitor row is created by the deploy.\n\n` +
|
|
226
|
+
` celilo module config set ${target} health_check_interval 15m # watch it every 15 minutes\n` +
|
|
227
|
+
` celilo module config set ${target} health_check_interval manual # stop watching it\n` +
|
|
228
|
+
` celilo module config unset ${target} health_check_interval # follow the manifest again`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
/** The floor comes from the alerting sweep's own tick, never a written-down number. */
|
|
233
|
+
function validateMonitorCadence(value: string): string | null {
|
|
234
|
+
const result = cadenceSchema({ floorMinutes: MONITOR_INTERVAL_FLOOR_MINUTES }).safeParse(value);
|
|
235
|
+
if (result.success) return null;
|
|
236
|
+
return `Invalid interval "${value}".\n\n${result.error.issues.map((i) => i.message).join('\n')}`;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* `celilo monitor set-interval <check> <cadence>` — re-cadence a built-in check
|
|
241
|
+
* without removing and recreating it, which would orphan its alert history.
|
|
242
|
+
*/
|
|
243
|
+
function handleSetInterval(args: string[]): CommandResult {
|
|
244
|
+
const [target, cadence] = args;
|
|
245
|
+
if (!target || !cadence) {
|
|
246
|
+
return { success: false, error: 'Usage: celilo monitor set-interval <check> <cadence>' };
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const db = getDb();
|
|
250
|
+
const monitor = findMonitorByTarget(db, target);
|
|
251
|
+
if (!monitor) return { success: false, error: `No monitor for "${target}".` };
|
|
252
|
+
if (monitor.kind === 'module_hook') {
|
|
253
|
+
return { success: false, error: moduleCadenceRedirect(target, 'monitor set-interval') };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
const invalid = validateMonitorCadence(cadence);
|
|
257
|
+
if (invalid) return { success: false, error: invalid };
|
|
258
|
+
const parsed = parseCadence(cadence);
|
|
259
|
+
if (parsed === null || parsed === 'manual') {
|
|
260
|
+
return {
|
|
261
|
+
success: false,
|
|
262
|
+
error: `A built-in check has no "manual" — disable it instead: celilo monitor disable ${target}`,
|
|
263
|
+
};
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
updateMonitorInterval(db, monitor.id, parsed.minutes);
|
|
267
|
+
return { success: true, message: `${target} now runs every ${formatCadence(parsed)}` };
|
|
268
|
+
}
|
|
269
|
+
|
|
175
270
|
export async function handleMonitor(
|
|
176
271
|
subcommand: string | undefined,
|
|
177
272
|
args: string[],
|
|
@@ -189,10 +284,12 @@ export async function handleMonitor(
|
|
|
189
284
|
return handleToggle(args, true);
|
|
190
285
|
case 'disable':
|
|
191
286
|
return handleToggle(args, false);
|
|
287
|
+
case 'set-interval':
|
|
288
|
+
return handleSetInterval(args);
|
|
192
289
|
default:
|
|
193
290
|
return {
|
|
194
291
|
success: false,
|
|
195
|
-
error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, enable, disable`,
|
|
292
|
+
error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, set-interval, enable, disable`,
|
|
196
293
|
};
|
|
197
294
|
}
|
|
198
295
|
}
|
|
@@ -14,6 +14,7 @@ import { createDbClient, findMigrationsFolder, getDb } from '../../db/client';
|
|
|
14
14
|
import { runMigrationsOn } from '../../db/migrate';
|
|
15
15
|
import { getMigrationStatus } from '../../db/migration-status';
|
|
16
16
|
import { findSchemaDrift } from '../../db/schema-introspection';
|
|
17
|
+
import { migrateMonitorCadences } from '../../services/alerting/cadence-migration';
|
|
17
18
|
import { ensureBackupSweepSubscriber } from '../../services/backup-sweep';
|
|
18
19
|
import { ensureOperationsSweepSubscriber } from '../../services/module-operations';
|
|
19
20
|
import type { CommandResult } from '../types';
|
|
@@ -141,6 +142,13 @@ export async function handleSystemMigrate(
|
|
|
141
142
|
|
|
142
143
|
ensureCoreSubscribers();
|
|
143
144
|
|
|
145
|
+
// A health-check cadence used to live on the monitor row and now resolves
|
|
146
|
+
// from `module_configs`. Without carrying the existing rows over, this very
|
|
147
|
+
// upgrade would silently revert every operator's cadence to the manifest's
|
|
148
|
+
// suggestion and resume watching modules they deliberately stopped watching.
|
|
149
|
+
// Idempotent: writes only where no override exists.
|
|
150
|
+
const cadences = migrateMonitorCadences(db);
|
|
151
|
+
|
|
144
152
|
// Name the latest migration, not just a table count: "35 tables" reads the
|
|
145
153
|
// same whether a column migration applied or silently did nothing (celilo#604).
|
|
146
154
|
const status = getMigrationStatus(sqlite, findMigrationsFolder());
|
|
@@ -148,6 +156,12 @@ export async function handleSystemMigrate(
|
|
|
148
156
|
applied > 0 ? `Applied ${applied} migration(s).` : 'Schema already up to date.',
|
|
149
157
|
`Applied migrations: ${status.appliedCount} (latest: ${status.latestApplied ?? 'none'})`,
|
|
150
158
|
`Schema current: ${drift.tableCount} tables, ${drift.columnCount} columns.`,
|
|
159
|
+
...(cadences.written.size > 0
|
|
160
|
+
? [
|
|
161
|
+
`Carried ${cadences.written.size} health-check cadence(s) into module config:`,
|
|
162
|
+
...[...cadences.written].map(([moduleId, value]) => ` ${moduleId}: ${value}`),
|
|
163
|
+
]
|
|
164
|
+
: []),
|
|
151
165
|
];
|
|
152
166
|
return { success: true, message: lines.join('\n'), data: status };
|
|
153
167
|
}
|
|
@@ -31,8 +31,9 @@ import { fetchLatestCliVersion } from '../../services/audit/cli-version';
|
|
|
31
31
|
import { unusedPublicDnsProbe } from '../../services/audit/public-dns';
|
|
32
32
|
import { makeJournalReader, readAppliedMigrations } from '../../services/audit/schema';
|
|
33
33
|
import { createModuleBackup, createSystemStateBackup } from '../../services/backup-create';
|
|
34
|
+
import { BACKUP_SCHEDULE_CONFIG_KEY } from '../../services/backup-schedule';
|
|
34
35
|
import { runAllHealthChecks, runModuleHealthCheck } from '../../services/health-runner';
|
|
35
|
-
import { parseStoredConfigValue } from '../../services/module-config';
|
|
36
|
+
import { configOverride, parseStoredConfigValue } from '../../services/module-config';
|
|
36
37
|
import { deployModule } from '../../services/module-deploy';
|
|
37
38
|
import { buildModuleGraph } from '../../services/update/dep-graph';
|
|
38
39
|
import {
|
|
@@ -566,6 +567,7 @@ export async function handleSystemUpdate(
|
|
|
566
567
|
id: m.id,
|
|
567
568
|
state: m.state,
|
|
568
569
|
manifest: m.manifestData as ModuleManifest,
|
|
570
|
+
scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY),
|
|
569
571
|
lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
|
|
570
572
|
})),
|
|
571
573
|
},
|
|
@@ -794,6 +796,7 @@ export function rebuildAuditDepsForRerun(
|
|
|
794
796
|
id: m.id,
|
|
795
797
|
state: m.state,
|
|
796
798
|
manifest: m.manifestData as ModuleManifest,
|
|
799
|
+
scheduleOverride: configOverride(configsByModule.get(m.id), BACKUP_SCHEDULE_CONFIG_KEY),
|
|
797
800
|
lastSuccessfulBackupAt: priorBackupByModule.get(m.id) ?? null,
|
|
798
801
|
})),
|
|
799
802
|
},
|
package/src/cli/completion.ts
CHANGED
|
@@ -13,6 +13,7 @@ import { listBackups } from '../services/backup-metadata';
|
|
|
13
13
|
import { listBackupStorages } from '../services/backup-storage';
|
|
14
14
|
import { listContainerServices } from '../services/container-service';
|
|
15
15
|
import { listMachines } from '../services/machine-pool';
|
|
16
|
+
import { FRAMEWORK_CONFIG_KEYS } from './commands/module-config';
|
|
16
17
|
|
|
17
18
|
/**
|
|
18
19
|
* Get completion suggestions based on current command context
|
|
@@ -41,6 +42,7 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
41
42
|
'alerts',
|
|
42
43
|
'escalation-policy',
|
|
43
44
|
'events',
|
|
45
|
+
'firewall',
|
|
44
46
|
'help',
|
|
45
47
|
'hook',
|
|
46
48
|
'ipam',
|
|
@@ -200,9 +202,9 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
200
202
|
return filterSuggestions(subcommands, args[1] || '');
|
|
201
203
|
}
|
|
202
204
|
|
|
203
|
-
// Module config subcommands (celilo module config set/get)
|
|
205
|
+
// Module config subcommands (celilo module config set/get/unset)
|
|
204
206
|
if (command === 'module' && args[1] === 'config' && currentIndex === 2) {
|
|
205
|
-
const subcommands = ['set', 'get'];
|
|
207
|
+
const subcommands = ['set', 'get', 'unset'];
|
|
206
208
|
return filterSuggestions(subcommands, args[2] || '');
|
|
207
209
|
}
|
|
208
210
|
|
|
@@ -218,11 +220,11 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
218
220
|
return filterSuggestions(subcommands, args[2] || '');
|
|
219
221
|
}
|
|
220
222
|
|
|
221
|
-
// Module config set/get - complete with module IDs
|
|
223
|
+
// Module config set/get/unset - complete with module IDs
|
|
222
224
|
if (
|
|
223
225
|
command === 'module' &&
|
|
224
226
|
args[1] === 'config' &&
|
|
225
|
-
(args[2] === 'set' || args[2] === 'get') &&
|
|
227
|
+
(args[2] === 'set' || args[2] === 'get' || args[2] === 'unset') &&
|
|
226
228
|
currentIndex === 3
|
|
227
229
|
) {
|
|
228
230
|
const db = getDb();
|
|
@@ -231,11 +233,16 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
231
233
|
return filterSuggestions(moduleIds, args[3] || '');
|
|
232
234
|
}
|
|
233
235
|
|
|
234
|
-
// Module config set/get <module-id> - complete with config
|
|
236
|
+
// Module config set/get/unset <module-id> - complete with config key names.
|
|
237
|
+
// Framework keys (backup cadence, upgrade policy…) are offered on EVERY
|
|
238
|
+
// module: they describe how celilo treats a module, so a module never
|
|
239
|
+
// declares them and completion built from the manifest alone could not see
|
|
240
|
+
// them — which is why `auto_upgrade` and `upgrade_policy` were uncompletable
|
|
241
|
+
// for as long as they have existed.
|
|
235
242
|
if (
|
|
236
243
|
command === 'module' &&
|
|
237
244
|
args[1] === 'config' &&
|
|
238
|
-
(args[2] === 'set' || args[2] === 'get') &&
|
|
245
|
+
(args[2] === 'set' || args[2] === 'get' || args[2] === 'unset') &&
|
|
239
246
|
currentIndex === 4
|
|
240
247
|
) {
|
|
241
248
|
const db = getDb();
|
|
@@ -244,13 +251,15 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
244
251
|
.from(modules)
|
|
245
252
|
.where(eq(modules.id, args[3] || ''))
|
|
246
253
|
.get();
|
|
254
|
+
const frameworkKeys = Object.keys(FRAMEWORK_CONFIG_KEYS);
|
|
247
255
|
if (module?.manifestData) {
|
|
248
256
|
const manifest = module.manifestData as ModuleManifest;
|
|
249
257
|
const varNames = (manifest.variables?.owns || [])
|
|
250
258
|
.filter((v) => v.source === 'user' || !v.source)
|
|
251
259
|
.map((v) => v.name);
|
|
252
|
-
return filterSuggestions(varNames, args[4] || '');
|
|
260
|
+
return filterSuggestions([...varNames, ...frameworkKeys], args[4] || '');
|
|
253
261
|
}
|
|
262
|
+
return filterSuggestions(frameworkKeys, args[4] || '');
|
|
254
263
|
}
|
|
255
264
|
|
|
256
265
|
// Module secret subcommands (celilo module secret set/list)
|
|
@@ -395,6 +404,16 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
395
404
|
}
|
|
396
405
|
|
|
397
406
|
// Machine subcommands
|
|
407
|
+
if (command === 'firewall' && currentIndex === 1) {
|
|
408
|
+
// `interface` is the only group. `acknowledge` and `enforce` were deleted in
|
|
409
|
+
// the design amendments — there is no policy toggle to complete.
|
|
410
|
+
return filterSuggestions(['interface'], args[1] || '');
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
if (command === 'firewall' && args[1] === 'interface' && currentIndex === 2) {
|
|
414
|
+
return filterSuggestions(['list'], args[2] || '');
|
|
415
|
+
}
|
|
416
|
+
|
|
398
417
|
if (command === 'machine' && currentIndex === 1) {
|
|
399
418
|
const subcommands = ['add', 'list', 'status', 'remove', 'earmark', 'detect'];
|
|
400
419
|
return filterSuggestions(subcommands, args[1] || '');
|
|
@@ -484,7 +503,10 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
484
503
|
|
|
485
504
|
// Monitor subcommands
|
|
486
505
|
if (command === 'monitor' && currentIndex === 1) {
|
|
487
|
-
return filterSuggestions(
|
|
506
|
+
return filterSuggestions(
|
|
507
|
+
['list', 'add', 'run', 'set-interval', 'enable', 'disable'],
|
|
508
|
+
args[1] || '',
|
|
509
|
+
);
|
|
488
510
|
}
|
|
489
511
|
|
|
490
512
|
// Monitor targets - a module ID or one of celilo's own schedulable checks.
|
|
@@ -492,7 +514,11 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
492
514
|
// a new built-in check is completable the moment it is schedulable.
|
|
493
515
|
if (
|
|
494
516
|
command === 'monitor' &&
|
|
495
|
-
(args[1] === 'add' ||
|
|
517
|
+
(args[1] === 'add' ||
|
|
518
|
+
args[1] === 'run' ||
|
|
519
|
+
args[1] === 'set-interval' ||
|
|
520
|
+
args[1] === 'enable' ||
|
|
521
|
+
args[1] === 'disable') &&
|
|
496
522
|
currentIndex === 2
|
|
497
523
|
) {
|
|
498
524
|
const db = getDb();
|