@celilo/cli 0.14.4 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +18 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +3 -30
- package/src/cli/completion.ts +18 -1
- package/src/cli/index.ts +11 -0
- package/src/db/schema.ts +5 -3
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +17 -2
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +44 -8
- package/src/services/alerting/inbound-poller.ts +65 -28
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo module operations` — see and release the module-operation lock.
|
|
3
|
+
*
|
|
4
|
+
* Backup and restore refuse to run while another operation is in flight.
|
|
5
|
+
* When that refusal is wrong, the operator previously had no way to see
|
|
6
|
+
* the lock at all, let alone clear it: the error said "wait for it to
|
|
7
|
+
* complete", which for a suspended process is advice that can never come
|
|
8
|
+
* true. A `module deploy` Ctrl-Z'd on a lost terminal blocked every
|
|
9
|
+
* backup on the fleet for 20 days on exactly that advice.
|
|
10
|
+
*
|
|
11
|
+
* `clear` marks rows failed rather than deleting them — the history of
|
|
12
|
+
* what was abandoned, and when, is worth more than a tidy table.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { and, eq } from 'drizzle-orm';
|
|
16
|
+
import { getDb } from '../../db/client';
|
|
17
|
+
import { type ModuleOperation, moduleOperations } from '../../db/schema';
|
|
18
|
+
import { OPERATION_TTL_MS, isPidRunnable } from '../../services/module-operations';
|
|
19
|
+
import type { CommandResult } from '../types';
|
|
20
|
+
|
|
21
|
+
/** Why a row is not holding the lock, or null when it still is. */
|
|
22
|
+
function abandonedReason(row: ModuleOperation, now: number): string | null {
|
|
23
|
+
if (now - row.startedAt.getTime() > OPERATION_TTL_MS) return 'expired';
|
|
24
|
+
if (!isPidRunnable(row.pid)) return 'not running';
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function formatAge(ms: number): string {
|
|
29
|
+
const minutes = Math.floor(ms / 60_000);
|
|
30
|
+
if (minutes < 60) return `${minutes}m`;
|
|
31
|
+
const hours = Math.floor(minutes / 60);
|
|
32
|
+
if (hours < 24) return `${hours}h`;
|
|
33
|
+
return `${Math.floor(hours / 24)}d`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function inProgressRows(): ModuleOperation[] {
|
|
37
|
+
return getDb()
|
|
38
|
+
.select()
|
|
39
|
+
.from(moduleOperations)
|
|
40
|
+
.where(eq(moduleOperations.status, 'in_progress'))
|
|
41
|
+
.all();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function handleList(): CommandResult {
|
|
45
|
+
const now = Date.now();
|
|
46
|
+
const rows = inProgressRows();
|
|
47
|
+
|
|
48
|
+
if (rows.length === 0) {
|
|
49
|
+
console.log('\nNo module operations in progress.\n');
|
|
50
|
+
return { success: true, message: 'no operations in progress' };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log('\nModule operations in progress:\n');
|
|
54
|
+
let holding = 0;
|
|
55
|
+
for (const row of rows) {
|
|
56
|
+
const reason = abandonedReason(row, now);
|
|
57
|
+
if (!reason) holding++;
|
|
58
|
+
const age = formatAge(now - row.startedAt.getTime());
|
|
59
|
+
const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
|
|
60
|
+
console.log(
|
|
61
|
+
` ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago ${status}`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const abandoned = rows.length - holding;
|
|
66
|
+
console.log('');
|
|
67
|
+
if (abandoned > 0) {
|
|
68
|
+
console.log(`${abandoned} abandoned row(s) — "celilo module operations clear" sweeps them.\n`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
success: true,
|
|
73
|
+
message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Release abandoned rows. `--all` also releases rows whose process still
|
|
79
|
+
* looks alive.
|
|
80
|
+
*
|
|
81
|
+
* The permissive form exists because the pathological case is precisely
|
|
82
|
+
* the one where our liveness detection was wrong — a pid that has been
|
|
83
|
+
* recycled by an unrelated process reads as perfectly healthy. Refusing
|
|
84
|
+
* to clear it would recreate the outage this command exists to end. It
|
|
85
|
+
* is opt-in and names what it is overriding.
|
|
86
|
+
*/
|
|
87
|
+
function handleClear(flags: Record<string, boolean | string>): CommandResult {
|
|
88
|
+
const db = getDb();
|
|
89
|
+
const now = Date.now();
|
|
90
|
+
const force = flags.all === true;
|
|
91
|
+
const rows = inProgressRows();
|
|
92
|
+
|
|
93
|
+
const targets = force ? rows : rows.filter((row) => abandonedReason(row, now) !== null);
|
|
94
|
+
|
|
95
|
+
if (targets.length === 0) {
|
|
96
|
+
const held = rows.length;
|
|
97
|
+
if (held > 0) {
|
|
98
|
+
return {
|
|
99
|
+
success: true,
|
|
100
|
+
message: `Nothing to clear — ${held} operation(s) still look genuinely in flight. Use --all to release them anyway.`,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return { success: true, message: 'Nothing to clear — no operations in progress.' };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const row of targets) {
|
|
107
|
+
db.update(moduleOperations)
|
|
108
|
+
.set({
|
|
109
|
+
status: 'failed',
|
|
110
|
+
completedAt: new Date(),
|
|
111
|
+
errorMessage: 'abandoned — released by "celilo module operations clear"',
|
|
112
|
+
})
|
|
113
|
+
.where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
|
|
114
|
+
.run();
|
|
115
|
+
console.log(` released ${row.operation} of ${row.moduleId} (pid ${row.pid})`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return { success: true, message: `Released ${targets.length} operation(s)` };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function handleModuleOperations(
|
|
122
|
+
args: string[],
|
|
123
|
+
flags: Record<string, boolean | string>,
|
|
124
|
+
): CommandResult {
|
|
125
|
+
const action = args[0];
|
|
126
|
+
|
|
127
|
+
if (!action || action === 'list') return handleList();
|
|
128
|
+
if (action === 'clear') return handleClear(flags);
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
success: false,
|
|
132
|
+
error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
@@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test';
|
|
|
2
2
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
3
3
|
import {
|
|
4
4
|
type PollCandidate,
|
|
5
|
+
isPollInvocation,
|
|
5
6
|
needsPreUpgradeBackup,
|
|
6
7
|
pickAutoUpgrade,
|
|
7
8
|
pickUpgradePolicy,
|
|
@@ -15,39 +16,50 @@ function manifest(hooks?: ModuleManifest['hooks']): ModuleManifest {
|
|
|
15
16
|
const withBackupHook = manifest({ on_backup: { script: './scripts/backup.ts', timeout: 300000 } });
|
|
16
17
|
const noBackupHook = manifest({ on_install: { script: './scripts/setup.ts', timeout: 180000 } });
|
|
17
18
|
|
|
18
|
-
describe('pickUpgradePolicy (ISS-0138 — config
|
|
19
|
-
test('operator config
|
|
20
|
-
expect(pickUpgradePolicy('always-safe'
|
|
19
|
+
describe('pickUpgradePolicy (ISS-0138 — operator config, else by-semver)', () => {
|
|
20
|
+
test('operator config decides', () => {
|
|
21
|
+
expect(pickUpgradePolicy('always-safe')).toBe('always-safe');
|
|
22
|
+
expect(pickUpgradePolicy('always-fast')).toBe('always-fast');
|
|
21
23
|
});
|
|
22
24
|
|
|
23
|
-
test('
|
|
24
|
-
expect(pickUpgradePolicy(undefined
|
|
25
|
+
test('defaults to by-semver when unset', () => {
|
|
26
|
+
expect(pickUpgradePolicy(undefined)).toBe('by-semver');
|
|
25
27
|
});
|
|
26
28
|
|
|
27
|
-
test('
|
|
28
|
-
expect(pickUpgradePolicy(
|
|
29
|
+
test('an unknown value falls back to by-semver (no crash on bad input)', () => {
|
|
30
|
+
expect(pickUpgradePolicy('bogus')).toBe('by-semver');
|
|
29
31
|
});
|
|
32
|
+
});
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
expect(
|
|
34
|
+
describe('pickAutoUpgrade (ISS-0139 — opt-in via operator config, default off)', () => {
|
|
35
|
+
test('config value (string or boolean) decides', () => {
|
|
36
|
+
expect(pickAutoUpgrade('true')).toBe(true);
|
|
37
|
+
expect(pickAutoUpgrade('false')).toBe(false);
|
|
38
|
+
expect(pickAutoUpgrade(true)).toBe(true);
|
|
39
|
+
expect(pickAutoUpgrade(false)).toBe(false);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('defaults to OFF (opt-in) when unset', () => {
|
|
43
|
+
expect(pickAutoUpgrade(undefined)).toBe(false);
|
|
34
44
|
});
|
|
35
45
|
});
|
|
36
46
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
// The dispatcher spawns a subprocess handler as `<handler> <event_id>`
|
|
48
|
+
// (openspec/specs/event-bus/spec.md), so the registry-poll subscription's
|
|
49
|
+
// handler arrives as `celilo module upgrade --poll 5517`. Without --poll the
|
|
50
|
+
// event id landed in the module-name slot and every 15m tick died with
|
|
51
|
+
// "Module not found: 5517" — the poll never ran on celilo-mgr for weeks.
|
|
52
|
+
describe('isPollInvocation (the CD poll must survive the appended event id)', () => {
|
|
53
|
+
test('--poll wins over a positional (the dispatcher-appended event id)', () => {
|
|
54
|
+
expect(isPollInvocation(['5517'], { poll: true })).toBe(true);
|
|
42
55
|
});
|
|
43
56
|
|
|
44
|
-
test('
|
|
45
|
-
expect(
|
|
46
|
-
expect(pickAutoUpgrade(undefined, false)).toBe(false);
|
|
57
|
+
test('bare `module upgrade` is still the poll', () => {
|
|
58
|
+
expect(isPollInvocation([], {})).toBe(true);
|
|
47
59
|
});
|
|
48
60
|
|
|
49
|
-
test('
|
|
50
|
-
expect(
|
|
61
|
+
test('a named module without --poll is a single-module upgrade', () => {
|
|
62
|
+
expect(isPollInvocation(['lunacycle'], {})).toBe(false);
|
|
51
63
|
});
|
|
52
64
|
});
|
|
53
65
|
|
|
@@ -37,33 +37,29 @@ import { classifyVersionChange, fetchAndUpdate } from './module-update';
|
|
|
37
37
|
const VALID_POLICIES: readonly UpgradePolicy[] = ['by-semver', 'always-safe', 'always-fast'];
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
|
-
* Pick the effective upgrade policy. Pure (Rule 10): operator config
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* Pick the effective upgrade policy. Pure (Rule 10): operator config decides;
|
|
41
|
+
* an unknown/absent value falls back to `by-semver`.
|
|
42
|
+
*
|
|
43
|
+
* Config-only by construction — the manifest schema is `.strict()` and declares
|
|
44
|
+
* no `upgrade_policy`, so a manifest that set one could never pass validation.
|
|
45
|
+
* (The old manifest-default branch read a key no valid manifest can carry —
|
|
46
|
+
* Rule 3.9 dead code, deleted rather than left as a corpse.)
|
|
43
47
|
*/
|
|
44
|
-
export function pickUpgradePolicy(
|
|
45
|
-
fromConfig
|
|
46
|
-
|
|
47
|
-
): UpgradePolicy {
|
|
48
|
-
const candidate = fromConfig ?? fromManifest;
|
|
49
|
-
return VALID_POLICIES.includes(candidate as UpgradePolicy)
|
|
50
|
-
? (candidate as UpgradePolicy)
|
|
48
|
+
export function pickUpgradePolicy(fromConfig: string | undefined): UpgradePolicy {
|
|
49
|
+
return VALID_POLICIES.includes(fromConfig as UpgradePolicy)
|
|
50
|
+
? (fromConfig as UpgradePolicy)
|
|
51
51
|
: 'by-semver';
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
55
|
* Resolve whether a module opts into auto-upgrade by the poll. Pure (Rule 10):
|
|
56
|
-
* operator config
|
|
57
|
-
*
|
|
56
|
+
* operator config only (same `.strict()` reason as pickUpgradePolicy), default
|
|
57
|
+
* OFF — a production module isn't auto-upgraded unless the operator chose it:
|
|
58
|
+
* `celilo module config set <module> auto_upgrade true`.
|
|
58
59
|
*/
|
|
59
|
-
export function pickAutoUpgrade(
|
|
60
|
-
fromConfig: string | boolean | undefined,
|
|
61
|
-
fromManifest: boolean | undefined,
|
|
62
|
-
): boolean {
|
|
60
|
+
export function pickAutoUpgrade(fromConfig: string | boolean | undefined): boolean {
|
|
63
61
|
if (typeof fromConfig === 'boolean') return fromConfig;
|
|
64
|
-
|
|
65
|
-
if (fromConfig === 'false') return false;
|
|
66
|
-
return fromManifest ?? false;
|
|
62
|
+
return fromConfig === 'true';
|
|
67
63
|
}
|
|
68
64
|
|
|
69
65
|
/**
|
|
@@ -113,14 +109,11 @@ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] {
|
|
|
113
109
|
return targets;
|
|
114
110
|
}
|
|
115
111
|
|
|
116
|
-
/** Resolve a module's auto_upgrade
|
|
117
|
-
function resolveAutoUpgrade(moduleId: string
|
|
112
|
+
/** Resolve a module's auto_upgrade opt-in from operator config. */
|
|
113
|
+
function resolveAutoUpgrade(moduleId: string): boolean {
|
|
118
114
|
const cfg = getModuleConfigValue(moduleId, 'auto_upgrade');
|
|
119
|
-
const fromConfig =
|
|
120
|
-
typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined;
|
|
121
115
|
return pickAutoUpgrade(
|
|
122
|
-
|
|
123
|
-
(manifest as ModuleManifest & { auto_upgrade?: boolean }).auto_upgrade,
|
|
116
|
+
typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined,
|
|
124
117
|
);
|
|
125
118
|
}
|
|
126
119
|
|
|
@@ -158,13 +151,11 @@ async function upgradeOneModule(
|
|
|
158
151
|
(updatedRow?.manifestData as ModuleManifest | undefined) ??
|
|
159
152
|
(mod.manifestData as ModuleManifest);
|
|
160
153
|
|
|
161
|
-
// Posture. Version delta is installed→target; the policy comes from
|
|
162
|
-
//
|
|
163
|
-
// with an operator config override still winning.
|
|
154
|
+
// Posture. Version delta is installed→target; the policy comes from operator
|
|
155
|
+
// config (`celilo module config set <m> upgrade_policy …`), else by-semver.
|
|
164
156
|
const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy');
|
|
165
157
|
const modulePolicy = pickUpgradePolicy(
|
|
166
158
|
typeof configPolicy?.value === 'string' ? configPolicy.value : undefined,
|
|
167
|
-
(targetManifest as ModuleManifest & { upgrade_policy?: string }).upgrade_policy,
|
|
168
159
|
);
|
|
169
160
|
// Per-release deploy_posture override lives in the .netapp release metadata;
|
|
170
161
|
// reading it requires fetching the package first. Deferred — the classifier
|
|
@@ -244,7 +235,7 @@ async function runRegistryPoll(
|
|
|
244
235
|
moduleId: mod.id,
|
|
245
236
|
installed: mod.version,
|
|
246
237
|
latest: await latestRegistryVersion(client, mod.id),
|
|
247
|
-
autoUpgrade: resolveAutoUpgrade(mod.id
|
|
238
|
+
autoUpgrade: resolveAutoUpgrade(mod.id),
|
|
248
239
|
});
|
|
249
240
|
}
|
|
250
241
|
|
|
@@ -278,6 +269,20 @@ async function runRegistryPoll(
|
|
|
278
269
|
};
|
|
279
270
|
}
|
|
280
271
|
|
|
272
|
+
/**
|
|
273
|
+
* Pure (Rule 10.1): is this the CD poll rather than a single-module upgrade?
|
|
274
|
+
*
|
|
275
|
+
* `--poll` is what a bus subscription MUST use. The dispatcher spawns a
|
|
276
|
+
* subprocess handler as `<handler> <event_id>` (openspec/specs/event-bus/spec.md),
|
|
277
|
+
* so a bare `celilo module upgrade` handler arrives as `celilo module upgrade
|
|
278
|
+
* 5517` — the event id lands in the optional module-name slot and the poll dies
|
|
279
|
+
* with "Module not found: 5517" every tick. An explicit flag makes the poll
|
|
280
|
+
* invocation immune to the appended id instead of relying on argv position.
|
|
281
|
+
*/
|
|
282
|
+
export function isPollInvocation(args: string[], flags: Record<string, string | boolean>): boolean {
|
|
283
|
+
return Boolean(flags.poll) || !getArg(args, 0);
|
|
284
|
+
}
|
|
285
|
+
|
|
281
286
|
export async function handleModuleUpgrade(
|
|
282
287
|
args: string[],
|
|
283
288
|
flags: Record<string, string | boolean> = {},
|
|
@@ -285,8 +290,8 @@ export async function handleModuleUpgrade(
|
|
|
285
290
|
const db = getDb();
|
|
286
291
|
const moduleId = getArg(args, 0);
|
|
287
292
|
|
|
288
|
-
//
|
|
289
|
-
if (!moduleId) {
|
|
293
|
+
// `--poll` or no name → the CD poll over all auto_upgrade modules.
|
|
294
|
+
if (isPollInvocation(args, flags) || !moduleId) {
|
|
290
295
|
return runRegistryPoll(db, flags);
|
|
291
296
|
}
|
|
292
297
|
|
|
@@ -11,7 +11,10 @@ import { getEventBusPath } from '../../config/paths';
|
|
|
11
11
|
import { getDb } from '../../db/client';
|
|
12
12
|
import type { MonitorKind } from '../../db/schema';
|
|
13
13
|
import { parseIntervalMinutes } from '../../manifest/schema';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
isSchedulableBuiltin,
|
|
16
|
+
runBuiltinCheckForMonitor,
|
|
17
|
+
} from '../../services/alerting/builtin-source';
|
|
15
18
|
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
16
19
|
import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
|
|
17
20
|
import {
|
|
@@ -21,6 +24,7 @@ import {
|
|
|
21
24
|
listMonitors,
|
|
22
25
|
setMonitorEnabled,
|
|
23
26
|
} from '../../services/alerting/monitors';
|
|
27
|
+
import { listPolicies } from '../../services/alerting/people';
|
|
24
28
|
import { runOneMonitor } from '../../services/alerting/run-monitor';
|
|
25
29
|
import { promoteReadyAlerts } from '../../services/alerting/store';
|
|
26
30
|
import type { DriftCategory } from '../../services/audit/types';
|
|
@@ -37,7 +41,7 @@ function buildDeps() {
|
|
|
37
41
|
return {
|
|
38
42
|
runModuleCheck: (moduleId: string) =>
|
|
39
43
|
runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
|
|
40
|
-
runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category),
|
|
44
|
+
runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db),
|
|
41
45
|
loadModuleCoverage: () => loadModuleCoverage(db),
|
|
42
46
|
now: () => new Date(),
|
|
43
47
|
graceMs: DEFAULT_GRACE_MS,
|
|
@@ -53,15 +57,26 @@ function handleList(): CommandResult {
|
|
|
53
57
|
return { success: true, message: 'No monitors configured' };
|
|
54
58
|
}
|
|
55
59
|
|
|
60
|
+
// The POLICY column is the answer to "why did nothing page me". A monitor
|
|
61
|
+
// with no policy is one whose alerts reach nobody, and until this column
|
|
62
|
+
// existed there was no way to see that from the CLI at all — `assign`
|
|
63
|
+
// reported success and nothing anywhere reflected the result (#481).
|
|
64
|
+
const policies = new Map(listPolicies(getDb()).map((p) => [p.id, p.name]));
|
|
65
|
+
const policyOf = (id: string | null) =>
|
|
66
|
+
id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
|
|
67
|
+
|
|
56
68
|
const width = Math.max(6, ...rows.map((r) => r.target.length));
|
|
69
|
+
const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
|
|
57
70
|
console.log('');
|
|
58
|
-
console.log(
|
|
71
|
+
console.log(
|
|
72
|
+
`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(6)} ${'POLICY'.padEnd(policyWidth)} STATE`,
|
|
73
|
+
);
|
|
59
74
|
for (const monitor of rows) {
|
|
60
75
|
const state = monitor.enabled ? 'enabled' : 'disabled';
|
|
61
76
|
const suffix = monitor.lastRunAt ? '' : ' (never run)';
|
|
62
77
|
const every = `${monitor.intervalMinutes}m`;
|
|
63
78
|
console.log(
|
|
64
|
-
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${state}${suffix}`,
|
|
79
|
+
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)} ${state}${suffix}`,
|
|
65
80
|
);
|
|
66
81
|
}
|
|
67
82
|
console.log('');
|
|
@@ -89,9 +104,14 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
|
|
|
89
104
|
}
|
|
90
105
|
|
|
91
106
|
// A target naming an audit category is a built-in check; anything else is a
|
|
92
|
-
// module's health_check hook.
|
|
107
|
+
// module's health_check hook. `isSchedulableBuiltin` is checked explicitly
|
|
108
|
+
// because not every category is snake_case — `backups` is one word, and the
|
|
109
|
+
// underscore heuristic alone would file it as a module hook against a module
|
|
110
|
+
// that does not exist.
|
|
93
111
|
const kind: MonitorKind =
|
|
94
|
-
target === HEALTH_COVERAGE_CHECK || target.includes('_')
|
|
112
|
+
target === HEALTH_COVERAGE_CHECK || isSchedulableBuiltin(target) || target.includes('_')
|
|
113
|
+
? 'builtin_check'
|
|
114
|
+
: 'module_hook';
|
|
95
115
|
|
|
96
116
|
createMonitor(db, { kind, target, intervalMinutes });
|
|
97
117
|
|
|
@@ -20,12 +20,11 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
20
20
|
import { dirname, join } from 'node:path';
|
|
21
21
|
import { fileURLToPath } from 'node:url';
|
|
22
22
|
import { promisify } from 'node:util';
|
|
23
|
-
import {
|
|
23
|
+
import { isNotNull } from 'drizzle-orm';
|
|
24
24
|
import { testDigitalOceanConnection } from '../../api-clients/digitalocean';
|
|
25
25
|
import { testProxmoxConnection } from '../../api-clients/proxmox';
|
|
26
26
|
import { getDb } from '../../db/client';
|
|
27
27
|
import {
|
|
28
|
-
backups,
|
|
29
28
|
capabilitySecrets,
|
|
30
29
|
moduleConfigs as moduleConfigsTbl,
|
|
31
30
|
modules,
|
|
@@ -38,6 +37,7 @@ import { decryptSecret } from '../../secrets/encryption';
|
|
|
38
37
|
import { getOrCreateMasterKey } from '../../secrets/master-key';
|
|
39
38
|
import { runAudit } from '../../services/audit';
|
|
40
39
|
import type { DriftFinding, SystemAuditReport } from '../../services/audit';
|
|
40
|
+
import { loadBackupAuditInfo } from '../../services/audit/backup-source';
|
|
41
41
|
import {
|
|
42
42
|
type LatestCliVersionFetcher,
|
|
43
43
|
fetchLatestCliVersion,
|
|
@@ -154,28 +154,6 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
154
154
|
|
|
155
155
|
const registryClient = new RegistryClient();
|
|
156
156
|
|
|
157
|
-
// Most recent successful backup per module.
|
|
158
|
-
// The `backups` table is added via inline ALTER statements in db/client.ts
|
|
159
|
-
// for upgraded databases; a freshly-initialized DB created from the
|
|
160
|
-
// drizzle journal alone may not have it yet. Treat absence as "no
|
|
161
|
-
// backups recorded" rather than crashing the audit.
|
|
162
|
-
const latestBackupByModule = new Map<string, number>();
|
|
163
|
-
try {
|
|
164
|
-
const successfulBackups = db
|
|
165
|
-
.select()
|
|
166
|
-
.from(backups)
|
|
167
|
-
.where(eq(backups.status, 'completed'))
|
|
168
|
-
.all();
|
|
169
|
-
for (const b of successfulBackups) {
|
|
170
|
-
if (!b.moduleId || !b.completedAt) continue;
|
|
171
|
-
const ts = b.completedAt.getTime();
|
|
172
|
-
const prev = latestBackupByModule.get(b.moduleId);
|
|
173
|
-
if (prev === undefined || ts > prev) latestBackupByModule.set(b.moduleId, ts);
|
|
174
|
-
}
|
|
175
|
-
} catch {
|
|
176
|
-
// backups table missing — leave map empty
|
|
177
|
-
}
|
|
178
|
-
|
|
179
157
|
// Build per-module config map for module-configs check.
|
|
180
158
|
const allConfigs = db
|
|
181
159
|
.select()
|
|
@@ -196,12 +174,7 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
|
|
|
196
174
|
configs: configsByModule.get(m.id) ?? {},
|
|
197
175
|
}));
|
|
198
176
|
|
|
199
|
-
const installedBackupInfo =
|
|
200
|
-
id: m.id,
|
|
201
|
-
state: m.state,
|
|
202
|
-
manifest: m.manifestData as ModuleManifest,
|
|
203
|
-
lastSuccessfulBackupAt: latestBackupByModule.get(m.id) ?? null,
|
|
204
|
-
}));
|
|
177
|
+
const installedBackupInfo = loadBackupAuditInfo(db);
|
|
205
178
|
|
|
206
179
|
// Compose per-module TF_VAR_* env vars in parallel — each call hits
|
|
207
180
|
// the secret store / DB so they're not free, but they're all
|
package/src/cli/completion.ts
CHANGED
|
@@ -185,6 +185,7 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
185
185
|
'secret',
|
|
186
186
|
'status',
|
|
187
187
|
'where',
|
|
188
|
+
'operations',
|
|
188
189
|
'terraform-unlock',
|
|
189
190
|
'types',
|
|
190
191
|
'validate',
|
|
@@ -198,6 +199,12 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
198
199
|
return filterSuggestions(subcommands, args[2] || '');
|
|
199
200
|
}
|
|
200
201
|
|
|
202
|
+
// Module operations subcommands (celilo module operations list/clear)
|
|
203
|
+
if (command === 'module' && args[1] === 'operations' && currentIndex === 2) {
|
|
204
|
+
const subcommands = ['list', 'clear'];
|
|
205
|
+
return filterSuggestions(subcommands, args[2] || '');
|
|
206
|
+
}
|
|
207
|
+
|
|
201
208
|
// Module types subcommands (celilo module types generate/check)
|
|
202
209
|
if (command === 'module' && args[1] === 'types' && currentIndex === 2) {
|
|
203
210
|
const subcommands = ['generate', 'check'];
|
|
@@ -500,7 +507,17 @@ export async function getCompletions(words: string[], current: number): Promise<
|
|
|
500
507
|
|
|
501
508
|
// Backup subcommands
|
|
502
509
|
if (command === 'backup' && currentIndex === 1) {
|
|
503
|
-
const subcommands = [
|
|
510
|
+
const subcommands = [
|
|
511
|
+
'create',
|
|
512
|
+
'sweep',
|
|
513
|
+
'list',
|
|
514
|
+
'restore',
|
|
515
|
+
'delete',
|
|
516
|
+
'prune',
|
|
517
|
+
'name',
|
|
518
|
+
'import',
|
|
519
|
+
'pull',
|
|
520
|
+
];
|
|
504
521
|
return filterSuggestions(subcommands, args[1] || '');
|
|
505
522
|
}
|
|
506
523
|
|
package/src/cli/index.ts
CHANGED
|
@@ -66,6 +66,7 @@ import { handleModuleHealth } from './commands/module-health';
|
|
|
66
66
|
import { handleModuleImport } from './commands/module-import';
|
|
67
67
|
import { handleModuleList } from './commands/module-list';
|
|
68
68
|
import { handleModuleLogs } from './commands/module-logs';
|
|
69
|
+
import { handleModuleOperations } from './commands/module-operations';
|
|
69
70
|
import { handleModulePublish } from './commands/module-publish';
|
|
70
71
|
import { handleModuleRemove } from './commands/module-remove';
|
|
71
72
|
import { handleModuleSearch } from './commands/module-search';
|
|
@@ -747,6 +748,9 @@ Subcommands:
|
|
|
747
748
|
--storage <id> Use specific storage destination
|
|
748
749
|
--no-interactive Non-interactive mode (for cron)
|
|
749
750
|
|
|
751
|
+
sweep Back up every module whose declared schedule is due
|
|
752
|
+
(run by the event bus on timer.tick.1h, not by hand)
|
|
753
|
+
|
|
750
754
|
list [module-id] List available backups
|
|
751
755
|
Options:
|
|
752
756
|
--limit <n> Number of backups to show (default: 20)
|
|
@@ -1466,6 +1470,8 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1466
1470
|
return handleModuleLogs(parsed.args, parsed.flags);
|
|
1467
1471
|
case 'health':
|
|
1468
1472
|
return handleModuleHealth(parsed.args, parsed.flags);
|
|
1473
|
+
case 'operations':
|
|
1474
|
+
return handleModuleOperations(parsed.args, parsed.flags);
|
|
1469
1475
|
case 'remove':
|
|
1470
1476
|
return handleModuleRemove(parsed.args, parsed.flags);
|
|
1471
1477
|
case 'update':
|
|
@@ -1864,6 +1870,11 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
|
|
|
1864
1870
|
return handleBackupCreate(parsed.args, parsed.flags);
|
|
1865
1871
|
}
|
|
1866
1872
|
|
|
1873
|
+
if (parsed.subcommand === 'sweep') {
|
|
1874
|
+
const { handleBackupSweep } = await import('./commands/backup-sweep');
|
|
1875
|
+
return handleBackupSweep();
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1867
1878
|
if (parsed.subcommand === 'list') {
|
|
1868
1879
|
const { handleBackupList } = await import('./commands/backup-list');
|
|
1869
1880
|
return handleBackupList(parsed.args, parsed.flags);
|
package/src/db/schema.ts
CHANGED
|
@@ -1048,9 +1048,11 @@ export const alerts = sqliteTable(
|
|
|
1048
1048
|
deferredRouteId: text('deferred_route_id').references(() => routes.id, {
|
|
1049
1049
|
onDelete: 'set null',
|
|
1050
1050
|
}),
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1051
|
+
// No escalation policy is stored here on purpose. It lives on the MONITOR
|
|
1052
|
+
// and is resolved fresh on every sweep, so assigning a policy takes effect
|
|
1053
|
+
// on alerts that are already firing. Snapshotting it at alert creation made
|
|
1054
|
+
// `escalation-policy assign` a no-op for exactly the alert an operator was
|
|
1055
|
+
// trying to route — the one already paging nobody (#481).
|
|
1054
1056
|
message: text('message').notNull(),
|
|
1055
1057
|
details: text('details'),
|
|
1056
1058
|
resolvedAt: integer('resolved_at', { mode: 'timestamp' }),
|
package/src/manifest/schema.ts
CHANGED
|
@@ -694,7 +694,10 @@ export const ModuleManifestSchema = z
|
|
|
694
694
|
|
|
695
695
|
backup: z
|
|
696
696
|
.object({
|
|
697
|
-
|
|
697
|
+
// Absent means `daily`. Opting out of backups entirely is a real
|
|
698
|
+
// decision and takes an explicit `manual` — see
|
|
699
|
+
// [[services/backup-schedule.ts]].
|
|
700
|
+
schedule: z.enum(['hourly', 'daily', 'weekly', 'monthly', 'manual']).default('daily'),
|
|
698
701
|
retention: z
|
|
699
702
|
.object({
|
|
700
703
|
count: z.number().int().positive().default(7),
|
|
@@ -59,6 +59,10 @@ const EXCLUDE_PATTERNS = [
|
|
|
59
59
|
'.DS_Store',
|
|
60
60
|
'*.netapp',
|
|
61
61
|
'*.test.ts',
|
|
62
|
+
// Dev-only, same bucket as the tests: scripts/tsconfig.json exists so tsc can
|
|
63
|
+
// check hooks in CI. Nothing on a target ever runs tsc, and shipping it would
|
|
64
|
+
// make every module's packaged content change whenever the shared base moves.
|
|
65
|
+
'tsconfig.json',
|
|
62
66
|
'checksums.json',
|
|
63
67
|
'signature.sig',
|
|
64
68
|
];
|
|
@@ -12,22 +12,37 @@
|
|
|
12
12
|
* as "nothing is wrong".
|
|
13
13
|
*/
|
|
14
14
|
|
|
15
|
+
import type { DbClient } from '../../db/client';
|
|
16
|
+
import { loadBackupAuditInfo } from '../audit/backup-source';
|
|
17
|
+
import { auditBackups } from '../audit/backups';
|
|
15
18
|
import { auditMachinesReachable } from '../audit/machines-reachable';
|
|
16
19
|
import type { DriftCategory, DriftFinding } from '../audit/types';
|
|
17
20
|
import { probeMachines } from '../machine-probe';
|
|
18
21
|
|
|
19
22
|
/** Categories a monitor can currently schedule. */
|
|
20
|
-
export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
|
|
23
|
+
export const SCHEDULABLE_BUILTIN_CHECKS: readonly DriftCategory[] = [
|
|
24
|
+
'machines_reachable',
|
|
25
|
+
'backups',
|
|
26
|
+
];
|
|
21
27
|
|
|
22
28
|
export function isSchedulableBuiltin(category: string): category is DriftCategory {
|
|
23
29
|
return (SCHEDULABLE_BUILTIN_CHECKS as readonly string[]).includes(category);
|
|
24
30
|
}
|
|
25
31
|
|
|
26
|
-
export async function runBuiltinCheckForMonitor(
|
|
32
|
+
export async function runBuiltinCheckForMonitor(
|
|
33
|
+
category: DriftCategory,
|
|
34
|
+
db: DbClient,
|
|
35
|
+
): Promise<DriftFinding[]> {
|
|
27
36
|
if (category === 'machines_reachable') {
|
|
28
37
|
return auditMachinesReachable({ results: await probeMachines() });
|
|
29
38
|
}
|
|
30
39
|
|
|
40
|
+
// Local DB reads only — cheap enough to run on every sweep, which is
|
|
41
|
+
// the whole reason this category is schedulable and most are not.
|
|
42
|
+
if (category === 'backups') {
|
|
43
|
+
return auditBackups({ modules: loadBackupAuditInfo(db) });
|
|
44
|
+
}
|
|
45
|
+
|
|
31
46
|
throw new Error(
|
|
32
47
|
`Built-in check "${category}" is not schedulable yet. Schedulable: ${SCHEDULABLE_BUILTIN_CHECKS.join(', ')}.`,
|
|
33
48
|
);
|
|
@@ -68,6 +68,11 @@ beforeAll(async () => {
|
|
|
68
68
|
...process.env,
|
|
69
69
|
SIGNAL_RPC_PORT: String(PORT),
|
|
70
70
|
SIGNAL_KNOWN_RECIPIENTS: `${PETER},${WIFE}`,
|
|
71
|
+
// The flag the module's systemd unit must pass. The simulator defaults
|
|
72
|
+
// to signal-cli's real default (`on-start`), where the daemon drains
|
|
73
|
+
// replies into its own SSE stream and REFUSES `receive` — so a loop test
|
|
74
|
+
// may only read replies from a daemon started the way celilo deploys it.
|
|
75
|
+
SIGNAL_RECEIVE_MODE: 'manual',
|
|
71
76
|
},
|
|
72
77
|
stdout: 'pipe',
|
|
73
78
|
stderr: 'pipe',
|
|
@@ -169,7 +174,6 @@ describe('delivery loop against the signal-cli simulator', () => {
|
|
|
169
174
|
silencedUntil: null,
|
|
170
175
|
escalationStep: 0,
|
|
171
176
|
nextEscalationAt: null,
|
|
172
|
-
escalationPolicyId: null,
|
|
173
177
|
message: '/var 94% used',
|
|
174
178
|
details: null,
|
|
175
179
|
resolvedAt: null,
|