@celilo/cli 0.14.4 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +19 -2
- package/drizzle/0018_drop_alert_policy_snapshot.sql +46 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/cli/commands/alerts-list.ts +10 -0
- package/src/cli/commands/alerts-poll.ts +12 -6
- package/src/cli/commands/alerts-sweep.ts +22 -81
- package/src/cli/commands/backup-sweep.ts +65 -0
- package/src/cli/commands/module-config.test.ts +77 -1
- package/src/cli/commands/module-config.ts +45 -3
- package/src/cli/commands/module-journal.test.ts +47 -0
- package/src/cli/commands/module-journal.ts +98 -0
- package/src/cli/commands/module-operations.test.ts +93 -0
- package/src/cli/commands/module-operations.ts +134 -0
- package/src/cli/commands/module-upgrade.test.ts +32 -20
- package/src/cli/commands/module-upgrade.ts +37 -32
- package/src/cli/commands/monitor.ts +26 -6
- package/src/cli/commands/system-audit.ts +3 -30
- package/src/cli/completion.ts +20 -1
- package/src/cli/generate-zsh-completion.ts +4 -0
- package/src/cli/index.ts +14 -0
- package/src/db/schema.ts +5 -3
- package/src/manifest/schema.ts +4 -1
- package/src/module/packaging/build.ts +4 -0
- package/src/services/alerting/builtin-source.ts +17 -2
- package/src/services/alerting/delivery-loop.test.ts +5 -1
- package/src/services/alerting/format.test.ts +0 -1
- package/src/services/alerting/inbound-poller.test.ts +235 -8
- package/src/services/alerting/inbound-poller.ts +95 -34
- package/src/services/alerting/inbound.test.ts +213 -2
- package/src/services/alerting/inbound.ts +161 -32
- package/src/services/alerting/interview-responder.test.ts +0 -32
- package/src/services/alerting/interview-responder.ts +6 -17
- package/src/services/alerting/notify-deps.ts +113 -0
- package/src/services/alerting/run-monitor.ts +0 -1
- package/src/services/alerting/store.test.ts +1 -1
- package/src/services/alerting/store.ts +0 -2
- package/src/services/alerting/sweep-runner.test.ts +11 -2
- package/src/services/alerting/sweep-runner.ts +14 -7
- package/src/services/alerting/tokens.ts +39 -1
- package/src/services/audit/backup-source.ts +54 -0
- package/src/services/audit/backups.test.ts +7 -2
- package/src/services/audit/backups.ts +10 -18
- package/src/services/backup-cipher.test.ts +188 -0
- package/src/services/backup-cipher.ts +178 -0
- package/src/services/backup-create.ts +20 -30
- package/src/services/backup-envelope-roundtrip.test.ts +6 -26
- package/src/services/backup-restore.ts +10 -16
- package/src/services/backup-schedule.ts +35 -0
- package/src/services/backup-sweep.test.ts +148 -0
- package/src/services/backup-sweep.ts +124 -0
- package/src/services/deploy-posture.ts +15 -2
- package/src/services/module-journal.test.ts +302 -0
- package/src/services/module-journal.ts +160 -0
- package/src/services/module-operations.test.ts +67 -6
- package/src/services/module-operations.ts +69 -19
- package/src/services/module-subscriptions.test.ts +33 -2
- package/src/services/module-subscriptions.ts +10 -1
- package/src/services/module-validator/typescript-build.test.ts +20 -1
- package/src/services/module-validator/typescript-build.ts +9 -5
- package/src/services/restore-from-file.ts +6 -21
- package/src/templates/generator.test.ts +88 -0
- package/src/templates/generator.ts +119 -16
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { JournalReport } from '../../services/module-journal';
|
|
3
|
+
import { formatJournalReport } from './module-journal';
|
|
4
|
+
|
|
5
|
+
const system = (over: Partial<JournalReport['systems'][number]> = {}) => ({
|
|
6
|
+
system: 'signal',
|
|
7
|
+
hostname: 'signal',
|
|
8
|
+
ipv4Address: '10.0.20.40',
|
|
9
|
+
unit: 'signal*',
|
|
10
|
+
ok: true,
|
|
11
|
+
lines: [] as string[],
|
|
12
|
+
...over,
|
|
13
|
+
});
|
|
14
|
+
|
|
15
|
+
describe('formatJournalReport', () => {
|
|
16
|
+
test('renders the journal lines under their host', () => {
|
|
17
|
+
const { message, allOk } = formatJournalReport({
|
|
18
|
+
moduleId: 'signal',
|
|
19
|
+
systems: [system({ lines: ['Received sync sent message'] })],
|
|
20
|
+
});
|
|
21
|
+
expect(allOk).toBe(true);
|
|
22
|
+
expect(message).toContain('signal (10.0.20.40)');
|
|
23
|
+
expect(message).toContain('Received sync sent message');
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
// The defect class of #501: unreadable must not present as quiet.
|
|
27
|
+
test('an unreadable host and an empty journal do not look the same', () => {
|
|
28
|
+
const empty = formatJournalReport({ moduleId: 'signal', systems: [system()] });
|
|
29
|
+
const broken = formatJournalReport({
|
|
30
|
+
moduleId: 'signal',
|
|
31
|
+
systems: [system({ ok: false, error: 'Connection timed out' })],
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
expect(empty.allOk).toBe(true);
|
|
35
|
+
expect(broken.allOk).toBe(false);
|
|
36
|
+
expect(broken.message).not.toBe(empty.message);
|
|
37
|
+
expect(broken.message).toContain('UNREADABLE');
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
test('one unreadable host among several fails the whole read', () => {
|
|
41
|
+
const { allOk } = formatJournalReport({
|
|
42
|
+
moduleId: 'forgejo',
|
|
43
|
+
systems: [system({ lines: ['ok'] }), system({ ok: false, error: 'no route to host' })],
|
|
44
|
+
});
|
|
45
|
+
expect(allOk).toBe(false);
|
|
46
|
+
});
|
|
47
|
+
});
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Module journal command — read a deployed module's runtime daemon logs
|
|
3
|
+
* (`journalctl -u <unit>`) through celilo, without SSH to the host.
|
|
4
|
+
*
|
|
5
|
+
* Sibling to `celilo module logs`, not an extension of it: `logs` reads the
|
|
6
|
+
* LOCAL deploy log written by Ansible on celilo-mgr, `journal` reads the
|
|
7
|
+
* REMOTE daemon's journal on the host that runs the module. Different source,
|
|
8
|
+
* different failure modes (a host can be unreachable; a log file cannot), and
|
|
9
|
+
* different answers to different questions. Folding them together would have
|
|
10
|
+
* meant one command whose meaning depended on a flag.
|
|
11
|
+
*
|
|
12
|
+
* Read-only by construction — see services/module-journal.ts.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { getDb } from '../../db/client';
|
|
16
|
+
import { type JournalReport, readModuleJournal } from '../../services/module-journal';
|
|
17
|
+
import { getArg, hasFlag, validateRequiredArgs } from '../parser';
|
|
18
|
+
import type { CommandResult } from '../types';
|
|
19
|
+
|
|
20
|
+
const USAGE =
|
|
21
|
+
'Usage: celilo module journal <id> [--unit <name>] [--lines <n>] [--since <when>] [--grep <pattern>] [--json]';
|
|
22
|
+
|
|
23
|
+
function numericFlag(value: string | boolean | undefined): number | undefined {
|
|
24
|
+
return typeof value === 'string' ? Number(value) : undefined;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function stringFlag(value: string | boolean | undefined): string | undefined {
|
|
28
|
+
return typeof value === 'string' ? value : undefined;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* Handle module journal command.
|
|
33
|
+
*/
|
|
34
|
+
export async function handleModuleJournal(
|
|
35
|
+
args: string[],
|
|
36
|
+
flags: Record<string, string | boolean> = {},
|
|
37
|
+
): Promise<CommandResult> {
|
|
38
|
+
const argError = validateRequiredArgs(args, 1);
|
|
39
|
+
if (argError) {
|
|
40
|
+
return { success: false, error: `${argError}\n\n${USAGE}` };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
const moduleId = getArg(args, 0);
|
|
44
|
+
if (!moduleId) {
|
|
45
|
+
return { success: false, error: 'Module ID is required' };
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const report = readModuleJournal(
|
|
49
|
+
{
|
|
50
|
+
moduleId,
|
|
51
|
+
unit: stringFlag(flags.unit),
|
|
52
|
+
lines: numericFlag(flags.lines),
|
|
53
|
+
since: stringFlag(flags.since),
|
|
54
|
+
grep: stringFlag(flags.grep),
|
|
55
|
+
},
|
|
56
|
+
getDb(),
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
if ('error' in report) {
|
|
60
|
+
return { success: false, error: report.error };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (hasFlag(flags, 'json')) {
|
|
64
|
+
return {
|
|
65
|
+
success: true,
|
|
66
|
+
message: JSON.stringify(report, null, 2),
|
|
67
|
+
rawOutput: true,
|
|
68
|
+
data: report,
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
const { message, allOk } = formatJournalReport(report);
|
|
73
|
+
return allOk ? { success: true, message } : { success: false, error: message };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Presentation. A host celilo could not read is a FAILURE, not a quiet success
|
|
78
|
+
* — the defect this whole change exists to fix is unreadable and empty being
|
|
79
|
+
* byte-identical to the operator.
|
|
80
|
+
*/
|
|
81
|
+
export function formatJournalReport(report: JournalReport): { message: string; allOk: boolean } {
|
|
82
|
+
const out: string[] = [];
|
|
83
|
+
for (const sys of report.systems) {
|
|
84
|
+
out.push(`── ${sys.hostname} (${sys.ipv4Address}) · unit ${sys.unit}`);
|
|
85
|
+
if (!sys.ok) {
|
|
86
|
+
out.push(` ✗ UNREADABLE: ${sys.error}`);
|
|
87
|
+
} else if (sys.lines.length === 0) {
|
|
88
|
+
out.push(' (no matching journal lines)');
|
|
89
|
+
} else {
|
|
90
|
+
out.push(...sys.lines.map((line) => ` ${line}`));
|
|
91
|
+
}
|
|
92
|
+
out.push('');
|
|
93
|
+
}
|
|
94
|
+
return {
|
|
95
|
+
message: out.join('\n').trimEnd(),
|
|
96
|
+
allOk: report.systems.every((s) => s.ok),
|
|
97
|
+
};
|
|
98
|
+
}
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
|
|
2
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { eq } from 'drizzle-orm';
|
|
6
|
+
import { closeDb, getDb } from '../../db/client';
|
|
7
|
+
import { runMigrations } from '../../db/migrate';
|
|
8
|
+
import { moduleOperations } from '../../db/schema';
|
|
9
|
+
import { OPERATION_TTL_MS } from '../../services/module-operations';
|
|
10
|
+
import { handleModuleOperations } from './module-operations';
|
|
11
|
+
|
|
12
|
+
describe('celilo module operations', () => {
|
|
13
|
+
let dir: string;
|
|
14
|
+
|
|
15
|
+
beforeEach(async () => {
|
|
16
|
+
dir = mkdtempSync(join(tmpdir(), 'celilo-ops-cmd-test-'));
|
|
17
|
+
process.env.CELILO_DB_PATH = join(dir, 'celilo.db');
|
|
18
|
+
await runMigrations(process.env.CELILO_DB_PATH);
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
closeDb();
|
|
23
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
24
|
+
rmSync(dir, { recursive: true, force: true });
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
function insert(id: string, pid: number, ageMs: number): void {
|
|
28
|
+
getDb()
|
|
29
|
+
.insert(moduleOperations)
|
|
30
|
+
.values({
|
|
31
|
+
id,
|
|
32
|
+
moduleId: 'byoi',
|
|
33
|
+
operation: 'deploy',
|
|
34
|
+
status: 'in_progress',
|
|
35
|
+
pid,
|
|
36
|
+
startedAt: new Date(Date.now() - ageMs),
|
|
37
|
+
})
|
|
38
|
+
.run();
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function statusOf(id: string): string | undefined {
|
|
42
|
+
return getDb().select().from(moduleOperations).where(eq(moduleOperations.id, id)).get()?.status;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
it('reports nothing to clear when the lock is genuinely held', () => {
|
|
46
|
+
insert('live', process.pid, 60_000);
|
|
47
|
+
|
|
48
|
+
const result = handleModuleOperations(['clear'], {});
|
|
49
|
+
|
|
50
|
+
if (!result.success) throw new Error(`expected success, got: ${result.error}`);
|
|
51
|
+
expect(result.message).toContain('still look genuinely in flight');
|
|
52
|
+
expect(statusOf('live')).toBe('in_progress');
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
it('releases an expired row without touching a live one', () => {
|
|
56
|
+
insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
|
|
57
|
+
insert('live', process.pid, 60_000);
|
|
58
|
+
|
|
59
|
+
const result = handleModuleOperations(['clear'], {});
|
|
60
|
+
|
|
61
|
+
expect(result.success).toBe(true);
|
|
62
|
+
expect(statusOf('expired')).toBe('failed');
|
|
63
|
+
expect(statusOf('live')).toBe('in_progress');
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
// The escape hatch's reason for existing: when liveness detection is wrong
|
|
67
|
+
// — a recycled pid reads as perfectly healthy — refusing to clear would
|
|
68
|
+
// recreate the outage the command exists to end.
|
|
69
|
+
it('--all releases a row whose process is still alive', () => {
|
|
70
|
+
insert('live', process.pid, 60_000);
|
|
71
|
+
|
|
72
|
+
const result = handleModuleOperations(['clear'], { all: true });
|
|
73
|
+
|
|
74
|
+
expect(result.success).toBe(true);
|
|
75
|
+
expect(statusOf('live')).toBe('failed');
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('lists without mutating anything', () => {
|
|
79
|
+
insert('expired', process.pid, OPERATION_TTL_MS + 60_000);
|
|
80
|
+
|
|
81
|
+
const result = handleModuleOperations([], {});
|
|
82
|
+
|
|
83
|
+
if (!result.success) throw new Error(`expected success, got: ${result.error}`);
|
|
84
|
+
expect(result.message).toContain('1 abandoned');
|
|
85
|
+
expect(statusOf('expired')).toBe('in_progress');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
it('rejects an unknown action rather than silently listing', () => {
|
|
89
|
+
const result = handleModuleOperations(['nuke'], {});
|
|
90
|
+
if (result.success) throw new Error('expected an unknown action to fail');
|
|
91
|
+
expect(result.error).toContain('Unknown action');
|
|
92
|
+
});
|
|
93
|
+
});
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo module operations` — see and release the module-operation lock.
|
|
3
|
+
*
|
|
4
|
+
* Backup and restore refuse to run while another operation is in flight.
|
|
5
|
+
* When that refusal is wrong, the operator previously had no way to see
|
|
6
|
+
* the lock at all, let alone clear it: the error said "wait for it to
|
|
7
|
+
* complete", which for a suspended process is advice that can never come
|
|
8
|
+
* true. A `module deploy` Ctrl-Z'd on a lost terminal blocked every
|
|
9
|
+
* backup on the fleet for 20 days on exactly that advice.
|
|
10
|
+
*
|
|
11
|
+
* `clear` marks rows failed rather than deleting them — the history of
|
|
12
|
+
* what was abandoned, and when, is worth more than a tidy table.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { and, eq } from 'drizzle-orm';
|
|
16
|
+
import { getDb } from '../../db/client';
|
|
17
|
+
import { type ModuleOperation, moduleOperations } from '../../db/schema';
|
|
18
|
+
import { OPERATION_TTL_MS, isPidRunnable } from '../../services/module-operations';
|
|
19
|
+
import type { CommandResult } from '../types';
|
|
20
|
+
|
|
21
|
+
/** Why a row is not holding the lock, or null when it still is. */
|
|
22
|
+
function abandonedReason(row: ModuleOperation, now: number): string | null {
|
|
23
|
+
if (now - row.startedAt.getTime() > OPERATION_TTL_MS) return 'expired';
|
|
24
|
+
if (!isPidRunnable(row.pid)) return 'not running';
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function formatAge(ms: number): string {
|
|
29
|
+
const minutes = Math.floor(ms / 60_000);
|
|
30
|
+
if (minutes < 60) return `${minutes}m`;
|
|
31
|
+
const hours = Math.floor(minutes / 60);
|
|
32
|
+
if (hours < 24) return `${hours}h`;
|
|
33
|
+
return `${Math.floor(hours / 24)}d`;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function inProgressRows(): ModuleOperation[] {
|
|
37
|
+
return getDb()
|
|
38
|
+
.select()
|
|
39
|
+
.from(moduleOperations)
|
|
40
|
+
.where(eq(moduleOperations.status, 'in_progress'))
|
|
41
|
+
.all();
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function handleList(): CommandResult {
|
|
45
|
+
const now = Date.now();
|
|
46
|
+
const rows = inProgressRows();
|
|
47
|
+
|
|
48
|
+
if (rows.length === 0) {
|
|
49
|
+
console.log('\nNo module operations in progress.\n');
|
|
50
|
+
return { success: true, message: 'no operations in progress' };
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log('\nModule operations in progress:\n');
|
|
54
|
+
let holding = 0;
|
|
55
|
+
for (const row of rows) {
|
|
56
|
+
const reason = abandonedReason(row, now);
|
|
57
|
+
if (!reason) holding++;
|
|
58
|
+
const age = formatAge(now - row.startedAt.getTime());
|
|
59
|
+
const status = reason ? `abandoned (${reason})` : 'HOLDING LOCK';
|
|
60
|
+
console.log(
|
|
61
|
+
` ${row.operation.padEnd(9)} ${row.moduleId.padEnd(16)} pid ${String(row.pid).padEnd(8)} ${age.padStart(4)} ago ${status}`,
|
|
62
|
+
);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
const abandoned = rows.length - holding;
|
|
66
|
+
console.log('');
|
|
67
|
+
if (abandoned > 0) {
|
|
68
|
+
console.log(`${abandoned} abandoned row(s) — "celilo module operations clear" sweeps them.\n`);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return {
|
|
72
|
+
success: true,
|
|
73
|
+
message: `${rows.length} in progress (${holding} holding the lock, ${abandoned} abandoned)`,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Release abandoned rows. `--all` also releases rows whose process still
|
|
79
|
+
* looks alive.
|
|
80
|
+
*
|
|
81
|
+
* The permissive form exists because the pathological case is precisely
|
|
82
|
+
* the one where our liveness detection was wrong — a pid that has been
|
|
83
|
+
* recycled by an unrelated process reads as perfectly healthy. Refusing
|
|
84
|
+
* to clear it would recreate the outage this command exists to end. It
|
|
85
|
+
* is opt-in and names what it is overriding.
|
|
86
|
+
*/
|
|
87
|
+
function handleClear(flags: Record<string, boolean | string>): CommandResult {
|
|
88
|
+
const db = getDb();
|
|
89
|
+
const now = Date.now();
|
|
90
|
+
const force = flags.all === true;
|
|
91
|
+
const rows = inProgressRows();
|
|
92
|
+
|
|
93
|
+
const targets = force ? rows : rows.filter((row) => abandonedReason(row, now) !== null);
|
|
94
|
+
|
|
95
|
+
if (targets.length === 0) {
|
|
96
|
+
const held = rows.length;
|
|
97
|
+
if (held > 0) {
|
|
98
|
+
return {
|
|
99
|
+
success: true,
|
|
100
|
+
message: `Nothing to clear — ${held} operation(s) still look genuinely in flight. Use --all to release them anyway.`,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
return { success: true, message: 'Nothing to clear — no operations in progress.' };
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const row of targets) {
|
|
107
|
+
db.update(moduleOperations)
|
|
108
|
+
.set({
|
|
109
|
+
status: 'failed',
|
|
110
|
+
completedAt: new Date(),
|
|
111
|
+
errorMessage: 'abandoned — released by "celilo module operations clear"',
|
|
112
|
+
})
|
|
113
|
+
.where(and(eq(moduleOperations.id, row.id), eq(moduleOperations.status, 'in_progress')))
|
|
114
|
+
.run();
|
|
115
|
+
console.log(` released ${row.operation} of ${row.moduleId} (pid ${row.pid})`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return { success: true, message: `Released ${targets.length} operation(s)` };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function handleModuleOperations(
|
|
122
|
+
args: string[],
|
|
123
|
+
flags: Record<string, boolean | string>,
|
|
124
|
+
): CommandResult {
|
|
125
|
+
const action = args[0];
|
|
126
|
+
|
|
127
|
+
if (!action || action === 'list') return handleList();
|
|
128
|
+
if (action === 'clear') return handleClear(flags);
|
|
129
|
+
|
|
130
|
+
return {
|
|
131
|
+
success: false,
|
|
132
|
+
error: `Unknown action "${action}"\n\nUsage: celilo module operations [list|clear] [--all]`,
|
|
133
|
+
};
|
|
134
|
+
}
|
|
@@ -2,6 +2,7 @@ import { describe, expect, test } from 'bun:test';
|
|
|
2
2
|
import type { ModuleManifest } from '../../manifest/schema';
|
|
3
3
|
import {
|
|
4
4
|
type PollCandidate,
|
|
5
|
+
isPollInvocation,
|
|
5
6
|
needsPreUpgradeBackup,
|
|
6
7
|
pickAutoUpgrade,
|
|
7
8
|
pickUpgradePolicy,
|
|
@@ -15,39 +16,50 @@ function manifest(hooks?: ModuleManifest['hooks']): ModuleManifest {
|
|
|
15
16
|
const withBackupHook = manifest({ on_backup: { script: './scripts/backup.ts', timeout: 300000 } });
|
|
16
17
|
const noBackupHook = manifest({ on_install: { script: './scripts/setup.ts', timeout: 180000 } });
|
|
17
18
|
|
|
18
|
-
describe('pickUpgradePolicy (ISS-0138 — config
|
|
19
|
-
test('operator config
|
|
20
|
-
expect(pickUpgradePolicy('always-safe'
|
|
19
|
+
describe('pickUpgradePolicy (ISS-0138 — operator config, else by-semver)', () => {
|
|
20
|
+
test('operator config decides', () => {
|
|
21
|
+
expect(pickUpgradePolicy('always-safe')).toBe('always-safe');
|
|
22
|
+
expect(pickUpgradePolicy('always-fast')).toBe('always-fast');
|
|
21
23
|
});
|
|
22
24
|
|
|
23
|
-
test('
|
|
24
|
-
expect(pickUpgradePolicy(undefined
|
|
25
|
+
test('defaults to by-semver when unset', () => {
|
|
26
|
+
expect(pickUpgradePolicy(undefined)).toBe('by-semver');
|
|
25
27
|
});
|
|
26
28
|
|
|
27
|
-
test('
|
|
28
|
-
expect(pickUpgradePolicy(
|
|
29
|
+
test('an unknown value falls back to by-semver (no crash on bad input)', () => {
|
|
30
|
+
expect(pickUpgradePolicy('bogus')).toBe('by-semver');
|
|
29
31
|
});
|
|
32
|
+
});
|
|
30
33
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
expect(
|
|
34
|
+
describe('pickAutoUpgrade (ISS-0139 — opt-in via operator config, default off)', () => {
|
|
35
|
+
test('config value (string or boolean) decides', () => {
|
|
36
|
+
expect(pickAutoUpgrade('true')).toBe(true);
|
|
37
|
+
expect(pickAutoUpgrade('false')).toBe(false);
|
|
38
|
+
expect(pickAutoUpgrade(true)).toBe(true);
|
|
39
|
+
expect(pickAutoUpgrade(false)).toBe(false);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test('defaults to OFF (opt-in) when unset', () => {
|
|
43
|
+
expect(pickAutoUpgrade(undefined)).toBe(false);
|
|
34
44
|
});
|
|
35
45
|
});
|
|
36
46
|
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
47
|
+
// The dispatcher spawns a subprocess handler as `<handler> <event_id>`
|
|
48
|
+
// (openspec/specs/event-bus/spec.md), so the registry-poll subscription's
|
|
49
|
+
// handler arrives as `celilo module upgrade --poll 5517`. Without --poll the
|
|
50
|
+
// event id landed in the module-name slot and every 15m tick died with
|
|
51
|
+
// "Module not found: 5517" — the poll never ran on celilo-mgr for weeks.
|
|
52
|
+
describe('isPollInvocation (the CD poll must survive the appended event id)', () => {
|
|
53
|
+
test('--poll wins over a positional (the dispatcher-appended event id)', () => {
|
|
54
|
+
expect(isPollInvocation(['5517'], { poll: true })).toBe(true);
|
|
42
55
|
});
|
|
43
56
|
|
|
44
|
-
test('
|
|
45
|
-
expect(
|
|
46
|
-
expect(pickAutoUpgrade(undefined, false)).toBe(false);
|
|
57
|
+
test('bare `module upgrade` is still the poll', () => {
|
|
58
|
+
expect(isPollInvocation([], {})).toBe(true);
|
|
47
59
|
});
|
|
48
60
|
|
|
49
|
-
test('
|
|
50
|
-
expect(
|
|
61
|
+
test('a named module without --poll is a single-module upgrade', () => {
|
|
62
|
+
expect(isPollInvocation(['lunacycle'], {})).toBe(false);
|
|
51
63
|
});
|
|
52
64
|
});
|
|
53
65
|
|
|
@@ -37,33 +37,29 @@ import { classifyVersionChange, fetchAndUpdate } from './module-update';
|
|
|
37
37
|
const VALID_POLICIES: readonly UpgradePolicy[] = ['by-semver', 'always-safe', 'always-fast'];
|
|
38
38
|
|
|
39
39
|
/**
|
|
40
|
-
* Pick the effective upgrade policy. Pure (Rule 10): operator config
|
|
41
|
-
*
|
|
42
|
-
*
|
|
40
|
+
* Pick the effective upgrade policy. Pure (Rule 10): operator config decides;
|
|
41
|
+
* an unknown/absent value falls back to `by-semver`.
|
|
42
|
+
*
|
|
43
|
+
* Config-only by construction — the manifest schema is `.strict()` and declares
|
|
44
|
+
* no `upgrade_policy`, so a manifest that set one could never pass validation.
|
|
45
|
+
* (The old manifest-default branch read a key no valid manifest can carry —
|
|
46
|
+
* Rule 3.9 dead code, deleted rather than left as a corpse.)
|
|
43
47
|
*/
|
|
44
|
-
export function pickUpgradePolicy(
|
|
45
|
-
fromConfig
|
|
46
|
-
|
|
47
|
-
): UpgradePolicy {
|
|
48
|
-
const candidate = fromConfig ?? fromManifest;
|
|
49
|
-
return VALID_POLICIES.includes(candidate as UpgradePolicy)
|
|
50
|
-
? (candidate as UpgradePolicy)
|
|
48
|
+
export function pickUpgradePolicy(fromConfig: string | undefined): UpgradePolicy {
|
|
49
|
+
return VALID_POLICIES.includes(fromConfig as UpgradePolicy)
|
|
50
|
+
? (fromConfig as UpgradePolicy)
|
|
51
51
|
: 'by-semver';
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
/**
|
|
55
55
|
* Resolve whether a module opts into auto-upgrade by the poll. Pure (Rule 10):
|
|
56
|
-
* operator config
|
|
57
|
-
*
|
|
56
|
+
* operator config only (same `.strict()` reason as pickUpgradePolicy), default
|
|
57
|
+
* OFF — a production module isn't auto-upgraded unless the operator chose it:
|
|
58
|
+
* `celilo module config set <module> auto_upgrade true`.
|
|
58
59
|
*/
|
|
59
|
-
export function pickAutoUpgrade(
|
|
60
|
-
fromConfig: string | boolean | undefined,
|
|
61
|
-
fromManifest: boolean | undefined,
|
|
62
|
-
): boolean {
|
|
60
|
+
export function pickAutoUpgrade(fromConfig: string | boolean | undefined): boolean {
|
|
63
61
|
if (typeof fromConfig === 'boolean') return fromConfig;
|
|
64
|
-
|
|
65
|
-
if (fromConfig === 'false') return false;
|
|
66
|
-
return fromManifest ?? false;
|
|
62
|
+
return fromConfig === 'true';
|
|
67
63
|
}
|
|
68
64
|
|
|
69
65
|
/**
|
|
@@ -113,14 +109,11 @@ export function selectPollTargets(candidates: PollCandidate[]): PollTarget[] {
|
|
|
113
109
|
return targets;
|
|
114
110
|
}
|
|
115
111
|
|
|
116
|
-
/** Resolve a module's auto_upgrade
|
|
117
|
-
function resolveAutoUpgrade(moduleId: string
|
|
112
|
+
/** Resolve a module's auto_upgrade opt-in from operator config. */
|
|
113
|
+
function resolveAutoUpgrade(moduleId: string): boolean {
|
|
118
114
|
const cfg = getModuleConfigValue(moduleId, 'auto_upgrade');
|
|
119
|
-
const fromConfig =
|
|
120
|
-
typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined;
|
|
121
115
|
return pickAutoUpgrade(
|
|
122
|
-
|
|
123
|
-
(manifest as ModuleManifest & { auto_upgrade?: boolean }).auto_upgrade,
|
|
116
|
+
typeof cfg?.value === 'string' || typeof cfg?.value === 'boolean' ? cfg.value : undefined,
|
|
124
117
|
);
|
|
125
118
|
}
|
|
126
119
|
|
|
@@ -158,13 +151,11 @@ async function upgradeOneModule(
|
|
|
158
151
|
(updatedRow?.manifestData as ModuleManifest | undefined) ??
|
|
159
152
|
(mod.manifestData as ModuleManifest);
|
|
160
153
|
|
|
161
|
-
// Posture. Version delta is installed→target; the policy comes from
|
|
162
|
-
//
|
|
163
|
-
// with an operator config override still winning.
|
|
154
|
+
// Posture. Version delta is installed→target; the policy comes from operator
|
|
155
|
+
// config (`celilo module config set <m> upgrade_policy …`), else by-semver.
|
|
164
156
|
const configPolicy = getModuleConfigValue(moduleId, 'upgrade_policy');
|
|
165
157
|
const modulePolicy = pickUpgradePolicy(
|
|
166
158
|
typeof configPolicy?.value === 'string' ? configPolicy.value : undefined,
|
|
167
|
-
(targetManifest as ModuleManifest & { upgrade_policy?: string }).upgrade_policy,
|
|
168
159
|
);
|
|
169
160
|
// Per-release deploy_posture override lives in the .netapp release metadata;
|
|
170
161
|
// reading it requires fetching the package first. Deferred — the classifier
|
|
@@ -244,7 +235,7 @@ async function runRegistryPoll(
|
|
|
244
235
|
moduleId: mod.id,
|
|
245
236
|
installed: mod.version,
|
|
246
237
|
latest: await latestRegistryVersion(client, mod.id),
|
|
247
|
-
autoUpgrade: resolveAutoUpgrade(mod.id
|
|
238
|
+
autoUpgrade: resolveAutoUpgrade(mod.id),
|
|
248
239
|
});
|
|
249
240
|
}
|
|
250
241
|
|
|
@@ -278,6 +269,20 @@ async function runRegistryPoll(
|
|
|
278
269
|
};
|
|
279
270
|
}
|
|
280
271
|
|
|
272
|
+
/**
|
|
273
|
+
* Pure (Rule 10.1): is this the CD poll rather than a single-module upgrade?
|
|
274
|
+
*
|
|
275
|
+
* `--poll` is what a bus subscription MUST use. The dispatcher spawns a
|
|
276
|
+
* subprocess handler as `<handler> <event_id>` (openspec/specs/event-bus/spec.md),
|
|
277
|
+
* so a bare `celilo module upgrade` handler arrives as `celilo module upgrade
|
|
278
|
+
* 5517` — the event id lands in the optional module-name slot and the poll dies
|
|
279
|
+
* with "Module not found: 5517" every tick. An explicit flag makes the poll
|
|
280
|
+
* invocation immune to the appended id instead of relying on argv position.
|
|
281
|
+
*/
|
|
282
|
+
export function isPollInvocation(args: string[], flags: Record<string, string | boolean>): boolean {
|
|
283
|
+
return Boolean(flags.poll) || !getArg(args, 0);
|
|
284
|
+
}
|
|
285
|
+
|
|
281
286
|
export async function handleModuleUpgrade(
|
|
282
287
|
args: string[],
|
|
283
288
|
flags: Record<string, string | boolean> = {},
|
|
@@ -285,8 +290,8 @@ export async function handleModuleUpgrade(
|
|
|
285
290
|
const db = getDb();
|
|
286
291
|
const moduleId = getArg(args, 0);
|
|
287
292
|
|
|
288
|
-
//
|
|
289
|
-
if (!moduleId) {
|
|
293
|
+
// `--poll` or no name → the CD poll over all auto_upgrade modules.
|
|
294
|
+
if (isPollInvocation(args, flags) || !moduleId) {
|
|
290
295
|
return runRegistryPoll(db, flags);
|
|
291
296
|
}
|
|
292
297
|
|
|
@@ -11,7 +11,10 @@ import { getEventBusPath } from '../../config/paths';
|
|
|
11
11
|
import { getDb } from '../../db/client';
|
|
12
12
|
import type { MonitorKind } from '../../db/schema';
|
|
13
13
|
import { parseIntervalMinutes } from '../../manifest/schema';
|
|
14
|
-
import {
|
|
14
|
+
import {
|
|
15
|
+
isSchedulableBuiltin,
|
|
16
|
+
runBuiltinCheckForMonitor,
|
|
17
|
+
} from '../../services/alerting/builtin-source';
|
|
15
18
|
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
16
19
|
import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
|
|
17
20
|
import {
|
|
@@ -21,6 +24,7 @@ import {
|
|
|
21
24
|
listMonitors,
|
|
22
25
|
setMonitorEnabled,
|
|
23
26
|
} from '../../services/alerting/monitors';
|
|
27
|
+
import { listPolicies } from '../../services/alerting/people';
|
|
24
28
|
import { runOneMonitor } from '../../services/alerting/run-monitor';
|
|
25
29
|
import { promoteReadyAlerts } from '../../services/alerting/store';
|
|
26
30
|
import type { DriftCategory } from '../../services/audit/types';
|
|
@@ -37,7 +41,7 @@ function buildDeps() {
|
|
|
37
41
|
return {
|
|
38
42
|
runModuleCheck: (moduleId: string) =>
|
|
39
43
|
runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
|
|
40
|
-
runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category),
|
|
44
|
+
runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category, db),
|
|
41
45
|
loadModuleCoverage: () => loadModuleCoverage(db),
|
|
42
46
|
now: () => new Date(),
|
|
43
47
|
graceMs: DEFAULT_GRACE_MS,
|
|
@@ -53,15 +57,26 @@ function handleList(): CommandResult {
|
|
|
53
57
|
return { success: true, message: 'No monitors configured' };
|
|
54
58
|
}
|
|
55
59
|
|
|
60
|
+
// The POLICY column is the answer to "why did nothing page me". A monitor
|
|
61
|
+
// with no policy is one whose alerts reach nobody, and until this column
|
|
62
|
+
// existed there was no way to see that from the CLI at all — `assign`
|
|
63
|
+
// reported success and nothing anywhere reflected the result (#481).
|
|
64
|
+
const policies = new Map(listPolicies(getDb()).map((p) => [p.id, p.name]));
|
|
65
|
+
const policyOf = (id: string | null) =>
|
|
66
|
+
id ? (policies.get(id) ?? '(deleted policy)') : '— pages nobody';
|
|
67
|
+
|
|
56
68
|
const width = Math.max(6, ...rows.map((r) => r.target.length));
|
|
69
|
+
const policyWidth = Math.max(6, ...rows.map((r) => policyOf(r.escalationPolicyId).length));
|
|
57
70
|
console.log('');
|
|
58
|
-
console.log(
|
|
71
|
+
console.log(
|
|
72
|
+
`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(6)} ${'POLICY'.padEnd(policyWidth)} STATE`,
|
|
73
|
+
);
|
|
59
74
|
for (const monitor of rows) {
|
|
60
75
|
const state = monitor.enabled ? 'enabled' : 'disabled';
|
|
61
76
|
const suffix = monitor.lastRunAt ? '' : ' (never run)';
|
|
62
77
|
const every = `${monitor.intervalMinutes}m`;
|
|
63
78
|
console.log(
|
|
64
|
-
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${state}${suffix}`,
|
|
79
|
+
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${policyOf(monitor.escalationPolicyId).padEnd(policyWidth)} ${state}${suffix}`,
|
|
65
80
|
);
|
|
66
81
|
}
|
|
67
82
|
console.log('');
|
|
@@ -89,9 +104,14 @@ function handleAdd(args: string[], flags: Record<string, boolean | string>): Com
|
|
|
89
104
|
}
|
|
90
105
|
|
|
91
106
|
// A target naming an audit category is a built-in check; anything else is a
|
|
92
|
-
// module's health_check hook.
|
|
107
|
+
// module's health_check hook. `isSchedulableBuiltin` is checked explicitly
|
|
108
|
+
// because not every category is snake_case — `backups` is one word, and the
|
|
109
|
+
// underscore heuristic alone would file it as a module hook against a module
|
|
110
|
+
// that does not exist.
|
|
93
111
|
const kind: MonitorKind =
|
|
94
|
-
target === HEALTH_COVERAGE_CHECK || target.includes('_')
|
|
112
|
+
target === HEALTH_COVERAGE_CHECK || isSchedulableBuiltin(target) || target.includes('_')
|
|
113
|
+
? 'builtin_check'
|
|
114
|
+
: 'module_hook';
|
|
95
115
|
|
|
96
116
|
createMonitor(db, { kind, target, intervalMinutes });
|
|
97
117
|
|