@celilo/cli 0.13.3 → 0.14.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +3 -0
- package/CELILO_SUBSYSTEMS.md +71 -2
- package/docs/ALERTING.md +298 -0
- package/docs/INDEX.md +103 -0
- package/drizzle/0016_trusted_sources.sql +10 -0
- package/drizzle/0017_alerting.sql +127 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -2
- package/schemas/system_config.json +9 -0
- package/src/capabilities/well-known.ts +11 -1
- package/src/cli/commands/alerts-act.ts +107 -0
- package/src/cli/commands/alerts-list.ts +62 -0
- package/src/cli/commands/alerts-poll.ts +129 -0
- package/src/cli/commands/alerts-sweep.ts +156 -0
- package/src/cli/commands/module-list.ts +50 -3
- package/src/cli/commands/module-show.ts +11 -3
- package/src/cli/commands/monitor.ts +178 -0
- package/src/cli/commands/notify-config.ts +453 -0
- package/src/cli/commands/system-audit.ts +2 -0
- package/src/cli/commands/system-update.ts +1 -0
- package/src/cli/completion.ts +26 -0
- package/src/cli/generate-zsh-completion.ts +2 -0
- package/src/cli/index.ts +58 -0
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/db/schema.ts +371 -2
- package/src/hooks/capability-loader.ts +158 -46
- package/src/hooks/capability-map-coverage.test.ts +101 -0
- package/src/manifest/schema.ts +77 -4
- package/src/services/alerting/ack.test.ts +212 -0
- package/src/services/alerting/ack.ts +119 -0
- package/src/services/alerting/builtin-monitors.test.ts +132 -0
- package/src/services/alerting/builtin-monitors.ts +84 -0
- package/src/services/alerting/builtin-source.ts +82 -0
- package/src/services/alerting/coverage-source.ts +38 -0
- package/src/services/alerting/deferral.test.ts +161 -0
- package/src/services/alerting/delivery-loop.test.ts +396 -0
- package/src/services/alerting/deploy-hooks.test.ts +125 -0
- package/src/services/alerting/deploy-hooks.ts +111 -0
- package/src/services/alerting/escalation.test.ts +207 -0
- package/src/services/alerting/escalation.ts +151 -0
- package/src/services/alerting/format.test.ts +193 -0
- package/src/services/alerting/format.ts +150 -0
- package/src/services/alerting/health-coverage.ts +81 -0
- package/src/services/alerting/inbound-poller.test.ts +298 -0
- package/src/services/alerting/inbound-poller.ts +236 -0
- package/src/services/alerting/inbound.test.ts +201 -0
- package/src/services/alerting/inbound.ts +112 -0
- package/src/services/alerting/interview-responder.test.ts +169 -0
- package/src/services/alerting/interview-responder.ts +158 -0
- package/src/services/alerting/keys.test.ts +155 -0
- package/src/services/alerting/keys.ts +190 -0
- package/src/services/alerting/monitors.ts +185 -0
- package/src/services/alerting/notification-responder.test.ts +290 -0
- package/src/services/alerting/notification-responder.ts +260 -0
- package/src/services/alerting/notifier.ts +219 -0
- package/src/services/alerting/people.ts +178 -0
- package/src/services/alerting/quiet-hours.test.ts +140 -0
- package/src/services/alerting/quiet-hours.ts +99 -0
- package/src/services/alerting/reconcile.test.ts +190 -0
- package/src/services/alerting/reconcile.ts +166 -0
- package/src/services/alerting/run-monitor.test.ts +185 -0
- package/src/services/alerting/run-monitor.ts +177 -0
- package/src/services/alerting/store.test.ts +222 -0
- package/src/services/alerting/store.ts +289 -0
- package/src/services/alerting/suppression.test.ts +228 -0
- package/src/services/alerting/suppression.ts +142 -0
- package/src/services/alerting/sweep-runner.test.ts +229 -0
- package/src/services/alerting/sweep-runner.ts +204 -0
- package/src/services/alerting/sweep.test.ts +61 -0
- package/src/services/alerting/sweep.ts +41 -0
- package/src/services/alerting/tokens.test.ts +152 -0
- package/src/services/alerting/tokens.ts +119 -0
- package/src/services/alerting/transport-loader.ts +48 -0
- package/src/services/audit/index.test.ts +1 -0
- package/src/services/audit/index.ts +3 -0
- package/src/services/audit/trusted-sources.test.ts +137 -0
- package/src/services/audit/trusted-sources.ts +124 -0
- package/src/services/audit/types.ts +2 -1
- package/src/services/firewall-reach.ts +83 -0
- package/src/services/health-runner.test.ts +50 -0
- package/src/services/health-runner.ts +116 -82
- package/src/services/machine-pool.ts +2 -1
- package/src/services/module-deploy.ts +17 -0
- package/src/services/system-config-validator.test.ts +31 -1
- package/src/services/trusted-sources.test.ts +221 -0
- package/src/services/trusted-sources.ts +159 -0
- package/src/services/update/orchestrator.test.ts +1 -0
- package/src/templates/generator.ts +6 -29
|
@@ -1,5 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { eq } from 'drizzle-orm';
|
|
2
|
+
import { type DbClient, getDb } from '../../db/client';
|
|
3
|
+
import { modules, modules as modulesTable, monitors } from '../../db/schema';
|
|
4
|
+
import { moduleHealthCell } from '../../services/alerting/format';
|
|
5
|
+
import { loadAllLiveAlerts, summariseByModule } from '../../services/alerting/store';
|
|
3
6
|
import { hasFlag } from '../parser';
|
|
4
7
|
import type { CommandResult } from '../types';
|
|
5
8
|
|
|
@@ -36,10 +39,18 @@ export async function handleModuleList(
|
|
|
36
39
|
};
|
|
37
40
|
}
|
|
38
41
|
|
|
42
|
+
// Observed health, derived from live alerts rather than from module state.
|
|
43
|
+
// The two answer different questions: `state` says whether someone
|
|
44
|
+
// deliberately verified this module, `health` says whether it is working
|
|
45
|
+
// right now. "not observed" is a finding, not a blank — see design D15.
|
|
46
|
+
const health = loadObservedHealth(db);
|
|
47
|
+
|
|
39
48
|
// Format module list
|
|
40
49
|
const lines = ['Installed modules:', ''];
|
|
41
50
|
for (const module of moduleRows) {
|
|
42
|
-
|
|
51
|
+
const observed = health.get(module.id);
|
|
52
|
+
const healthNote = observed ? ` [${observed}]` : '';
|
|
53
|
+
lines.push(`${module.id} (v${module.version}) - ${module.state}${healthNote}`);
|
|
43
54
|
if (module.description) {
|
|
44
55
|
lines.push(` ${module.description}`);
|
|
45
56
|
}
|
|
@@ -55,3 +66,39 @@ export async function handleModuleList(
|
|
|
55
66
|
data: moduleRows,
|
|
56
67
|
};
|
|
57
68
|
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* Observed health per module: `ok`, `N firing`, `suppressed`, or
|
|
72
|
+
* `not observed`.
|
|
73
|
+
*
|
|
74
|
+
* Only computed for DEPLOYED modules — a module still being imported has
|
|
75
|
+
* nothing to observe, and reporting it as unwatched would be noise rather
|
|
76
|
+
* than a finding.
|
|
77
|
+
*/
|
|
78
|
+
function loadObservedHealth(db: DbClient): Map<string, string> {
|
|
79
|
+
const monitored = new Set(
|
|
80
|
+
db
|
|
81
|
+
.select({ target: monitors.target })
|
|
82
|
+
.from(monitors)
|
|
83
|
+
.where(eq(monitors.enabled, true))
|
|
84
|
+
.all()
|
|
85
|
+
.map((m) => m.target),
|
|
86
|
+
);
|
|
87
|
+
const byModule = summariseByModule(loadAllLiveAlerts(db));
|
|
88
|
+
|
|
89
|
+
const result = new Map<string, string>();
|
|
90
|
+
for (const module of db.select().from(modulesTable).all()) {
|
|
91
|
+
if (module.state !== 'INSTALLED' && module.state !== 'VERIFIED') continue;
|
|
92
|
+
|
|
93
|
+
const summary = byModule.get(module.id) ?? [];
|
|
94
|
+
result.set(
|
|
95
|
+
module.id,
|
|
96
|
+
moduleHealthCell({
|
|
97
|
+
monitored: monitored.has(module.id),
|
|
98
|
+
firingCount: summary.filter((s) => !s.suppressed).length,
|
|
99
|
+
suppressed: summary.length > 0 && summary.every((s) => s.suppressed),
|
|
100
|
+
}),
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
return result;
|
|
104
|
+
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import { eq } from 'drizzle-orm';
|
|
7
7
|
import { getDb } from '../../db/client';
|
|
8
|
-
import { modules } from '../../db/schema';
|
|
8
|
+
import { type NetworkZone, modules } from '../../db/schema';
|
|
9
9
|
import { type ModuleManifest, getSingularSystemSpec } from '../../manifest/schema';
|
|
10
10
|
import { buildResolutionContext } from '../../variables/context';
|
|
11
11
|
import { getArg, validateRequiredArgs } from '../parser';
|
|
@@ -161,14 +161,22 @@ export async function handleModuleShowZone(args: string[]): Promise<CommandResul
|
|
|
161
161
|
};
|
|
162
162
|
}
|
|
163
163
|
|
|
164
|
-
|
|
164
|
+
// Keyed by NetworkZone, not string: a new zone added to NETWORK_ZONES becomes a
|
|
165
|
+
// compile error here rather than silently rendering as "Unknown zone". Both
|
|
166
|
+
// `internal` and `secure-mgmt` were missing from the previous string-keyed map.
|
|
167
|
+
const zoneDescriptions: Record<NetworkZone, string> = {
|
|
168
|
+
internal: 'Internal (Semi-trusted network behind the firewall)',
|
|
165
169
|
dmz: 'DMZ (Public-facing services)',
|
|
166
170
|
app: 'Application (Internal services)',
|
|
167
171
|
secure: 'Secure (Authentication/Database)',
|
|
172
|
+
'secure-mgmt': "Secure-Mgmt (celilo's own control plane)",
|
|
168
173
|
external: 'External (VPS/Cloud)',
|
|
169
174
|
};
|
|
170
175
|
|
|
171
|
-
|
|
176
|
+
// Cast at the lookup, not the declaration: `zone` comes from config and may be
|
|
177
|
+
// any string, so the runtime fallback stays — but the map above still has to
|
|
178
|
+
// cover every NetworkZone.
|
|
179
|
+
const description = zoneDescriptions[zone as NetworkZone] ?? 'Unknown zone';
|
|
172
180
|
|
|
173
181
|
const lines = [`Module: ${moduleId}`, `Zone: ${zone} - ${description}`, ''];
|
|
174
182
|
|
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `celilo monitor` — what celilo is watching, and running a check on demand.
|
|
3
|
+
*
|
|
4
|
+
* Thin adapter (Rule 10.5): parse, delegate, format. The scheduling decisions
|
|
5
|
+
* live in services/alerting/, and this command composes the real health runner
|
|
6
|
+
* and audit checks into the injectable deps `runOneMonitor` expects.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { defineEvents, openBus } from '@celilo/event-bus';
|
|
10
|
+
import { getEventBusPath } from '../../config/paths';
|
|
11
|
+
import { getDb } from '../../db/client';
|
|
12
|
+
import type { MonitorKind } from '../../db/schema';
|
|
13
|
+
import { parseIntervalMinutes } from '../../manifest/schema';
|
|
14
|
+
import { runBuiltinCheckForMonitor } from '../../services/alerting/builtin-source';
|
|
15
|
+
import { loadModuleCoverage } from '../../services/alerting/coverage-source';
|
|
16
|
+
import { HEALTH_COVERAGE_CHECK } from '../../services/alerting/health-coverage';
|
|
17
|
+
import {
|
|
18
|
+
createMonitor,
|
|
19
|
+
ensureSweepSubscriber,
|
|
20
|
+
findMonitorByTarget,
|
|
21
|
+
listMonitors,
|
|
22
|
+
setMonitorEnabled,
|
|
23
|
+
} from '../../services/alerting/monitors';
|
|
24
|
+
import { runOneMonitor } from '../../services/alerting/run-monitor';
|
|
25
|
+
import { promoteReadyAlerts } from '../../services/alerting/store';
|
|
26
|
+
import type { DriftCategory } from '../../services/audit/types';
|
|
27
|
+
import { runModuleHealthCheck } from '../../services/health-runner';
|
|
28
|
+
import type { CommandResult } from '../types';
|
|
29
|
+
|
|
30
|
+
const NO_SCHEMAS = defineEvents({});
|
|
31
|
+
|
|
32
|
+
/** Grace window before a newly-fired alert may notify. */
|
|
33
|
+
const DEFAULT_GRACE_MS = 60_000;
|
|
34
|
+
|
|
35
|
+
function buildDeps() {
|
|
36
|
+
const db = getDb();
|
|
37
|
+
return {
|
|
38
|
+
runModuleCheck: (moduleId: string) =>
|
|
39
|
+
runModuleHealthCheck(moduleId, db, { unattended: true, noInteractive: true }),
|
|
40
|
+
runBuiltinCheck: (category: DriftCategory) => runBuiltinCheckForMonitor(category),
|
|
41
|
+
loadModuleCoverage: () => loadModuleCoverage(db),
|
|
42
|
+
now: () => new Date(),
|
|
43
|
+
graceMs: DEFAULT_GRACE_MS,
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function handleList(): CommandResult {
|
|
48
|
+
const rows = listMonitors(getDb());
|
|
49
|
+
if (rows.length === 0) {
|
|
50
|
+
console.log('\nNo monitors configured.\n');
|
|
51
|
+
console.log('A module declaring hooks.health_check.interval gets one on deploy,');
|
|
52
|
+
console.log('or add one directly: celilo monitor add <module> --interval 15m\n');
|
|
53
|
+
return { success: true, message: 'No monitors configured' };
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const width = Math.max(6, ...rows.map((r) => r.target.length));
|
|
57
|
+
console.log('');
|
|
58
|
+
console.log(`${'TARGET'.padEnd(width)} ${'KIND'.padEnd(14)} ${'EVERY'.padEnd(6)} STATE`);
|
|
59
|
+
for (const monitor of rows) {
|
|
60
|
+
const state = monitor.enabled ? 'enabled' : 'disabled';
|
|
61
|
+
const suffix = monitor.lastRunAt ? '' : ' (never run)';
|
|
62
|
+
const every = `${monitor.intervalMinutes}m`;
|
|
63
|
+
console.log(
|
|
64
|
+
`${monitor.target.padEnd(width)} ${monitor.kind.padEnd(14)} ${every.padEnd(6)} ${state}${suffix}`,
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
console.log('');
|
|
68
|
+
return { success: true, message: `${rows.length} monitor(s)` };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function handleAdd(args: string[], flags: Record<string, boolean | string>): CommandResult {
|
|
72
|
+
const target = args[0];
|
|
73
|
+
if (!target) {
|
|
74
|
+
return { success: false, error: 'Usage: celilo monitor add <target> [--interval 15m]' };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const db = getDb();
|
|
78
|
+
if (findMonitorByTarget(db, target)) {
|
|
79
|
+
return { success: false, error: `A monitor for "${target}" already exists.` };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const interval = typeof flags.interval === 'string' ? flags.interval : '15m';
|
|
83
|
+
const intervalMinutes = parseIntervalMinutes(interval);
|
|
84
|
+
if (intervalMinutes === null) {
|
|
85
|
+
return {
|
|
86
|
+
success: false,
|
|
87
|
+
error: `Invalid --interval "${interval}". Use a duration like "15m", "1h", or "1d".`,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// A target naming an audit category is a built-in check; anything else is a
|
|
92
|
+
// module's health_check hook.
|
|
93
|
+
const kind: MonitorKind =
|
|
94
|
+
target === HEALTH_COVERAGE_CHECK || target.includes('_') ? 'builtin_check' : 'module_hook';
|
|
95
|
+
|
|
96
|
+
createMonitor(db, { kind, target, intervalMinutes });
|
|
97
|
+
|
|
98
|
+
// Creating the first monitor is also what switches the sweep on. Registering
|
|
99
|
+
// here rather than at install means a celilo with no monitors carries no
|
|
100
|
+
// subscriber and does no periodic work.
|
|
101
|
+
const bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
102
|
+
try {
|
|
103
|
+
ensureSweepSubscriber(bus);
|
|
104
|
+
} finally {
|
|
105
|
+
bus.close();
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return {
|
|
109
|
+
success: true,
|
|
110
|
+
message: `Monitoring ${target} every ${interval} (sweep runs on timer.tick.5m)`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
async function handleRun(args: string[]): Promise<CommandResult> {
|
|
115
|
+
const target = args[0];
|
|
116
|
+
if (!target) return { success: false, error: 'Usage: celilo monitor run <target>' };
|
|
117
|
+
|
|
118
|
+
const db = getDb();
|
|
119
|
+
const monitor = findMonitorByTarget(db, target);
|
|
120
|
+
if (!monitor) {
|
|
121
|
+
return {
|
|
122
|
+
success: false,
|
|
123
|
+
error: `No monitor for "${target}".\n\nRun "celilo monitor list" to see configured monitors.`,
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
const summary = await runOneMonitor(db, monitor, buildDeps());
|
|
128
|
+
promoteReadyAlerts(db, new Date());
|
|
129
|
+
|
|
130
|
+
if (summary.outcome === 'error') {
|
|
131
|
+
return {
|
|
132
|
+
success: true,
|
|
133
|
+
message: `${target}: check could not run — ${summary.errorMessage ?? 'unknown error'}`,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
return {
|
|
137
|
+
success: true,
|
|
138
|
+
message: `${target}: ${summary.failingKeyCount} failing, ${summary.resolvedIds.length} resolved`,
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function handleToggle(args: string[], enabled: boolean): CommandResult {
|
|
143
|
+
const target = args[0];
|
|
144
|
+
const verb = enabled ? 'enable' : 'disable';
|
|
145
|
+
if (!target) return { success: false, error: `Usage: celilo monitor ${verb} <target>` };
|
|
146
|
+
|
|
147
|
+
const db = getDb();
|
|
148
|
+
const monitor = findMonitorByTarget(db, target);
|
|
149
|
+
if (!monitor) return { success: false, error: `No monitor for "${target}".` };
|
|
150
|
+
|
|
151
|
+
setMonitorEnabled(db, monitor.id, enabled, new Date());
|
|
152
|
+
return { success: true, message: `Monitor for ${target} ${enabled ? 'enabled' : 'disabled'}` };
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export async function handleMonitor(
|
|
156
|
+
subcommand: string | undefined,
|
|
157
|
+
args: string[],
|
|
158
|
+
flags: Record<string, boolean | string> = {},
|
|
159
|
+
): Promise<CommandResult> {
|
|
160
|
+
switch (subcommand) {
|
|
161
|
+
case undefined:
|
|
162
|
+
case 'list':
|
|
163
|
+
return handleList();
|
|
164
|
+
case 'add':
|
|
165
|
+
return handleAdd(args, flags);
|
|
166
|
+
case 'run':
|
|
167
|
+
return handleRun(args);
|
|
168
|
+
case 'enable':
|
|
169
|
+
return handleToggle(args, true);
|
|
170
|
+
case 'disable':
|
|
171
|
+
return handleToggle(args, false);
|
|
172
|
+
default:
|
|
173
|
+
return {
|
|
174
|
+
success: false,
|
|
175
|
+
error: `Unknown monitor subcommand: ${subcommand}\n\nUse: list, add, run, enable, disable`,
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
}
|