@celilo/cli 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -0
- package/CELILO_SUBSYSTEMS.md +1 -1
- package/README.md +1 -1
- package/drizzle/0032_module_jail_policies.sql +28 -0
- package/drizzle/0033_build_bus_hook_runs.sql +37 -0
- package/drizzle/meta/_journal.json +15 -1
- package/package.json +3 -3
- package/schemas/system_config.json +5 -0
- package/src/__integration__/container-services-cli.integration.test.ts +1 -1
- package/src/ansible/inventory.test.ts +2 -1
- package/src/api/sessions.test.ts +5 -4
- package/src/api-clients/proxmox.ts +10 -7
- package/src/cli/backup-rename.test.ts +5 -4
- package/src/cli/cli.test.ts +3 -2
- package/src/cli/commands/console-get-chain.test.ts +2 -1
- package/src/cli/commands/events.test.ts +2 -2
- package/src/cli/commands/firewall-interface-list.test.ts +4 -3
- package/src/cli/commands/machine-list.test.ts +57 -0
- package/src/cli/commands/machine-list.ts +35 -4
- package/src/cli/commands/module-config.test.ts +2 -1
- package/src/cli/commands/module-deploy.ts +8 -2
- package/src/cli/commands/module-generate.test.ts +53 -0
- package/src/cli/commands/module-generate.ts +31 -26
- package/src/cli/commands/module-health.ts +1 -0
- package/src/cli/commands/module-import-aspect.test.ts +2 -1
- package/src/cli/commands/module-import-registry.test.ts +3 -2
- package/src/cli/commands/module-jail.test.ts +242 -0
- package/src/cli/commands/module-jail.ts +227 -0
- package/src/cli/commands/module-list-jail.test.ts +136 -0
- package/src/cli/commands/module-list.ts +30 -3
- package/src/cli/commands/module-operations.test.ts +2 -1
- package/src/cli/commands/module-publish.test.ts +11 -18
- package/src/cli/commands/module-update.test.ts +28 -12
- package/src/cli/commands/module-update.ts +4 -1
- package/src/cli/commands/module-upgrade.test.ts +130 -1
- package/src/cli/commands/module-upgrade.ts +77 -3
- package/src/cli/commands/module-verify.test.ts +3 -4
- package/src/cli/commands/module-verify.ts +0 -1
- package/src/cli/commands/notify-config.test.ts +5 -3
- package/src/cli/commands/publish/execute.ts +4 -1
- package/src/cli/commands/publish/index.ts +11 -1
- package/src/cli/commands/publish/module-registry.test.ts +24 -1
- package/src/cli/commands/publish/module-registry.ts +23 -4
- package/src/cli/commands/publish/plan.ts +1 -1
- package/src/cli/commands/publish/types.ts +7 -0
- package/src/cli/commands/registry-owner.test.ts +3 -2
- package/src/cli/commands/registry-token.test.ts +3 -2
- package/src/cli/commands/system-apply-config-equivalence.test.ts +5 -7
- package/src/cli/commands/system-audit.ts +20 -6
- package/src/cli/commands/system-config.test.ts +148 -0
- package/src/cli/commands/system-config.ts +26 -1
- package/src/cli/commands/system-doctor-remediation-gate.test.ts +295 -0
- package/src/cli/commands/system-doctor.test.ts +143 -6
- package/src/cli/commands/system-doctor.ts +142 -25
- package/src/cli/commands/system-init-deprecation.test.ts +7 -4
- package/src/cli/commands/system-migrate.test.ts +2 -1
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +8 -2
- package/src/cli/flag-surface-gate.test.ts +279 -0
- package/src/cli/index.ts +70 -3
- package/src/cli/parser.test.ts +37 -1
- package/src/cli/restore-command.test.ts +5 -4
- package/src/cli/restore-migration-failure.test.ts +4 -3
- package/src/cli/tui/audit-state.ts +2 -0
- package/src/config/paths.test.ts +22 -22
- package/src/db/client.test.ts +46 -1
- package/src/db/client.ts +26 -0
- package/src/db/migrate.test.ts +2 -1
- package/src/db/migrate.ts +16 -16
- package/src/db/schema.ts +63 -0
- package/src/hooks/capability-loader-firewall.test.ts +9 -1
- package/src/hooks/capability-loader.test.ts +78 -0
- package/src/hooks/capability-loader.ts +45 -2
- package/src/hooks/executor.test.ts +161 -1
- package/src/hooks/executor.ts +87 -21
- package/src/hooks/hook-jail-unreachability.test.ts +17 -18
- package/src/hooks/hook-store.test.ts +6 -0
- package/src/hooks/hook-trespass.test.ts +26 -9
- package/src/hooks/jail.test.ts +152 -5
- package/src/hooks/jail.ts +93 -4
- package/src/manifest/contracts/v1.ts +13 -0
- package/src/module/packaging/audit.ts +9 -26
- package/src/module/packaging/build-paths.test.ts +127 -0
- package/src/module/packaging/build-paths.ts +175 -0
- package/src/module/packaging/build.test.ts +71 -1
- package/src/module/packaging/build.ts +60 -0
- package/src/module/packaging/extract.ts +1 -5
- package/src/policy/module-script-scan.ts +51 -107
- package/src/policy/no-hand-built-ssh.test.ts +5 -1
- package/src/policy/no-swallowed-refusal.test.ts +14 -15
- package/src/registry/client.test.ts +2 -2
- package/src/secrets/storage.test.ts +1 -1
- package/src/services/alerting/ack.test.ts +2 -1
- package/src/services/alerting/cadence-migration.test.ts +3 -2
- package/src/services/alerting/coverage-source.test.ts +2 -1
- package/src/services/alerting/deferral.test.ts +2 -1
- package/src/services/alerting/delivery-loop.test.ts +2 -1
- package/src/services/alerting/deploy-hooks.test.ts +2 -1
- package/src/services/alerting/inbound-poller.test.ts +2 -1
- package/src/services/alerting/inbound.test.ts +2 -1
- package/src/services/alerting/keys.test.ts +4 -0
- package/src/services/alerting/keys.ts +12 -2
- package/src/services/alerting/notification-responder.test.ts +2 -1
- package/src/services/alerting/run-monitor.test.ts +2 -1
- package/src/services/alerting/store.test.ts +2 -1
- package/src/services/alerting/sweep-runner.test.ts +2 -1
- package/src/services/alerting/tokens.test.ts +2 -1
- package/src/services/aspect-approvals.test.ts +2 -1
- package/src/services/aspect-reconcile.test.ts +4 -3
- package/src/services/aspect-runner.test.ts +2 -1
- package/src/services/audit/health.test.ts +97 -2
- package/src/services/audit/health.ts +64 -2
- package/src/services/audit/index.test.ts +22 -0
- package/src/services/audit/index.ts +8 -2
- package/src/services/audit/jail-exemptions.test.ts +42 -0
- package/src/services/audit/jail-exemptions.ts +44 -0
- package/src/services/audit/module-integrity.test.ts +23 -22
- package/src/services/audit/module-integrity.ts +7 -16
- package/src/services/audit/types.ts +2 -1
- package/src/services/backup-age-agreement.test.ts +2 -1
- package/src/services/backup-create.ts +9 -0
- package/src/services/backup-envelope-roundtrip.test.ts +3 -2
- package/src/services/backup-in-flight-refusal.test.ts +3 -2
- package/src/services/build-bus/hook-dispatch-mgmt.test.ts +48 -0
- package/src/services/build-bus/hook-dispatcher.ts +49 -1
- package/src/services/bus-ensure-flow.test.ts +3 -2
- package/src/services/bus-interview-park.test.ts +4 -3
- package/src/services/bus-interview.test.ts +2 -2
- package/src/services/bus-secret-flow.test.ts +3 -2
- package/src/services/capability-table-rows.test.ts +2 -1
- package/src/services/celilo-events.test.ts +1 -1
- package/src/services/celilo-mgmt-hooks.test.ts +23 -5
- package/src/services/consumer-cleanup.test.ts +3 -2
- package/src/services/container-service.test.ts +3 -2
- package/src/services/control-plane-bootstrap.test.ts +2 -1
- package/src/services/cross-module-read.test.ts +4 -3
- package/src/services/deploy-preflight.ts +7 -0
- package/src/services/deploy-validation.test.ts +3 -2
- package/src/services/deployed-systems.ts +1 -1
- package/src/services/dns-internal-records.test.ts +3 -2
- package/src/services/dns-provider-backfill.test.ts +2 -1
- package/src/services/dns-registrations.test.ts +2 -1
- package/src/services/ensure-interview.test.ts +3 -2
- package/src/services/fleet-checks.test.ts +126 -6
- package/src/services/fleet-checks.ts +176 -3
- package/src/services/fleet-key.test.ts +3 -2
- package/src/services/health-runner.test.ts +87 -2
- package/src/services/health-runner.ts +40 -11
- package/src/services/infrastructure-selector.test.ts +3 -2
- package/src/services/infrastructure-variable-resolver.test.ts +2 -1
- package/src/services/jail-exemptions.test.ts +125 -0
- package/src/services/jail-exemptions.ts +81 -0
- package/src/services/machine-pool.test.ts +3 -2
- package/src/services/module-config.test.ts +2 -1
- package/src/services/module-deploy.dns-repoint.test.ts +2 -1
- package/src/services/module-deploy.ts +48 -24
- package/src/services/module-operations.test.ts +2 -1
- package/src/services/module-subscriptions.test.ts +4 -3
- package/src/services/network-discovery.test.ts +64 -1
- package/src/services/network-discovery.ts +30 -5
- package/src/services/port-forwards.test.ts +2 -1
- package/src/services/programmatic-responder.aspect.test.ts +2 -1
- package/src/services/proxmox-reconcile.test.ts +2 -1
- package/src/services/responder-probe.test.ts +1 -1
- package/src/services/restore-from-file.test.ts +7 -6
- package/src/services/restore-preflight.test.ts +3 -2
- package/src/services/secret-schema-loader.test.ts +2 -1
- package/src/services/ssh-key-manager.test.ts +4 -3
- package/src/services/static-content-converge.test.ts +5 -4
- package/src/services/static-content-converge.ts +33 -10
- package/src/services/system-state-stage.test.ts +4 -3
- package/src/services/trusted-sources.test.ts +3 -2
- package/src/services/update/orchestrator.test.ts +2 -0
- package/src/services/zone-detector.test.ts +2 -1
- package/src/test-utils/bus-responder.ts +6 -4
- package/src/test-utils/db-path.ts +25 -0
- package/src/test-utils/integration.ts +7 -0
- package/src/test-utils/module-fixtures.ts +5 -6
- package/src/module/packaging/generated-plane.test.ts +0 -79
- package/src/module/packaging/generated-plane.ts +0 -134
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import type { DbClient } from '../../db/client';
|
|
6
6
|
import { type Route, alerts, modules, monitors } from '../../db/schema';
|
|
7
7
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
8
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
8
9
|
import { acknowledgeAlert, findLiveAlertByKey, resolveAlertManually, silenceAlert } from './ack';
|
|
9
10
|
import { moduleCheckAlertKey } from './keys';
|
|
10
11
|
import { createPerson, createRoute } from './people';
|
|
@@ -79,7 +80,7 @@ describe('ack / silence / resolve', () => {
|
|
|
79
80
|
|
|
80
81
|
afterEach(() => {
|
|
81
82
|
db.$client.close();
|
|
82
|
-
|
|
83
|
+
resetTestDbPath();
|
|
83
84
|
try {
|
|
84
85
|
rmSync(dir, { recursive: true, force: true });
|
|
85
86
|
} catch {
|
|
@@ -16,6 +16,7 @@ import { join } from 'node:path';
|
|
|
16
16
|
import { closeDb, getDb } from '../../db/client';
|
|
17
17
|
import { runMigrations } from '../../db/migrate';
|
|
18
18
|
import { modules, monitors } from '../../db/schema';
|
|
19
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
19
20
|
import { getModuleConfigValue, upsertModuleConfig } from '../module-config';
|
|
20
21
|
import { migrateMonitorCadences } from './cadence-migration';
|
|
21
22
|
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, loadModuleHealthCadences } from './health-cadence';
|
|
@@ -66,7 +67,7 @@ describe('migrateMonitorCadences', () => {
|
|
|
66
67
|
|
|
67
68
|
afterEach(() => {
|
|
68
69
|
closeDb();
|
|
69
|
-
|
|
70
|
+
resetTestDbPath();
|
|
70
71
|
rmSync(dir, { recursive: true, force: true });
|
|
71
72
|
});
|
|
72
73
|
|
|
@@ -132,7 +133,7 @@ describe("a module_hook row's stored cadence is not consulted", () => {
|
|
|
132
133
|
|
|
133
134
|
afterEach(() => {
|
|
134
135
|
closeDb();
|
|
135
|
-
|
|
136
|
+
resetTestDbPath();
|
|
136
137
|
rmSync(dir, { recursive: true, force: true });
|
|
137
138
|
});
|
|
138
139
|
|
|
@@ -20,6 +20,7 @@ import { join } from 'node:path';
|
|
|
20
20
|
import type { DbClient } from '../../db/client';
|
|
21
21
|
import { modules } from '../../db/schema';
|
|
22
22
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
23
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
23
24
|
import { CONTROL_PLANE_MODULE_ID } from '../deployed-systems';
|
|
24
25
|
import { loadModuleCoverage } from './coverage-source';
|
|
25
26
|
import { healthCoverageFailingKeys } from './health-coverage';
|
|
@@ -37,7 +38,7 @@ describe('health coverage and the control plane', () => {
|
|
|
37
38
|
|
|
38
39
|
afterEach(() => {
|
|
39
40
|
db.$client.close();
|
|
40
|
-
|
|
41
|
+
resetTestDbPath();
|
|
41
42
|
try {
|
|
42
43
|
rmSync(dir, { recursive: true, force: true });
|
|
43
44
|
} catch {
|
|
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
|
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
7
|
import { type Alert, type Route, alerts, modules, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
9
10
|
import { moduleCheckAlertKey } from './keys';
|
|
10
11
|
import { deliverDeferred } from './notifier';
|
|
11
12
|
import { createPerson, createRoute } from './people';
|
|
@@ -80,7 +81,7 @@ describe('quiet-hours deferral', () => {
|
|
|
80
81
|
|
|
81
82
|
afterEach(() => {
|
|
82
83
|
db.$client.close();
|
|
83
|
-
|
|
84
|
+
resetTestDbPath();
|
|
84
85
|
try {
|
|
85
86
|
rmSync(dir, { recursive: true, force: true });
|
|
86
87
|
} catch {
|
|
@@ -21,6 +21,7 @@ import { join } from 'node:path';
|
|
|
21
21
|
import type { DbClient } from '../../db/client';
|
|
22
22
|
import { type Alert, type Route, modules } from '../../db/schema';
|
|
23
23
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
24
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
24
25
|
import type { EscalationStep, RouteForEscalation } from './escalation';
|
|
25
26
|
import { interpretInbound } from './inbound';
|
|
26
27
|
import {
|
|
@@ -249,7 +250,7 @@ describe('delivery loop against the signal-cli simulator', () => {
|
|
|
249
250
|
|
|
250
251
|
afterEach(() => {
|
|
251
252
|
db.$client.close();
|
|
252
|
-
|
|
253
|
+
resetTestDbPath();
|
|
253
254
|
try {
|
|
254
255
|
rmSync(dir, { recursive: true, force: true });
|
|
255
256
|
} catch {
|
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import type { DbClient } from '../../db/client';
|
|
6
6
|
import { modules, monitors } from '../../db/schema';
|
|
7
7
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
8
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
8
9
|
import {
|
|
9
10
|
closeDeployWindows,
|
|
10
11
|
ensureMonitorOnDeploy,
|
|
@@ -34,7 +35,7 @@ describe('deploy hooks', () => {
|
|
|
34
35
|
|
|
35
36
|
afterEach(() => {
|
|
36
37
|
db.$client.close();
|
|
37
|
-
|
|
38
|
+
resetTestDbPath();
|
|
38
39
|
try {
|
|
39
40
|
rmSync(dir, { recursive: true, force: true });
|
|
40
41
|
} catch {
|
|
@@ -7,6 +7,7 @@ import { eq } from 'drizzle-orm';
|
|
|
7
7
|
import type { DbClient } from '../../db/client';
|
|
8
8
|
import { type Route, alerts, modules, monitors, notificationDeliveries } from '../../db/schema';
|
|
9
9
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
10
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
10
11
|
import {
|
|
11
12
|
type InboundPollDeps,
|
|
12
13
|
type TransportReadRecord,
|
|
@@ -105,7 +106,7 @@ describe('pollInbound', () => {
|
|
|
105
106
|
|
|
106
107
|
afterEach(() => {
|
|
107
108
|
db.$client.close();
|
|
108
|
-
|
|
109
|
+
resetTestDbPath();
|
|
109
110
|
try {
|
|
110
111
|
rmSync(dir, { recursive: true, force: true });
|
|
111
112
|
} catch {
|
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import type { DbClient } from '../../db/client';
|
|
6
6
|
import { type NotificationDelivery, modules, people, routes } from '../../db/schema';
|
|
7
7
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
8
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
8
9
|
import { interpretInbound, parseInbound } from './inbound';
|
|
9
10
|
import { mintDelivery } from './tokens';
|
|
10
11
|
|
|
@@ -107,7 +108,7 @@ describe('interpretInbound', () => {
|
|
|
107
108
|
|
|
108
109
|
afterEach(() => {
|
|
109
110
|
db.$client.close();
|
|
110
|
-
|
|
111
|
+
resetTestDbPath();
|
|
111
112
|
try {
|
|
112
113
|
rmSync(dir, { recursive: true, force: true });
|
|
113
114
|
} catch {
|
|
@@ -116,6 +116,10 @@ describe('severityForItemStatus', () => {
|
|
|
116
116
|
expect(severityForItemStatus('fail', 'critical')).toBe('critical');
|
|
117
117
|
expect(severityForItemStatus('fail', 'warning')).toBe('warning');
|
|
118
118
|
});
|
|
119
|
+
|
|
120
|
+
test('skip produces no alert — a check that could not run is not a failure', () => {
|
|
121
|
+
expect(severityForItemStatus('skip', 'critical')).toBeNull();
|
|
122
|
+
});
|
|
119
123
|
});
|
|
120
124
|
|
|
121
125
|
describe('failingKeysFromHealthItems', () => {
|
|
@@ -134,14 +134,24 @@ export function parentModuleKey(key: string): string | null {
|
|
|
134
134
|
* and never pages. That gives module authors somewhere to put "cert expires in
|
|
135
135
|
* 20 days" without anyone having to decide whether it is a 3am problem.
|
|
136
136
|
*
|
|
137
|
+
* `skip` produces no alert either: a check that could not perform its probe
|
|
138
|
+
* is not a failure. Note the honest cost — a previously-failing key whose
|
|
139
|
+
* item now reports skip leaves this run's failing set, so the reconciler
|
|
140
|
+
* reads "recovered" off a run that measured nothing. That is the same hazard
|
|
141
|
+
* class design D5 handles at run level (a run that could not execute is the
|
|
142
|
+
* caller's branch); there is no per-item sticky-key mechanism here, so a
|
|
143
|
+
* module whose credential disappeared needs the audit's `unmeasured` finding
|
|
144
|
+
* to be seen (celilo#1326).
|
|
145
|
+
*
|
|
137
146
|
* See design D6.
|
|
138
147
|
*/
|
|
139
148
|
export function severityForItemStatus(
|
|
140
|
-
itemStatus: 'pass' | 'warn' | 'fail',
|
|
149
|
+
itemStatus: 'pass' | 'warn' | 'fail' | 'skip',
|
|
141
150
|
monitorSeverity: AlertSeverity,
|
|
142
151
|
): AlertSeverity | null {
|
|
143
152
|
if (itemStatus === 'pass') return null;
|
|
144
153
|
if (itemStatus === 'warn') return 'warning';
|
|
154
|
+
if (itemStatus === 'skip') return null;
|
|
145
155
|
return monitorSeverity;
|
|
146
156
|
}
|
|
147
157
|
|
|
@@ -155,7 +165,7 @@ export interface FailingKey {
|
|
|
155
165
|
|
|
156
166
|
export interface HealthCheckItemLike {
|
|
157
167
|
name: string;
|
|
158
|
-
status: 'pass' | 'warn' | 'fail';
|
|
168
|
+
status: 'pass' | 'warn' | 'fail' | 'skip';
|
|
159
169
|
message: string;
|
|
160
170
|
details?: string;
|
|
161
171
|
}
|
|
@@ -6,6 +6,7 @@ import { defineEvents, openBus } from '@celilo/event-bus';
|
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
7
|
import { type Route, modules } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
9
10
|
import {
|
|
10
11
|
type NotificationResponderHandle,
|
|
11
12
|
startNotificationResponder,
|
|
@@ -88,7 +89,7 @@ describe('notification responder', () => {
|
|
|
88
89
|
handle?.stop();
|
|
89
90
|
handle = undefined;
|
|
90
91
|
db.$client.close();
|
|
91
|
-
|
|
92
|
+
resetTestDbPath();
|
|
92
93
|
try {
|
|
93
94
|
rmSync(dir, { recursive: true, force: true });
|
|
94
95
|
} catch {
|
|
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
|
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
7
|
import { type Monitor, alerts, monitorRuns, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
9
10
|
import type { HealthCheckResult } from '../health-runner';
|
|
10
11
|
import { HOOK_JAIL_CHECK, type HookJailState } from './hook-jail';
|
|
11
12
|
import { moduleAlertKey, moduleCheckAlertKey } from './keys';
|
|
@@ -71,7 +72,7 @@ describe('runOneMonitor', () => {
|
|
|
71
72
|
|
|
72
73
|
afterEach(() => {
|
|
73
74
|
db.$client.close();
|
|
74
|
-
|
|
75
|
+
resetTestDbPath();
|
|
75
76
|
try {
|
|
76
77
|
rmSync(dir, { recursive: true, force: true });
|
|
77
78
|
} catch {
|
|
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
|
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
7
|
import { alerts, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
9
10
|
import { moduleAlertKey, moduleCheckAlertKey } from './keys';
|
|
10
11
|
import type { ReconcileAction } from './reconcile';
|
|
11
12
|
import {
|
|
@@ -55,7 +56,7 @@ describe('alert store', () => {
|
|
|
55
56
|
|
|
56
57
|
afterEach(() => {
|
|
57
58
|
db.$client.close();
|
|
58
|
-
|
|
59
|
+
resetTestDbPath();
|
|
59
60
|
try {
|
|
60
61
|
rmSync(dir, { recursive: true, force: true });
|
|
61
62
|
} catch {
|
|
@@ -6,6 +6,7 @@ import { eq } from 'drizzle-orm';
|
|
|
6
6
|
import type { DbClient } from '../../db/client';
|
|
7
7
|
import { type Alert, type Monitor, alerts, modules, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
9
10
|
import type { HealthCheckResult } from '../health-runner';
|
|
10
11
|
import { deleteModuleConfig, upsertModuleConfig } from '../module-config';
|
|
11
12
|
import { HEALTH_CHECK_INTERVAL_CONFIG_KEY, reconcileModuleWatchState } from './health-cadence';
|
|
@@ -99,7 +100,7 @@ describe('runSweep', () => {
|
|
|
99
100
|
|
|
100
101
|
afterEach(() => {
|
|
101
102
|
db.$client.close();
|
|
102
|
-
|
|
103
|
+
resetTestDbPath();
|
|
103
104
|
try {
|
|
104
105
|
rmSync(dir, { recursive: true, force: true });
|
|
105
106
|
} catch {
|
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import type { DbClient } from '../../db/client';
|
|
6
6
|
import { modules, people, routes } from '../../db/schema';
|
|
7
7
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
8
|
+
import { resetTestDbPath } from '../../test-utils/db-path';
|
|
8
9
|
import {
|
|
9
10
|
consumeDelivery,
|
|
10
11
|
deliveriesForAlert,
|
|
@@ -75,7 +76,7 @@ describe('delivery records', () => {
|
|
|
75
76
|
|
|
76
77
|
afterEach(() => {
|
|
77
78
|
db.$client.close();
|
|
78
|
-
|
|
79
|
+
resetTestDbPath();
|
|
79
80
|
try {
|
|
80
81
|
rmSync(dir, { recursive: true, force: true });
|
|
81
82
|
} catch {
|
|
@@ -6,6 +6,7 @@ import { closeDb, getDb } from '../db/client';
|
|
|
6
6
|
import { runMigrations } from '../db/migrate';
|
|
7
7
|
import { modules } from '../db/schema';
|
|
8
8
|
import type { BaseModuleAspect } from '../manifest/schema';
|
|
9
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
9
10
|
import {
|
|
10
11
|
checkAspectApproval,
|
|
11
12
|
computeAspectScopeHash,
|
|
@@ -44,7 +45,7 @@ describe('aspect-approvals', () => {
|
|
|
44
45
|
|
|
45
46
|
afterEach(() => {
|
|
46
47
|
closeDb();
|
|
47
|
-
|
|
48
|
+
resetTestDbPath();
|
|
48
49
|
try {
|
|
49
50
|
rmSync(dir, { recursive: true, force: true });
|
|
50
51
|
} catch {
|
|
@@ -20,6 +20,7 @@ import { closeDb, getDb } from '../db/client';
|
|
|
20
20
|
import { runMigrations } from '../db/migrate';
|
|
21
21
|
import { type ModuleState, modules } from '../db/schema';
|
|
22
22
|
import type { BaseModuleAspect } from '../manifest/schema';
|
|
23
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
23
24
|
import {
|
|
24
25
|
computeAspectScopeHash,
|
|
25
26
|
recordAspectApproval,
|
|
@@ -159,7 +160,7 @@ describe('reconcileAspectsForSystems', () => {
|
|
|
159
160
|
|
|
160
161
|
afterEach(() => {
|
|
161
162
|
closeDb();
|
|
162
|
-
|
|
163
|
+
resetTestDbPath();
|
|
163
164
|
try {
|
|
164
165
|
rmSync(dir, { recursive: true, force: true });
|
|
165
166
|
} catch {
|
|
@@ -310,7 +311,7 @@ describe('planAspectFanOut onlyHostnames', () => {
|
|
|
310
311
|
|
|
311
312
|
afterEach(() => {
|
|
312
313
|
closeDb();
|
|
313
|
-
|
|
314
|
+
resetTestDbPath();
|
|
314
315
|
try {
|
|
315
316
|
rmSync(dir, { recursive: true, force: true });
|
|
316
317
|
} catch {
|
|
@@ -352,7 +353,7 @@ describe('verifyAspectCoverage', () => {
|
|
|
352
353
|
|
|
353
354
|
afterEach(() => {
|
|
354
355
|
closeDb();
|
|
355
|
-
|
|
356
|
+
resetTestDbPath();
|
|
356
357
|
try {
|
|
357
358
|
rmSync(dir, { recursive: true, force: true });
|
|
358
359
|
} catch {
|
|
@@ -20,6 +20,7 @@ import { closeDb, getDb } from '../db/client';
|
|
|
20
20
|
import { runMigrations } from '../db/migrate';
|
|
21
21
|
import { modules } from '../db/schema';
|
|
22
22
|
import type { BaseModuleAspect, ModuleManifest } from '../manifest/schema';
|
|
23
|
+
import { resetTestDbPath } from '../test-utils/db-path';
|
|
23
24
|
import {
|
|
24
25
|
computeAspectScopeHash,
|
|
25
26
|
findAspectApproval,
|
|
@@ -109,7 +110,7 @@ describe('aspect-runner', () => {
|
|
|
109
110
|
|
|
110
111
|
afterEach(() => {
|
|
111
112
|
closeDb();
|
|
112
|
-
|
|
113
|
+
resetTestDbPath();
|
|
113
114
|
try {
|
|
114
115
|
rmSync(dir, { recursive: true, force: true });
|
|
115
116
|
} catch {
|
|
@@ -1,14 +1,109 @@
|
|
|
1
1
|
import { describe, expect, test } from 'bun:test';
|
|
2
2
|
import type { HealthCheckResult } from '../health-runner';
|
|
3
3
|
import { auditHealth } from './health';
|
|
4
|
+
import { computeVerdict } from './types';
|
|
4
5
|
|
|
5
6
|
describe('auditHealth', () => {
|
|
6
|
-
test('
|
|
7
|
+
test('a healthy module produces no finding; a module with no hook is unmeasured, never green', async () => {
|
|
7
8
|
const results: HealthCheckResult[] = [
|
|
8
9
|
{ moduleId: 'caddy', status: 'healthy', checks: [] },
|
|
9
10
|
{ moduleId: 'iptables', status: 'no-checks', checks: [] },
|
|
10
11
|
];
|
|
11
|
-
|
|
12
|
+
const result = await auditHealth({ results });
|
|
13
|
+
expect(result).toHaveLength(1);
|
|
14
|
+
expect(result[0]).toMatchObject({
|
|
15
|
+
category: 'health',
|
|
16
|
+
severity: 'unmeasured',
|
|
17
|
+
code: 'health_unmeasured',
|
|
18
|
+
subject: 'iptables',
|
|
19
|
+
actionable: false,
|
|
20
|
+
});
|
|
21
|
+
expect(result[0].message).toContain('no health_check hook');
|
|
22
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
test('a module whose checks all report skip is unmeasured and names the skipped checks', async () => {
|
|
26
|
+
// The smoketest_bot_token incident, as an audit finding: the hook ran,
|
|
27
|
+
// every meaningful check refused to guess, and the fleet must not read
|
|
28
|
+
// READY off that.
|
|
29
|
+
const results: HealthCheckResult[] = [
|
|
30
|
+
{
|
|
31
|
+
moduleId: 'smoketest',
|
|
32
|
+
status: 'no-checks',
|
|
33
|
+
checks: [
|
|
34
|
+
{
|
|
35
|
+
name: 'bot_token_valid',
|
|
36
|
+
status: 'skip',
|
|
37
|
+
message: 'smoketest_bot_token not yet provisioned',
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
name: 'webhook_reachable',
|
|
41
|
+
status: 'skip',
|
|
42
|
+
message: 'smoketest_bot_token not yet provisioned',
|
|
43
|
+
},
|
|
44
|
+
],
|
|
45
|
+
},
|
|
46
|
+
];
|
|
47
|
+
const result = await auditHealth({ results });
|
|
48
|
+
expect(result).toHaveLength(1);
|
|
49
|
+
expect(result[0]).toMatchObject({
|
|
50
|
+
severity: 'unmeasured',
|
|
51
|
+
code: 'health_unmeasured',
|
|
52
|
+
subject: 'smoketest',
|
|
53
|
+
actionable: true,
|
|
54
|
+
remediation: 'celilo module health smoketest --debug',
|
|
55
|
+
});
|
|
56
|
+
expect(result[0].message).toContain('all 2 health checks skipped');
|
|
57
|
+
expect(result[0].details).toContain('bot_token_valid');
|
|
58
|
+
expect(result[0].details).toContain('not yet provisioned');
|
|
59
|
+
});
|
|
60
|
+
|
|
61
|
+
test('skipped checks on an otherwise healthy module render unmeasured', async () => {
|
|
62
|
+
const results: HealthCheckResult[] = [
|
|
63
|
+
{
|
|
64
|
+
moduleId: 'caddy',
|
|
65
|
+
status: 'healthy',
|
|
66
|
+
checks: [
|
|
67
|
+
{ name: 'config_valid', status: 'pass', message: 'config parses' },
|
|
68
|
+
{
|
|
69
|
+
name: 'origin_reachable',
|
|
70
|
+
status: 'skip',
|
|
71
|
+
message: 'peer unreachable from control plane',
|
|
72
|
+
},
|
|
73
|
+
],
|
|
74
|
+
},
|
|
75
|
+
];
|
|
76
|
+
const result = await auditHealth({ results });
|
|
77
|
+
expect(result).toHaveLength(1);
|
|
78
|
+
expect(result[0]).toMatchObject({
|
|
79
|
+
severity: 'unmeasured',
|
|
80
|
+
code: 'health_checks_skipped',
|
|
81
|
+
subject: 'caddy',
|
|
82
|
+
});
|
|
83
|
+
expect(result[0].message).toContain('1 of 2');
|
|
84
|
+
expect(result[0].details).toContain('origin_reachable');
|
|
85
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
86
|
+
});
|
|
87
|
+
|
|
88
|
+
test('a degraded module with a skipped check yields both a drift and an unmeasured finding', async () => {
|
|
89
|
+
const results: HealthCheckResult[] = [
|
|
90
|
+
{
|
|
91
|
+
moduleId: 'iptables',
|
|
92
|
+
status: 'degraded',
|
|
93
|
+
checks: [
|
|
94
|
+
{
|
|
95
|
+
name: 'persistence_drift',
|
|
96
|
+
status: 'warn',
|
|
97
|
+
message: '1 live rule(s) absent from rules.v4',
|
|
98
|
+
},
|
|
99
|
+
{ name: 'ssh_access', status: 'skip', message: 'fw01 unreachable' },
|
|
100
|
+
],
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
const result = await auditHealth({ results });
|
|
104
|
+
expect(result).toHaveLength(2);
|
|
105
|
+
expect(result.map((f) => f.severity).sort()).toEqual(['drift', 'unmeasured']);
|
|
106
|
+
expect(computeVerdict(result)).toBe('UNKNOWN');
|
|
12
107
|
});
|
|
13
108
|
|
|
14
109
|
test('drift finding for unhealthy module', async () => {
|
|
@@ -5,6 +5,14 @@
|
|
|
5
5
|
* informational (DRIFT, not BLOCKED) — the user might want to update
|
|
6
6
|
* even if some module is currently unhealthy (an update could fix it).
|
|
7
7
|
*
|
|
8
|
+
* Unmeasured is reported, never folded into green: a module with no
|
|
9
|
+
* health_check hook, and a module whose checks all reported skip, have
|
|
10
|
+
* contributed no measurement at all, and both render the fleet UNKNOWN
|
|
11
|
+
* rather than READY (celilo#1326). The same rule reaches one level down —
|
|
12
|
+
* a module that measured something but skipped a check gets an
|
|
13
|
+
* `unmeasured` finding beside its verdict, so a VERIFIED module whose
|
|
14
|
+
* meaningful probe was skipped is still visible.
|
|
15
|
+
*
|
|
8
16
|
* The health-runner result is passed in directly so this file stays
|
|
9
17
|
* pure for testing. The aggregator wires `runAllHealthChecks` to it.
|
|
10
18
|
*/
|
|
@@ -16,13 +24,67 @@ export interface HealthAuditDeps {
|
|
|
16
24
|
results: HealthCheckResult[];
|
|
17
25
|
}
|
|
18
26
|
|
|
27
|
+
function skippedChecksFinding(r: HealthCheckResult, code: string, message: string): DriftFinding {
|
|
28
|
+
const details = r.checks
|
|
29
|
+
.filter((c) => c.status === 'skip')
|
|
30
|
+
.map((c) => ` • ${c.name}: ${c.message}`)
|
|
31
|
+
.join('\n');
|
|
32
|
+
|
|
33
|
+
return {
|
|
34
|
+
category: 'health',
|
|
35
|
+
severity: 'unmeasured',
|
|
36
|
+
code,
|
|
37
|
+
message,
|
|
38
|
+
details: details || undefined,
|
|
39
|
+
remediation: `celilo module health ${r.moduleId} --debug`,
|
|
40
|
+
actionable: true,
|
|
41
|
+
subject: r.moduleId,
|
|
42
|
+
};
|
|
43
|
+
}
|
|
44
|
+
|
|
19
45
|
export async function auditHealth(deps: HealthAuditDeps): Promise<DriftFinding[]> {
|
|
20
46
|
const findings: DriftFinding[] = [];
|
|
21
47
|
|
|
22
48
|
for (const r of deps.results) {
|
|
23
|
-
|
|
49
|
+
const skipped = r.checks.filter((c) => c.status === 'skip');
|
|
50
|
+
const measured = r.checks.filter((c) => c.status !== 'skip');
|
|
51
|
+
|
|
52
|
+
if (r.status === 'no-checks') {
|
|
53
|
+
// Either the module declares no health_check hook, or the hook ran
|
|
54
|
+
// and every check reported skip. Neither measured anything.
|
|
55
|
+
findings.push(
|
|
56
|
+
skipped.length > 0
|
|
57
|
+
? skippedChecksFinding(
|
|
58
|
+
r,
|
|
59
|
+
'health_unmeasured',
|
|
60
|
+
`${r.moduleId}: all ${skipped.length} health checks skipped — nothing measured`,
|
|
61
|
+
)
|
|
62
|
+
: {
|
|
63
|
+
category: 'health',
|
|
64
|
+
severity: 'unmeasured',
|
|
65
|
+
code: 'health_unmeasured',
|
|
66
|
+
message: `${r.moduleId}: no health_check hook — never measured`,
|
|
67
|
+
remediation: `Add a health_check hook to the ${r.moduleId} module manifest`,
|
|
68
|
+
actionable: false,
|
|
69
|
+
subject: r.moduleId,
|
|
70
|
+
},
|
|
71
|
+
);
|
|
72
|
+
continue;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
if (skipped.length > 0) {
|
|
76
|
+
findings.push(
|
|
77
|
+
skippedChecksFinding(
|
|
78
|
+
r,
|
|
79
|
+
'health_checks_skipped',
|
|
80
|
+
`${r.moduleId}: ${skipped.length} of ${r.checks.length} health checks skipped`,
|
|
81
|
+
),
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (r.status === 'healthy') continue;
|
|
24
86
|
|
|
25
|
-
const failingChecks =
|
|
87
|
+
const failingChecks = measured
|
|
26
88
|
.filter((c) => c.status === 'fail' || c.status === 'warn')
|
|
27
89
|
.map((c) => ` • ${c.name}: ${c.message}`)
|
|
28
90
|
.join('\n');
|
|
@@ -32,6 +32,7 @@ const emptyDeps = {
|
|
|
32
32
|
moduleConfigs: { modules: [] },
|
|
33
33
|
moduleIntegrity: { results: [] },
|
|
34
34
|
detectWithoutConverge: { modules: [] },
|
|
35
|
+
jailExemptions: { exemptions: [] },
|
|
35
36
|
health: { results: [] },
|
|
36
37
|
backups: { modules: [] },
|
|
37
38
|
abandonedOperations: { records: [] },
|
|
@@ -41,6 +42,7 @@ const emptyDeps = {
|
|
|
41
42
|
secretsDecryptable: { results: [] },
|
|
42
43
|
servicesReachable: { results: [] },
|
|
43
44
|
machinesReachable: { results: [] },
|
|
45
|
+
diskSpace: { results: [] },
|
|
44
46
|
transportReads: { statuses: [], now: new Date(), staleAfterMs: 30 * 60_000 },
|
|
45
47
|
trustedSources: { firewalls: [] },
|
|
46
48
|
publicDns: { records: [], probe: unusedPublicDnsProbe },
|
|
@@ -88,6 +90,26 @@ describe('runAudit', () => {
|
|
|
88
90
|
expect(report.findings.some((f) => f.category === 'schema')).toBe(true);
|
|
89
91
|
});
|
|
90
92
|
|
|
93
|
+
test('disk_space is wired into the report (celilo#1336)', async () => {
|
|
94
|
+
const events: string[] = [];
|
|
95
|
+
const report = await runAudit(
|
|
96
|
+
{
|
|
97
|
+
...emptyDeps,
|
|
98
|
+
diskSpace: {
|
|
99
|
+
results: [{ hostname: 'celilo-mgr', ipAddress: '10.0.0.5', usedPercent: 96 }],
|
|
100
|
+
},
|
|
101
|
+
},
|
|
102
|
+
(event) => events.push(`${event.category}:${event.phase}`),
|
|
103
|
+
);
|
|
104
|
+
|
|
105
|
+
// 96% is at the blocked threshold, so the wired category alone flips
|
|
106
|
+
// the fleet verdict — which is the point of wiring it.
|
|
107
|
+
expect(report.verdict).toBe('BLOCKED');
|
|
108
|
+
expect(report.findings.map((f) => f.category)).toContain('disk_space');
|
|
109
|
+
expect(events).toContain('disk_space:start');
|
|
110
|
+
expect(events).toContain('disk_space:end');
|
|
111
|
+
});
|
|
112
|
+
|
|
91
113
|
test('aggregates findings across all categories', async () => {
|
|
92
114
|
const report = await runAudit({
|
|
93
115
|
...emptyDeps,
|
|
@@ -22,7 +22,9 @@ import {
|
|
|
22
22
|
type DetectWithoutConvergeAuditDeps,
|
|
23
23
|
auditDetectWithoutConverge,
|
|
24
24
|
} from './detect-without-converge';
|
|
25
|
+
import { type DiskSpaceAuditDeps, auditDiskSpace } from './disk-space';
|
|
25
26
|
import { type HealthAuditDeps, auditHealth } from './health';
|
|
27
|
+
import { type JailExemptionsAuditDeps, auditJailExemptions } from './jail-exemptions';
|
|
26
28
|
import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
|
|
27
29
|
import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
|
|
28
30
|
import { type ModuleIntegrityAuditDeps, auditModuleIntegrity } from './module-integrity';
|
|
@@ -60,6 +62,7 @@ export interface AuditDeps {
|
|
|
60
62
|
moduleConfigs: ModuleConfigsAuditDeps;
|
|
61
63
|
moduleIntegrity: ModuleIntegrityAuditDeps;
|
|
62
64
|
detectWithoutConverge: DetectWithoutConvergeAuditDeps;
|
|
65
|
+
jailExemptions: JailExemptionsAuditDeps;
|
|
63
66
|
health: HealthAuditDeps;
|
|
64
67
|
backups: BackupsAuditDeps;
|
|
65
68
|
abandonedOperations: AbandonedOperationsAuditDeps;
|
|
@@ -70,6 +73,7 @@ export interface AuditDeps {
|
|
|
70
73
|
servicesReachable: ServicesReachableAuditDeps;
|
|
71
74
|
machinesReachable: MachinesReachableAuditDeps;
|
|
72
75
|
publicDns: PublicDnsAuditDeps;
|
|
76
|
+
diskSpace: DiskSpaceAuditDeps;
|
|
73
77
|
transportReads: TransportReadsAuditDeps;
|
|
74
78
|
trustedSources: TrustedSourcesAuditDeps;
|
|
75
79
|
/** Defaults to `Date.now()`-based ISO string. */
|
|
@@ -95,8 +99,8 @@ export async function runAudit(
|
|
|
95
99
|
): Promise<SystemAuditReport> {
|
|
96
100
|
// Wrap each category's promise so the caller sees per-category
|
|
97
101
|
// start/end events. `start` fires synchronously when this function
|
|
98
|
-
// schedules the category; in practice all
|
|
99
|
-
// since Promise.all spawns them in parallel.
|
|
102
|
+
// schedules the category; in practice all of them fire roughly at
|
|
103
|
+
// once since Promise.all spawns them in parallel.
|
|
100
104
|
const wrap = (category: DriftCategory, p: Promise<DriftFinding[]>): Promise<DriftFinding[]> => {
|
|
101
105
|
onProgress?.({ category, phase: 'start' });
|
|
102
106
|
return p.then((findings) => {
|
|
@@ -119,6 +123,7 @@ export async function runAudit(
|
|
|
119
123
|
'detect_without_converge',
|
|
120
124
|
Promise.resolve(auditDetectWithoutConverge(deps.detectWithoutConverge)),
|
|
121
125
|
),
|
|
126
|
+
wrap('jail_exemptions', Promise.resolve(auditJailExemptions(deps.jailExemptions))),
|
|
122
127
|
wrap('health', auditHealth(deps.health)),
|
|
123
128
|
wrap('backups', auditBackups(deps.backups)),
|
|
124
129
|
wrap(
|
|
@@ -132,6 +137,7 @@ export async function runAudit(
|
|
|
132
137
|
wrap('services_reachable', auditServicesReachable(deps.servicesReachable)),
|
|
133
138
|
wrap('machines_reachable', auditMachinesReachable(deps.machinesReachable)),
|
|
134
139
|
wrap('public_dns', auditPublicDns(deps.publicDns)),
|
|
140
|
+
wrap('disk_space', Promise.resolve(auditDiskSpace(deps.diskSpace))),
|
|
135
141
|
wrap('transport_reads', auditTransportReads(deps.transportReads)),
|
|
136
142
|
wrap('trusted_sources', auditTrustedSources(deps.trustedSources)),
|
|
137
143
|
]);
|