@celilo/cli 2.0.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/CELILO_SUBSYSTEMS.md +1 -0
- package/package.json +2 -2
- package/src/hooks/capability-loader.ts +1 -2
- package/src/hooks/executor.ts +6 -0
- package/src/hooks/hook-jail-toolchain-reach.test.ts +76 -27
- package/src/hooks/hook-jail-unreachability.test.ts +51 -26
- package/src/hooks/hook-trespass.test.ts +65 -9
- package/src/hooks/jail.ts +180 -25
- package/src/hooks/mount-set.test.ts +88 -1
- package/src/hooks/mount-set.ts +155 -0
- package/src/hooks/unjailed-lint.test.ts +5 -2
- package/src/services/alerting/coverage-source.test.ts +86 -0
- package/src/services/alerting/coverage-source.ts +11 -1
- package/src/services/control-plane-bootstrap.test.ts +177 -0
- package/src/services/control-plane-bootstrap.ts +176 -0
- package/src/services/control-plane-health.test.ts +66 -0
- package/src/services/control-plane-health.ts +67 -0
- package/src/services/deployed-systems.ts +12 -0
- package/src/services/dns-discovery.test.ts +93 -0
- package/src/services/dns-discovery.ts +92 -0
- package/src/services/fleet-checks.ts +6 -2
- package/src/services/health-runner.ts +34 -3
- package/src/services/module-deploy.ts +51 -0
- package/src/services/remote-access.test.ts +88 -4
- package/src/services/remote-access.ts +52 -1
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The control-plane bootstrap, which replaced celilo-mgmt's `on_install`
|
|
3
|
+
* (celilo#1225).
|
|
4
|
+
*
|
|
5
|
+
* Every dependency is injected, so these run against a real database and no
|
|
6
|
+
* host: no resolver file is read, no ssh-keygen runs, no bus is opened. That is
|
|
7
|
+
* the property the hook could not have. Its equivalents shelled out to the CLI,
|
|
8
|
+
* so the only way to exercise them was to have a celilo on PATH — which is also
|
|
9
|
+
* why the jail broke them and no unit test noticed.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
13
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
14
|
+
import { tmpdir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { eq } from 'drizzle-orm';
|
|
17
|
+
import type { DbClient } from '../db/client';
|
|
18
|
+
import { systemConfig } from '../db/schema';
|
|
19
|
+
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
20
|
+
import {
|
|
21
|
+
type ControlPlaneBootstrapOptions,
|
|
22
|
+
bootstrapControlPlane,
|
|
23
|
+
} from './control-plane-bootstrap';
|
|
24
|
+
import type { FleetFinding } from './fleet-checks';
|
|
25
|
+
|
|
26
|
+
const DNS = { primary: '9.9.9.9', fallback: '8.8.4.4' };
|
|
27
|
+
const PUBLIC_KEY = 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI test celilo-fleet';
|
|
28
|
+
|
|
29
|
+
function dispatcher(status: FleetFinding['status']): FleetFinding {
|
|
30
|
+
return {
|
|
31
|
+
id: 'dispatcher',
|
|
32
|
+
title: 'Event dispatcher',
|
|
33
|
+
status,
|
|
34
|
+
summary: `dispatcher ${status}`,
|
|
35
|
+
detail: [],
|
|
36
|
+
remediation: status === 'ok' ? null : 'start the dispatcher',
|
|
37
|
+
autoFixable: false,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function configValue(db: DbClient, key: string): string | undefined {
|
|
42
|
+
return db.select().from(systemConfig).where(eq(systemConfig.key, key)).get()?.value ?? undefined;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('bootstrapControlPlane', () => {
|
|
46
|
+
let dir: string;
|
|
47
|
+
let db: DbClient;
|
|
48
|
+
/** Every sleep the poll performed, so waiting is asserted rather than timed. */
|
|
49
|
+
let sleeps: number[];
|
|
50
|
+
|
|
51
|
+
beforeEach(async () => {
|
|
52
|
+
dir = mkdtempSync(join(tmpdir(), 'cpb-'));
|
|
53
|
+
const dbPath = join(dir, 'celilo.db');
|
|
54
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
55
|
+
db = await setupTestDatabaseAt(dbPath);
|
|
56
|
+
sleeps = [];
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterEach(() => {
|
|
60
|
+
db.$client.close();
|
|
61
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
62
|
+
try {
|
|
63
|
+
rmSync(dir, { recursive: true, force: true });
|
|
64
|
+
} catch {
|
|
65
|
+
/* ignore */
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
function run(overrides: Partial<ControlPlaneBootstrapOptions> = {}) {
|
|
70
|
+
return bootstrapControlPlane({
|
|
71
|
+
db,
|
|
72
|
+
discoverDnsImpl: () => DNS,
|
|
73
|
+
ensureFleetKeyImpl: () => ({ publicKey: PUBLIC_KEY, created: true }),
|
|
74
|
+
discoverNetworkImpl: () => ({ applied: ['network.internal.subnet = 10.0.0.0/24'] }),
|
|
75
|
+
probeDispatcher: () => dispatcher('ok'),
|
|
76
|
+
sleep: async (ms) => {
|
|
77
|
+
sleeps.push(ms);
|
|
78
|
+
},
|
|
79
|
+
...overrides,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
test('records the discovered DNS and the fleet public key in system config', async () => {
|
|
84
|
+
await run();
|
|
85
|
+
|
|
86
|
+
expect(configValue(db, 'dns.primary')).toBe('9.9.9.9');
|
|
87
|
+
expect(configValue(db, 'dns.fallback')).toBe('8.8.4.4');
|
|
88
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('returns what each step found, so the caller renders and this does not', async () => {
|
|
92
|
+
const result = await run();
|
|
93
|
+
|
|
94
|
+
expect(result.dns).toEqual(DNS);
|
|
95
|
+
expect(result.fleetKey).toEqual({ publicKey: PUBLIC_KEY, created: true });
|
|
96
|
+
expect(result.network.applied).toEqual(['network.internal.subnet = 10.0.0.0/24']);
|
|
97
|
+
expect(result.dispatcher.status).toBe('ok');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('a dispatcher already up is probed once and never waited for', async () => {
|
|
101
|
+
let probes = 0;
|
|
102
|
+
await run({
|
|
103
|
+
probeDispatcher: () => {
|
|
104
|
+
probes++;
|
|
105
|
+
return dispatcher('ok');
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(probes).toBe(1);
|
|
110
|
+
expect(sleeps).toEqual([]);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test('a dispatcher still starting is polled until it answers', async () => {
|
|
114
|
+
// The role enables the supervisor unit moments before this runs, so the
|
|
115
|
+
// first probe legitimately finds nothing. Two failures then success.
|
|
116
|
+
const readings: FleetFinding['status'][] = ['fail', 'fail', 'ok'];
|
|
117
|
+
let probes = 0;
|
|
118
|
+
const result = await run({
|
|
119
|
+
probeDispatcher: () => dispatcher(readings[probes++] ?? 'ok'),
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
expect(probes).toBe(3);
|
|
123
|
+
expect(sleeps).toEqual([1000, 1000]);
|
|
124
|
+
expect(result.dispatcher.status).toBe('ok');
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('a dispatcher that never answers gives up after the attempt budget', async () => {
|
|
128
|
+
let probes = 0;
|
|
129
|
+
const result = await run({
|
|
130
|
+
probeDispatcher: () => {
|
|
131
|
+
probes++;
|
|
132
|
+
return dispatcher('fail');
|
|
133
|
+
},
|
|
134
|
+
dispatcherAttempts: 5,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
expect(probes).toBe(5);
|
|
138
|
+
expect(sleeps).toHaveLength(4);
|
|
139
|
+
// Reported, not thrown. The caller decides how loud to be, which is what
|
|
140
|
+
// lets `module deploy` fail the install and a later caller merely warn.
|
|
141
|
+
expect(result.dispatcher.status).toBe('fail');
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('a warn reading is accepted rather than polled against', async () => {
|
|
145
|
+
let probes = 0;
|
|
146
|
+
await run({
|
|
147
|
+
probeDispatcher: () => {
|
|
148
|
+
probes++;
|
|
149
|
+
return dispatcher('warn');
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
expect(probes).toBe(1);
|
|
154
|
+
expect(sleeps).toEqual([]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('a box with no default route still gets its DNS and key recorded', async () => {
|
|
158
|
+
// `discoverAndRecordNetwork` reports a skip rather than throwing, and a
|
|
159
|
+
// box celilo cannot read a network from must not fail the whole deploy.
|
|
160
|
+
const result = await run({
|
|
161
|
+
discoverNetworkImpl: () => ({ applied: [], skipped: 'no default route' }),
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
expect(result.network.skipped).toBe('no default route');
|
|
165
|
+
expect(configValue(db, 'dns.primary')).toBe('9.9.9.9');
|
|
166
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test('an existing fleet key is recorded without being reminted', async () => {
|
|
170
|
+
const result = await run({
|
|
171
|
+
ensureFleetKeyImpl: () => ({ publicKey: PUBLIC_KEY, created: false }),
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
expect(result.fleetKey.created).toBe(false);
|
|
175
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Initialise the management box's own celilo state, during the celilo-mgmt
|
|
3
|
+
* deploy.
|
|
4
|
+
*
|
|
5
|
+
* ## Why this is here and not in a hook
|
|
6
|
+
*
|
|
7
|
+
* It was a hook until celilo#1225. `modules/celilo-mgmt/scripts/on_install.ts`
|
|
8
|
+
* reached every one of these operations by spawning the `celilo` CLI, which a
|
|
9
|
+
* jailed hook cannot do: the jail's mount set binds no `/usr/bin`, no
|
|
10
|
+
* `/usr/local/bin` and no shell, so `execFileSync` dies on the missing binary
|
|
11
|
+
* and `execSync` dies one step earlier on the missing `/bin/sh`. The deploy ran
|
|
12
|
+
* Ansible to completion and then failed in its own install hook, on every host
|
|
13
|
+
* where a jail backend exists.
|
|
14
|
+
*
|
|
15
|
+
* The fix is not to let the hook reach the CLI. It is that none of this wanted
|
|
16
|
+
* to be in a hook. celilo-mgmt is never deployed to a remote box (ruled
|
|
17
|
+
* 2026-09-02), so the host being configured is always the host celilo runs on,
|
|
18
|
+
* and every step below is celilo acting on itself: read this box's resolvers,
|
|
19
|
+
* mint celilo's own fleet key, record celilo's own network, check celilo's own
|
|
20
|
+
* event bus. A hook was a process boundary with celilo on both sides of it.
|
|
21
|
+
*
|
|
22
|
+
* ## What it does NOT do
|
|
23
|
+
*
|
|
24
|
+
* It does not decide whether to run. The deploy path does that, from the
|
|
25
|
+
* control-plane module id, which celilo already keys on in three other places
|
|
26
|
+
* (`manifest/validate.ts`'s privileged allow-list, `capability-loader.ts`'s
|
|
27
|
+
* `CONTROL_PLANE_MODULE_ID`, `fleet-checks.ts`'s `CONTROL_PLANE_MODULE`).
|
|
28
|
+
*
|
|
29
|
+
* It also does not print. It returns what happened and the caller renders it,
|
|
30
|
+
* so the same function serves a deploy, a test and any later caller without
|
|
31
|
+
* carrying a logger's opinions (Rule 10.1).
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
|
|
35
|
+
import { getEventBusPath } from '../config/paths';
|
|
36
|
+
import type { DbClient } from '../db/client';
|
|
37
|
+
import { type DnsServers, discoverDns } from './dns-discovery';
|
|
38
|
+
import { type FleetFinding, checkDispatcher } from './fleet-checks';
|
|
39
|
+
import { type FleetKey, ensureFleetKey } from './fleet-key';
|
|
40
|
+
import { type NetworkDiscoveryResult, discoverAndRecordNetwork } from './network-discovery';
|
|
41
|
+
import { initializeSystem } from './system-init';
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* How long to give a dispatcher that is still coming up.
|
|
45
|
+
*
|
|
46
|
+
* The Ansible role enables its supervisor unit moments before this runs, so a
|
|
47
|
+
* first probe legitimately finds nothing. Five one-second attempts is what the
|
|
48
|
+
* hook this replaced used, and no case has argued for more.
|
|
49
|
+
*/
|
|
50
|
+
const DISPATCHER_ATTEMPTS = 5;
|
|
51
|
+
const DISPATCHER_INTERVAL_MS = 1_000;
|
|
52
|
+
|
|
53
|
+
export interface ControlPlaneBootstrapOptions {
|
|
54
|
+
db: DbClient;
|
|
55
|
+
/** Injectable so tests drive the whole sequence without a host or a bus. */
|
|
56
|
+
discoverDnsImpl?: () => DnsServers;
|
|
57
|
+
ensureFleetKeyImpl?: () => FleetKey;
|
|
58
|
+
discoverNetworkImpl?: (db: DbClient) => NetworkDiscoveryResult;
|
|
59
|
+
/**
|
|
60
|
+
* One dispatcher reading. Injectable so a test needs no bus at all; the
|
|
61
|
+
* default opens celilo's own and runs the four-part check against it.
|
|
62
|
+
*/
|
|
63
|
+
probeDispatcher?: () => FleetFinding;
|
|
64
|
+
sleep?: (ms: number) => Promise<void>;
|
|
65
|
+
dispatcherAttempts?: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ControlPlaneBootstrapResult {
|
|
69
|
+
dns: DnsServers;
|
|
70
|
+
fleetKey: FleetKey;
|
|
71
|
+
network: NetworkDiscoveryResult;
|
|
72
|
+
/** The last dispatcher reading. `fail` means no dispatcher ever answered. */
|
|
73
|
+
dispatcher: FleetFinding;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The control plane reads the bus's own health tables and emits nothing, so it
|
|
78
|
+
* registers no event schemas — the same empty registry `celilo events` opens
|
|
79
|
+
* with.
|
|
80
|
+
*/
|
|
81
|
+
const NO_SCHEMAS = defineEvents({});
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read the dispatcher's state from celilo's own bus.
|
|
85
|
+
*
|
|
86
|
+
* `checkDispatcher` is deliberately used in place of the "is the process up?"
|
|
87
|
+
* read the hook did. Its docblock records why: a naive liveness check reports
|
|
88
|
+
* green on a dispatcher that is running, unsupervised, on stale code, and
|
|
89
|
+
* delivering nothing.
|
|
90
|
+
*/
|
|
91
|
+
/**
|
|
92
|
+
* Open celilo's own bus, do one thing with it, close it.
|
|
93
|
+
*
|
|
94
|
+
* There is an async twin below and the two are NOT interchangeable: handing an
|
|
95
|
+
* async function to this one closes the bus when the promise is CREATED, not
|
|
96
|
+
* when it settles, so the work runs against a closed handle.
|
|
97
|
+
*/
|
|
98
|
+
export function withCeliloBus<T>(fn: (bus: Bus) => T): T {
|
|
99
|
+
const bus: Bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
100
|
+
try {
|
|
101
|
+
return fn(bus);
|
|
102
|
+
} finally {
|
|
103
|
+
bus.close();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The async twin. See the warning on `withCeliloBus`. */
|
|
108
|
+
export async function withCeliloBusAsync<T>(fn: (bus: Bus) => Promise<T>): Promise<T> {
|
|
109
|
+
const bus: Bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
110
|
+
try {
|
|
111
|
+
return await fn(bus);
|
|
112
|
+
} finally {
|
|
113
|
+
bus.close();
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function probeOwnDispatcher(): FleetFinding {
|
|
118
|
+
return withCeliloBus((bus) => checkDispatcher(bus));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Poll until a dispatcher answers, or the attempts run out. */
|
|
122
|
+
async function waitForDispatcher(
|
|
123
|
+
probe: () => FleetFinding,
|
|
124
|
+
attempts: number,
|
|
125
|
+
sleep: (ms: number) => Promise<void>,
|
|
126
|
+
): Promise<FleetFinding> {
|
|
127
|
+
let finding = probe();
|
|
128
|
+
for (let attempt = 1; attempt < attempts && finding.status === 'fail'; attempt++) {
|
|
129
|
+
await sleep(DISPATCHER_INTERVAL_MS);
|
|
130
|
+
finding = probe();
|
|
131
|
+
}
|
|
132
|
+
return finding;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Record what celilo needs to know about the box it runs on.
|
|
137
|
+
*
|
|
138
|
+
* Ordering is not cosmetic. The fleet key and the DNS servers are written in
|
|
139
|
+
* ONE `initializeSystem` call, because that function applies defaults and the
|
|
140
|
+
* init sentinel alongside the keys it is given, and calling it twice makes the
|
|
141
|
+
* second call re-derive everything the first already settled. The hook this
|
|
142
|
+
* replaced made two round trips through the CLI for exactly that reason and
|
|
143
|
+
* had no way not to.
|
|
144
|
+
*
|
|
145
|
+
* The network is recorded after, not in the same call: `system apply-config`
|
|
146
|
+
* REFUSES `network.*` keys outright, so a network is not something you write
|
|
147
|
+
* through the config surface at all. `discoverAndRecordNetwork` is the only
|
|
148
|
+
* path, it is idempotent, and it never overwrites an operator's value.
|
|
149
|
+
*/
|
|
150
|
+
export async function bootstrapControlPlane(
|
|
151
|
+
options: ControlPlaneBootstrapOptions,
|
|
152
|
+
): Promise<ControlPlaneBootstrapResult> {
|
|
153
|
+
const {
|
|
154
|
+
db,
|
|
155
|
+
discoverDnsImpl = discoverDns,
|
|
156
|
+
ensureFleetKeyImpl = ensureFleetKey,
|
|
157
|
+
discoverNetworkImpl = discoverAndRecordNetwork,
|
|
158
|
+
probeDispatcher = probeOwnDispatcher,
|
|
159
|
+
sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms)),
|
|
160
|
+
dispatcherAttempts = DISPATCHER_ATTEMPTS,
|
|
161
|
+
} = options;
|
|
162
|
+
|
|
163
|
+
const dns = discoverDnsImpl();
|
|
164
|
+
const fleetKey = ensureFleetKeyImpl();
|
|
165
|
+
|
|
166
|
+
initializeSystem(db, {
|
|
167
|
+
'dns.primary': dns.primary,
|
|
168
|
+
'dns.fallback': dns.fallback,
|
|
169
|
+
'ssh.public_key': fleetKey.publicKey,
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
const network = discoverNetworkImpl(db);
|
|
173
|
+
const dispatcher = await waitForDispatcher(probeDispatcher, dispatcherAttempts, sleep);
|
|
174
|
+
|
|
175
|
+
return { dns, fleetKey, network, dispatcher };
|
|
176
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The control plane's health check (celilo#1225).
|
|
3
|
+
*
|
|
4
|
+
* The mapping is what these pin. The failure that made this code exist was not
|
|
5
|
+
* a wrong assertion — the hook's `existsSync(db_path)` was a perfectly correct
|
|
6
|
+
* question — it was a correct question asked from inside a jail that does not
|
|
7
|
+
* bind the directory, so a present database read as absent. Nothing in a unit
|
|
8
|
+
* test can catch that shape; what a unit test CAN hold is that every finding
|
|
9
|
+
* reaches the operator with its status and its remediation intact, because the
|
|
10
|
+
* unattended path is the one nobody is watching.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { describe, expect, test } from 'bun:test';
|
|
14
|
+
import { findingAsHealthCheck } from './control-plane-health';
|
|
15
|
+
import type { FleetFinding } from './fleet-checks';
|
|
16
|
+
|
|
17
|
+
function finding(over: Partial<FleetFinding> = {}): FleetFinding {
|
|
18
|
+
return {
|
|
19
|
+
id: 'dispatcher',
|
|
20
|
+
title: 'Event dispatcher',
|
|
21
|
+
status: 'ok',
|
|
22
|
+
summary: 'a dispatcher is live',
|
|
23
|
+
detail: [],
|
|
24
|
+
remediation: null,
|
|
25
|
+
autoFixable: false,
|
|
26
|
+
...over,
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
describe('findingAsHealthCheck', () => {
|
|
31
|
+
test("ok becomes pass — celilo's vocabulary is not the health path's", () => {
|
|
32
|
+
expect(findingAsHealthCheck(finding({ status: 'ok' })).status).toBe('pass');
|
|
33
|
+
expect(findingAsHealthCheck(finding({ status: 'warn' })).status).toBe('warn');
|
|
34
|
+
expect(findingAsHealthCheck(finding({ status: 'fail' })).status).toBe('fail');
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
test('the check is named by the finding id, so an operator can grep for it', () => {
|
|
38
|
+
const item = findingAsHealthCheck(finding({ id: 'schema_drift' }));
|
|
39
|
+
expect(item.name).toBe('schema_drift');
|
|
40
|
+
expect(item.message).toBe('a dispatcher is live');
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test('the remediation survives the mapping', () => {
|
|
44
|
+
// This is the line that tells an operator what to DO, and the health path
|
|
45
|
+
// is the one that runs unattended every fifteen minutes. Dropping it here
|
|
46
|
+
// would leave a failing check with no next step, quietly.
|
|
47
|
+
const item = findingAsHealthCheck(
|
|
48
|
+
finding({
|
|
49
|
+
status: 'fail',
|
|
50
|
+
detail: ['no live dispatcher — heartbeat absent'],
|
|
51
|
+
remediation: 'systemctl enable --now celilo-events.service',
|
|
52
|
+
}),
|
|
53
|
+
);
|
|
54
|
+
expect(item.details).toContain('no live dispatcher — heartbeat absent');
|
|
55
|
+
expect(item.details).toContain('Fix: systemctl enable --now celilo-events.service');
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test('a healthy finding carries no empty details field', () => {
|
|
59
|
+
expect(findingAsHealthCheck(finding()).details).toBeUndefined();
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('detail lines survive without a remediation', () => {
|
|
63
|
+
const item = findingAsHealthCheck(finding({ status: 'warn', detail: ['two dispatchers'] }));
|
|
64
|
+
expect(item.details).toBe('two dispatchers');
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The control plane's health check, run by celilo rather than by a hook.
|
|
3
|
+
*
|
|
4
|
+
* ## Why this is not a hook
|
|
5
|
+
*
|
|
6
|
+
* It was `modules/celilo-mgmt/scripts/health-check.ts` until celilo#1225, and
|
|
7
|
+
* it failed under the hook jail in the way that is worst: **it reported a
|
|
8
|
+
* healthy fleet's database missing.** The hook asked `existsSync(db_path)`,
|
|
9
|
+
* the jail does not bind celilo's data directory (deliberately — that is where
|
|
10
|
+
* `master.key` and `celilo.db` live), so the answer was `false` and the deploy
|
|
11
|
+
* failed on a file that was sitting right there. Its other two checks shelled
|
|
12
|
+
* out to the CLI, which a jailed hook cannot reach at all.
|
|
13
|
+
*
|
|
14
|
+
* Every one of those was celilo inspecting itself. Deleting `on_install` for
|
|
15
|
+
* that reason and leaving this behind would have fixed one hook and left the
|
|
16
|
+
* deploy failing on the next.
|
|
17
|
+
*
|
|
18
|
+
* ## What replaced it is stronger, not merely relocated
|
|
19
|
+
*
|
|
20
|
+
* The hook asked two questions: does a file exist, and is this hostname in
|
|
21
|
+
* `celilo machine list`. `runFleetChecks` is what celilo already runs for
|
|
22
|
+
* `celilo system doctor`, and it asks eight, including the four-part dispatcher
|
|
23
|
+
* probe whose whole point is that "a process is up" reports green on a
|
|
24
|
+
* dispatcher that is unsupervised, on stale code, and delivering nothing.
|
|
25
|
+
*
|
|
26
|
+
* So the module gets a better check by having none of its own. That is the
|
|
27
|
+
* outcome to keep in mind before anyone adds a hook back: a health_check hook
|
|
28
|
+
* here can only re-ask, worse, questions celilo can answer directly.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
import type { DbClient } from '../db/client';
|
|
32
|
+
import { withCeliloBusAsync } from './control-plane-bootstrap';
|
|
33
|
+
import { type FleetFinding, runFleetChecks } from './fleet-checks';
|
|
34
|
+
import type { HealthCheckItem } from './health-runner';
|
|
35
|
+
|
|
36
|
+
/** `FleetFinding`'s three statuses in `HealthCheckItem`'s vocabulary. */
|
|
37
|
+
const STATUS: Record<FleetFinding['status'], HealthCheckItem['status']> = {
|
|
38
|
+
ok: 'pass',
|
|
39
|
+
warn: 'warn',
|
|
40
|
+
fail: 'fail',
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* One finding as a health item.
|
|
45
|
+
*
|
|
46
|
+
* `detail` and `remediation` are folded into `details` rather than dropped:
|
|
47
|
+
* the health path is what runs unattended every fifteen minutes, so the line
|
|
48
|
+
* telling an operator what to DO has to survive the mapping.
|
|
49
|
+
*/
|
|
50
|
+
export function findingAsHealthCheck(finding: FleetFinding): HealthCheckItem {
|
|
51
|
+
const details = [
|
|
52
|
+
...finding.detail,
|
|
53
|
+
...(finding.remediation ? [`Fix: ${finding.remediation}`] : []),
|
|
54
|
+
];
|
|
55
|
+
return {
|
|
56
|
+
name: finding.id,
|
|
57
|
+
status: STATUS[finding.status],
|
|
58
|
+
message: finding.summary,
|
|
59
|
+
...(details.length > 0 ? { details: details.join('\n') } : {}),
|
|
60
|
+
};
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Run celilo's own fleet checks and report them as the control plane's health. */
|
|
64
|
+
export async function controlPlaneHealthChecks(db: DbClient): Promise<HealthCheckItem[]> {
|
|
65
|
+
const findings = await withCeliloBusAsync((bus) => runFleetChecks(bus, db));
|
|
66
|
+
return findings.map(findingAsHealthCheck);
|
|
67
|
+
}
|
|
@@ -48,6 +48,18 @@ function rowToSystem(row: typeof moduleSystems.$inferSelect): DeployedSystem {
|
|
|
48
48
|
* Returns [] for API-only modules (e.g. namecheap) — that is a modeled state,
|
|
49
49
|
* not an error.
|
|
50
50
|
*/
|
|
51
|
+
/**
|
|
52
|
+
* The module that IS celilo's control plane.
|
|
53
|
+
*
|
|
54
|
+
* celilo knows this module by name in several places — its privileged
|
|
55
|
+
* capability allow-list, the subnet it trusts, the fleet checks that ask where
|
|
56
|
+
* the control plane runs, and the deploy step that initialises the box's own
|
|
57
|
+
* state. The id lives here because every one of those already reaches for
|
|
58
|
+
* `getModuleSystems`, and three separate private copies of the string is how
|
|
59
|
+
* they drift.
|
|
60
|
+
*/
|
|
61
|
+
export const CONTROL_PLANE_MODULE_ID = 'celilo-mgmt';
|
|
62
|
+
|
|
51
63
|
export function getModuleSystems(moduleId: string, db: DbClient): DeployedSystem[] {
|
|
52
64
|
const rows = db.select().from(moduleSystems).where(eq(moduleSystems.moduleId, moduleId)).all();
|
|
53
65
|
return rows.map(rowToSystem).sort((a, b) => a.name.localeCompare(b.name));
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Moved from `modules/celilo-mgmt/scripts/discovery.test.ts` with celilo#1225.
|
|
3
|
+
*
|
|
4
|
+
* The parser cases came across unchanged. The `discoverDns` cases are new and
|
|
5
|
+
* could not have been written there: the module's version read the filesystem
|
|
6
|
+
* directly, so the file-preference rule — the part with the actual failure mode
|
|
7
|
+
* — had no seam to test through.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test } from 'bun:test';
|
|
11
|
+
import { type ResolverFileReader, chooseDns, discoverDns, parseNameservers } from './dns-discovery';
|
|
12
|
+
|
|
13
|
+
const SYSTEMD = '/run/systemd/resolve/resolv.conf';
|
|
14
|
+
const ETC = '/etc/resolv.conf';
|
|
15
|
+
|
|
16
|
+
/** A host offering exactly the files given, and nothing else. */
|
|
17
|
+
function host(files: Record<string, string>): ResolverFileReader {
|
|
18
|
+
return (path) => files[path] ?? null;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
describe('parseNameservers', () => {
|
|
22
|
+
test('extracts nameserver IPs in order', () => {
|
|
23
|
+
const resolv = ['# comment', 'nameserver 9.9.9.9', 'nameserver 8.8.4.4', 'search lan'].join(
|
|
24
|
+
'\n',
|
|
25
|
+
);
|
|
26
|
+
expect(parseNameservers(resolv)).toEqual(['9.9.9.9', '8.8.4.4']);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test('returns empty for no nameservers', () => {
|
|
30
|
+
expect(parseNameservers('search lan\noptions edns0')).toEqual([]);
|
|
31
|
+
});
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe('chooseDns', () => {
|
|
35
|
+
test('drops loopback stubs (systemd-resolved 127.0.0.53)', () => {
|
|
36
|
+
expect(chooseDns(['127.0.0.53'])).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test('uses the first two real upstreams', () => {
|
|
40
|
+
expect(chooseDns(['9.9.9.9', '8.8.4.4', '1.0.0.1'])).toEqual({
|
|
41
|
+
primary: '9.9.9.9',
|
|
42
|
+
fallback: '8.8.4.4',
|
|
43
|
+
});
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
test('falls back to 1.1.1.1 for the second when only one upstream', () => {
|
|
47
|
+
expect(chooseDns(['9.9.9.9'])).toEqual({ primary: '9.9.9.9', fallback: '1.1.1.1' });
|
|
48
|
+
});
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
describe('discoverDns', () => {
|
|
52
|
+
test('prefers the systemd upstream file over the stub in /etc/resolv.conf', () => {
|
|
53
|
+
// The case the file order exists for. Reading only /etc/resolv.conf here
|
|
54
|
+
// discovers 127.0.0.53, which no other box on the network can reach.
|
|
55
|
+
const servers = discoverDns(
|
|
56
|
+
host({
|
|
57
|
+
[SYSTEMD]: 'nameserver 9.9.9.9\nnameserver 8.8.4.4\n',
|
|
58
|
+
[ETC]: 'nameserver 127.0.0.53\n',
|
|
59
|
+
}),
|
|
60
|
+
);
|
|
61
|
+
expect(servers).toEqual({ primary: '9.9.9.9', fallback: '8.8.4.4' });
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('falls through to /etc/resolv.conf when the systemd file yields nothing real', () => {
|
|
65
|
+
const servers = discoverDns(
|
|
66
|
+
host({
|
|
67
|
+
[SYSTEMD]: 'nameserver 127.0.0.53\n',
|
|
68
|
+
[ETC]: 'nameserver 192.0.2.10\nnameserver 192.0.2.11\n',
|
|
69
|
+
}),
|
|
70
|
+
);
|
|
71
|
+
expect(servers).toEqual({ primary: '192.0.2.10', fallback: '192.0.2.11' });
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
test('reads /etc/resolv.conf on a host with no systemd resolver file', () => {
|
|
75
|
+
expect(discoverDns(host({ [ETC]: 'nameserver 192.0.2.10\n' }))).toEqual({
|
|
76
|
+
primary: '192.0.2.10',
|
|
77
|
+
fallback: '1.1.1.1',
|
|
78
|
+
});
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('accepts a loopback-only /etc/resolv.conf as the public fallback, not as an error', () => {
|
|
82
|
+
// Last resort: whatever it gives is taken, and a stub-only host yields the
|
|
83
|
+
// public fallback rather than an unreachable address.
|
|
84
|
+
expect(discoverDns(host({ [ETC]: 'nameserver 127.0.0.53\n' }))).toEqual({
|
|
85
|
+
primary: '1.1.1.1',
|
|
86
|
+
fallback: '1.1.1.1',
|
|
87
|
+
});
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test('returns the public fallback on a host with no resolver files at all', () => {
|
|
91
|
+
expect(discoverDns(host({}))).toEqual({ primary: '1.1.1.1', fallback: '1.1.1.1' });
|
|
92
|
+
});
|
|
93
|
+
});
|