@celilo/cli 1.14.0 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +2 -2
- package/CELILO_SUBSYSTEMS.md +26 -3
- package/README.md +0 -2
- package/drizzle/0030_drop_module_builds_environment.sql +8 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +3 -3
- package/src/capabilities/public-web-publish.test.ts +18 -0
- package/src/cli/commands/alerts-sweep.ts +3 -0
- package/src/cli/commands/monitor.ts +15 -2
- package/src/cli/commands/system-doctor.test.ts +121 -1
- package/src/cli/commands/system-doctor.ts +151 -1
- package/src/cli/completion.ts +9 -2
- package/src/cli/index.ts +1 -1
- package/src/console/control-plane-boundary.test.ts +82 -4
- package/src/db/schema.ts +0 -1
- package/src/hooks/capability-loader.ts +15 -2
- package/src/hooks/executor.ts +116 -17
- package/src/hooks/hook-jail-toolchain-reach.test.ts +273 -0
- package/src/hooks/hook-jail-unreachability.test.ts +77 -26
- package/src/hooks/hook-protocol.ts +44 -0
- package/src/hooks/hook-runner-entry.ts +23 -0
- package/src/hooks/hook-runner.ts +10 -0
- package/src/hooks/hook-trespass.test.ts +74 -12
- package/src/hooks/jail-browser-launch-flags.test.ts +34 -0
- package/src/hooks/jail.test.ts +92 -0
- package/src/hooks/jail.ts +304 -32
- package/src/hooks/mount-set.test.ts +116 -7
- package/src/hooks/mount-set.ts +189 -20
- package/src/hooks/remote-broker.test.ts +350 -0
- package/src/hooks/remote-broker.ts +404 -0
- package/src/hooks/run-named-hook.ts +2 -0
- package/src/hooks/test-fixtures/jail-probe-hook.ts +14 -1
- package/src/hooks/test-fixtures/jail-toolchain-hook.ts +227 -0
- package/src/hooks/test-fixtures/remote-bridge-probe.ts +82 -0
- package/src/hooks/unjailed-lint.test.ts +254 -0
- package/src/hooks/unjailed-lint.ts +395 -0
- package/src/policy/module-business-baseline.ts +13 -1
- package/src/policy/module-script-scan.ts +60 -1
- package/src/policy/no-hand-built-ssh.test.ts +39 -1
- package/src/policy/no-module-business-in-core.test.ts +1 -1
- package/src/services/alerting/coverage-source.test.ts +86 -0
- package/src/services/alerting/coverage-source.ts +11 -1
- package/src/services/alerting/hook-jail.test.ts +66 -0
- package/src/services/alerting/hook-jail.ts +70 -0
- package/src/services/alerting/run-monitor.test.ts +62 -0
- package/src/services/alerting/run-monitor.ts +12 -0
- package/src/services/alerting/sweep-runner.test.ts +1 -0
- package/src/services/backup-create.ts +3 -0
- package/src/services/backup-restore.ts +2 -0
- package/src/services/control-plane-bootstrap.test.ts +177 -0
- package/src/services/control-plane-bootstrap.ts +176 -0
- package/src/services/control-plane-health.test.ts +66 -0
- package/src/services/control-plane-health.ts +67 -0
- package/src/services/deploy-ansible.ts +9 -1
- package/src/services/deployed-systems.ts +12 -0
- package/src/services/dns-discovery.test.ts +93 -0
- package/src/services/dns-discovery.ts +92 -0
- package/src/services/fleet-checks.ts +6 -2
- package/src/services/health-runner.ts +36 -3
- package/src/services/module-build.test.ts +1 -64
- package/src/services/module-build.ts +10 -86
- package/src/services/module-deploy.ts +71 -0
- package/src/services/remote-access.test.ts +223 -0
- package/src/services/remote-access.ts +149 -0
- package/src/services/restore-from-file.ts +6 -1
- package/src/services/static-content-converge.test.ts +338 -0
- package/src/services/static-content-converge.ts +299 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* D8's three states, asserted one branch at a time: only "used to jail and has
|
|
3
|
+
* stopped" raises. Steady-state unjailed — a Mac that never jailed — is a
|
|
4
|
+
* configuration fact the doctor reports, never an alert.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { describe, expect, test } from 'bun:test';
|
|
8
|
+
import type { JailModeRecord } from '../../hooks/jail';
|
|
9
|
+
import { hookJailFailingKeys } from './hook-jail';
|
|
10
|
+
|
|
11
|
+
const HOST = 'celilo-mgr';
|
|
12
|
+
|
|
13
|
+
const regressed: JailModeRecord = {
|
|
14
|
+
mode: 'unjailed',
|
|
15
|
+
backend: 'none',
|
|
16
|
+
reason: 'bubblewrap is installed but could not build a namespace, so hooks run unjailed.',
|
|
17
|
+
host: HOST,
|
|
18
|
+
recordedAt: '2026-08-28T09:00:00.000Z',
|
|
19
|
+
lastJailed: { backend: 'bubblewrap', recordedAt: '2026-08-27T09:00:00.000Z' },
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
describe('hookJailFailingKeys', () => {
|
|
23
|
+
test('a host that used to jail and has stopped raises, naming the host and the reason', () => {
|
|
24
|
+
const failing = hookJailFailingKeys({ record: regressed, host: HOST }, 'critical');
|
|
25
|
+
expect(failing).toHaveLength(1);
|
|
26
|
+
expect(failing[0]?.key).toBe(`builtin:hook_jail/host:${HOST}`);
|
|
27
|
+
expect(failing[0]?.severity).toBe('critical');
|
|
28
|
+
expect(failing[0]?.message).toContain(HOST);
|
|
29
|
+
expect(failing[0]?.message).toContain('until 2026-08-27T09:00:00.000Z');
|
|
30
|
+
expect(failing[0]?.message).toContain('could not build a namespace');
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
test('steady-state unjailed is not an event', () => {
|
|
34
|
+
const neverJailed: JailModeRecord = {
|
|
35
|
+
mode: 'unjailed',
|
|
36
|
+
backend: 'none',
|
|
37
|
+
reason: 'macOS has no hook jail yet',
|
|
38
|
+
host: HOST,
|
|
39
|
+
recordedAt: '2026-08-28T09:00:00.000Z',
|
|
40
|
+
};
|
|
41
|
+
expect(hookJailFailingKeys({ record: neverJailed, host: HOST }, 'critical')).toEqual([]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
test('a jailed host raises nothing, which is also how the alert resolves', () => {
|
|
45
|
+
const healthy: JailModeRecord = {
|
|
46
|
+
mode: 'jailed',
|
|
47
|
+
backend: 'bubblewrap',
|
|
48
|
+
host: HOST,
|
|
49
|
+
recordedAt: '2026-08-28T09:00:00.000Z',
|
|
50
|
+
};
|
|
51
|
+
expect(hookJailFailingKeys({ record: healthy, host: HOST }, 'critical')).toEqual([]);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('no record yet raises nothing', () => {
|
|
55
|
+
expect(hookJailFailingKeys({ record: undefined, host: HOST }, 'critical')).toEqual([]);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("a record written by another host is a move, not this host's regression", () => {
|
|
59
|
+
expect(hookJailFailingKeys({ record: regressed, host: 'a-new-box' }, 'critical')).toEqual([]);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test("the monitor's severity is the alert's severity", () => {
|
|
63
|
+
const failing = hookJailFailingKeys({ record: regressed, host: HOST }, 'warning');
|
|
64
|
+
expect(failing[0]?.severity).toBe('warning');
|
|
65
|
+
});
|
|
66
|
+
});
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Hook-jail regression — a host that used to jail its hooks and has stopped.
|
|
3
|
+
*
|
|
4
|
+
* Design D8's three states want three different things (see
|
|
5
|
+
* openspec/changes/hook-process-boundary/design.md): jailed is normal and
|
|
6
|
+
* raises nothing; a host that never jailed (a Mac) is a configuration fact
|
|
7
|
+
* that `celilo system doctor` reports and is NOT an event; a host that ran
|
|
8
|
+
* hooks jailed yesterday and runs them unjailed today is a kernel upgrade, an
|
|
9
|
+
* AppArmor policy change, or a changed container profile, and is exactly the
|
|
10
|
+
* failure that looks like nothing. Only the third state raises here.
|
|
11
|
+
*
|
|
12
|
+
* The state this reads is `hook-jail-mode.json`, written by `recordJailMode`
|
|
13
|
+
* on every real module-hook run. The transition survives the record being
|
|
14
|
+
* overwritten because an unjailed record carries `lastJailed` — the jailed
|
|
15
|
+
* record it replaced on the same host.
|
|
16
|
+
*
|
|
17
|
+
* The monitor scheduling this check is a SELF-monitor: `celilo monitor add
|
|
18
|
+
* hook_jail` creates it `suppressible: false`, so per
|
|
19
|
+
* openspec/specs/alerting/spec.md no ancestor alert and no deploy window can
|
|
20
|
+
* silence it. Losing confinement during a broad outage is precisely when it
|
|
21
|
+
* must still page.
|
|
22
|
+
*
|
|
23
|
+
* Deliberately cheap: one local file read, no remote contact, safe on every
|
|
24
|
+
* sweep.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import type { AlertSeverity } from '../../db/schema';
|
|
28
|
+
import type { JailModeRecord } from '../../hooks/jail';
|
|
29
|
+
import { type FailingKey, builtinAlertKey } from './keys';
|
|
30
|
+
|
|
31
|
+
export const HOOK_JAIL_CHECK = 'hook_jail';
|
|
32
|
+
|
|
33
|
+
export interface HookJailState {
|
|
34
|
+
/** The recorded mode, or undefined when no hook has recorded one yet. */
|
|
35
|
+
record: JailModeRecord | undefined;
|
|
36
|
+
/** The host this check runs on, compared against the record's own. */
|
|
37
|
+
host: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* One failing key when this host used to jail and has stopped; empty
|
|
42
|
+
* otherwise. The message names the host and the reason, which is what the
|
|
43
|
+
* security-model spec requires the alert to carry.
|
|
44
|
+
*
|
|
45
|
+
* A record written by another host is ignored rather than compared: a database
|
|
46
|
+
* or data directory restored onto a new box is a move, not a regression, and
|
|
47
|
+
* the new box has its own history to make.
|
|
48
|
+
*/
|
|
49
|
+
export function hookJailFailingKeys(state: HookJailState, severity: AlertSeverity): FailingKey[] {
|
|
50
|
+
const { record, host } = state;
|
|
51
|
+
if (!record || record.host !== host) return [];
|
|
52
|
+
if (record.mode !== 'unjailed' || !record.lastJailed) return [];
|
|
53
|
+
|
|
54
|
+
return [
|
|
55
|
+
{
|
|
56
|
+
key: builtinAlertKey(HOOK_JAIL_CHECK, 'host', host),
|
|
57
|
+
severity,
|
|
58
|
+
message:
|
|
59
|
+
`Hooks on ${host} ran jailed (${record.lastJailed.backend}) until ` +
|
|
60
|
+
`${record.lastJailed.recordedAt} and now run unjailed: ` +
|
|
61
|
+
`${record.reason ?? 'no reason was recorded'}`,
|
|
62
|
+
details:
|
|
63
|
+
'A host that quietly stops confining hooks is a failure that otherwise\n' +
|
|
64
|
+
'looks like nothing — the usual causes are a kernel upgrade, an AppArmor\n' +
|
|
65
|
+
'policy change, or a changed container profile. `celilo system doctor`\n' +
|
|
66
|
+
'reports what the host can do right now. The alert resolves when a hook\n' +
|
|
67
|
+
'runs jailed on this host again.',
|
|
68
|
+
},
|
|
69
|
+
];
|
|
70
|
+
}
|
|
@@ -7,6 +7,7 @@ import type { DbClient } from '../../db/client';
|
|
|
7
7
|
import { type Monitor, alerts, monitorRuns, monitors } from '../../db/schema';
|
|
8
8
|
import { setupTestDatabaseAt } from '../../test-utils/database';
|
|
9
9
|
import type { HealthCheckResult } from '../health-runner';
|
|
10
|
+
import { HOOK_JAIL_CHECK, type HookJailState } from './hook-jail';
|
|
10
11
|
import { moduleAlertKey, moduleCheckAlertKey } from './keys';
|
|
11
12
|
import { type MonitorRunDeps, runOneMonitor } from './run-monitor';
|
|
12
13
|
import { loadLiveAlerts, markUnsuppressed } from './store';
|
|
@@ -40,6 +41,7 @@ describe('runOneMonitor', () => {
|
|
|
40
41
|
runModuleCheck: async () => result,
|
|
41
42
|
runBuiltinCheck: async () => [],
|
|
42
43
|
loadModuleCoverage: () => [],
|
|
44
|
+
loadJailState: () => ({ record: undefined, host: 'test-host' }),
|
|
43
45
|
now: () => now,
|
|
44
46
|
graceMs: 60_000,
|
|
45
47
|
};
|
|
@@ -182,4 +184,64 @@ describe('runOneMonitor', () => {
|
|
|
182
184
|
expect(summary.outcome).toBe('success');
|
|
183
185
|
expect(liveKeys()).toEqual([]);
|
|
184
186
|
});
|
|
187
|
+
|
|
188
|
+
describe('the hook-jail self-monitor (hook-process-boundary task 4.4)', () => {
|
|
189
|
+
const HOST = 'celilo-mgr';
|
|
190
|
+
const regressed: HookJailState = {
|
|
191
|
+
host: HOST,
|
|
192
|
+
record: {
|
|
193
|
+
mode: 'unjailed',
|
|
194
|
+
backend: 'none',
|
|
195
|
+
reason: 'bubblewrap is installed but could not build a namespace',
|
|
196
|
+
host: HOST,
|
|
197
|
+
recordedAt: NOW.toISOString(),
|
|
198
|
+
lastJailed: { backend: 'bubblewrap', recordedAt: '2026-07-27T03:00:00.000Z' },
|
|
199
|
+
},
|
|
200
|
+
};
|
|
201
|
+
const healed: HookJailState = {
|
|
202
|
+
host: HOST,
|
|
203
|
+
record: {
|
|
204
|
+
mode: 'jailed',
|
|
205
|
+
backend: 'bubblewrap',
|
|
206
|
+
host: HOST,
|
|
207
|
+
recordedAt: LATER.toISOString(),
|
|
208
|
+
},
|
|
209
|
+
};
|
|
210
|
+
|
|
211
|
+
let jailMonitor: Monitor;
|
|
212
|
+
|
|
213
|
+
beforeEach(() => {
|
|
214
|
+
db.insert(monitors)
|
|
215
|
+
.values({
|
|
216
|
+
id: 'mon-jail',
|
|
217
|
+
kind: 'builtin_check',
|
|
218
|
+
target: HOOK_JAIL_CHECK,
|
|
219
|
+
intervalMinutes: 15,
|
|
220
|
+
severity: 'critical',
|
|
221
|
+
suppressible: false,
|
|
222
|
+
})
|
|
223
|
+
.run();
|
|
224
|
+
jailMonitor = db.select().from(monitors).where(eq(monitors.id, 'mon-jail')).get() as Monitor;
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
const jailDeps = (state: HookJailState, now: Date = NOW): MonitorRunDeps => ({
|
|
228
|
+
...deps(healthy, now),
|
|
229
|
+
loadJailState: () => state,
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
test('a jailed-to-unjailed transition fires an alert naming the host', async () => {
|
|
233
|
+
const summary = await runOneMonitor(db, jailMonitor, jailDeps(regressed));
|
|
234
|
+
expect(summary.outcome).toBe('success');
|
|
235
|
+
const live = loadLiveAlerts(db, jailMonitor.id);
|
|
236
|
+
expect(live.map((a) => a.key)).toEqual([`builtin:hook_jail/host:${HOST}`]);
|
|
237
|
+
const row = db.select().from(alerts).where(eq(alerts.monitorId, jailMonitor.id)).get();
|
|
238
|
+
expect(row?.message).toContain(HOST);
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
test('re-jailing resolves it by set difference', async () => {
|
|
242
|
+
await runOneMonitor(db, jailMonitor, jailDeps(regressed));
|
|
243
|
+
await runOneMonitor(db, jailMonitor, jailDeps(healed, LATER));
|
|
244
|
+
expect(loadLiveAlerts(db, jailMonitor.id)).toEqual([]);
|
|
245
|
+
});
|
|
246
|
+
});
|
|
185
247
|
});
|
|
@@ -19,6 +19,7 @@ import type { HealthCheckResult } from '../health-runner';
|
|
|
19
19
|
import { failingKeysFromFindings } from './builtin-monitors';
|
|
20
20
|
import { HEALTH_COVERAGE_CHECK, healthCoverageFailingKeys } from './health-coverage';
|
|
21
21
|
import type { ModuleCoverageInput } from './health-coverage';
|
|
22
|
+
import { HOOK_JAIL_CHECK, type HookJailState, hookJailFailingKeys } from './hook-jail';
|
|
22
23
|
import {
|
|
23
24
|
type FailingKey,
|
|
24
25
|
builtinMonitorKey,
|
|
@@ -35,6 +36,8 @@ export interface MonitorRunDeps {
|
|
|
35
36
|
runBuiltinCheck(category: DriftCategory): Promise<DriftFinding[]>;
|
|
36
37
|
/** Module roster for the health-coverage check. */
|
|
37
38
|
loadModuleCoverage(): ModuleCoverageInput[];
|
|
39
|
+
/** Recorded jail mode and current host, for the hook-jail self-monitor. */
|
|
40
|
+
loadJailState(): HookJailState;
|
|
38
41
|
now(): Date;
|
|
39
42
|
/** Grace window before a new alert may notify. */
|
|
40
43
|
graceMs: number;
|
|
@@ -95,6 +98,15 @@ async function runCheck(
|
|
|
95
98
|
};
|
|
96
99
|
}
|
|
97
100
|
|
|
101
|
+
// Same shape: one local file read. This is the self-monitor for D8's third
|
|
102
|
+
// state — a host that used to jail its hooks and has stopped.
|
|
103
|
+
if (monitor.target === HOOK_JAIL_CHECK) {
|
|
104
|
+
return {
|
|
105
|
+
outcome: 'success',
|
|
106
|
+
failingKeys: hookJailFailingKeys(deps.loadJailState(), severity),
|
|
107
|
+
};
|
|
108
|
+
}
|
|
109
|
+
|
|
98
110
|
const category = monitor.target as DriftCategory;
|
|
99
111
|
try {
|
|
100
112
|
const findings = await deps.runBuiltinCheck(category);
|
|
@@ -41,6 +41,7 @@ import {
|
|
|
41
41
|
refuseIfInFlight,
|
|
42
42
|
startOperation,
|
|
43
43
|
} from './module-operations';
|
|
44
|
+
import { remoteAccessPolicy } from './remote-access';
|
|
44
45
|
import { stageSystemState } from './system-state-stage';
|
|
45
46
|
|
|
46
47
|
export interface BackupCreateOptions {
|
|
@@ -399,6 +400,7 @@ export async function createModuleBackup(
|
|
|
399
400
|
{
|
|
400
401
|
debug: false,
|
|
401
402
|
systems: getModuleSystems(moduleId, db),
|
|
403
|
+
remoteAccess: remoteAccessPolicy(moduleId, db),
|
|
402
404
|
},
|
|
403
405
|
);
|
|
404
406
|
|
|
@@ -563,6 +565,7 @@ export async function importModuleBackup(
|
|
|
563
565
|
{
|
|
564
566
|
debug: false,
|
|
565
567
|
systems: getModuleSystems(moduleId, db),
|
|
568
|
+
remoteAccess: remoteAccessPolicy(moduleId, db),
|
|
566
569
|
},
|
|
567
570
|
);
|
|
568
571
|
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
refuseIfInFlight,
|
|
31
31
|
startOperation,
|
|
32
32
|
} from './module-operations';
|
|
33
|
+
import { remoteAccessPolicy } from './remote-access';
|
|
33
34
|
|
|
34
35
|
export interface RestoreResult {
|
|
35
36
|
success: boolean;
|
|
@@ -288,6 +289,7 @@ export async function restoreModuleBackup(
|
|
|
288
289
|
{
|
|
289
290
|
debug: false,
|
|
290
291
|
systems: getModuleSystems(backup.moduleId, db),
|
|
292
|
+
remoteAccess: remoteAccessPolicy(backup.moduleId, db),
|
|
291
293
|
},
|
|
292
294
|
);
|
|
293
295
|
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The control-plane bootstrap, which replaced celilo-mgmt's `on_install`
|
|
3
|
+
* (celilo#1225).
|
|
4
|
+
*
|
|
5
|
+
* Every dependency is injected, so these run against a real database and no
|
|
6
|
+
* host: no resolver file is read, no ssh-keygen runs, no bus is opened. That is
|
|
7
|
+
* the property the hook could not have. Its equivalents shelled out to the CLI,
|
|
8
|
+
* so the only way to exercise them was to have a celilo on PATH — which is also
|
|
9
|
+
* why the jail broke them and no unit test noticed.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
|
|
13
|
+
import { mkdtempSync, rmSync } from 'node:fs';
|
|
14
|
+
import { tmpdir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
import { eq } from 'drizzle-orm';
|
|
17
|
+
import type { DbClient } from '../db/client';
|
|
18
|
+
import { systemConfig } from '../db/schema';
|
|
19
|
+
import { setupTestDatabaseAt } from '../test-utils/database';
|
|
20
|
+
import {
|
|
21
|
+
type ControlPlaneBootstrapOptions,
|
|
22
|
+
bootstrapControlPlane,
|
|
23
|
+
} from './control-plane-bootstrap';
|
|
24
|
+
import type { FleetFinding } from './fleet-checks';
|
|
25
|
+
|
|
26
|
+
const DNS = { primary: '9.9.9.9', fallback: '8.8.4.4' };
|
|
27
|
+
const PUBLIC_KEY = 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAI test celilo-fleet';
|
|
28
|
+
|
|
29
|
+
function dispatcher(status: FleetFinding['status']): FleetFinding {
|
|
30
|
+
return {
|
|
31
|
+
id: 'dispatcher',
|
|
32
|
+
title: 'Event dispatcher',
|
|
33
|
+
status,
|
|
34
|
+
summary: `dispatcher ${status}`,
|
|
35
|
+
detail: [],
|
|
36
|
+
remediation: status === 'ok' ? null : 'start the dispatcher',
|
|
37
|
+
autoFixable: false,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function configValue(db: DbClient, key: string): string | undefined {
|
|
42
|
+
return db.select().from(systemConfig).where(eq(systemConfig.key, key)).get()?.value ?? undefined;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
describe('bootstrapControlPlane', () => {
|
|
46
|
+
let dir: string;
|
|
47
|
+
let db: DbClient;
|
|
48
|
+
/** Every sleep the poll performed, so waiting is asserted rather than timed. */
|
|
49
|
+
let sleeps: number[];
|
|
50
|
+
|
|
51
|
+
beforeEach(async () => {
|
|
52
|
+
dir = mkdtempSync(join(tmpdir(), 'cpb-'));
|
|
53
|
+
const dbPath = join(dir, 'celilo.db');
|
|
54
|
+
process.env.CELILO_DB_PATH = dbPath;
|
|
55
|
+
db = await setupTestDatabaseAt(dbPath);
|
|
56
|
+
sleeps = [];
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
afterEach(() => {
|
|
60
|
+
db.$client.close();
|
|
61
|
+
process.env.CELILO_DB_PATH = undefined;
|
|
62
|
+
try {
|
|
63
|
+
rmSync(dir, { recursive: true, force: true });
|
|
64
|
+
} catch {
|
|
65
|
+
/* ignore */
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
69
|
+
function run(overrides: Partial<ControlPlaneBootstrapOptions> = {}) {
|
|
70
|
+
return bootstrapControlPlane({
|
|
71
|
+
db,
|
|
72
|
+
discoverDnsImpl: () => DNS,
|
|
73
|
+
ensureFleetKeyImpl: () => ({ publicKey: PUBLIC_KEY, created: true }),
|
|
74
|
+
discoverNetworkImpl: () => ({ applied: ['network.internal.subnet = 10.0.0.0/24'] }),
|
|
75
|
+
probeDispatcher: () => dispatcher('ok'),
|
|
76
|
+
sleep: async (ms) => {
|
|
77
|
+
sleeps.push(ms);
|
|
78
|
+
},
|
|
79
|
+
...overrides,
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
test('records the discovered DNS and the fleet public key in system config', async () => {
|
|
84
|
+
await run();
|
|
85
|
+
|
|
86
|
+
expect(configValue(db, 'dns.primary')).toBe('9.9.9.9');
|
|
87
|
+
expect(configValue(db, 'dns.fallback')).toBe('8.8.4.4');
|
|
88
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test('returns what each step found, so the caller renders and this does not', async () => {
|
|
92
|
+
const result = await run();
|
|
93
|
+
|
|
94
|
+
expect(result.dns).toEqual(DNS);
|
|
95
|
+
expect(result.fleetKey).toEqual({ publicKey: PUBLIC_KEY, created: true });
|
|
96
|
+
expect(result.network.applied).toEqual(['network.internal.subnet = 10.0.0.0/24']);
|
|
97
|
+
expect(result.dispatcher.status).toBe('ok');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test('a dispatcher already up is probed once and never waited for', async () => {
|
|
101
|
+
let probes = 0;
|
|
102
|
+
await run({
|
|
103
|
+
probeDispatcher: () => {
|
|
104
|
+
probes++;
|
|
105
|
+
return dispatcher('ok');
|
|
106
|
+
},
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
expect(probes).toBe(1);
|
|
110
|
+
expect(sleeps).toEqual([]);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test('a dispatcher still starting is polled until it answers', async () => {
|
|
114
|
+
// The role enables the supervisor unit moments before this runs, so the
|
|
115
|
+
// first probe legitimately finds nothing. Two failures then success.
|
|
116
|
+
const readings: FleetFinding['status'][] = ['fail', 'fail', 'ok'];
|
|
117
|
+
let probes = 0;
|
|
118
|
+
const result = await run({
|
|
119
|
+
probeDispatcher: () => dispatcher(readings[probes++] ?? 'ok'),
|
|
120
|
+
});
|
|
121
|
+
|
|
122
|
+
expect(probes).toBe(3);
|
|
123
|
+
expect(sleeps).toEqual([1000, 1000]);
|
|
124
|
+
expect(result.dispatcher.status).toBe('ok');
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test('a dispatcher that never answers gives up after the attempt budget', async () => {
|
|
128
|
+
let probes = 0;
|
|
129
|
+
const result = await run({
|
|
130
|
+
probeDispatcher: () => {
|
|
131
|
+
probes++;
|
|
132
|
+
return dispatcher('fail');
|
|
133
|
+
},
|
|
134
|
+
dispatcherAttempts: 5,
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
expect(probes).toBe(5);
|
|
138
|
+
expect(sleeps).toHaveLength(4);
|
|
139
|
+
// Reported, not thrown. The caller decides how loud to be, which is what
|
|
140
|
+
// lets `module deploy` fail the install and a later caller merely warn.
|
|
141
|
+
expect(result.dispatcher.status).toBe('fail');
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
test('a warn reading is accepted rather than polled against', async () => {
|
|
145
|
+
let probes = 0;
|
|
146
|
+
await run({
|
|
147
|
+
probeDispatcher: () => {
|
|
148
|
+
probes++;
|
|
149
|
+
return dispatcher('warn');
|
|
150
|
+
},
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
expect(probes).toBe(1);
|
|
154
|
+
expect(sleeps).toEqual([]);
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test('a box with no default route still gets its DNS and key recorded', async () => {
|
|
158
|
+
// `discoverAndRecordNetwork` reports a skip rather than throwing, and a
|
|
159
|
+
// box celilo cannot read a network from must not fail the whole deploy.
|
|
160
|
+
const result = await run({
|
|
161
|
+
discoverNetworkImpl: () => ({ applied: [], skipped: 'no default route' }),
|
|
162
|
+
});
|
|
163
|
+
|
|
164
|
+
expect(result.network.skipped).toBe('no default route');
|
|
165
|
+
expect(configValue(db, 'dns.primary')).toBe('9.9.9.9');
|
|
166
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
167
|
+
});
|
|
168
|
+
|
|
169
|
+
test('an existing fleet key is recorded without being reminted', async () => {
|
|
170
|
+
const result = await run({
|
|
171
|
+
ensureFleetKeyImpl: () => ({ publicKey: PUBLIC_KEY, created: false }),
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
expect(result.fleetKey.created).toBe(false);
|
|
175
|
+
expect(configValue(db, 'ssh.public_key')).toBe(PUBLIC_KEY);
|
|
176
|
+
});
|
|
177
|
+
});
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Initialise the management box's own celilo state, during the celilo-mgmt
|
|
3
|
+
* deploy.
|
|
4
|
+
*
|
|
5
|
+
* ## Why this is here and not in a hook
|
|
6
|
+
*
|
|
7
|
+
* It was a hook until celilo#1225. `modules/celilo-mgmt/scripts/on_install.ts`
|
|
8
|
+
* reached every one of these operations by spawning the `celilo` CLI, which a
|
|
9
|
+
* jailed hook cannot do: the jail's mount set binds no `/usr/bin`, no
|
|
10
|
+
* `/usr/local/bin` and no shell, so `execFileSync` dies on the missing binary
|
|
11
|
+
* and `execSync` dies one step earlier on the missing `/bin/sh`. The deploy ran
|
|
12
|
+
* Ansible to completion and then failed in its own install hook, on every host
|
|
13
|
+
* where a jail backend exists.
|
|
14
|
+
*
|
|
15
|
+
* The fix is not to let the hook reach the CLI. It is that none of this wanted
|
|
16
|
+
* to be in a hook. celilo-mgmt is never deployed to a remote box (ruled
|
|
17
|
+
* 2026-09-02), so the host being configured is always the host celilo runs on,
|
|
18
|
+
* and every step below is celilo acting on itself: read this box's resolvers,
|
|
19
|
+
* mint celilo's own fleet key, record celilo's own network, check celilo's own
|
|
20
|
+
* event bus. A hook was a process boundary with celilo on both sides of it.
|
|
21
|
+
*
|
|
22
|
+
* ## What it does NOT do
|
|
23
|
+
*
|
|
24
|
+
* It does not decide whether to run. The deploy path does that, from the
|
|
25
|
+
* control-plane module id, which celilo already keys on in three other places
|
|
26
|
+
* (`manifest/validate.ts`'s privileged allow-list, `capability-loader.ts`'s
|
|
27
|
+
* `CONTROL_PLANE_MODULE_ID`, `fleet-checks.ts`'s `CONTROL_PLANE_MODULE`).
|
|
28
|
+
*
|
|
29
|
+
* It also does not print. It returns what happened and the caller renders it,
|
|
30
|
+
* so the same function serves a deploy, a test and any later caller without
|
|
31
|
+
* carrying a logger's opinions (Rule 10.1).
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
|
|
35
|
+
import { getEventBusPath } from '../config/paths';
|
|
36
|
+
import type { DbClient } from '../db/client';
|
|
37
|
+
import { type DnsServers, discoverDns } from './dns-discovery';
|
|
38
|
+
import { type FleetFinding, checkDispatcher } from './fleet-checks';
|
|
39
|
+
import { type FleetKey, ensureFleetKey } from './fleet-key';
|
|
40
|
+
import { type NetworkDiscoveryResult, discoverAndRecordNetwork } from './network-discovery';
|
|
41
|
+
import { initializeSystem } from './system-init';
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* How long to give a dispatcher that is still coming up.
|
|
45
|
+
*
|
|
46
|
+
* The Ansible role enables its supervisor unit moments before this runs, so a
|
|
47
|
+
* first probe legitimately finds nothing. Five one-second attempts is what the
|
|
48
|
+
* hook this replaced used, and no case has argued for more.
|
|
49
|
+
*/
|
|
50
|
+
const DISPATCHER_ATTEMPTS = 5;
|
|
51
|
+
const DISPATCHER_INTERVAL_MS = 1_000;
|
|
52
|
+
|
|
53
|
+
export interface ControlPlaneBootstrapOptions {
|
|
54
|
+
db: DbClient;
|
|
55
|
+
/** Injectable so tests drive the whole sequence without a host or a bus. */
|
|
56
|
+
discoverDnsImpl?: () => DnsServers;
|
|
57
|
+
ensureFleetKeyImpl?: () => FleetKey;
|
|
58
|
+
discoverNetworkImpl?: (db: DbClient) => NetworkDiscoveryResult;
|
|
59
|
+
/**
|
|
60
|
+
* One dispatcher reading. Injectable so a test needs no bus at all; the
|
|
61
|
+
* default opens celilo's own and runs the four-part check against it.
|
|
62
|
+
*/
|
|
63
|
+
probeDispatcher?: () => FleetFinding;
|
|
64
|
+
sleep?: (ms: number) => Promise<void>;
|
|
65
|
+
dispatcherAttempts?: number;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export interface ControlPlaneBootstrapResult {
|
|
69
|
+
dns: DnsServers;
|
|
70
|
+
fleetKey: FleetKey;
|
|
71
|
+
network: NetworkDiscoveryResult;
|
|
72
|
+
/** The last dispatcher reading. `fail` means no dispatcher ever answered. */
|
|
73
|
+
dispatcher: FleetFinding;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* The control plane reads the bus's own health tables and emits nothing, so it
|
|
78
|
+
* registers no event schemas — the same empty registry `celilo events` opens
|
|
79
|
+
* with.
|
|
80
|
+
*/
|
|
81
|
+
const NO_SCHEMAS = defineEvents({});
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* Read the dispatcher's state from celilo's own bus.
|
|
85
|
+
*
|
|
86
|
+
* `checkDispatcher` is deliberately used in place of the "is the process up?"
|
|
87
|
+
* read the hook did. Its docblock records why: a naive liveness check reports
|
|
88
|
+
* green on a dispatcher that is running, unsupervised, on stale code, and
|
|
89
|
+
* delivering nothing.
|
|
90
|
+
*/
|
|
91
|
+
/**
|
|
92
|
+
* Open celilo's own bus, do one thing with it, close it.
|
|
93
|
+
*
|
|
94
|
+
* There is an async twin below and the two are NOT interchangeable: handing an
|
|
95
|
+
* async function to this one closes the bus when the promise is CREATED, not
|
|
96
|
+
* when it settles, so the work runs against a closed handle.
|
|
97
|
+
*/
|
|
98
|
+
export function withCeliloBus<T>(fn: (bus: Bus) => T): T {
|
|
99
|
+
const bus: Bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
100
|
+
try {
|
|
101
|
+
return fn(bus);
|
|
102
|
+
} finally {
|
|
103
|
+
bus.close();
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** The async twin. See the warning on `withCeliloBus`. */
|
|
108
|
+
export async function withCeliloBusAsync<T>(fn: (bus: Bus) => Promise<T>): Promise<T> {
|
|
109
|
+
const bus: Bus = openBus({ dbPath: getEventBusPath(), events: NO_SCHEMAS });
|
|
110
|
+
try {
|
|
111
|
+
return await fn(bus);
|
|
112
|
+
} finally {
|
|
113
|
+
bus.close();
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function probeOwnDispatcher(): FleetFinding {
|
|
118
|
+
return withCeliloBus((bus) => checkDispatcher(bus));
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/** Poll until a dispatcher answers, or the attempts run out. */
|
|
122
|
+
async function waitForDispatcher(
|
|
123
|
+
probe: () => FleetFinding,
|
|
124
|
+
attempts: number,
|
|
125
|
+
sleep: (ms: number) => Promise<void>,
|
|
126
|
+
): Promise<FleetFinding> {
|
|
127
|
+
let finding = probe();
|
|
128
|
+
for (let attempt = 1; attempt < attempts && finding.status === 'fail'; attempt++) {
|
|
129
|
+
await sleep(DISPATCHER_INTERVAL_MS);
|
|
130
|
+
finding = probe();
|
|
131
|
+
}
|
|
132
|
+
return finding;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Record what celilo needs to know about the box it runs on.
|
|
137
|
+
*
|
|
138
|
+
* Ordering is not cosmetic. The fleet key and the DNS servers are written in
|
|
139
|
+
* ONE `initializeSystem` call, because that function applies defaults and the
|
|
140
|
+
* init sentinel alongside the keys it is given, and calling it twice makes the
|
|
141
|
+
* second call re-derive everything the first already settled. The hook this
|
|
142
|
+
* replaced made two round trips through the CLI for exactly that reason and
|
|
143
|
+
* had no way not to.
|
|
144
|
+
*
|
|
145
|
+
* The network is recorded after, not in the same call: `system apply-config`
|
|
146
|
+
* REFUSES `network.*` keys outright, so a network is not something you write
|
|
147
|
+
* through the config surface at all. `discoverAndRecordNetwork` is the only
|
|
148
|
+
* path, it is idempotent, and it never overwrites an operator's value.
|
|
149
|
+
*/
|
|
150
|
+
export async function bootstrapControlPlane(
|
|
151
|
+
options: ControlPlaneBootstrapOptions,
|
|
152
|
+
): Promise<ControlPlaneBootstrapResult> {
|
|
153
|
+
const {
|
|
154
|
+
db,
|
|
155
|
+
discoverDnsImpl = discoverDns,
|
|
156
|
+
ensureFleetKeyImpl = ensureFleetKey,
|
|
157
|
+
discoverNetworkImpl = discoverAndRecordNetwork,
|
|
158
|
+
probeDispatcher = probeOwnDispatcher,
|
|
159
|
+
sleep = (ms: number) => new Promise<void>((r) => setTimeout(r, ms)),
|
|
160
|
+
dispatcherAttempts = DISPATCHER_ATTEMPTS,
|
|
161
|
+
} = options;
|
|
162
|
+
|
|
163
|
+
const dns = discoverDnsImpl();
|
|
164
|
+
const fleetKey = ensureFleetKeyImpl();
|
|
165
|
+
|
|
166
|
+
initializeSystem(db, {
|
|
167
|
+
'dns.primary': dns.primary,
|
|
168
|
+
'dns.fallback': dns.fallback,
|
|
169
|
+
'ssh.public_key': fleetKey.publicKey,
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
const network = discoverNetworkImpl(db);
|
|
173
|
+
const dispatcher = await waitForDispatcher(probeDispatcher, dispatcherAttempts, sleep);
|
|
174
|
+
|
|
175
|
+
return { dns, fleetKey, network, dispatcher };
|
|
176
|
+
}
|