@celilo/cli 2.3.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CELILO_CORE_MODULES.md +1 -1
- package/package.json +3 -3
- package/src/api/serve.ts +13 -1
- package/src/cli/commands/module-health.test.ts +35 -0
- package/src/cli/commands/module-health.ts +11 -3
- package/src/cli/commands/module-publish.test.ts +22 -0
- package/src/cli/commands/module-publish.ts +14 -12
- package/src/cli/commands/module-update.ts +69 -17
- package/src/cli/commands/module-upgrade-gate.test.ts +154 -0
- package/src/cli/commands/module-upgrade.ts +43 -3
- package/src/cli/commands/publish/helpers.ts +4 -3
- package/src/cli/commands/publish/index.ts +13 -1
- package/src/cli/commands/publish/plan.test.ts +64 -0
- package/src/cli/commands/publish/plan.ts +52 -19
- package/src/cli/commands/publish/types.ts +16 -3
- package/src/cli/commands/subscribers-install-daemon.test.ts +44 -0
- package/src/cli/commands/subscribers-install-daemon.ts +107 -0
- package/src/cli/commands/subscribers-serve.test.ts +22 -0
- package/src/cli/commands/subscribers-serve.ts +22 -4
- package/src/cli/commands/system-audit.ts +14 -5
- package/src/cli/commands/system-update.ts +11 -6
- package/src/cli/completion.ts +10 -1
- package/src/cli/fuel-gauge.ts +12 -4
- package/src/cli/index.ts +12 -0
- package/src/cli/json-output.test.ts +81 -0
- package/src/cli/types.ts +9 -0
- package/src/hooks/broker.test.ts +9 -3
- package/src/hooks/executor.test.ts +5 -2
- package/src/hooks/hook-jail-toolchain-reach.test.ts +8 -3
- package/src/hooks/hook-jail-unreachability.test.ts +10 -2
- package/src/hooks/hook-trespass.test.ts +13 -4
- package/src/hooks/run-named-hook.ts +19 -16
- package/src/hooks/test-fixtures/artifact-writing-hook.ts +0 -1
- package/src/hooks/test-fixtures/capability-calling-hook.ts +20 -13
- package/src/hooks/test-fixtures/jail-probe-hook.ts +9 -1
- package/src/hooks/test-fixtures/jail-toolchain-hook.ts +7 -2
- package/src/hooks/test-fixtures/runaway-hook.ts +0 -1
- package/src/hooks/test-fixtures/sigterm-ignoring-hook.ts +0 -1
- package/src/hooks/test-fixtures/silent-hook.ts +0 -1
- package/src/hooks/test-fixtures/store-writing-hook.ts +20 -11
- package/src/hooks/test-fixtures/success-hook.ts +4 -4
- package/src/manifest/contracts/v1.ts +19 -14
- package/src/manifest/json-schema-roundtrip.test.ts +1 -1
- package/src/manifest/schema.ts +66 -16
- package/src/manifest/validate.test.ts +47 -0
- package/src/policy/capability-shape-baseline.ts +63 -21
- package/src/policy/capability-shape-drift.test.ts +53 -1
- package/src/policy/capability-shape.test.ts +105 -0
- package/src/policy/capability-shape.ts +283 -2
- package/src/registry/client.test.ts +65 -0
- package/src/registry/client.ts +7 -7
- package/src/secrets/storage.test.ts +69 -3
- package/src/secrets/storage.ts +71 -1
- package/src/services/audit/health.test.ts +58 -0
- package/src/services/audit/health.ts +15 -3
- package/src/services/audit/index.test.ts +1 -1
- package/src/services/audit/interface-classification.test.ts +16 -5
- package/src/services/audit/interface-classification.ts +25 -2
- package/src/services/audit/module-versions.ts +5 -1
- package/src/services/audit/public-dns.test.ts +20 -0
- package/src/services/audit/public-dns.ts +7 -2
- package/src/services/audit/recurrence-gate.test.ts +225 -0
- package/src/services/audit/trusted-sources.test.ts +17 -0
- package/src/services/audit/trusted-sources.ts +21 -0
- package/src/services/build-bus/hook-dispatch-executor.test.ts +269 -0
- package/src/services/build-bus/hook-dispatch-mgmt.test.ts +102 -150
- package/src/services/build-bus/hook-dispatch-path.test.ts +95 -0
- package/src/services/build-bus/hook-dispatch.test.ts +86 -116
- package/src/services/build-bus/hook-dispatch.ts +99 -121
- package/src/services/build-bus/hook-dispatcher.ts +106 -28
- package/src/services/build-bus/receiver-daemon.test.ts +189 -0
- package/src/services/build-bus/receiver-daemon.ts +355 -0
- package/src/services/build-bus/self-update.ts +156 -0
- package/src/services/capability-compat.test.ts +90 -0
- package/src/services/capability-compat.ts +128 -0
- package/src/services/deploy-terraform.ts +38 -1
- package/src/services/events-daemon.test.ts +57 -0
- package/src/services/events-daemon.ts +76 -0
- package/src/services/firewall-reach.ts +21 -8
- package/src/services/fleet-checks.test.ts +37 -1
- package/src/services/fleet-checks.ts +31 -1
- package/src/services/health-runner.ts +43 -5
- package/src/services/module-deploy-prune.test.ts +89 -0
- package/src/services/module-deploy.ts +72 -125
- package/src/services/module-types-drift.test.ts +1 -1
- package/src/services/module-validator/capability-versions.test.ts +13 -2
- package/src/services/terraform-safety.test.ts +83 -0
- package/src/services/terraform-safety.ts +53 -0
- package/src/services/update/orchestrator.test.ts +1 -1
- package/tsconfig.json +2 -13
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Capability compatibility between a module's declared requirements and the
|
|
3
|
+
* providers actually deployed in the fleet.
|
|
4
|
+
*
|
|
5
|
+
* The pure half mirrors the runtime lookup's semantics (see
|
|
6
|
+
* `deploy-preflight.ts` section 2): a requirement is served when ANY deployed
|
|
7
|
+
* provider satisfies it under `compareConsumerToProvider`. The reason
|
|
8
|
+
* `deploy-preflight` and the upgrade path must agree is the failure
|
|
9
|
+
* celilo#1361 recorded: celilo-website was upgraded against a caddy whose
|
|
10
|
+
* bundled provider rejected its publish calls, and every attempt burned a
|
|
11
|
+
* release. The upgrade path now asks this question BEFORE updating, and
|
|
12
|
+
* defers instead of deploying into a wall.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
import { compareConsumerToProvider } from '@celilo/capabilities';
|
|
16
|
+
import type { DbClient } from '../db/client';
|
|
17
|
+
import { capabilities } from '../db/schema';
|
|
18
|
+
import type { ModuleManifest } from '../manifest/schema';
|
|
19
|
+
import { isPrivilegedCapability } from '../manifest/validate';
|
|
20
|
+
|
|
21
|
+
export interface CapabilityRequirement {
|
|
22
|
+
name: string;
|
|
23
|
+
version: string;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface DeployedCapabilityProvider {
|
|
27
|
+
capabilityName: string;
|
|
28
|
+
moduleId: string;
|
|
29
|
+
version: string;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export interface UnservedCapability {
|
|
33
|
+
capability: string;
|
|
34
|
+
/** The version the module's manifest requires. */
|
|
35
|
+
required: string;
|
|
36
|
+
/** Example provider that does not serve it (the first mismatch found). */
|
|
37
|
+
providerModuleId: string | null;
|
|
38
|
+
providerVersion: string | null;
|
|
39
|
+
reason:
|
|
40
|
+
| 'no_provider'
|
|
41
|
+
| 'major_mismatch_higher'
|
|
42
|
+
| 'major_mismatch_lower'
|
|
43
|
+
| 'minor_mismatch_higher'
|
|
44
|
+
| 'caller_minor_too_old';
|
|
45
|
+
message: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Which of a module's capability requirements no deployed provider serves.
|
|
50
|
+
* Pure (Rule 10): the I/O — reading the capabilities table — happens in
|
|
51
|
+
* `capabilityBlockersForManifest`.
|
|
52
|
+
*/
|
|
53
|
+
export function unservedCapabilityRequirements(
|
|
54
|
+
requirements: CapabilityRequirement[] | undefined,
|
|
55
|
+
providers: DeployedCapabilityProvider[],
|
|
56
|
+
): UnservedCapability[] {
|
|
57
|
+
const unserved: UnservedCapability[] = [];
|
|
58
|
+
for (const cap of requirements ?? []) {
|
|
59
|
+
// Framework-granted privileges (e.g. cross_module_read) are not
|
|
60
|
+
// provider-backed — the same skip `deploy-preflight` makes.
|
|
61
|
+
if (isPrivilegedCapability(cap.name)) continue;
|
|
62
|
+
|
|
63
|
+
const installed = providers.filter((p) => p.capabilityName === cap.name);
|
|
64
|
+
if (installed.length === 0) {
|
|
65
|
+
unserved.push({
|
|
66
|
+
capability: cap.name,
|
|
67
|
+
required: cap.version,
|
|
68
|
+
providerModuleId: null,
|
|
69
|
+
providerVersion: null,
|
|
70
|
+
reason: 'no_provider',
|
|
71
|
+
message: `no deployed provider for '${cap.name}'`,
|
|
72
|
+
});
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
let anyCompatible = false;
|
|
76
|
+
let example: DeployedCapabilityProvider | undefined;
|
|
77
|
+
let reason: UnservedCapability['reason'] = 'major_mismatch_higher';
|
|
78
|
+
for (const p of installed) {
|
|
79
|
+
const result = compareConsumerToProvider(cap.version, p.version);
|
|
80
|
+
if (result.compatible) {
|
|
81
|
+
anyCompatible = true;
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
if (!example) {
|
|
85
|
+
example = p;
|
|
86
|
+
reason = result.reason;
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
if (anyCompatible || !example) continue;
|
|
90
|
+
unserved.push({
|
|
91
|
+
capability: cap.name,
|
|
92
|
+
required: cap.version,
|
|
93
|
+
providerModuleId: example.moduleId,
|
|
94
|
+
providerVersion: example.version,
|
|
95
|
+
reason,
|
|
96
|
+
message: `requires ${cap.name}@${cap.version} but deployed provider '${example.moduleId}' provides ${cap.name}@${example.version} (${reason})`,
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
return unserved;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Every capability provider currently registered in the fleet. */
|
|
103
|
+
export function readDeployedProviders(db: DbClient): DeployedCapabilityProvider[] {
|
|
104
|
+
return db
|
|
105
|
+
.select({
|
|
106
|
+
capabilityName: capabilities.capabilityName,
|
|
107
|
+
moduleId: capabilities.moduleId,
|
|
108
|
+
version: capabilities.version,
|
|
109
|
+
})
|
|
110
|
+
.from(capabilities)
|
|
111
|
+
.all();
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The upgrade-path entry point: which of a target manifest's capability
|
|
116
|
+
* requirements the deployed fleet cannot serve. A non-empty result means the
|
|
117
|
+
* upgrade would deploy into a wall the preflight would then reject — defer
|
|
118
|
+
* instead of burning the attempt.
|
|
119
|
+
*/
|
|
120
|
+
export function capabilityBlockersForManifest(
|
|
121
|
+
db: DbClient,
|
|
122
|
+
manifest: Pick<ModuleManifest, 'requires' | 'id'>,
|
|
123
|
+
): UnservedCapability[] {
|
|
124
|
+
return unservedCapabilityRequirements(
|
|
125
|
+
manifest.requires?.capabilities as CapabilityRequirement[] | undefined,
|
|
126
|
+
readDeployedProviders(db),
|
|
127
|
+
);
|
|
128
|
+
}
|
|
@@ -2,7 +2,7 @@ import { existsSync, readFileSync } from 'node:fs';
|
|
|
2
2
|
import { join } from 'node:path';
|
|
3
3
|
import { log } from '../cli/prompts';
|
|
4
4
|
import { executeBuildWithProgress } from './build-stream';
|
|
5
|
-
import { validateTerraformPlanSafety } from './terraform-safety';
|
|
5
|
+
import { planAfterImportVerdict, validateTerraformPlanSafety } from './terraform-safety';
|
|
6
6
|
|
|
7
7
|
export interface TerraformResult {
|
|
8
8
|
success: boolean;
|
|
@@ -173,6 +173,43 @@ export async function executeTerraform(
|
|
|
173
173
|
|
|
174
174
|
if (importResult.success) {
|
|
175
175
|
log.success('Resource imported into Terraform state');
|
|
176
|
+
|
|
177
|
+
// The import REPLACED the plan validated above. That plan described a
|
|
178
|
+
// create; the plan terraform computes now can describe a REPLACE,
|
|
179
|
+
// because `ostemplate` and `rootfs[0].storage` do not round-trip
|
|
180
|
+
// through an import and each forces one. Re-plan and re-check, or the
|
|
181
|
+
// retry applies a destroy the safety gate exists to forbid — which is
|
|
182
|
+
// how a redeploy destroyed signal's LXC on a converged fleet
|
|
183
|
+
// (celilo#1374).
|
|
184
|
+
const rePlan = await executeBuildWithProgress({
|
|
185
|
+
command: 'terraform',
|
|
186
|
+
args: ['plan', '-detailed-exitcode', '-no-color'],
|
|
187
|
+
cwd: terraformDir,
|
|
188
|
+
title: 'Re-planning after import',
|
|
189
|
+
env: terraformEnv,
|
|
190
|
+
noInteractive,
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
const verdict = planAfterImportVerdict(rePlan.exitCode, rePlan.output);
|
|
194
|
+
|
|
195
|
+
if (verdict.action === 'done') {
|
|
196
|
+
// The import alone reconciled state with reality: adopt and stop.
|
|
197
|
+
if (phases) phases.terraformApply = true;
|
|
198
|
+
log.success('Infrastructure adopted by import (no changes needed)');
|
|
199
|
+
return { success: true, output: rePlan.output, exitCode: 0 };
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
if (verdict.action === 'refuse') {
|
|
203
|
+
if (phases) phases.terraformApply = false;
|
|
204
|
+
log.error('Refusing to apply the plan computed after import');
|
|
205
|
+
return {
|
|
206
|
+
success: false,
|
|
207
|
+
output: rePlan.output,
|
|
208
|
+
error: verdict.error,
|
|
209
|
+
exitCode: 2,
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
|
|
176
213
|
log.message(' Retrying deployment...');
|
|
177
214
|
|
|
178
215
|
// Retry apply after successful import
|
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
readInstalledUnit,
|
|
11
11
|
renderLaunchdPlist,
|
|
12
12
|
renderSystemdUnit,
|
|
13
|
+
resolveDaemonPathEnv,
|
|
13
14
|
resolveRestartScope,
|
|
14
15
|
resolveRunAsUser,
|
|
15
16
|
restartDaemon,
|
|
@@ -27,11 +28,15 @@ describe('renderSystemdUnit', () => {
|
|
|
27
28
|
concurrency: 4,
|
|
28
29
|
home: '/home/op',
|
|
29
30
|
scope: 'user',
|
|
31
|
+
pathEnv: '/bun/bin:/usr/bin:/bin',
|
|
30
32
|
});
|
|
31
33
|
expect(out).toContain(
|
|
32
34
|
'ExecStart=/usr/local/bin/celilo events run --poll-ms 1000 --concurrency 4',
|
|
33
35
|
);
|
|
34
36
|
expect(out).toContain('Environment=EVENT_BUS_DB=/var/lib/celilo/events.db');
|
|
37
|
+
// celilo#1373 — the unit's environment must resolve bun; the global celilo
|
|
38
|
+
// wrapper is a bash script whose first act is `command -v bun`.
|
|
39
|
+
expect(out).toContain('Environment=PATH=/bun/bin:/usr/bin:/bin');
|
|
35
40
|
expect(out).toContain('Restart=on-failure');
|
|
36
41
|
expect(out).toContain('WantedBy=default.target');
|
|
37
42
|
expect(out).not.toContain('User=');
|
|
@@ -45,6 +50,7 @@ describe('renderSystemdUnit', () => {
|
|
|
45
50
|
concurrency: 8,
|
|
46
51
|
home: '/h',
|
|
47
52
|
scope: 'user',
|
|
53
|
+
pathEnv: '/bun/bin:/usr/bin',
|
|
48
54
|
});
|
|
49
55
|
expect(out).toContain('--poll-ms 250 --concurrency 8');
|
|
50
56
|
});
|
|
@@ -57,6 +63,7 @@ describe('renderSystemdUnit', () => {
|
|
|
57
63
|
concurrency: 4,
|
|
58
64
|
home: '/root',
|
|
59
65
|
scope: 'system',
|
|
66
|
+
pathEnv: '/bun/bin:/usr/bin',
|
|
60
67
|
runAsUser: 'celilo',
|
|
61
68
|
});
|
|
62
69
|
expect(out).toContain('User=celilo');
|
|
@@ -75,6 +82,7 @@ describe('renderLaunchdPlist', () => {
|
|
|
75
82
|
concurrency: 4,
|
|
76
83
|
home: '/Users/op',
|
|
77
84
|
scope: 'user',
|
|
85
|
+
pathEnv: '/bun/bin:/usr/bin:/bin',
|
|
78
86
|
});
|
|
79
87
|
expect(out).toContain('<string>/Users/op/Library/Logs/celilo-events.out.log</string>');
|
|
80
88
|
expect(out).toContain('<string>/Users/op/Library/Logs/celilo-events.err.log</string>');
|
|
@@ -85,6 +93,29 @@ describe('renderLaunchdPlist', () => {
|
|
|
85
93
|
expect(out).not.toContain('<key>UserName</key>');
|
|
86
94
|
});
|
|
87
95
|
|
|
96
|
+
// celilo#1373 — launchd starts jobs with its own default PATH (no bun),
|
|
97
|
+
// and the global celilo wrapper is a bash script whose first act is
|
|
98
|
+
// `command -v bun`. The rendered plist MUST carry a PATH that resolves
|
|
99
|
+
// bun, inside EnvironmentVariables, or the unit crash-loops forever.
|
|
100
|
+
it('carries a PATH in EnvironmentVariables that resolves bun', () => {
|
|
101
|
+
const out = renderLaunchdPlist({
|
|
102
|
+
celiloPath: '/Users/op/.bun/bin/celilo',
|
|
103
|
+
busDbPath: '/db',
|
|
104
|
+
pollMs: 1000,
|
|
105
|
+
concurrency: 4,
|
|
106
|
+
home: '/Users/op',
|
|
107
|
+
scope: 'user',
|
|
108
|
+
pathEnv: '/bun/bin:/usr/local/bin:/usr/bin:/bin',
|
|
109
|
+
});
|
|
110
|
+
expect(out).toContain('<key>PATH</key>');
|
|
111
|
+
expect(out).toContain('<string>/bun/bin:/usr/local/bin:/usr/bin:/bin</string>');
|
|
112
|
+
// PATH lives INSIDE the EnvironmentVariables dict, after EVENT_BUS_DB.
|
|
113
|
+
const envIdx = out.indexOf('<key>EnvironmentVariables</key>');
|
|
114
|
+
const pathIdx = out.indexOf('<key>PATH</key>');
|
|
115
|
+
expect(envIdx).toBeGreaterThan(-1);
|
|
116
|
+
expect(pathIdx).toBeGreaterThan(envIdx);
|
|
117
|
+
});
|
|
118
|
+
|
|
88
119
|
it('system scope sets UserName and logs under /Library/Logs', () => {
|
|
89
120
|
const out = renderLaunchdPlist({
|
|
90
121
|
celiloPath: '/c',
|
|
@@ -93,6 +124,7 @@ describe('renderLaunchdPlist', () => {
|
|
|
93
124
|
concurrency: 4,
|
|
94
125
|
home: '/Users/op',
|
|
95
126
|
scope: 'system',
|
|
127
|
+
pathEnv: '/bun/bin:/usr/bin',
|
|
96
128
|
runAsUser: 'celilo',
|
|
97
129
|
});
|
|
98
130
|
expect(out).toContain('<key>UserName</key>');
|
|
@@ -103,6 +135,31 @@ describe('renderLaunchdPlist', () => {
|
|
|
103
135
|
});
|
|
104
136
|
});
|
|
105
137
|
|
|
138
|
+
describe('resolveDaemonPathEnv', () => {
|
|
139
|
+
// celilo#1373 — the whole point of the PATH key is that the bun the
|
|
140
|
+
// wrapper needs is actually in the directory we wrote. First colon
|
|
141
|
+
// segment, and a bun executable must exist in it.
|
|
142
|
+
it('starts with a directory that contains a bun executable', () => {
|
|
143
|
+
const firstDir = resolveDaemonPathEnv().split(':')[0] ?? '';
|
|
144
|
+
expect(firstDir).not.toBe('');
|
|
145
|
+
expect(existsSync(join(firstDir, 'bun'))).toBe(true);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
it('planDaemonInstall pins the PATH into the rendered unit', () => {
|
|
149
|
+
const dir = mkdtempSync(join(tmpdir(), 'celilo-pathenv-'));
|
|
150
|
+
const celiloPath = join(dir, 'fake-celilo');
|
|
151
|
+
writeFileSync(celiloPath, '#!/bin/sh\n', { mode: 0o755 });
|
|
152
|
+
const plan = planDaemonInstall({
|
|
153
|
+
platform: 'darwin',
|
|
154
|
+
scope: 'user',
|
|
155
|
+
home: join(dir, 'home'),
|
|
156
|
+
celiloPath,
|
|
157
|
+
pathEnv: '/bun/bin:/usr/bin:/bin',
|
|
158
|
+
});
|
|
159
|
+
expect(plan.unitContent).toContain('<string>/bun/bin:/usr/bin:/bin</string>');
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
|
|
106
163
|
describe('getDaemonUnitPath', () => {
|
|
107
164
|
it('returns the systemd user path on linux', () => {
|
|
108
165
|
expect(getDaemonUnitPath('linux', '/home/op')).toBe(
|
|
@@ -48,6 +48,12 @@ export interface InstallDaemonOptions {
|
|
|
48
48
|
platform?: SupervisorPlatform;
|
|
49
49
|
/** Override the bus DB path. Defaults to celilo's getEventBusPath(). */
|
|
50
50
|
busDbPath?: string;
|
|
51
|
+
/**
|
|
52
|
+
* Override the PATH written into the unit's environment. Defaults to
|
|
53
|
+
* resolveDaemonPathEnv() captured at render time. Test seam — pin it so
|
|
54
|
+
* assertions don't depend on the machine's bun install.
|
|
55
|
+
*/
|
|
56
|
+
pathEnv?: string;
|
|
51
57
|
/** Override the home directory used to compute install paths. */
|
|
52
58
|
home?: string;
|
|
53
59
|
/** Prefix for system-scope paths. Test seam — see getDaemonUnitPath. */
|
|
@@ -159,6 +165,47 @@ export function getDaemonUnitPath(
|
|
|
159
165
|
: join(home, 'Library', 'LaunchAgents', `${LAUNCHD_LABEL}.plist`);
|
|
160
166
|
}
|
|
161
167
|
|
|
168
|
+
export interface LaunchdUnitStatus {
|
|
169
|
+
/** The pid launchd currently runs the unit under, or null when it isn't running. */
|
|
170
|
+
pid: number | null;
|
|
171
|
+
/**
|
|
172
|
+
* The unit's last exit status in wait4 form (256 = exited 1, 15 = killed by
|
|
173
|
+
* SIGTERM), or null when launchd has no record of one.
|
|
174
|
+
*/
|
|
175
|
+
lastExitStatus: number | null;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
/**
|
|
179
|
+
* Probe launchd for the unit's health. Uses `launchctl list <label>` —
|
|
180
|
+
* on current macOS the richer `launchctl print gui/<uid>/<label>` answers
|
|
181
|
+
* "Domain does not support specified action" for legacy-loaded jobs, while
|
|
182
|
+
* `list <label>` prints a legacy-style dict carrying `PID` (absent when not
|
|
183
|
+
* running) and `LastExitStatus`.
|
|
184
|
+
*
|
|
185
|
+
* Returns null when launchd doesn't answer at all (not loaded, linux, the
|
|
186
|
+
* binary missing) — null is "don't know", never "bad". Only the user scope
|
|
187
|
+
* is probed: the system domain needs root, and a mac running a system-scope
|
|
188
|
+
* celilo daemon is not a supported shape (see supervisorCommands).
|
|
189
|
+
*/
|
|
190
|
+
export function launchdUnitStatus(scope: SupervisorScope = 'user'): LaunchdUnitStatus | null {
|
|
191
|
+
if (scope !== 'user' || detectPlatform() !== 'darwin') return null;
|
|
192
|
+
let out: string;
|
|
193
|
+
try {
|
|
194
|
+
out = execFileSync('launchctl', ['list', LAUNCHD_LABEL], {
|
|
195
|
+
encoding: 'utf-8',
|
|
196
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
197
|
+
});
|
|
198
|
+
} catch {
|
|
199
|
+
return null;
|
|
200
|
+
}
|
|
201
|
+
const pidMatch = out.match(/"PID" = (\d+)/);
|
|
202
|
+
const exitMatch = out.match(/"LastExitStatus" = (-?\d+)/);
|
|
203
|
+
return {
|
|
204
|
+
pid: pidMatch ? Number(pidMatch[1]) : null,
|
|
205
|
+
lastExitStatus: exitMatch ? Number(exitMatch[1]) : null,
|
|
206
|
+
};
|
|
207
|
+
}
|
|
208
|
+
|
|
162
209
|
/**
|
|
163
210
|
* Find the celilo executable. Priority:
|
|
164
211
|
* 1. Explicit override
|
|
@@ -185,6 +232,25 @@ export function resolveCeliloPath(override?: string): string {
|
|
|
185
232
|
return found;
|
|
186
233
|
}
|
|
187
234
|
|
|
235
|
+
/**
|
|
236
|
+
* A PATH that resolves bun, for a supervisor unit's environment.
|
|
237
|
+
*
|
|
238
|
+
* launchd starts jobs with its own default PATH, and systemd runs
|
|
239
|
+
* ExecStart with a similarly minimal one. Neither includes bun, and the
|
|
240
|
+
* global `celilo` entry is a bash wrapper whose first act is
|
|
241
|
+
* `command -v bun` — so a unit without a PATH key crash-loops forever on
|
|
242
|
+
* "Error: celilo requires bun" (celilo#1373: 22680 respawns on one Mac).
|
|
243
|
+
* The PATH is captured at RENDER time — the moment the unit is written —
|
|
244
|
+
* from Bun.which('bun'), falling back to process.execPath, which IS the
|
|
245
|
+
* bun binary under the bun runtime. Standard dirs follow so the
|
|
246
|
+
* dispatcher's own subprocesses keep working.
|
|
247
|
+
*/
|
|
248
|
+
export function resolveDaemonPathEnv(): string {
|
|
249
|
+
const bun =
|
|
250
|
+
(Bun as unknown as { which: (cmd: string) => string | null }).which('bun') ?? process.execPath;
|
|
251
|
+
return `${dirname(bun)}:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin`;
|
|
252
|
+
}
|
|
253
|
+
|
|
188
254
|
/**
|
|
189
255
|
* Run-as user for a system-scope unit: the owner of the celilo state
|
|
190
256
|
* dir (the bus DB's directory). Falls back to the current user when
|
|
@@ -211,6 +277,11 @@ interface UnitInputs {
|
|
|
211
277
|
concurrency: number;
|
|
212
278
|
home: string;
|
|
213
279
|
scope: SupervisorScope;
|
|
280
|
+
/**
|
|
281
|
+
* PATH injected into the unit's environment (celilo#1373). Required:
|
|
282
|
+
* a unit without it cannot resolve bun and crash-loops.
|
|
283
|
+
*/
|
|
284
|
+
pathEnv: string;
|
|
214
285
|
/** Required for system scope; ignored for user scope. */
|
|
215
286
|
runAsUser?: string;
|
|
216
287
|
}
|
|
@@ -232,6 +303,7 @@ ${userLine}ExecStart=${input.celiloPath} events run --poll-ms ${input.pollMs} --
|
|
|
232
303
|
Restart=on-failure
|
|
233
304
|
RestartSec=10s
|
|
234
305
|
Environment=EVENT_BUS_DB=${input.busDbPath}
|
|
306
|
+
Environment=PATH=${input.pathEnv}
|
|
235
307
|
# stdout/stderr are captured by ${journalHint}.
|
|
236
308
|
StandardOutput=journal
|
|
237
309
|
StandardError=journal
|
|
@@ -271,6 +343,8 @@ ${userNameBlock} <key>ProgramArguments</key>
|
|
|
271
343
|
<dict>
|
|
272
344
|
<key>EVENT_BUS_DB</key>
|
|
273
345
|
<string>${input.busDbPath}</string>
|
|
346
|
+
<key>PATH</key>
|
|
347
|
+
<string>${input.pathEnv}</string>
|
|
274
348
|
</dict>
|
|
275
349
|
<key>RunAtLoad</key>
|
|
276
350
|
<true/>
|
|
@@ -334,6 +408,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
|
|
|
334
408
|
const home = opts.home ?? homedir();
|
|
335
409
|
const celiloPath = resolveCeliloPath(opts.celiloPath);
|
|
336
410
|
const busDbPath = opts.busDbPath ?? getEventBusPath();
|
|
411
|
+
const pathEnv = opts.pathEnv ?? resolveDaemonPathEnv();
|
|
337
412
|
const pollMs = opts.pollMs ?? 1000;
|
|
338
413
|
const concurrency = opts.concurrency ?? 4;
|
|
339
414
|
const runAsUser = scope === 'system' ? resolveRunAsUser(busDbPath, opts.runAsUser) : undefined;
|
|
@@ -346,6 +421,7 @@ export function planDaemonInstall(opts: InstallDaemonOptions = {}): InstallDaemo
|
|
|
346
421
|
concurrency,
|
|
347
422
|
home,
|
|
348
423
|
scope,
|
|
424
|
+
pathEnv,
|
|
349
425
|
runAsUser,
|
|
350
426
|
};
|
|
351
427
|
const unitContent =
|
|
@@ -57,27 +57,40 @@ export function loadZoneSubnets(db: DbClient): string[] {
|
|
|
57
57
|
|
|
58
58
|
/**
|
|
59
59
|
* Read each firewall's live ruleset and pair it with celilo's composed trusted
|
|
60
|
-
* set.
|
|
61
|
-
*
|
|
60
|
+
* set. Firewalls whose ruleset cannot be read are reported separately, as
|
|
61
|
+
* `unreachable` — the audit turns each into an `unmeasured` finding, because
|
|
62
|
+
* an unreadable box is unknown, not clean (D7).
|
|
62
63
|
*/
|
|
63
|
-
export
|
|
64
|
+
export interface FirewallReachCollection {
|
|
65
|
+
firewalls: FirewallReachState[];
|
|
66
|
+
unreachableFirewalls: string[];
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function collectFirewallReach(
|
|
70
|
+
db: DbClient,
|
|
71
|
+
run: Runner = execRunner,
|
|
72
|
+
): FirewallReachCollection {
|
|
64
73
|
const zoneSubnets = loadZoneSubnets(db);
|
|
65
|
-
if (zoneSubnets.length === 0) return [];
|
|
74
|
+
if (zoneSubnets.length === 0) return { firewalls: [], unreachableFirewalls: [] };
|
|
66
75
|
|
|
67
|
-
const
|
|
76
|
+
const firewalls: FirewallReachState[] = [];
|
|
77
|
+
const unreachableFirewalls: string[] = [];
|
|
68
78
|
for (const firewallIp of listFirewallIps(db)) {
|
|
69
79
|
// escape-hatch: reads the running filter table (`iptables-save`), which is
|
|
70
80
|
// the only place an out-of-band rule appears. Read-only, no capability.
|
|
71
81
|
const result = runAppCommand({ ipv4_address: firewallIp }, 'iptables-save -t filter', run, {
|
|
72
82
|
timeoutMs: IPTABLES_SAVE_TIMEOUT_MS,
|
|
73
83
|
});
|
|
74
|
-
if (!result.ok)
|
|
84
|
+
if (!result.ok) {
|
|
85
|
+
unreachableFirewalls.push(firewallIp);
|
|
86
|
+
continue;
|
|
87
|
+
}
|
|
75
88
|
|
|
76
|
-
|
|
89
|
+
firewalls.push({
|
|
77
90
|
firewallIp,
|
|
78
91
|
live: parseLiveReachRules(result.stdout, zoneSubnets),
|
|
79
92
|
known: loadTrustedSubnets(db, firewallIp).map((e) => e.subnet),
|
|
80
93
|
});
|
|
81
94
|
}
|
|
82
|
-
return
|
|
95
|
+
return { firewalls, unreachableFirewalls };
|
|
83
96
|
}
|
|
@@ -114,6 +114,41 @@ describe('checkDispatcher', () => {
|
|
|
114
114
|
expect(f.remediation).toContain('enable --now celilo-events.service');
|
|
115
115
|
});
|
|
116
116
|
|
|
117
|
+
// celilo#1373 — the unit is installed and launchd keeps respawning it, but
|
|
118
|
+
// it dies before it can heartbeat (no PATH → the wrapper can't find
|
|
119
|
+
// bun). The finding must NAME the crash loop, not tell the operator to
|
|
120
|
+
// "start the dispatcher".
|
|
121
|
+
it('names a crash-looping launchd unit when no dispatcher is live (darwin)', () => {
|
|
122
|
+
installFakeUnit(home);
|
|
123
|
+
const f = checkDispatcher(bus, {
|
|
124
|
+
now,
|
|
125
|
+
home,
|
|
126
|
+
platform: 'darwin',
|
|
127
|
+
launchdProbe: () => ({ pid: null, lastExitStatus: 256 }),
|
|
128
|
+
});
|
|
129
|
+
expect(f.status).toBe('fail');
|
|
130
|
+
expect(f.detail.join(' ')).toContain('crash-looping');
|
|
131
|
+
expect(f.detail.join(' ')).toContain('exit code 1');
|
|
132
|
+
expect(f.remediation).toContain('celilo events install-daemon');
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it('does not claim a crash loop when launchd runs the unit or cannot answer', () => {
|
|
136
|
+
installFakeUnit(home);
|
|
137
|
+
for (const probe of [
|
|
138
|
+
() => ({ pid: 4242, lastExitStatus: 256 }), // running despite an old failure
|
|
139
|
+
() => null, // launchd didn't answer — ignorance, not evidence
|
|
140
|
+
]) {
|
|
141
|
+
const f = checkDispatcher(bus, {
|
|
142
|
+
now,
|
|
143
|
+
home,
|
|
144
|
+
platform: 'darwin',
|
|
145
|
+
launchdProbe: probe,
|
|
146
|
+
});
|
|
147
|
+
expect(f.status).toBe('fail');
|
|
148
|
+
expect(f.detail.join(' ')).not.toContain('crash-looping');
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
|
|
117
152
|
it('passes when running, supervised, current, and not behind on timer ticks', () => {
|
|
118
153
|
seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
|
|
119
154
|
installFakeUnit(home);
|
|
@@ -358,7 +393,8 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
|
|
|
358
393
|
const f = checkBuildBusPublishing(bus, db, { now });
|
|
359
394
|
expect(f.status).toBe('warn');
|
|
360
395
|
expect(f.summary).toContain('no build-bus publish has ever arrived');
|
|
361
|
-
expect(f.remediation).toContain('subscribers
|
|
396
|
+
expect(f.remediation).toContain('subscribers install-daemon');
|
|
397
|
+
expect(f.remediation).toContain('CELILO_BUS_SECRET');
|
|
362
398
|
});
|
|
363
399
|
|
|
364
400
|
it('warns when publishes arrive but no hook has ever run', () => {
|
|
@@ -39,9 +39,11 @@ import {
|
|
|
39
39
|
} from './deployed-systems';
|
|
40
40
|
import { listDnsInternalRecords } from './dns-internal-records';
|
|
41
41
|
import {
|
|
42
|
+
type LaunchdUnitStatus,
|
|
42
43
|
SUPERVISOR_SCOPES,
|
|
43
44
|
type SupervisorPlatform,
|
|
44
45
|
type SupervisorScope,
|
|
46
|
+
launchdUnitStatus,
|
|
45
47
|
readInstalledUnit,
|
|
46
48
|
unitMainPid,
|
|
47
49
|
} from './events-daemon';
|
|
@@ -196,6 +198,11 @@ export interface DispatcherCheckOptions {
|
|
|
196
198
|
* no supervision claim rather than guessing.
|
|
197
199
|
*/
|
|
198
200
|
unitMainPid?: (scope: SupervisorScope) => number | null;
|
|
201
|
+
/**
|
|
202
|
+
* Probe launchd for the unit's health (darwin only — see launchdUnitStatus).
|
|
203
|
+
* Injected so the crash-loop aspect is testable without launchd.
|
|
204
|
+
*/
|
|
205
|
+
launchdProbe?: (scope: SupervisorScope) => LaunchdUnitStatus | null;
|
|
199
206
|
}
|
|
200
207
|
|
|
201
208
|
/**
|
|
@@ -228,6 +235,29 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
|
|
|
228
235
|
remediations.push(
|
|
229
236
|
'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
|
|
230
237
|
);
|
|
238
|
+
// A unit can be INSTALLED and still dead: the supervisor respawns a
|
|
239
|
+
// program that cannot start in the unit's environment and it dies again,
|
|
240
|
+
// forever. On macOS the classic cause is a PATH-less
|
|
241
|
+
// EnvironmentVariables dict — the global celilo wrapper needs
|
|
242
|
+
// `command -v bun`, and launchd's default PATH has none (celilo#1373:
|
|
243
|
+
// 22680 respawns on one Mac). Name the crash loop instead of telling
|
|
244
|
+
// the operator to "start the dispatcher".
|
|
245
|
+
if (opts.platform === 'darwin') {
|
|
246
|
+
const probe = opts.launchdProbe ?? launchdUnitStatus;
|
|
247
|
+
const st = probe('user');
|
|
248
|
+
if (st && st.pid === null && st.lastExitStatus != null && st.lastExitStatus !== 0) {
|
|
249
|
+
const fate =
|
|
250
|
+
st.lastExitStatus & 0xff
|
|
251
|
+
? `signal ${st.lastExitStatus & 0xff}`
|
|
252
|
+
: `exit code ${st.lastExitStatus >> 8}`;
|
|
253
|
+
detail.push(
|
|
254
|
+
`the installed launchd unit is crash-looping (last ${fate}) — launchd respawns it and it dies again, so no dispatcher ever serves the bus`,
|
|
255
|
+
);
|
|
256
|
+
remediations.push(
|
|
257
|
+
"read the unit's stderr (`tail ~/Library/Logs/celilo-events.err.log`) and reinstall with `celilo events install-daemon` so the unit carries an environment that resolves bun (celilo#1373)",
|
|
258
|
+
);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
231
261
|
// Without a heartbeat there's nothing more to assert about it.
|
|
232
262
|
return {
|
|
233
263
|
id: 'dispatcher',
|
|
@@ -449,7 +479,7 @@ export function checkBuildBusPublishing(
|
|
|
449
479
|
[
|
|
450
480
|
'The webhook receiver (`celilo subscribers serve`) either is not running here or was never sent a verified publish — and nothing in the deploy installs it.',
|
|
451
481
|
],
|
|
452
|
-
"
|
|
482
|
+
"install the receiver under a supervisor unit: `celilo subscribers install-daemon --secret $CELILO_BUS_SECRET` (systemd/launchd; `--print` renders for Ansible), enable it per its next-steps output, and register this host in the publisher's build-bus-subscribers.json with the same secret",
|
|
453
483
|
);
|
|
454
484
|
}
|
|
455
485
|
|