@celilo/cli 4.0.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +2 -2
- package/CELILO_SUBSYSTEMS.md +1 -0
- package/drizzle/0034_private_routes.sql +41 -0
- package/drizzle/meta/_journal.json +8 -1
- package/package.json +5 -4
- package/src/api/sessions.test.ts +1 -1
- package/src/capabilities/registration.test.ts +11 -11
- package/src/capabilities/secrets.test.ts +18 -18
- package/src/capabilities/validation.test.ts +24 -24
- package/src/cli/cli.test.ts +3 -3
- package/src/cli/command-tree-parser.test.ts +18 -0
- package/src/cli/command-tree-parser.ts +12 -0
- package/src/cli/commands/api.ts +67 -14
- package/src/cli/commands/events.ts +5 -2
- package/src/cli/commands/machine-reclassify.ts +79 -0
- package/src/cli/commands/module-changeset.test.ts +1 -1
- package/src/cli/commands/module-generate.test.ts +2 -2
- package/src/cli/commands/module-import-aspect.test.ts +2 -2
- package/src/cli/commands/module-update.test.ts +14 -14
- package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
- package/src/cli/commands/module-upgrade.test.ts +3 -3
- package/src/cli/commands/module-upgrade.ts +12 -0
- package/src/cli/commands/module-version.test.ts +1 -1
- package/src/cli/commands/system-audit.ts +22 -0
- package/src/cli/commands/system-reboot.ts +126 -0
- package/src/cli/commands/system-update.ts +5 -0
- package/src/cli/completion.ts +3 -2
- package/src/cli/index.ts +41 -2
- package/src/cli/restore-migration-failure.test.ts +1 -1
- package/src/cli/stdout-pipe-flush.test.ts +35 -0
- package/src/cli/tui/audit-state.ts +4 -0
- package/src/db/private-routes.test.ts +159 -0
- package/src/db/schema.ts +49 -0
- package/src/hooks/broker.ts +31 -0
- package/src/hooks/capability-loader.ts +185 -12
- package/src/hooks/executor.test.ts +13 -13
- package/src/hooks/executor.ts +20 -2
- package/src/hooks/hook-state-dir.test.ts +2 -2
- package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
- package/src/hooks/test-fixtures/store-backed.ts +5 -3
- package/src/hooks/timeout-names-provider.test.ts +87 -0
- package/src/manifest/contracts/index.ts +28 -7
- package/src/manifest/contracts/v2.ts +32 -0
- package/src/manifest/icon-schema.test.ts +1 -1
- package/src/manifest/schema.ts +13 -2
- package/src/manifest/template-validator.test.ts +1 -1
- package/src/manifest/validate-privileged.test.ts +1 -1
- package/src/manifest/validate.test.ts +68 -29
- package/src/manifest/validate.ts +14 -1
- package/src/module/import.test.ts +7 -7
- package/src/module/packaging/build.test.ts +1 -1
- package/src/packaging/nfpm-depends.test.ts +42 -0
- package/src/policy/capability-shape-baseline.ts +7 -1
- package/src/policy/fixture-capability-coverage.test.ts +5 -5
- package/src/policy/module-business-baseline.ts +33 -4
- package/src/services/api-access.test.ts +44 -0
- package/src/services/aspect-approvals.test.ts +2 -2
- package/src/services/aspect-reconcile.test.ts +2 -2
- package/src/services/aspect-runner.test.ts +4 -4
- package/src/services/audit/backups.test.ts +1 -1
- package/src/services/audit/capability-abi.test.ts +1 -1
- package/src/services/audit/index.test.ts +7 -0
- package/src/services/audit/index.ts +15 -0
- package/src/services/audit/machine-interface-zones.test.ts +30 -0
- package/src/services/audit/machine-interface-zones.ts +43 -0
- package/src/services/audit/module-configs.test.ts +1 -1
- package/src/services/audit/reboot-pending.test.ts +63 -0
- package/src/services/audit/reboot-pending.ts +85 -0
- package/src/services/audit/recurrence-gate.test.ts +21 -0
- package/src/services/audit/server-bun-pin.test.ts +44 -0
- package/src/services/audit/server-bun-pin.ts +95 -0
- package/src/services/audit/types.ts +2 -0
- package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
- package/src/services/bus-interview-park.test.ts +1 -1
- package/src/services/bus-secret-flow.test.ts +1 -1
- package/src/services/capability-compat.test.ts +22 -0
- package/src/services/capability-compat.ts +15 -2
- package/src/services/capability-table-rows.test.ts +20 -2
- package/src/services/celilo-events.test.ts +119 -6
- package/src/services/celilo-events.ts +156 -16
- package/src/services/celilo-mgmt-hooks.test.ts +6 -3
- package/src/services/cross-module-read.test.ts +2 -2
- package/src/services/deploy-preflight.ts +12 -3
- package/src/services/deploy-validation.test.ts +86 -7
- package/src/services/deploy-validation.ts +19 -0
- package/src/services/dns-provider-backfill.test.ts +2 -2
- package/src/services/dns-registrations.test.ts +1 -1
- package/src/services/fleet-checks.test.ts +2 -2
- package/src/services/fleet-checks.ts +40 -10
- package/src/services/health-runner.test.ts +1 -1
- package/src/services/infrastructure-variable-resolver.test.ts +11 -11
- package/src/services/instance-ops.test.ts +2 -2
- package/src/services/machine-interface-zones.test.ts +73 -0
- package/src/services/machine-interface-zones.ts +101 -0
- package/src/services/module-deploy.dns-repoint.test.ts +1 -1
- package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
- package/src/services/module-deploy.ts +31 -10
- package/src/services/module-pause-quiescence.test.ts +1 -1
- package/src/services/module-pause.test.ts +1 -1
- package/src/services/module-subscriptions.test.ts +2 -2
- package/src/services/module-types-generator.test.ts +1 -1
- package/src/services/module-validator/contract-version.test.ts +3 -3
- package/src/services/network-ensure.test.ts +1 -1
- package/src/services/provider-arrival.test.ts +62 -46
- package/src/services/provider-arrival.ts +41 -10
- package/src/services/proxmox-reconcile.test.ts +2 -2
- package/src/services/remove-guard.test.ts +1 -1
- package/src/services/system-reboot.test.ts +231 -0
- package/src/services/system-reboot.ts +357 -0
- package/src/services/update/dep-graph.test.ts +1 -1
- package/src/services/update/orchestrator.test.ts +165 -1
- package/src/services/update/orchestrator.ts +43 -12
- package/src/services/update/types.ts +10 -1
- package/src/templates/generator.test.ts +15 -4
- package/src/templates/generator.ts +12 -4
- package/src/variables/capability-self-ref.test.ts +2 -2
- package/src/variables/computed/computed-integration.test.ts +1 -1
- package/src/variables/context.ts +14 -1
- package/src/variables/declarative-derivation.test.ts +19 -19
|
@@ -5,6 +5,7 @@ import { join } from 'node:path';
|
|
|
5
5
|
import { skipIntegration } from '../test-utils/integration-guard';
|
|
6
6
|
|
|
7
7
|
import type { HookContext } from '@celilo/capabilities';
|
|
8
|
+
import { configStore, secretStore } from '../hooks/test-fixtures/store-backed';
|
|
8
9
|
|
|
9
10
|
// Hooks under test — imported via the scripts dir adjacent to the
|
|
10
11
|
// modules/celilo-mgmt/manifest.yml. The dynamic import keeps the test
|
|
@@ -46,7 +47,7 @@ function buildContext(extras: Record<string, unknown>): HookContext {
|
|
|
46
47
|
const { logger } = buildLogger();
|
|
47
48
|
return {
|
|
48
49
|
logger,
|
|
49
|
-
config: {
|
|
50
|
+
config: configStore({
|
|
50
51
|
db_path: '', // populated by caller via env
|
|
51
52
|
hostname: 'test-mgmt',
|
|
52
53
|
target_ip: '127.0.0.1',
|
|
@@ -61,9 +62,11 @@ function buildContext(extras: Record<string, unknown>): HookContext {
|
|
|
61
62
|
ssh_public_key: '',
|
|
62
63
|
install_docker: false,
|
|
63
64
|
install_terraform: false,
|
|
64
|
-
}
|
|
65
|
-
secrets:
|
|
65
|
+
}),
|
|
66
|
+
secrets: secretStore(),
|
|
66
67
|
capabilities: {},
|
|
68
|
+
systems: [],
|
|
69
|
+
stateDir: '',
|
|
67
70
|
debug: false,
|
|
68
71
|
screenshotDir: '',
|
|
69
72
|
...extras,
|
|
@@ -25,7 +25,7 @@ import {
|
|
|
25
25
|
|
|
26
26
|
function buildManifest(caps: Array<{ name: string; version: string }>): ModuleManifest {
|
|
27
27
|
return {
|
|
28
|
-
celilo_contract: '
|
|
28
|
+
celilo_contract: '2.0',
|
|
29
29
|
id: 'm',
|
|
30
30
|
name: 'm',
|
|
31
31
|
version: '0.0.1',
|
|
@@ -88,7 +88,7 @@ describe('materializeCrossModuleRoot', () => {
|
|
|
88
88
|
name: id,
|
|
89
89
|
version,
|
|
90
90
|
sourcePath: join(dir, 'sources', id),
|
|
91
|
-
manifestData: { id, name: id, version, celilo_contract: '
|
|
91
|
+
manifestData: { id, name: id, version, celilo_contract: '2.0', requires: {} } as Record<
|
|
92
92
|
string,
|
|
93
93
|
unknown
|
|
94
94
|
>,
|
|
@@ -112,8 +112,15 @@ export async function runPreflight(
|
|
|
112
112
|
// drift after deploy; this surface refuses up front so the
|
|
113
113
|
// operator gets an actionable error instead of a silent
|
|
114
114
|
// template-generation against an ABI the provider can't fulfil.
|
|
115
|
-
|
|
116
|
-
|
|
115
|
+
// An OPTIONAL capability joins the version check but not the presence check:
|
|
116
|
+
// its provider may legitimately be absent, but when one IS installed the
|
|
117
|
+
// module's hooks call it, so another major breaks at runtime (celilo#1384).
|
|
118
|
+
{
|
|
119
|
+
const consumed = [
|
|
120
|
+
...(manifest.requires?.capabilities ?? []).map((cap) => ({ cap, required: true })),
|
|
121
|
+
...(manifest.optional?.capabilities ?? []).map((cap) => ({ cap, required: false })),
|
|
122
|
+
];
|
|
123
|
+
for (const { cap, required } of consumed) {
|
|
117
124
|
// Framework-granted privileges (e.g. cross_module_read) aren't
|
|
118
125
|
// provider-backed — they're gated by the allow-list at import time,
|
|
119
126
|
// not satisfied by deploying another module. Skip them here.
|
|
@@ -126,6 +133,7 @@ export async function runPreflight(
|
|
|
126
133
|
.where(eq(capabilities.capabilityName, cap.name))
|
|
127
134
|
.all();
|
|
128
135
|
if (providers.length === 0) {
|
|
136
|
+
if (!required) continue;
|
|
129
137
|
errors.push({
|
|
130
138
|
category: 'missing-capability',
|
|
131
139
|
message: `Required capability '${cap.name}' has no provider installed`,
|
|
@@ -154,7 +162,8 @@ export async function runPreflight(
|
|
|
154
162
|
errors.push({
|
|
155
163
|
category: 'capability-version-mismatch',
|
|
156
164
|
message:
|
|
157
|
-
`Module '${moduleId}' requires
|
|
165
|
+
`Module '${moduleId}' ${required ? 'requires' : 'optionally uses'} ` +
|
|
166
|
+
`${cap.name}@${cap.version} but ` +
|
|
158
167
|
`provider '${example.moduleId}' offers ${cap.name}@${example.version}`,
|
|
159
168
|
suggestion:
|
|
160
169
|
reason === 'caller_minor_too_old'
|
|
@@ -19,15 +19,19 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
|
19
19
|
import { tmpdir } from 'node:os';
|
|
20
20
|
import { join } from 'node:path';
|
|
21
21
|
import { type DbClient, getDb } from '../db/client';
|
|
22
|
-
import { modules, secrets } from '../db/schema';
|
|
22
|
+
import { capabilities, modules, secrets } from '../db/schema';
|
|
23
23
|
import type { ModuleManifest } from '../manifest/schema';
|
|
24
24
|
import { resetTestDbPath } from '../test-utils/db-path';
|
|
25
25
|
import { findMissingSecrets } from './config-interview';
|
|
26
|
-
import {
|
|
26
|
+
import {
|
|
27
|
+
findMissingBuildArtifacts,
|
|
28
|
+
findMissingRequiredVariables,
|
|
29
|
+
validateAndPrepareDeployment,
|
|
30
|
+
} from './deploy-validation';
|
|
27
31
|
|
|
28
32
|
function manifestWithStringMapSecret(): ModuleManifest {
|
|
29
33
|
return {
|
|
30
|
-
celilo_contract: '
|
|
34
|
+
celilo_contract: '2.0',
|
|
31
35
|
id: 'testmod',
|
|
32
36
|
name: 'Test Module',
|
|
33
37
|
version: '1.0.0',
|
|
@@ -50,7 +54,7 @@ function manifestWithStringMapSecret(): ModuleManifest {
|
|
|
50
54
|
|
|
51
55
|
function manifestWithPlainStringSecret(): ModuleManifest {
|
|
52
56
|
return {
|
|
53
|
-
celilo_contract: '
|
|
57
|
+
celilo_contract: '2.0',
|
|
54
58
|
id: 'testmod',
|
|
55
59
|
name: 'Test Module',
|
|
56
60
|
version: '1.0.0',
|
|
@@ -136,7 +140,7 @@ describe('findMissingRequiredVariables (deploy path)', () => {
|
|
|
136
140
|
// for the interview, and not as a deploy-blocking error.
|
|
137
141
|
test('a source: hook variable is never reported missing, even required and unwritten', async () => {
|
|
138
142
|
const manifest = {
|
|
139
|
-
celilo_contract: '
|
|
143
|
+
celilo_contract: '2.0',
|
|
140
144
|
variables: {
|
|
141
145
|
owns: [
|
|
142
146
|
{
|
|
@@ -337,7 +341,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
|
|
|
337
341
|
|
|
338
342
|
function manifestWithArtifacts(artifacts: string[]): ModuleManifest {
|
|
339
343
|
return {
|
|
340
|
-
celilo_contract: '
|
|
344
|
+
celilo_contract: '2.0',
|
|
341
345
|
id: 'buildmod',
|
|
342
346
|
name: 'Build Module',
|
|
343
347
|
version: '1.0.0',
|
|
@@ -347,7 +351,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
|
|
|
347
351
|
}
|
|
348
352
|
|
|
349
353
|
test('returns [] when the module declares no build section', () => {
|
|
350
|
-
const manifest = { celilo_contract: '
|
|
354
|
+
const manifest = { celilo_contract: '2.0', id: 'm', version: '1.0.0' } as ModuleManifest;
|
|
351
355
|
expect(findMissingBuildArtifacts(manifest, dir)).toEqual([]);
|
|
352
356
|
});
|
|
353
357
|
|
|
@@ -366,3 +370,78 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
|
|
|
366
370
|
expect(findMissingBuildArtifacts(manifest, dir)).toEqual(['dist/index.html', 'dist/db.sqlite']);
|
|
367
371
|
});
|
|
368
372
|
});
|
|
373
|
+
|
|
374
|
+
/**
|
|
375
|
+
* celilo#1384. An OPTIONAL capability is optional in its PRESENCE, not in its
|
|
376
|
+
* contract: with a provider installed the module's hooks call it, so a
|
|
377
|
+
* provider on another major fails at runtime — and `on_install` failing is
|
|
378
|
+
* only a warning, so it fails silently. Measured: generic-cpanel 0.4.1+2
|
|
379
|
+
* (external_web 2.0.0) against tango-nexus 0.8.0 (optional external_web
|
|
380
|
+
* 1.0.0); nothing published, the site kept serving, nothing said so.
|
|
381
|
+
*/
|
|
382
|
+
describe('validateAndPrepareDeployment (optional capability version gate)', () => {
|
|
383
|
+
let tempDir: string;
|
|
384
|
+
let db: DbClient;
|
|
385
|
+
|
|
386
|
+
function seed(optionalVersion: string, providerVersion: string | null) {
|
|
387
|
+
db.insert(modules)
|
|
388
|
+
.values({
|
|
389
|
+
id: 'consumer',
|
|
390
|
+
name: 'Consumer',
|
|
391
|
+
sourcePath: tempDir,
|
|
392
|
+
version: '1.0.0',
|
|
393
|
+
manifestData: {
|
|
394
|
+
celilo_contract: '2.0',
|
|
395
|
+
id: 'consumer',
|
|
396
|
+
name: 'Consumer',
|
|
397
|
+
version: '1.0.0',
|
|
398
|
+
description: 'fixture',
|
|
399
|
+
optional: { capabilities: [{ name: 'external_web', version: optionalVersion }] },
|
|
400
|
+
},
|
|
401
|
+
})
|
|
402
|
+
.run();
|
|
403
|
+
if (providerVersion === null) return;
|
|
404
|
+
db.insert(modules)
|
|
405
|
+
.values({
|
|
406
|
+
id: 'provider',
|
|
407
|
+
name: 'Provider',
|
|
408
|
+
sourcePath: tempDir,
|
|
409
|
+
version: '1.0.0',
|
|
410
|
+
manifestData: {},
|
|
411
|
+
})
|
|
412
|
+
.run();
|
|
413
|
+
db.insert(capabilities)
|
|
414
|
+
.values({
|
|
415
|
+
moduleId: 'provider',
|
|
416
|
+
capabilityName: 'external_web',
|
|
417
|
+
version: providerVersion,
|
|
418
|
+
data: {},
|
|
419
|
+
})
|
|
420
|
+
.run();
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
beforeEach(() => {
|
|
424
|
+
tempDir = mkdtempSync(join(tmpdir(), 'celilo-optional-cap-'));
|
|
425
|
+
process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
|
|
426
|
+
db = getDb();
|
|
427
|
+
});
|
|
428
|
+
|
|
429
|
+
afterEach(() => {
|
|
430
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
431
|
+
resetTestDbPath();
|
|
432
|
+
});
|
|
433
|
+
|
|
434
|
+
test('refuses the deploy when the installed provider is another major', async () => {
|
|
435
|
+
seed('1.0.0', '2.0.0');
|
|
436
|
+
const result = await validateAndPrepareDeployment('consumer', db);
|
|
437
|
+
expect(result.success).toBe(false);
|
|
438
|
+
expect(result.error).toContain('external_web');
|
|
439
|
+
expect(result.error).toContain('provider');
|
|
440
|
+
});
|
|
441
|
+
|
|
442
|
+
test('does not refuse when nobody provides the optional capability', async () => {
|
|
443
|
+
seed('1.0.0', null);
|
|
444
|
+
const result = await validateAndPrepareDeployment('consumer', db);
|
|
445
|
+
expect(result.error).toBeUndefined();
|
|
446
|
+
});
|
|
447
|
+
});
|
|
@@ -113,6 +113,25 @@ export async function validateAndPrepareDeployment(
|
|
|
113
113
|
}
|
|
114
114
|
}
|
|
115
115
|
|
|
116
|
+
// An OPTIONAL capability is optional in its PRESENCE, not in its contract.
|
|
117
|
+
// When a provider is installed the module's hooks will call it, so a provider
|
|
118
|
+
// on another major breaks at runtime exactly as a required one would — except
|
|
119
|
+
// an `on_install` failure is only a warning, so it breaks silently
|
|
120
|
+
// (celilo#1384). No provider installed is still fine: the version check below
|
|
121
|
+
// skips a capability nobody provides.
|
|
122
|
+
if (manifest.optional?.capabilities) {
|
|
123
|
+
const optionalMismatches = await findCapabilityVersionMismatches(
|
|
124
|
+
manifest.optional.capabilities,
|
|
125
|
+
db,
|
|
126
|
+
);
|
|
127
|
+
if (optionalMismatches.length > 0) {
|
|
128
|
+
return {
|
|
129
|
+
success: false,
|
|
130
|
+
error: formatVersionMismatchError(moduleId, optionalMismatches),
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
116
135
|
// Check for missing required variables BEFORE generating — if any are missing
|
|
117
136
|
// we return them for the caller to interview, then the caller re-invokes deploy
|
|
118
137
|
// after the user has answered. This prevents generation from failing mid-way
|
|
@@ -42,7 +42,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
|
|
|
42
42
|
id,
|
|
43
43
|
name: id,
|
|
44
44
|
version: '1.0.0',
|
|
45
|
-
manifestData: { id, name: id, version: '1.0.0', celilo_contract: '
|
|
45
|
+
manifestData: { id, name: id, version: '1.0.0', celilo_contract: '2.0' },
|
|
46
46
|
sourcePath: `/tmp/${id}`,
|
|
47
47
|
})
|
|
48
48
|
.run();
|
|
@@ -73,7 +73,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
|
|
|
73
73
|
id: 'caddy',
|
|
74
74
|
name: 'caddy',
|
|
75
75
|
version: '1.0.0',
|
|
76
|
-
manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '
|
|
76
|
+
manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '2.0' },
|
|
77
77
|
sourcePath: '/tmp/caddy',
|
|
78
78
|
})
|
|
79
79
|
.onConflictDoNothing()
|
|
@@ -37,7 +37,7 @@ describe('dns_registrations ledger', () => {
|
|
|
37
37
|
name: id,
|
|
38
38
|
sourcePath: join(tempDir, id),
|
|
39
39
|
version: '1.0.0',
|
|
40
|
-
manifestData: { celilo_contract: '
|
|
40
|
+
manifestData: { celilo_contract: '2.0', id, name: id, version: '1.0.0' },
|
|
41
41
|
})
|
|
42
42
|
.run();
|
|
43
43
|
}
|
|
@@ -70,7 +70,7 @@ function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', system
|
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
const baseManifest = (overrides: Partial<ModuleManifest> = {}): ModuleManifest => ({
|
|
73
|
-
celilo_contract: '
|
|
73
|
+
celilo_contract: '2.0',
|
|
74
74
|
id: 'mod',
|
|
75
75
|
name: 'Mod',
|
|
76
76
|
version: '1.0.0',
|
|
@@ -340,7 +340,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
|
|
|
340
340
|
writeFileSync(
|
|
341
341
|
join(src, 'manifest.yml'),
|
|
342
342
|
[
|
|
343
|
-
'celilo_contract: "
|
|
343
|
+
'celilo_contract: "2.0"',
|
|
344
344
|
`id: ${id}`,
|
|
345
345
|
`name: ${id}`,
|
|
346
346
|
'version: 1.0.0',
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
|
|
20
20
|
import { readFileSync } from 'node:fs';
|
|
21
21
|
import { join } from 'node:path';
|
|
22
|
-
import { type Bus, describeError } from '@celilo/event-bus';
|
|
22
|
+
import { type Bus, type BusHealth, describeError } from '@celilo/event-bus';
|
|
23
23
|
import { desc, eq, inArray } from 'drizzle-orm';
|
|
24
24
|
import { parse as parseYaml } from 'yaml';
|
|
25
25
|
import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
|
|
@@ -203,6 +203,12 @@ export interface DispatcherCheckOptions {
|
|
|
203
203
|
* Injected so the crash-loop aspect is testable without launchd.
|
|
204
204
|
*/
|
|
205
205
|
launchdProbe?: (scope: SupervisorScope) => LaunchdUnitStatus | null;
|
|
206
|
+
/**
|
|
207
|
+
* A verdict already taken, normally `await bus.healthConfirmed()`. Callers
|
|
208
|
+
* that can await pass it so a load-starved dispatcher reads `lagging` rather
|
|
209
|
+
* than the `wedged` a single sample cannot distinguish it from (celilo#1399).
|
|
210
|
+
*/
|
|
211
|
+
health?: BusHealth;
|
|
206
212
|
}
|
|
207
213
|
|
|
208
214
|
/**
|
|
@@ -214,7 +220,7 @@ export interface DispatcherCheckOptions {
|
|
|
214
220
|
*/
|
|
215
221
|
export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): FleetFinding {
|
|
216
222
|
const now = opts.now ?? Date.now();
|
|
217
|
-
const health = bus.health();
|
|
223
|
+
const health = opts.health ?? bus.health();
|
|
218
224
|
const hb = bus.db
|
|
219
225
|
.query<HeartbeatRow, []>(
|
|
220
226
|
'SELECT dispatcher_id, last_heartbeat, started_at, pid, version FROM dispatcher_heartbeat ORDER BY last_heartbeat DESC LIMIT 1',
|
|
@@ -227,14 +233,25 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
|
|
|
227
233
|
|
|
228
234
|
// (1) running — a fresh heartbeat. health() already classifies a
|
|
229
235
|
// stale/absent heartbeat as no_dispatcher.
|
|
230
|
-
if (health.status === 'no_dispatcher' || !hb) {
|
|
236
|
+
if (health.status === 'no_dispatcher' || health.status === 'wedged' || !hb) {
|
|
231
237
|
statuses.push('fail');
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
+
if (health.status === 'wedged') {
|
|
239
|
+
// Registered, process alive, heartbeat stale, nothing running: it is present
|
|
240
|
+
// but dead, so "start the dispatcher" is the wrong instruction (celilo#1398).
|
|
241
|
+
detail.push(
|
|
242
|
+
`dispatcher WEDGED — pid ${hb?.pid ?? '?'} is alive and registered but its heartbeat is ${Math.round((health.lastHeartbeatAgeMs ?? 0) / 1000)}s old with nothing running (events are queueing, not delivered)`,
|
|
243
|
+
);
|
|
244
|
+
remediations.push(
|
|
245
|
+
health.remedy ?? 'restart the dispatcher: `celilo events restart-daemon --system`',
|
|
246
|
+
);
|
|
247
|
+
} else {
|
|
248
|
+
detail.push(
|
|
249
|
+
'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
|
|
250
|
+
);
|
|
251
|
+
remediations.push(
|
|
252
|
+
'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
|
|
253
|
+
);
|
|
254
|
+
}
|
|
238
255
|
// A unit can be INSTALLED and still dead: the supervisor respawns a
|
|
239
256
|
// program that cannot start in the unit's environment and it dies again,
|
|
240
257
|
// forever. On macOS the classic cause is a PATH-less
|
|
@@ -274,6 +291,15 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
|
|
|
274
291
|
detail.push(
|
|
275
292
|
`running (pid ${hb.pid}, heartbeat ${Math.round(ageMs / 1000)}s ago, code v${hb.version})`,
|
|
276
293
|
);
|
|
294
|
+
if (health.status === 'lagging') {
|
|
295
|
+
// Confirmed alive by a second beat: slow, not dead. Worth saying — an operator
|
|
296
|
+
// reading a 60s heartbeat wants to know it was checked — but not a failure,
|
|
297
|
+
// and explicitly not a restart (celilo#1399).
|
|
298
|
+
statuses.push('warn');
|
|
299
|
+
detail.push(
|
|
300
|
+
'heartbeat is stale but still advancing — the dispatcher is alive and starved for CPU, not wedged; it recovers when load drops',
|
|
301
|
+
);
|
|
302
|
+
}
|
|
277
303
|
if (health.status === 'stuck') {
|
|
278
304
|
statuses.push('warn');
|
|
279
305
|
detail.push(
|
|
@@ -1395,7 +1421,11 @@ export async function runFleetChecks(
|
|
|
1395
1421
|
const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
|
|
1396
1422
|
return [
|
|
1397
1423
|
checkSchemaDrift(db),
|
|
1398
|
-
checkDispatcher(bus, {
|
|
1424
|
+
checkDispatcher(bus, {
|
|
1425
|
+
now: opts.now,
|
|
1426
|
+
installedCodeMtimeMs: opts.installedCodeMtimeMs,
|
|
1427
|
+
health: await bus.healthConfirmed(),
|
|
1428
|
+
}),
|
|
1399
1429
|
checkBuildBusPublishing(bus, db, { now: opts.now }),
|
|
1400
1430
|
checkSubscribers(bus, db),
|
|
1401
1431
|
checkCapabilityProviders(db),
|
|
@@ -106,7 +106,7 @@ describe('runModuleHealthCheck — clearing a recorded failure (celilo#1363)', (
|
|
|
106
106
|
state: 'ERROR',
|
|
107
107
|
errorMessage: 'Upgrade to 1.0.0+6 failed to deploy: ssh: connection refused',
|
|
108
108
|
manifestData: {
|
|
109
|
-
celilo_contract: '
|
|
109
|
+
celilo_contract: '2.0',
|
|
110
110
|
id: 'celilo-mgmt',
|
|
111
111
|
name: 'Control plane',
|
|
112
112
|
version: '1.0.0',
|
|
@@ -59,7 +59,7 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
|
|
|
59
59
|
|
|
60
60
|
// Create manifest with infrastructure variables
|
|
61
61
|
const manifest: ModuleManifest = {
|
|
62
|
-
celilo_contract: '
|
|
62
|
+
celilo_contract: '2.0',
|
|
63
63
|
id: moduleId,
|
|
64
64
|
name: 'DNS External',
|
|
65
65
|
version: '1.0.0',
|
|
@@ -137,7 +137,7 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
|
|
|
137
137
|
const moduleId = 'test-module';
|
|
138
138
|
|
|
139
139
|
const manifest: ModuleManifest = {
|
|
140
|
-
celilo_contract: '
|
|
140
|
+
celilo_contract: '2.0',
|
|
141
141
|
id: moduleId,
|
|
142
142
|
name: 'Test',
|
|
143
143
|
version: '1.0.0',
|
|
@@ -203,7 +203,7 @@ describe('resolveInfrastructureVariables - Proxmox Container Service', () => {
|
|
|
203
203
|
const moduleId = 'homebridge';
|
|
204
204
|
|
|
205
205
|
const manifest: ModuleManifest = {
|
|
206
|
-
celilo_contract: '
|
|
206
|
+
celilo_contract: '2.0',
|
|
207
207
|
id: moduleId,
|
|
208
208
|
name: 'Homebridge',
|
|
209
209
|
version: '1.0.0',
|
|
@@ -287,7 +287,7 @@ describe('resolveInfrastructureVariables - Proxmox Container Service', () => {
|
|
|
287
287
|
const moduleId = 'test-module';
|
|
288
288
|
|
|
289
289
|
const manifest: ModuleManifest = {
|
|
290
|
-
celilo_contract: '
|
|
290
|
+
celilo_contract: '2.0',
|
|
291
291
|
id: moduleId,
|
|
292
292
|
name: 'Test',
|
|
293
293
|
version: '1.0.0',
|
|
@@ -350,7 +350,7 @@ describe('resolveInfrastructureVariables - Digital Ocean Container Service', ()
|
|
|
350
350
|
const moduleId = 'dns-external';
|
|
351
351
|
|
|
352
352
|
const manifest: ModuleManifest = {
|
|
353
|
-
celilo_contract: '
|
|
353
|
+
celilo_contract: '2.0',
|
|
354
354
|
id: moduleId,
|
|
355
355
|
name: 'DNS External',
|
|
356
356
|
version: '1.0.0',
|
|
@@ -436,7 +436,7 @@ describe('resolveInfrastructureVariables - User Override', () => {
|
|
|
436
436
|
const moduleId = 'dns-external';
|
|
437
437
|
|
|
438
438
|
const manifest: ModuleManifest = {
|
|
439
|
-
celilo_contract: '
|
|
439
|
+
celilo_contract: '2.0',
|
|
440
440
|
id: moduleId,
|
|
441
441
|
name: 'DNS External',
|
|
442
442
|
version: '1.0.0',
|
|
@@ -494,7 +494,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
|
|
|
494
494
|
const moduleId = 'test-module';
|
|
495
495
|
|
|
496
496
|
const manifest: ModuleManifest = {
|
|
497
|
-
celilo_contract: '
|
|
497
|
+
celilo_contract: '2.0',
|
|
498
498
|
id: moduleId,
|
|
499
499
|
name: 'Test',
|
|
500
500
|
version: '1.0.0',
|
|
@@ -544,7 +544,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
|
|
|
544
544
|
const moduleId = 'test-module';
|
|
545
545
|
|
|
546
546
|
const manifest: ModuleManifest = {
|
|
547
|
-
celilo_contract: '
|
|
547
|
+
celilo_contract: '2.0',
|
|
548
548
|
id: moduleId,
|
|
549
549
|
name: 'Test',
|
|
550
550
|
version: '1.0.0',
|
|
@@ -605,7 +605,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
|
|
|
605
605
|
describe('resolveInfrastructureVariables - Edge Cases', () => {
|
|
606
606
|
test('returns empty when no infrastructure variables declared', async () => {
|
|
607
607
|
const manifest: ModuleManifest = {
|
|
608
|
-
celilo_contract: '
|
|
608
|
+
celilo_contract: '2.0',
|
|
609
609
|
id: 'test-module',
|
|
610
610
|
name: 'Test',
|
|
611
611
|
version: '1.0.0',
|
|
@@ -632,7 +632,7 @@ describe('resolveInfrastructureVariables - Edge Cases', () => {
|
|
|
632
632
|
|
|
633
633
|
test('throws when no infrastructure selected', async () => {
|
|
634
634
|
const manifest: ModuleManifest = {
|
|
635
|
-
celilo_contract: '
|
|
635
|
+
celilo_contract: '2.0',
|
|
636
636
|
id: 'test-module',
|
|
637
637
|
name: 'Test',
|
|
638
638
|
version: '1.0.0',
|
|
@@ -660,7 +660,7 @@ describe('resolveInfrastructureVariables - Edge Cases', () => {
|
|
|
660
660
|
const moduleId = 'test-module';
|
|
661
661
|
|
|
662
662
|
const manifest: ModuleManifest = {
|
|
663
|
-
celilo_contract: '
|
|
663
|
+
celilo_contract: '2.0',
|
|
664
664
|
id: moduleId,
|
|
665
665
|
name: 'Test',
|
|
666
666
|
version: '1.0.0',
|
|
@@ -15,7 +15,7 @@ import { deriveInstanceModuleId } from './module-instances';
|
|
|
15
15
|
|
|
16
16
|
const manifest = (over: Record<string, unknown> = {}): ModuleManifest =>
|
|
17
17
|
({
|
|
18
|
-
celilo_contract: '
|
|
18
|
+
celilo_contract: '2.0',
|
|
19
19
|
id: 'lab',
|
|
20
20
|
name: 'BYOI Lab',
|
|
21
21
|
version: '1.0.0',
|
|
@@ -110,7 +110,7 @@ describe('createInstanceOps against a real database', () => {
|
|
|
110
110
|
let parentPath: string;
|
|
111
111
|
|
|
112
112
|
const SUBMODULE_YAML = `
|
|
113
|
-
celilo_contract: "
|
|
113
|
+
celilo_contract: "2.0"
|
|
114
114
|
id: lab
|
|
115
115
|
name: BYOI Lab
|
|
116
116
|
version: 1.0.0
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import type { NetworkZone } from '../db/schema';
|
|
3
|
+
import {
|
|
4
|
+
applyReclassification,
|
|
5
|
+
diffInterfaceZones,
|
|
6
|
+
explainMissingZoneInterface,
|
|
7
|
+
} from './machine-interface-zones';
|
|
8
|
+
|
|
9
|
+
/** The zone model as it stands today: 10.0.120.0/24 is secure-mgmt. */
|
|
10
|
+
const detect = async (ip: string): Promise<NetworkZone | 'unknown'> => {
|
|
11
|
+
if (ip.startsWith('10.0.120.')) return 'secure-mgmt';
|
|
12
|
+
if (ip.startsWith('10.0.10.')) return 'dmz';
|
|
13
|
+
return 'unknown';
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* celilo-mgr as celilo#1382 found it: added before `secure-mgmt` existed,
|
|
18
|
+
* while the detector labelled unmatched subnets `external`.
|
|
19
|
+
*/
|
|
20
|
+
const celiloMgr = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' }];
|
|
21
|
+
|
|
22
|
+
describe('diffInterfaceZones', () => {
|
|
23
|
+
test('reports a label the current zone model disagrees with', async () => {
|
|
24
|
+
expect(await diffInterfaceZones(celiloMgr, detect)).toEqual([
|
|
25
|
+
{ name: 'eth0', ipAddress: '10.0.120.10', from: 'external', to: 'secure-mgmt' },
|
|
26
|
+
]);
|
|
27
|
+
});
|
|
28
|
+
|
|
29
|
+
test('reports nothing when the stored labels already agree', async () => {
|
|
30
|
+
const current = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' }];
|
|
31
|
+
expect(await diffInterfaceZones(current, detect)).toEqual([]);
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
test('reports only the interfaces that changed', async () => {
|
|
35
|
+
const mixed = [
|
|
36
|
+
{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' },
|
|
37
|
+
{ name: 'eth1', ipAddress: '10.0.10.5', zone: 'dmz' },
|
|
38
|
+
];
|
|
39
|
+
const changes = await diffInterfaceZones(mixed, detect);
|
|
40
|
+
expect(changes.map((c) => c.name)).toEqual(['eth0']);
|
|
41
|
+
});
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
describe('applyReclassification', () => {
|
|
45
|
+
test('writes the new label and leaves the input untouched', async () => {
|
|
46
|
+
const changes = await diffInterfaceZones(celiloMgr, detect);
|
|
47
|
+
expect(applyReclassification(celiloMgr, changes)).toEqual([
|
|
48
|
+
{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' },
|
|
49
|
+
]);
|
|
50
|
+
expect(celiloMgr[0].zone).toBe('external');
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
describe('explainMissingZoneInterface', () => {
|
|
55
|
+
test('says MISLABELLED, and names the interface, when one would match', async () => {
|
|
56
|
+
const changes = await diffInterfaceZones(celiloMgr, detect);
|
|
57
|
+
const message = explainMissingZoneInterface(celiloMgr, 'secure-mgmt', changes);
|
|
58
|
+
expect(message).toContain('MISLABELLED');
|
|
59
|
+
expect(message).toContain('eth0');
|
|
60
|
+
expect(message).toContain('10.0.120.10');
|
|
61
|
+
expect(message).toContain('machine reclassify');
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test('says the interface is absent when no relabel would produce one', async () => {
|
|
65
|
+
const dmzOnly = [{ name: 'eth0', ipAddress: '10.0.10.5', zone: 'dmz' }];
|
|
66
|
+
const changes = await diffInterfaceZones(dmzOnly, detect);
|
|
67
|
+
const message = explainMissingZoneInterface(dmzOnly, 'secure-mgmt', changes);
|
|
68
|
+
expect(message).toContain("has no interface in zone 'secure-mgmt'");
|
|
69
|
+
expect(message).not.toContain('MISLABELLED');
|
|
70
|
+
// Names what it DOES hold, so the operator can see why.
|
|
71
|
+
expect(message).toContain('10.0.10.5');
|
|
72
|
+
});
|
|
73
|
+
});
|