@celilo/cli 4.0.0 → 5.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (119) hide show
  1. package/AGENTS.md +2 -2
  2. package/CELILO_SUBSYSTEMS.md +1 -0
  3. package/drizzle/0034_private_routes.sql +41 -0
  4. package/drizzle/meta/_journal.json +8 -1
  5. package/package.json +5 -4
  6. package/src/api/sessions.test.ts +1 -1
  7. package/src/capabilities/registration.test.ts +11 -11
  8. package/src/capabilities/secrets.test.ts +18 -18
  9. package/src/capabilities/validation.test.ts +24 -24
  10. package/src/cli/cli.test.ts +3 -3
  11. package/src/cli/command-tree-parser.test.ts +18 -0
  12. package/src/cli/command-tree-parser.ts +12 -0
  13. package/src/cli/commands/api.ts +67 -14
  14. package/src/cli/commands/events.ts +5 -2
  15. package/src/cli/commands/machine-reclassify.ts +79 -0
  16. package/src/cli/commands/module-changeset.test.ts +1 -1
  17. package/src/cli/commands/module-generate.test.ts +2 -2
  18. package/src/cli/commands/module-import-aspect.test.ts +2 -2
  19. package/src/cli/commands/module-update.test.ts +14 -14
  20. package/src/cli/commands/module-upgrade-gate.test.ts +26 -1
  21. package/src/cli/commands/module-upgrade.test.ts +3 -3
  22. package/src/cli/commands/module-upgrade.ts +12 -0
  23. package/src/cli/commands/module-version.test.ts +1 -1
  24. package/src/cli/commands/system-audit.ts +22 -0
  25. package/src/cli/commands/system-reboot.ts +126 -0
  26. package/src/cli/commands/system-update.ts +5 -0
  27. package/src/cli/completion.ts +3 -2
  28. package/src/cli/index.ts +41 -2
  29. package/src/cli/restore-migration-failure.test.ts +1 -1
  30. package/src/cli/stdout-pipe-flush.test.ts +35 -0
  31. package/src/cli/tui/audit-state.ts +4 -0
  32. package/src/db/private-routes.test.ts +159 -0
  33. package/src/db/schema.ts +49 -0
  34. package/src/hooks/broker.ts +31 -0
  35. package/src/hooks/capability-loader.ts +185 -12
  36. package/src/hooks/executor.test.ts +13 -13
  37. package/src/hooks/executor.ts +20 -2
  38. package/src/hooks/hook-state-dir.test.ts +2 -2
  39. package/src/hooks/test-fixtures/slow-capability-hook.ts +28 -0
  40. package/src/hooks/test-fixtures/store-backed.ts +5 -3
  41. package/src/hooks/timeout-names-provider.test.ts +87 -0
  42. package/src/manifest/contracts/index.ts +28 -7
  43. package/src/manifest/contracts/v2.ts +32 -0
  44. package/src/manifest/icon-schema.test.ts +1 -1
  45. package/src/manifest/schema.ts +13 -2
  46. package/src/manifest/template-validator.test.ts +1 -1
  47. package/src/manifest/validate-privileged.test.ts +1 -1
  48. package/src/manifest/validate.test.ts +68 -29
  49. package/src/manifest/validate.ts +14 -1
  50. package/src/module/import.test.ts +7 -7
  51. package/src/module/packaging/build.test.ts +1 -1
  52. package/src/packaging/nfpm-depends.test.ts +42 -0
  53. package/src/policy/capability-shape-baseline.ts +7 -1
  54. package/src/policy/fixture-capability-coverage.test.ts +5 -5
  55. package/src/policy/module-business-baseline.ts +33 -4
  56. package/src/services/api-access.test.ts +44 -0
  57. package/src/services/aspect-approvals.test.ts +2 -2
  58. package/src/services/aspect-reconcile.test.ts +2 -2
  59. package/src/services/aspect-runner.test.ts +4 -4
  60. package/src/services/audit/backups.test.ts +1 -1
  61. package/src/services/audit/capability-abi.test.ts +1 -1
  62. package/src/services/audit/index.test.ts +7 -0
  63. package/src/services/audit/index.ts +15 -0
  64. package/src/services/audit/machine-interface-zones.test.ts +30 -0
  65. package/src/services/audit/machine-interface-zones.ts +43 -0
  66. package/src/services/audit/module-configs.test.ts +1 -1
  67. package/src/services/audit/reboot-pending.test.ts +63 -0
  68. package/src/services/audit/reboot-pending.ts +85 -0
  69. package/src/services/audit/recurrence-gate.test.ts +21 -0
  70. package/src/services/audit/server-bun-pin.test.ts +44 -0
  71. package/src/services/audit/server-bun-pin.ts +95 -0
  72. package/src/services/audit/types.ts +2 -0
  73. package/src/services/build-bus/hook-dispatch-executor.test.ts +3 -3
  74. package/src/services/bus-interview-park.test.ts +1 -1
  75. package/src/services/bus-secret-flow.test.ts +1 -1
  76. package/src/services/capability-compat.test.ts +22 -0
  77. package/src/services/capability-compat.ts +15 -2
  78. package/src/services/capability-table-rows.test.ts +20 -2
  79. package/src/services/celilo-events.test.ts +119 -6
  80. package/src/services/celilo-events.ts +156 -16
  81. package/src/services/celilo-mgmt-hooks.test.ts +6 -3
  82. package/src/services/cross-module-read.test.ts +2 -2
  83. package/src/services/deploy-preflight.ts +12 -3
  84. package/src/services/deploy-validation.test.ts +86 -7
  85. package/src/services/deploy-validation.ts +19 -0
  86. package/src/services/dns-provider-backfill.test.ts +2 -2
  87. package/src/services/dns-registrations.test.ts +1 -1
  88. package/src/services/fleet-checks.test.ts +2 -2
  89. package/src/services/fleet-checks.ts +40 -10
  90. package/src/services/health-runner.test.ts +1 -1
  91. package/src/services/infrastructure-variable-resolver.test.ts +11 -11
  92. package/src/services/instance-ops.test.ts +2 -2
  93. package/src/services/machine-interface-zones.test.ts +73 -0
  94. package/src/services/machine-interface-zones.ts +101 -0
  95. package/src/services/module-deploy.dns-repoint.test.ts +1 -1
  96. package/src/services/module-deploy.stale-provider-guard.test.ts +98 -0
  97. package/src/services/module-deploy.ts +31 -10
  98. package/src/services/module-pause-quiescence.test.ts +1 -1
  99. package/src/services/module-pause.test.ts +1 -1
  100. package/src/services/module-subscriptions.test.ts +2 -2
  101. package/src/services/module-types-generator.test.ts +1 -1
  102. package/src/services/module-validator/contract-version.test.ts +3 -3
  103. package/src/services/network-ensure.test.ts +1 -1
  104. package/src/services/provider-arrival.test.ts +62 -46
  105. package/src/services/provider-arrival.ts +41 -10
  106. package/src/services/proxmox-reconcile.test.ts +2 -2
  107. package/src/services/remove-guard.test.ts +1 -1
  108. package/src/services/system-reboot.test.ts +231 -0
  109. package/src/services/system-reboot.ts +357 -0
  110. package/src/services/update/dep-graph.test.ts +1 -1
  111. package/src/services/update/orchestrator.test.ts +165 -1
  112. package/src/services/update/orchestrator.ts +43 -12
  113. package/src/services/update/types.ts +10 -1
  114. package/src/templates/generator.test.ts +15 -4
  115. package/src/templates/generator.ts +12 -4
  116. package/src/variables/capability-self-ref.test.ts +2 -2
  117. package/src/variables/computed/computed-integration.test.ts +1 -1
  118. package/src/variables/context.ts +14 -1
  119. package/src/variables/declarative-derivation.test.ts +19 -19
@@ -5,6 +5,7 @@ import { join } from 'node:path';
5
5
  import { skipIntegration } from '../test-utils/integration-guard';
6
6
 
7
7
  import type { HookContext } from '@celilo/capabilities';
8
+ import { configStore, secretStore } from '../hooks/test-fixtures/store-backed';
8
9
 
9
10
  // Hooks under test — imported via the scripts dir adjacent to the
10
11
  // modules/celilo-mgmt/manifest.yml. The dynamic import keeps the test
@@ -46,7 +47,7 @@ function buildContext(extras: Record<string, unknown>): HookContext {
46
47
  const { logger } = buildLogger();
47
48
  return {
48
49
  logger,
49
- config: {
50
+ config: configStore({
50
51
  db_path: '', // populated by caller via env
51
52
  hostname: 'test-mgmt',
52
53
  target_ip: '127.0.0.1',
@@ -61,9 +62,11 @@ function buildContext(extras: Record<string, unknown>): HookContext {
61
62
  ssh_public_key: '',
62
63
  install_docker: false,
63
64
  install_terraform: false,
64
- } as Record<string, unknown>,
65
- secrets: {},
65
+ }),
66
+ secrets: secretStore(),
66
67
  capabilities: {},
68
+ systems: [],
69
+ stateDir: '',
67
70
  debug: false,
68
71
  screenshotDir: '',
69
72
  ...extras,
@@ -25,7 +25,7 @@ import {
25
25
 
26
26
  function buildManifest(caps: Array<{ name: string; version: string }>): ModuleManifest {
27
27
  return {
28
- celilo_contract: '1.0',
28
+ celilo_contract: '2.0',
29
29
  id: 'm',
30
30
  name: 'm',
31
31
  version: '0.0.1',
@@ -88,7 +88,7 @@ describe('materializeCrossModuleRoot', () => {
88
88
  name: id,
89
89
  version,
90
90
  sourcePath: join(dir, 'sources', id),
91
- manifestData: { id, name: id, version, celilo_contract: '1.0', requires: {} } as Record<
91
+ manifestData: { id, name: id, version, celilo_contract: '2.0', requires: {} } as Record<
92
92
  string,
93
93
  unknown
94
94
  >,
@@ -112,8 +112,15 @@ export async function runPreflight(
112
112
  // drift after deploy; this surface refuses up front so the
113
113
  // operator gets an actionable error instead of a silent
114
114
  // template-generation against an ABI the provider can't fulfil.
115
- if (manifest.requires?.capabilities) {
116
- for (const cap of manifest.requires.capabilities) {
115
+ // An OPTIONAL capability joins the version check but not the presence check:
116
+ // its provider may legitimately be absent, but when one IS installed the
117
+ // module's hooks call it, so another major breaks at runtime (celilo#1384).
118
+ {
119
+ const consumed = [
120
+ ...(manifest.requires?.capabilities ?? []).map((cap) => ({ cap, required: true })),
121
+ ...(manifest.optional?.capabilities ?? []).map((cap) => ({ cap, required: false })),
122
+ ];
123
+ for (const { cap, required } of consumed) {
117
124
  // Framework-granted privileges (e.g. cross_module_read) aren't
118
125
  // provider-backed — they're gated by the allow-list at import time,
119
126
  // not satisfied by deploying another module. Skip them here.
@@ -126,6 +133,7 @@ export async function runPreflight(
126
133
  .where(eq(capabilities.capabilityName, cap.name))
127
134
  .all();
128
135
  if (providers.length === 0) {
136
+ if (!required) continue;
129
137
  errors.push({
130
138
  category: 'missing-capability',
131
139
  message: `Required capability '${cap.name}' has no provider installed`,
@@ -154,7 +162,8 @@ export async function runPreflight(
154
162
  errors.push({
155
163
  category: 'capability-version-mismatch',
156
164
  message:
157
- `Module '${moduleId}' requires ${cap.name}@${cap.version} but ` +
165
+ `Module '${moduleId}' ${required ? 'requires' : 'optionally uses'} ` +
166
+ `${cap.name}@${cap.version} but ` +
158
167
  `provider '${example.moduleId}' offers ${cap.name}@${example.version}`,
159
168
  suggestion:
160
169
  reason === 'caller_minor_too_old'
@@ -19,15 +19,19 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
19
19
  import { tmpdir } from 'node:os';
20
20
  import { join } from 'node:path';
21
21
  import { type DbClient, getDb } from '../db/client';
22
- import { modules, secrets } from '../db/schema';
22
+ import { capabilities, modules, secrets } from '../db/schema';
23
23
  import type { ModuleManifest } from '../manifest/schema';
24
24
  import { resetTestDbPath } from '../test-utils/db-path';
25
25
  import { findMissingSecrets } from './config-interview';
26
- import { findMissingBuildArtifacts, findMissingRequiredVariables } from './deploy-validation';
26
+ import {
27
+ findMissingBuildArtifacts,
28
+ findMissingRequiredVariables,
29
+ validateAndPrepareDeployment,
30
+ } from './deploy-validation';
27
31
 
28
32
  function manifestWithStringMapSecret(): ModuleManifest {
29
33
  return {
30
- celilo_contract: '1.0',
34
+ celilo_contract: '2.0',
31
35
  id: 'testmod',
32
36
  name: 'Test Module',
33
37
  version: '1.0.0',
@@ -50,7 +54,7 @@ function manifestWithStringMapSecret(): ModuleManifest {
50
54
 
51
55
  function manifestWithPlainStringSecret(): ModuleManifest {
52
56
  return {
53
- celilo_contract: '1.0',
57
+ celilo_contract: '2.0',
54
58
  id: 'testmod',
55
59
  name: 'Test Module',
56
60
  version: '1.0.0',
@@ -136,7 +140,7 @@ describe('findMissingRequiredVariables (deploy path)', () => {
136
140
  // for the interview, and not as a deploy-blocking error.
137
141
  test('a source: hook variable is never reported missing, even required and unwritten', async () => {
138
142
  const manifest = {
139
- celilo_contract: '1.0',
143
+ celilo_contract: '2.0',
140
144
  variables: {
141
145
  owns: [
142
146
  {
@@ -337,7 +341,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
337
341
 
338
342
  function manifestWithArtifacts(artifacts: string[]): ModuleManifest {
339
343
  return {
340
- celilo_contract: '1.0',
344
+ celilo_contract: '2.0',
341
345
  id: 'buildmod',
342
346
  name: 'Build Module',
343
347
  version: '1.0.0',
@@ -347,7 +351,7 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
347
351
  }
348
352
 
349
353
  test('returns [] when the module declares no build section', () => {
350
- const manifest = { celilo_contract: '1.0', id: 'm', version: '1.0.0' } as ModuleManifest;
354
+ const manifest = { celilo_contract: '2.0', id: 'm', version: '1.0.0' } as ModuleManifest;
351
355
  expect(findMissingBuildArtifacts(manifest, dir)).toEqual([]);
352
356
  });
353
357
 
@@ -366,3 +370,78 @@ describe('findMissingBuildArtifacts (deploy build-gate)', () => {
366
370
  expect(findMissingBuildArtifacts(manifest, dir)).toEqual(['dist/index.html', 'dist/db.sqlite']);
367
371
  });
368
372
  });
373
+
374
+ /**
375
+ * celilo#1384. An OPTIONAL capability is optional in its PRESENCE, not in its
376
+ * contract: with a provider installed the module's hooks call it, so a
377
+ * provider on another major fails at runtime — and `on_install` failing is
378
+ * only a warning, so it fails silently. Measured: generic-cpanel 0.4.1+2
379
+ * (external_web 2.0.0) against tango-nexus 0.8.0 (optional external_web
380
+ * 1.0.0); nothing published, the site kept serving, nothing said so.
381
+ */
382
+ describe('validateAndPrepareDeployment (optional capability version gate)', () => {
383
+ let tempDir: string;
384
+ let db: DbClient;
385
+
386
+ function seed(optionalVersion: string, providerVersion: string | null) {
387
+ db.insert(modules)
388
+ .values({
389
+ id: 'consumer',
390
+ name: 'Consumer',
391
+ sourcePath: tempDir,
392
+ version: '1.0.0',
393
+ manifestData: {
394
+ celilo_contract: '2.0',
395
+ id: 'consumer',
396
+ name: 'Consumer',
397
+ version: '1.0.0',
398
+ description: 'fixture',
399
+ optional: { capabilities: [{ name: 'external_web', version: optionalVersion }] },
400
+ },
401
+ })
402
+ .run();
403
+ if (providerVersion === null) return;
404
+ db.insert(modules)
405
+ .values({
406
+ id: 'provider',
407
+ name: 'Provider',
408
+ sourcePath: tempDir,
409
+ version: '1.0.0',
410
+ manifestData: {},
411
+ })
412
+ .run();
413
+ db.insert(capabilities)
414
+ .values({
415
+ moduleId: 'provider',
416
+ capabilityName: 'external_web',
417
+ version: providerVersion,
418
+ data: {},
419
+ })
420
+ .run();
421
+ }
422
+
423
+ beforeEach(() => {
424
+ tempDir = mkdtempSync(join(tmpdir(), 'celilo-optional-cap-'));
425
+ process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
426
+ db = getDb();
427
+ });
428
+
429
+ afterEach(() => {
430
+ rmSync(tempDir, { recursive: true, force: true });
431
+ resetTestDbPath();
432
+ });
433
+
434
+ test('refuses the deploy when the installed provider is another major', async () => {
435
+ seed('1.0.0', '2.0.0');
436
+ const result = await validateAndPrepareDeployment('consumer', db);
437
+ expect(result.success).toBe(false);
438
+ expect(result.error).toContain('external_web');
439
+ expect(result.error).toContain('provider');
440
+ });
441
+
442
+ test('does not refuse when nobody provides the optional capability', async () => {
443
+ seed('1.0.0', null);
444
+ const result = await validateAndPrepareDeployment('consumer', db);
445
+ expect(result.error).toBeUndefined();
446
+ });
447
+ });
@@ -113,6 +113,25 @@ export async function validateAndPrepareDeployment(
113
113
  }
114
114
  }
115
115
 
116
+ // An OPTIONAL capability is optional in its PRESENCE, not in its contract.
117
+ // When a provider is installed the module's hooks will call it, so a provider
118
+ // on another major breaks at runtime exactly as a required one would — except
119
+ // an `on_install` failure is only a warning, so it breaks silently
120
+ // (celilo#1384). No provider installed is still fine: the version check below
121
+ // skips a capability nobody provides.
122
+ if (manifest.optional?.capabilities) {
123
+ const optionalMismatches = await findCapabilityVersionMismatches(
124
+ manifest.optional.capabilities,
125
+ db,
126
+ );
127
+ if (optionalMismatches.length > 0) {
128
+ return {
129
+ success: false,
130
+ error: formatVersionMismatchError(moduleId, optionalMismatches),
131
+ };
132
+ }
133
+ }
134
+
116
135
  // Check for missing required variables BEFORE generating — if any are missing
117
136
  // we return them for the caller to interview, then the caller re-invokes deploy
118
137
  // after the user has answered. This prevents generation from failing mid-way
@@ -42,7 +42,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
42
42
  id,
43
43
  name: id,
44
44
  version: '1.0.0',
45
- manifestData: { id, name: id, version: '1.0.0', celilo_contract: '1.0' },
45
+ manifestData: { id, name: id, version: '1.0.0', celilo_contract: '2.0' },
46
46
  sourcePath: `/tmp/${id}`,
47
47
  })
48
48
  .run();
@@ -73,7 +73,7 @@ describe('backfillWebRouteDns (ISS-0029)', () => {
73
73
  id: 'caddy',
74
74
  name: 'caddy',
75
75
  version: '1.0.0',
76
- manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '1.0' },
76
+ manifestData: { id: 'caddy', name: 'caddy', version: '1.0.0', celilo_contract: '2.0' },
77
77
  sourcePath: '/tmp/caddy',
78
78
  })
79
79
  .onConflictDoNothing()
@@ -37,7 +37,7 @@ describe('dns_registrations ledger', () => {
37
37
  name: id,
38
38
  sourcePath: join(tempDir, id),
39
39
  version: '1.0.0',
40
- manifestData: { celilo_contract: '1.0', id, name: id, version: '1.0.0' },
40
+ manifestData: { celilo_contract: '2.0', id, name: id, version: '1.0.0' },
41
41
  })
42
42
  .run();
43
43
  }
@@ -70,7 +70,7 @@ function installFakeUnit(home: string, scope: 'user' | 'system' = 'user', system
70
70
  }
71
71
 
72
72
  const baseManifest = (overrides: Partial<ModuleManifest> = {}): ModuleManifest => ({
73
- celilo_contract: '1.0',
73
+ celilo_contract: '2.0',
74
74
  id: 'mod',
75
75
  name: 'Mod',
76
76
  version: '1.0.0',
@@ -340,7 +340,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
340
340
  writeFileSync(
341
341
  join(src, 'manifest.yml'),
342
342
  [
343
- 'celilo_contract: "1.0"',
343
+ 'celilo_contract: "2.0"',
344
344
  `id: ${id}`,
345
345
  `name: ${id}`,
346
346
  'version: 1.0.0',
@@ -19,7 +19,7 @@
19
19
 
20
20
  import { readFileSync } from 'node:fs';
21
21
  import { join } from 'node:path';
22
- import { type Bus, describeError } from '@celilo/event-bus';
22
+ import { type Bus, type BusHealth, describeError } from '@celilo/event-bus';
23
23
  import { desc, eq, inArray } from 'drizzle-orm';
24
24
  import { parse as parseYaml } from 'yaml';
25
25
  import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
@@ -203,6 +203,12 @@ export interface DispatcherCheckOptions {
203
203
  * Injected so the crash-loop aspect is testable without launchd.
204
204
  */
205
205
  launchdProbe?: (scope: SupervisorScope) => LaunchdUnitStatus | null;
206
+ /**
207
+ * A verdict already taken, normally `await bus.healthConfirmed()`. Callers
208
+ * that can await pass it so a load-starved dispatcher reads `lagging` rather
209
+ * than the `wedged` a single sample cannot distinguish it from (celilo#1399).
210
+ */
211
+ health?: BusHealth;
206
212
  }
207
213
 
208
214
  /**
@@ -214,7 +220,7 @@ export interface DispatcherCheckOptions {
214
220
  */
215
221
  export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): FleetFinding {
216
222
  const now = opts.now ?? Date.now();
217
- const health = bus.health();
223
+ const health = opts.health ?? bus.health();
218
224
  const hb = bus.db
219
225
  .query<HeartbeatRow, []>(
220
226
  'SELECT dispatcher_id, last_heartbeat, started_at, pid, version FROM dispatcher_heartbeat ORDER BY last_heartbeat DESC LIMIT 1',
@@ -227,14 +233,25 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
227
233
 
228
234
  // (1) running — a fresh heartbeat. health() already classifies a
229
235
  // stale/absent heartbeat as no_dispatcher.
230
- if (health.status === 'no_dispatcher' || !hb) {
236
+ if (health.status === 'no_dispatcher' || health.status === 'wedged' || !hb) {
231
237
  statuses.push('fail');
232
- detail.push(
233
- 'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
234
- );
235
- remediations.push(
236
- 'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
237
- );
238
+ if (health.status === 'wedged') {
239
+ // Registered, process alive, heartbeat stale, nothing running: it is present
240
+ // but dead, so "start the dispatcher" is the wrong instruction (celilo#1398).
241
+ detail.push(
242
+ `dispatcher WEDGED — pid ${hb?.pid ?? '?'} is alive and registered but its heartbeat is ${Math.round((health.lastHeartbeatAgeMs ?? 0) / 1000)}s old with nothing running (events are queueing, not delivered)`,
243
+ );
244
+ remediations.push(
245
+ health.remedy ?? 'restart the dispatcher: `celilo events restart-daemon --system`',
246
+ );
247
+ } else {
248
+ detail.push(
249
+ 'no live dispatcher — heartbeat absent or stale (events are queueing, not delivered)',
250
+ );
251
+ remediations.push(
252
+ 'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
253
+ );
254
+ }
238
255
  // A unit can be INSTALLED and still dead: the supervisor respawns a
239
256
  // program that cannot start in the unit's environment and it dies again,
240
257
  // forever. On macOS the classic cause is a PATH-less
@@ -274,6 +291,15 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
274
291
  detail.push(
275
292
  `running (pid ${hb.pid}, heartbeat ${Math.round(ageMs / 1000)}s ago, code v${hb.version})`,
276
293
  );
294
+ if (health.status === 'lagging') {
295
+ // Confirmed alive by a second beat: slow, not dead. Worth saying — an operator
296
+ // reading a 60s heartbeat wants to know it was checked — but not a failure,
297
+ // and explicitly not a restart (celilo#1399).
298
+ statuses.push('warn');
299
+ detail.push(
300
+ 'heartbeat is stale but still advancing — the dispatcher is alive and starved for CPU, not wedged; it recovers when load drops',
301
+ );
302
+ }
277
303
  if (health.status === 'stuck') {
278
304
  statuses.push('warn');
279
305
  detail.push(
@@ -1395,7 +1421,11 @@ export async function runFleetChecks(
1395
1421
  const hostLiveness = opts.hostLiveness ?? (() => collectHostLiveness(db));
1396
1422
  return [
1397
1423
  checkSchemaDrift(db),
1398
- checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
1424
+ checkDispatcher(bus, {
1425
+ now: opts.now,
1426
+ installedCodeMtimeMs: opts.installedCodeMtimeMs,
1427
+ health: await bus.healthConfirmed(),
1428
+ }),
1399
1429
  checkBuildBusPublishing(bus, db, { now: opts.now }),
1400
1430
  checkSubscribers(bus, db),
1401
1431
  checkCapabilityProviders(db),
@@ -106,7 +106,7 @@ describe('runModuleHealthCheck — clearing a recorded failure (celilo#1363)', (
106
106
  state: 'ERROR',
107
107
  errorMessage: 'Upgrade to 1.0.0+6 failed to deploy: ssh: connection refused',
108
108
  manifestData: {
109
- celilo_contract: '1.0',
109
+ celilo_contract: '2.0',
110
110
  id: 'celilo-mgmt',
111
111
  name: 'Control plane',
112
112
  version: '1.0.0',
@@ -59,7 +59,7 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
59
59
 
60
60
  // Create manifest with infrastructure variables
61
61
  const manifest: ModuleManifest = {
62
- celilo_contract: '1.0',
62
+ celilo_contract: '2.0',
63
63
  id: moduleId,
64
64
  name: 'DNS External',
65
65
  version: '1.0.0',
@@ -137,7 +137,7 @@ describe('resolveInfrastructureVariables - Machine Infrastructure', () => {
137
137
  const moduleId = 'test-module';
138
138
 
139
139
  const manifest: ModuleManifest = {
140
- celilo_contract: '1.0',
140
+ celilo_contract: '2.0',
141
141
  id: moduleId,
142
142
  name: 'Test',
143
143
  version: '1.0.0',
@@ -203,7 +203,7 @@ describe('resolveInfrastructureVariables - Proxmox Container Service', () => {
203
203
  const moduleId = 'homebridge';
204
204
 
205
205
  const manifest: ModuleManifest = {
206
- celilo_contract: '1.0',
206
+ celilo_contract: '2.0',
207
207
  id: moduleId,
208
208
  name: 'Homebridge',
209
209
  version: '1.0.0',
@@ -287,7 +287,7 @@ describe('resolveInfrastructureVariables - Proxmox Container Service', () => {
287
287
  const moduleId = 'test-module';
288
288
 
289
289
  const manifest: ModuleManifest = {
290
- celilo_contract: '1.0',
290
+ celilo_contract: '2.0',
291
291
  id: moduleId,
292
292
  name: 'Test',
293
293
  version: '1.0.0',
@@ -350,7 +350,7 @@ describe('resolveInfrastructureVariables - Digital Ocean Container Service', ()
350
350
  const moduleId = 'dns-external';
351
351
 
352
352
  const manifest: ModuleManifest = {
353
- celilo_contract: '1.0',
353
+ celilo_contract: '2.0',
354
354
  id: moduleId,
355
355
  name: 'DNS External',
356
356
  version: '1.0.0',
@@ -436,7 +436,7 @@ describe('resolveInfrastructureVariables - User Override', () => {
436
436
  const moduleId = 'dns-external';
437
437
 
438
438
  const manifest: ModuleManifest = {
439
- celilo_contract: '1.0',
439
+ celilo_contract: '2.0',
440
440
  id: moduleId,
441
441
  name: 'DNS External',
442
442
  version: '1.0.0',
@@ -494,7 +494,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
494
494
  const moduleId = 'test-module';
495
495
 
496
496
  const manifest: ModuleManifest = {
497
- celilo_contract: '1.0',
497
+ celilo_contract: '2.0',
498
498
  id: moduleId,
499
499
  name: 'Test',
500
500
  version: '1.0.0',
@@ -544,7 +544,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
544
544
  const moduleId = 'test-module';
545
545
 
546
546
  const manifest: ModuleManifest = {
547
- celilo_contract: '1.0',
547
+ celilo_contract: '2.0',
548
548
  id: moduleId,
549
549
  name: 'Test',
550
550
  version: '1.0.0',
@@ -605,7 +605,7 @@ describe('resolveInfrastructureVariables - Required vs Optional', () => {
605
605
  describe('resolveInfrastructureVariables - Edge Cases', () => {
606
606
  test('returns empty when no infrastructure variables declared', async () => {
607
607
  const manifest: ModuleManifest = {
608
- celilo_contract: '1.0',
608
+ celilo_contract: '2.0',
609
609
  id: 'test-module',
610
610
  name: 'Test',
611
611
  version: '1.0.0',
@@ -632,7 +632,7 @@ describe('resolveInfrastructureVariables - Edge Cases', () => {
632
632
 
633
633
  test('throws when no infrastructure selected', async () => {
634
634
  const manifest: ModuleManifest = {
635
- celilo_contract: '1.0',
635
+ celilo_contract: '2.0',
636
636
  id: 'test-module',
637
637
  name: 'Test',
638
638
  version: '1.0.0',
@@ -660,7 +660,7 @@ describe('resolveInfrastructureVariables - Edge Cases', () => {
660
660
  const moduleId = 'test-module';
661
661
 
662
662
  const manifest: ModuleManifest = {
663
- celilo_contract: '1.0',
663
+ celilo_contract: '2.0',
664
664
  id: moduleId,
665
665
  name: 'Test',
666
666
  version: '1.0.0',
@@ -15,7 +15,7 @@ import { deriveInstanceModuleId } from './module-instances';
15
15
 
16
16
  const manifest = (over: Record<string, unknown> = {}): ModuleManifest =>
17
17
  ({
18
- celilo_contract: '1.0',
18
+ celilo_contract: '2.0',
19
19
  id: 'lab',
20
20
  name: 'BYOI Lab',
21
21
  version: '1.0.0',
@@ -110,7 +110,7 @@ describe('createInstanceOps against a real database', () => {
110
110
  let parentPath: string;
111
111
 
112
112
  const SUBMODULE_YAML = `
113
- celilo_contract: "1.0"
113
+ celilo_contract: "2.0"
114
114
  id: lab
115
115
  name: BYOI Lab
116
116
  version: 1.0.0
@@ -0,0 +1,73 @@
1
+ import { describe, expect, test } from 'bun:test';
2
+ import type { NetworkZone } from '../db/schema';
3
+ import {
4
+ applyReclassification,
5
+ diffInterfaceZones,
6
+ explainMissingZoneInterface,
7
+ } from './machine-interface-zones';
8
+
9
+ /** The zone model as it stands today: 10.0.120.0/24 is secure-mgmt. */
10
+ const detect = async (ip: string): Promise<NetworkZone | 'unknown'> => {
11
+ if (ip.startsWith('10.0.120.')) return 'secure-mgmt';
12
+ if (ip.startsWith('10.0.10.')) return 'dmz';
13
+ return 'unknown';
14
+ };
15
+
16
+ /**
17
+ * celilo-mgr as celilo#1382 found it: added before `secure-mgmt` existed,
18
+ * while the detector labelled unmatched subnets `external`.
19
+ */
20
+ const celiloMgr = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' }];
21
+
22
+ describe('diffInterfaceZones', () => {
23
+ test('reports a label the current zone model disagrees with', async () => {
24
+ expect(await diffInterfaceZones(celiloMgr, detect)).toEqual([
25
+ { name: 'eth0', ipAddress: '10.0.120.10', from: 'external', to: 'secure-mgmt' },
26
+ ]);
27
+ });
28
+
29
+ test('reports nothing when the stored labels already agree', async () => {
30
+ const current = [{ name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' }];
31
+ expect(await diffInterfaceZones(current, detect)).toEqual([]);
32
+ });
33
+
34
+ test('reports only the interfaces that changed', async () => {
35
+ const mixed = [
36
+ { name: 'eth0', ipAddress: '10.0.120.10', zone: 'external' },
37
+ { name: 'eth1', ipAddress: '10.0.10.5', zone: 'dmz' },
38
+ ];
39
+ const changes = await diffInterfaceZones(mixed, detect);
40
+ expect(changes.map((c) => c.name)).toEqual(['eth0']);
41
+ });
42
+ });
43
+
44
+ describe('applyReclassification', () => {
45
+ test('writes the new label and leaves the input untouched', async () => {
46
+ const changes = await diffInterfaceZones(celiloMgr, detect);
47
+ expect(applyReclassification(celiloMgr, changes)).toEqual([
48
+ { name: 'eth0', ipAddress: '10.0.120.10', zone: 'secure-mgmt' },
49
+ ]);
50
+ expect(celiloMgr[0].zone).toBe('external');
51
+ });
52
+ });
53
+
54
+ describe('explainMissingZoneInterface', () => {
55
+ test('says MISLABELLED, and names the interface, when one would match', async () => {
56
+ const changes = await diffInterfaceZones(celiloMgr, detect);
57
+ const message = explainMissingZoneInterface(celiloMgr, 'secure-mgmt', changes);
58
+ expect(message).toContain('MISLABELLED');
59
+ expect(message).toContain('eth0');
60
+ expect(message).toContain('10.0.120.10');
61
+ expect(message).toContain('machine reclassify');
62
+ });
63
+
64
+ test('says the interface is absent when no relabel would produce one', async () => {
65
+ const dmzOnly = [{ name: 'eth0', ipAddress: '10.0.10.5', zone: 'dmz' }];
66
+ const changes = await diffInterfaceZones(dmzOnly, detect);
67
+ const message = explainMissingZoneInterface(dmzOnly, 'secure-mgmt', changes);
68
+ expect(message).toContain("has no interface in zone 'secure-mgmt'");
69
+ expect(message).not.toContain('MISLABELLED');
70
+ // Names what it DOES hold, so the operator can see why.
71
+ expect(message).toContain('10.0.10.5');
72
+ });
73
+ });