@celilo/cli 2.2.1 → 2.3.0

package/src/services/celilo-events.test.ts

@@ -29,7 +29,7 @@ describe('celilo lifecycle events', () => {
     process.env.EVENT_BUS_DB = dbPath;
   });
   afterEach(() => {
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/celilo-mgmt-hooks.test.ts

@@ -11,6 +11,14 @@ import type { HookContext } from '@celilo/capabilities';
 // runnable from the apps/celilo working dir.
 const HOOK_DIR = join(import.meta.dirname, '../../../../modules/celilo-mgmt/scripts');
 
+// on_backup reads its machine-pool snapshot from the context input the
+// framework stages (backup-create.ts, from listMachines()). These tests
+// inject a JSON string so the snapshot is deterministic on every host. The
+// pre-staging version of this hook shelled `celilo machine list --json` and
+// these tests had to inject a runner stub to avoid depending on the installed
+// CLI's age (celilo#1333); the staged input made that seam unnecessary.
+const STUB_MACHINE_POOL = [{ hostname: 'stub-machine', zone: 'dmz' }];
+
 interface BackupHookOutput {
   artifact_count: number;
   size_bytes: number;
@@ -116,6 +124,7 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
         backup_dir: backupDir,
         cross_module_root: crossModuleRoot,
         system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
       }),
     )) as BackupHookOutput;
 
@@ -135,7 +144,6 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
     expect(result.artifact_count).toBeGreaterThan(0);
     expect(result.size_bytes).toBeGreaterThan(0);
   });
-
   it('refuses when the framework did not stage celilo state', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
 
@@ -148,24 +156,34 @@ describe.skipIf(skipIntegration({ tools: ['wg'] }))('celilo-mgmt on_backup', ()
     ).rejects.toThrow(/system_state_root was not provided/);
   });
 
-  it('machine-pool.json is valid JSON (array)', async () => {
+  it('machine-pool.json is the staged snapshot (valid JSON array)', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
     await hook(
       buildContext({
         backup_dir: backupDir,
         cross_module_root: crossModuleRoot,
         system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
       }),
     );
 
-    const machinePool = JSON.parse(readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'));
-    expect(Array.isArray(machinePool)).toBe(true);
+    // Content, not just shape. A bare Array.isArray also passes when the
+    // hook swallowed a snapshot failure and wrote an empty pool, which is
+    // exactly the failure mode this stub exists to rule out.
+    const machinePool: unknown = JSON.parse(
+      readFileSync(join(backupDir, 'machine-pool.json'), 'utf-8'),
+    );
+    expect(machinePool).toEqual(STUB_MACHINE_POOL);
   });
 
   it('proceeds (with a warning) when cross_module_root is missing', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
     const result = (await hook(
-      buildContext({ backup_dir: backupDir, system_state_root: systemStateRoot }),
+      buildContext({
+        backup_dir: backupDir,
+        system_state_root: systemStateRoot,
+        machine_pool: JSON.stringify(STUB_MACHINE_POOL),
+      }),
     )) as BackupHookOutput;
 
     expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);

package/src/services/container-service.test.ts

@@ -55,7 +55,7 @@ describe('container-service', () => {
 
     // Clear environment variables
     resetTestDbPath();
-    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    delete process.env.CELILO_MASTER_KEY_PATH;
   });
 
   describe('addContainerService', () => {

package/src/services/cross-module-read.test.ts

@@ -72,7 +72,7 @@ describe('materializeCrossModuleRoot', () => {
   afterEach(() => {
     closeDb();
     resetTestDbPath();
-    process.env.CELILO_DATA_DIR = undefined;
+    delete process.env.CELILO_DATA_DIR;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -180,7 +180,7 @@ describe('applyCrossModuleWriteRoot', () => {
   });
 
   afterEach(() => {
-    process.env.CELILO_DATA_DIR = undefined;
+    delete process.env.CELILO_DATA_DIR;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/deploy-preflight.ts

@@ -27,6 +27,7 @@ import type { DbClient } from '../db/client';
 import { getDb } from '../db/client';
 import { capabilities, moduleConfigs, modules, secrets } from '../db/schema';
 import { type ModuleManifest, getSingularSystemSpec } from '../manifest/schema';
+import { isPrivilegedCapability } from '../manifest/validate';
 import { buildResolutionContext } from '../variables/context';
 import { E2E_CONFLICT_FIX, runningE2eContainers } from './e2e-guard';
 import { describeCapabilityProblem, findBrokenCapabilityDerivations } from './fleet-checks';
@@ -113,6 +114,12 @@ export async function runPreflight(
   //    template-generation against an ABI the provider can't fulfil.
   if (manifest.requires?.capabilities) {
     for (const cap of manifest.requires.capabilities) {
+      // Framework-granted privileges (e.g. cross_module_read) aren't
+      // provider-backed — they're gated by the allow-list at import time,
+      // not satisfied by deploying another module. Skip them here.
+      if (isPrivilegedCapability(cap.name)) {
+        continue;
+      }
       const providers = db
         .select()
         .from(capabilities)

package/src/services/deployed-systems.ts

@@ -242,7 +242,7 @@ export function upsertDeployedSystem(
     .run();
 }
 
-function asZone(value: string | null | undefined): NetworkZone | null {
+export function asZone(value: string | null | undefined): NetworkZone | null {
   return value && (NETWORK_ZONES as readonly string[]).includes(value)
     ? (value as NetworkZone)
     : null;

package/src/services/fleet-checks.test.ts

@@ -3,8 +3,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
+import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import {
+  buildBusHookRuns,
   capabilities as capabilitiesTable,
   dnsInternalRecords,
   moduleConfigs,
@@ -20,6 +22,7 @@ import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
   type HostLivenessInputs,
+  checkBuildBusPublishing,
   checkCapabilityProviders,
   checkControlPlaneNetwork,
   checkDispatcher,
@@ -96,7 +99,7 @@ describe('checkDispatcher', () => {
   });
   afterEach(() => {
     bus.close();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -271,7 +274,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     bus.close();
     db.$client.close();
     resetTestDbPath();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -292,6 +295,118 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
       .run();
   }
 
+  describe('checkBuildBusPublishing', () => {
+    const now = Date.now();
+
+    /** A module whose installed manifest really declares on_upstream_publish. */
+    function insertHookModule(id: string): void {
+      const src = join(dir, `${id}-src`);
+      mkdirSync(src, { recursive: true });
+      writeFileSync(
+        join(src, 'manifest.yml'),
+        [
+          'celilo_contract: "1.0"',
+          `id: ${id}`,
+          `name: ${id}`,
+          'version: 1.0.0',
+          'requires: { capabilities: [] }',
+          'provides: { capabilities: [] }',
+          'variables: { owns: [], imports: [] }',
+          'hooks:',
+          '  on_upstream_publish:',
+          '    - name: self-update',
+          '      match: { registry: npm, tag: latest, package_pattern: "@celilo/cli" }',
+          '      script: ./scripts/update.sh',
+        ].join('\n'),
+      );
+      insertModule(id, baseManifest({ id, name: id }), 'VERIFIED');
+      // insertModule pins sourcePath to /src/<id>; point it at the real manifest.
+      db.update(modules).set({ sourcePath: src }).where(eq(modules.id, id)).run();
+    }
+
+    function seedRun(overrides: Partial<typeof buildBusHookRuns.$inferInsert> = {}): void {
+      db.insert(buildBusHookRuns)
+        .values({
+          eventId: 'evt-1',
+          packageName: '@celilo/cli',
+          packageVersion: '2.2.1',
+          tag: 'latest',
+          moduleId: 'celilo-mgmt',
+          hookName: 'self-update',
+          scriptPath: '/src/celilo-mgmt/scripts/update.sh',
+          exitCode: 0,
+          timedOut: false,
+          durationMs: 1200,
+          // Default to the freshest time, not the describe-time `now`:
+          // emitRaw stamps the bus event with its own Date.now(), and a
+          // run stamped even 1ms earlier would read as "predates the
+          // newest publish".
+          ranAt: new Date(Date.now()),
+          ...overrides,
+        })
+        .run();
+    }
+
+    it('is ok when no installed module declares on_upstream_publish', () => {
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('not in use');
+    });
+
+    it('warns when a self-update module is installed but no publish has ever arrived', () => {
+      insertHookModule('celilo-mgmt');
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('no build-bus publish has ever arrived');
+      expect(f.remediation).toContain('subscribers serve');
+    });
+
+    it('warns when publishes arrive but no hook has ever run', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('no self-update hook has ever run');
+    });
+
+    it('warns with the stderr tail when the newest run failed', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun({ exitCode: 1, stderrTail: 'bun add -g failed; not promoting' });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('exited 1');
+      expect(f.detail.join('\n')).toContain('bun add -g failed');
+    });
+
+    it('warns when the newest run timed out', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun({ exitCode: null, timedOut: true, durationMs: 300000 });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('timed out');
+    });
+
+    it('warns when the newest run predates the newest publish — that publish dispatched nothing', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-2' });
+      seedRun({ ranAt: new Date(now - 60 * MINUTE) });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('dispatched nothing');
+    });
+
+    it('is ok when the newest run succeeded and is fresh', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun();
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('@celilo/cli@2.2.1');
+    });
+  });
+
   describe('checkSubscribers', () => {
     it('fails when a deployed module declares a subscription the bus is missing', () => {
       insertModule(
@@ -449,7 +564,11 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
         .run();
       const f = checkCapabilityProviders(db);
       expect(f.status).toBe('ok');
-      expect(f.detail.join(' ')).toContain('celilo capability chain');
+      // celilo#1308: `celilo capability chain` no longer exists, so the detail
+      // says the verification is by hand rather than naming a command that
+      // cannot run.
+      expect(f.detail.join(' ')).toContain('verify these by hand');
+      expect(f.detail.join(' ')).not.toContain('celilo capability chain');
     });
   });
 
@@ -980,7 +1099,7 @@ describe('runFleetChecks includes host liveness (#728)', () => {
     bus.close();
     db.$client.close();
     resetTestDbPath();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/fleet-checks.ts

@@ -17,13 +17,16 @@
  * rendering + `--fix` orchestration lives in the doctor command.
  */
 
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { type Bus, describeError } from '@celilo/event-bus';
-import { inArray } from 'drizzle-orm';
+import { desc, eq, inArray } from 'drizzle-orm';
+import { parse as parseYaml } from 'yaml';
 import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
 import { getMigrationStatus } from '../db/migration-status';
-import { capabilities as capabilitiesTable, modules } from '../db/schema';
+import { buildBusHookRuns, capabilities as capabilitiesTable, modules } from '../db/schema';
 import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
@@ -384,6 +387,172 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
   };
 }
 
+/**
+ * The build-bus self-update chain, asserted outcome-first like every check
+ * here. Four facts separate the ways `on_upstream_publish` silently does
+ * nothing (celilo#1304: the CLI on a management box aged in place while the
+ * doctor's dispatcher check read green — that check covers the event-bus
+ * dispatcher, which no build-bus hook ever travels through):
+ *
+ *   1. no module declares the hook        → nothing can self-update; not in use
+ *   2. `build-bus.publish` events arrive  → the webhook path works end to end
+ *   3. hook-run rows exist for an event   → the dispatcher ran matching hooks
+ *   4. the newest run exited 0            → the self-update itself succeeded
+ *
+ * Facts 3 and 4 read the `build_bus_hook_runs` ledger the dispatcher writes;
+ * before that ledger existed, a failed self-update existed only as daemon
+ * console output, which no surface reads.
+ */
+export function checkBuildBusPublishing(
+  bus: Bus,
+  db: DbClient,
+  opts: { now?: number } = {},
+): FleetFinding {
+  const now = opts.now ?? Date.now();
+
+  const hookModules = db
+    .select({ id: modules.id })
+    .from(modules)
+    .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
+    .all()
+    .filter((m) => hasUpstreamPublishHooks(db, m.id));
+
+  const finding = (
+    status: FleetFindingStatus,
+    summary: string,
+    detail: string[],
+    remediation: string | null,
+  ): FleetFinding => ({
+    id: 'build-bus-publishing',
+    title: 'build-bus webhooks arriving and self-update hooks running',
+    status,
+    summary,
+    detail,
+    remediation,
+    autoFixable: false,
+  });
+
+  if (hookModules.length === 0) {
+    return finding(
+      'ok',
+      'no installed module declares on_upstream_publish — build-bus self-update not in use here',
+      [],
+      null,
+    );
+  }
+
+  const newestPublish = bus.recentEvents({ type: 'build-bus.publish', limit: 1 })[0];
+  if (!newestPublish) {
+    return finding(
+      'warn',
+      `${hookModules.map((m) => m.id).join(', ')} self-update can never fire: no build-bus publish has ever arrived`,
+      [
+        'The webhook receiver (`celilo subscribers serve`) either is not running here or was never sent a verified publish — and nothing in the deploy installs it.',
+      ],
+      "start the receiver: `celilo subscribers serve --secret $CELILO_BUS_SECRET` under a supervisor unit, and register this host in the publisher's build-bus-subscribers.json with the same secret (export CELILO_BUS_SECRET in that unit)",
+    );
+  }
+
+  const runs = db
+    .select()
+    .from(buildBusHookRuns)
+    .orderBy(desc(buildBusHookRuns.id))
+    .limit(10)
+    .all();
+
+  const detail: string[] = [
+    `newest publish: ${newestPublish.emittedAt ? `${Math.round((now - newestPublish.emittedAt) / 60000)}min ago` : 'unknown age'}`,
+  ];
+
+  const newest = runs[0];
+  if (!newest) {
+    return finding(
+      'warn',
+      'publish events arrive but no self-update hook has ever run',
+      [
+        ...detail,
+        'The receiver emitted to the bus, yet the in-process hook dispatcher dispatched nothing — it was started with --no-dispatch, no module matched, or the running CLI predates the run ledger.',
+      ],
+      "check how the receiver daemon is started (drop --no-dispatch) and compare the publish events against the installed modules' on_upstream_publish match rules",
+    );
+  }
+
+  const runAgeMin = newest.ranAt ? Math.round((now - newest.ranAt.getTime()) / 60000) : null;
+  const runLine = `newest hook run: ${newest.packageName}@${newest.packageVersion} on ${newest.moduleId}${runAgeMin != null ? `, ${runAgeMin}min ago` : ''}`;
+
+  if (newest.timedOut) {
+    return finding(
+      'warn',
+      `self-update hook timed out after ${newest.durationMs}ms`,
+      [
+        ...detail,
+        runLine,
+        `  stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
+      ],
+      'inspect the ledger: `select * from build_bus_hook_runs order by id desc limit 10` in the celilo DB',
+    );
+  }
+  if (newest.exitCode !== 0) {
+    return finding(
+      'warn',
+      `self-update hook exited ${newest.exitCode ?? 'without an exit code (spawn failure)'}`,
+      [
+        ...detail,
+        runLine,
+        `  stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
+      ],
+      'fix the failing hook script, then re-publish or re-emit the build-bus.publish event',
+    );
+  }
+
+  // A success that predates the newest publish means the newest publish was
+  // dropped on the dispatch side even though earlier ones ran. The ledger's
+  // ran_at has second resolution (unixepoch), so the event's millisecond
+  // stamp is truncated to the same granularity before comparing — otherwise
+  // a run in the same second as its event reads as older than it.
+  const newestPublishSec = Math.floor((newestPublish.emittedAt ?? 0) / 1000) * 1000;
+  if (newest.ranAt && newest.ranAt.getTime() < newestPublishSec) {
+    return finding(
+      'warn',
+      'newest publish event predates the newest hook run — that publish dispatched nothing',
+      [...detail, runLine],
+      'check the receiver daemon logs for that event and the match rules against what the publisher sent',
+    );
+  }
+
+  return finding(
+    'ok',
+    `last self-update hook succeeded (${newest.packageName}@${newest.packageVersion})`,
+    [...detail, runLine],
+    null,
+  );
+}
+
+/**
+ * Does module `id`'s installed manifest declare an `on_upstream_publish`
+ * hook? Reads the manifest from disk exactly as the hook dispatcher's own
+ * loader does, so the check and the dispatcher cannot disagree about what
+ * "in use" means. A manifest that has vanished or fails to parse reads as
+ * "no hooks" — the dispatcher skips it silently too (Rule 6.2: the skip is
+ * visible here as fact 1 rather than as a crash).
+ */
+function hasUpstreamPublishHooks(db: DbClient, moduleId: string): boolean {
+  const row = db
+    .select({ sourcePath: modules.sourcePath })
+    .from(modules)
+    .where(eq(modules.id, moduleId))
+    .get();
+  if (!row?.sourcePath) return false;
+  try {
+    const parsed = parseYaml(readFileSync(join(row.sourcePath, 'manifest.yml'), 'utf-8')) as {
+      hooks?: { on_upstream_publish?: unknown[] };
+    };
+    return (parsed.hooks?.on_upstream_publish ?? []).length > 0;
+  } catch {
+    return false;
+  }
+}
+
 /** A deployed module + its parsed manifest (INSTALLED/VERIFIED only). */
 interface DeployedModule {
   id: string;
@@ -668,7 +837,10 @@ export function checkCapabilityProviders(db: DbClient): FleetFinding {
   }
   if (traceNeeded.length > 0) {
     detail.push(
-      `${traceNeeded.length} derived value(s) resolve through another capability — verify with \`celilo capability chain <module> <var>\` (ISS-0114): ${traceNeeded.join('; ')}`,
+      // celilo#1308: `celilo capability chain` no longer exists (the ISS-0115
+      // rework removed it), so prescribing it sent the operator to an
+      // "unknown command" error. No CLI command traces a chain; say so.
+      `${traceNeeded.length} derived value(s) resolve through another capability. No CLI command traces a capability chain, so verify these by hand (ISS-0114): ${traceNeeded.join('; ')}`,
     );
   }
 
@@ -1194,6 +1366,7 @@ export async function runFleetChecks(
   return [
     checkSchemaDrift(db),
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
+    checkBuildBusPublishing(bus, db, { now: opts.now }),
     checkSubscribers(bus, db),
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),

package/src/services/health-runner.test.ts

@@ -1,5 +1,14 @@
-import { describe, expect, test } from 'bun:test';
-import { nextModuleState } from './health-runner';
+import { describe, expect, spyOn, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { type DbClient, getDb } from '../db/client';
+import { modules } from '../db/schema';
+import { resetTestDbPath } from '../test-utils/db-path';
+import * as controlPlaneHealth from './control-plane-health';
+import type { HealthCheckItem } from './health-runner';
+import { deriveHealthStatus, nextModuleState, runModuleHealthCheck } from './health-runner';
 
 /**
  * The D15 rule: a scheduled (monitor-driven) health check observes only and
@@ -9,6 +18,34 @@ import { nextModuleState } from './health-runner';
  * throw, it just makes `celilo module list` flicker between INSTALLED and
  * VERIFIED every monitor interval.
  */
+describe('deriveHealthStatus', () => {
+  const item = (status: HealthCheckItem['status'], name = 'probe'): HealthCheckItem => ({
+    name,
+    status,
+    message: 'msg',
+  });
+
+  test('fail dominates everything', () => {
+    expect(deriveHealthStatus([item('pass'), item('fail'), item('warn')])).toBe('unhealthy');
+  });
+
+  test('warn without fail is degraded', () => {
+    expect(deriveHealthStatus([item('pass'), item('warn')])).toBe('degraded');
+  });
+
+  test('measured passes with a skip are healthy — the audit carries the skip', () => {
+    expect(deriveHealthStatus([item('pass'), item('skip')])).toBe('healthy');
+  });
+
+  test('only skips is no-checks — nothing was measured', () => {
+    expect(deriveHealthStatus([item('skip'), item('skip')])).toBe('no-checks');
+  });
+
+  test('empty checks is healthy (the hook ran and reported nothing)', () => {
+    expect(deriveHealthStatus([])).toBe('healthy');
+  });
+});
+
 describe('nextModuleState', () => {
   describe('operator-invoked run (unattended = false)', () => {
     test('passing check verifies an INSTALLED module', () => {
@@ -48,3 +85,51 @@ describe('nextModuleState', () => {
     });
   });
 });
+
+// celilo#1363: a failed upgrade leaves state ERROR with an errorMessage. The
+// recovery path is an attended health check passing, which moves ERROR →
+// VERIFIED. The message must move with the state, or `module list` keeps
+// printing the old `Error:` line beside a healthy row.
+describe('runModuleHealthCheck — clearing a recorded failure (celilo#1363)', () => {
+  test('an attended healthy check clears errorMessage along with the state', async () => {
+    const tempDir = mkdtempSync(join(tmpdir(), 'celilo-health-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+
+    try {
+      const db: DbClient = getDb();
+      db.insert(modules)
+        .values({
+          id: 'celilo-mgmt',
+          name: 'Control plane',
+          sourcePath: join(tempDir, 'celilo-mgmt'),
+          version: '1.0.0+5',
+          state: 'ERROR',
+          errorMessage: 'Upgrade to 1.0.0+6 failed to deploy: ssh: connection refused',
+          manifestData: {
+            celilo_contract: '1.0',
+            id: 'celilo-mgmt',
+            name: 'Control plane',
+            version: '1.0.0',
+          },
+        })
+        .run();
+
+      const checks: HealthCheckItem[] = [{ name: 'probe', status: 'pass', message: 'ok' }];
+      const spy = spyOn(controlPlaneHealth, 'controlPlaneHealthChecks').mockResolvedValue(checks);
+      try {
+        const result = await runModuleHealthCheck('celilo-mgmt', db, {});
+        expect(result.status).toBe('healthy');
+      } finally {
+        spy.mockRestore();
+      }
+
+      const row = db.select().from(modules).where(eq(modules.id, 'celilo-mgmt')).get();
+      if (!row) throw new Error('row missing after health check');
+      expect(row.state).toBe('VERIFIED');
+      expect(row.errorMessage).toBeNull();
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+      resetTestDbPath();
+    }
+  });
+});