@celilo/cli 2.2.1 → 3.0.0

package/src/services/fleet-checks.test.ts

@@ -3,8 +3,10 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { type Bus, defineEvents, openBus } from '@celilo/event-bus';
+import { eq } from 'drizzle-orm';
 import type { DbClient } from '../db/client';
 import {
+  buildBusHookRuns,
   capabilities as capabilitiesTable,
   dnsInternalRecords,
   moduleConfigs,
@@ -20,6 +22,7 @@ import { ensureBackupSweepSubscriber } from './backup-sweep';
 import { getDaemonUnitPath } from './events-daemon';
 import {
   type HostLivenessInputs,
+  checkBuildBusPublishing,
   checkCapabilityProviders,
   checkControlPlaneNetwork,
   checkDispatcher,
@@ -96,7 +99,7 @@ describe('checkDispatcher', () => {
   });
   afterEach(() => {
     bus.close();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -111,6 +114,41 @@ describe('checkDispatcher', () => {
     expect(f.remediation).toContain('enable --now celilo-events.service');
   });
 
+  // celilo#1373 — the unit is installed and launchd keeps respawning it, but
+  // it dies before it can heartbeat (no PATH → the wrapper can't find
+  // bun). The finding must NAME the crash loop, not tell the operator to
+  // "start the dispatcher".
+  it('names a crash-looping launchd unit when no dispatcher is live (darwin)', () => {
+    installFakeUnit(home);
+    const f = checkDispatcher(bus, {
+      now,
+      home,
+      platform: 'darwin',
+      launchdProbe: () => ({ pid: null, lastExitStatus: 256 }),
+    });
+    expect(f.status).toBe('fail');
+    expect(f.detail.join(' ')).toContain('crash-looping');
+    expect(f.detail.join(' ')).toContain('exit code 1');
+    expect(f.remediation).toContain('celilo events install-daemon');
+  });
+
+  it('does not claim a crash loop when launchd runs the unit or cannot answer', () => {
+    installFakeUnit(home);
+    for (const probe of [
+      () => ({ pid: 4242, lastExitStatus: 256 }), // running despite an old failure
+      () => null, // launchd didn't answer — ignorance, not evidence
+    ]) {
+      const f = checkDispatcher(bus, {
+        now,
+        home,
+        platform: 'darwin',
+        launchdProbe: probe,
+      });
+      expect(f.status).toBe('fail');
+      expect(f.detail.join(' ')).not.toContain('crash-looping');
+    }
+  });
+
   it('passes when running, supervised, current, and not behind on timer ticks', () => {
     seedHeartbeat(bus, { startedAt: now - MINUTE, lastHeartbeat: now - 1000 });
     installFakeUnit(home);
@@ -271,7 +309,7 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
     bus.close();
     db.$client.close();
     resetTestDbPath();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -292,6 +330,119 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
       .run();
   }
 
+  describe('checkBuildBusPublishing', () => {
+    const now = Date.now();
+
+    /** A module whose installed manifest really declares on_upstream_publish. */
+    function insertHookModule(id: string): void {
+      const src = join(dir, `${id}-src`);
+      mkdirSync(src, { recursive: true });
+      writeFileSync(
+        join(src, 'manifest.yml'),
+        [
+          'celilo_contract: "1.0"',
+          `id: ${id}`,
+          `name: ${id}`,
+          'version: 1.0.0',
+          'requires: { capabilities: [] }',
+          'provides: { capabilities: [] }',
+          'variables: { owns: [], imports: [] }',
+          'hooks:',
+          '  on_upstream_publish:',
+          '    - name: self-update',
+          '      match: { registry: npm, tag: latest, package_pattern: "@celilo/cli" }',
+          '      script: ./scripts/update.sh',
+        ].join('\n'),
+      );
+      insertModule(id, baseManifest({ id, name: id }), 'VERIFIED');
+      // insertModule pins sourcePath to /src/<id>; point it at the real manifest.
+      db.update(modules).set({ sourcePath: src }).where(eq(modules.id, id)).run();
+    }
+
+    function seedRun(overrides: Partial<typeof buildBusHookRuns.$inferInsert> = {}): void {
+      db.insert(buildBusHookRuns)
+        .values({
+          eventId: 'evt-1',
+          packageName: '@celilo/cli',
+          packageVersion: '2.2.1',
+          tag: 'latest',
+          moduleId: 'celilo-mgmt',
+          hookName: 'self-update',
+          scriptPath: '/src/celilo-mgmt/scripts/update.sh',
+          exitCode: 0,
+          timedOut: false,
+          durationMs: 1200,
+          // Default to the freshest time, not the describe-time `now`:
+          // emitRaw stamps the bus event with its own Date.now(), and a
+          // run stamped even 1ms earlier would read as "predates the
+          // newest publish".
+          ranAt: new Date(Date.now()),
+          ...overrides,
+        })
+        .run();
+    }
+
+    it('is ok when no installed module declares on_upstream_publish', () => {
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('not in use');
+    });
+
+    it('warns when a self-update module is installed but no publish has ever arrived', () => {
+      insertHookModule('celilo-mgmt');
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('no build-bus publish has ever arrived');
+      expect(f.remediation).toContain('subscribers install-daemon');
+      expect(f.remediation).toContain('CELILO_BUS_SECRET');
+    });
+
+    it('warns when publishes arrive but no hook has ever run', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('no self-update hook has ever run');
+    });
+
+    it('warns with the stderr tail when the newest run failed', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun({ exitCode: 1, stderrTail: 'bun add -g failed; not promoting' });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('exited 1');
+      expect(f.detail.join('\n')).toContain('bun add -g failed');
+    });
+
+    it('warns when the newest run timed out', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun({ exitCode: null, timedOut: true, durationMs: 300000 });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('timed out');
+    });
+
+    it('warns when the newest run predates the newest publish — that publish dispatched nothing', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-2' });
+      seedRun({ ranAt: new Date(now - 60 * MINUTE) });
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('warn');
+      expect(f.summary).toContain('dispatched nothing');
+    });
+
+    it('is ok when the newest run succeeded and is fresh', () => {
+      insertHookModule('celilo-mgmt');
+      bus.emitRaw('build-bus.publish', { eventId: 'evt-1' });
+      seedRun();
+      const f = checkBuildBusPublishing(bus, db, { now });
+      expect(f.status).toBe('ok');
+      expect(f.summary).toContain('@celilo/cli@2.2.1');
+    });
+  });
+
   describe('checkSubscribers', () => {
     it('fails when a deployed module declares a subscription the bus is missing', () => {
       insertModule(
@@ -449,7 +600,11 @@ describe('checkSubscribers + checkCapabilityProviders', () => {
         .run();
       const f = checkCapabilityProviders(db);
       expect(f.status).toBe('ok');
-      expect(f.detail.join(' ')).toContain('celilo capability chain');
+      // celilo#1308: `celilo capability chain` no longer exists, so the detail
+      // says the verification is by hand rather than naming a command that
+      // cannot run.
+      expect(f.detail.join(' ')).toContain('verify these by hand');
+      expect(f.detail.join(' ')).not.toContain('celilo capability chain');
     });
   });
 
@@ -980,7 +1135,7 @@ describe('runFleetChecks includes host liveness (#728)', () => {
     bus.close();
     db.$client.close();
     resetTestDbPath();
-    process.env.EVENT_BUS_DB = undefined;
+    delete process.env.EVENT_BUS_DB;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {

package/src/services/fleet-checks.ts

@@ -17,13 +17,16 @@
  * rendering + `--fix` orchestration lives in the doctor command.
  */
 
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
 import { type Bus, describeError } from '@celilo/event-bus';
-import { inArray } from 'drizzle-orm';
+import { desc, eq, inArray } from 'drizzle-orm';
+import { parse as parseYaml } from 'yaml';
 import { ProxmoxClient, type ProxmoxCredentials } from '../api-clients/proxmox';
 import { getModuleStoragePath } from '../config/paths';
 import { type DbClient, findMigrationsFolder } from '../db/client';
 import { getMigrationStatus } from '../db/migration-status';
-import { capabilities as capabilitiesTable, modules } from '../db/schema';
+import { buildBusHookRuns, capabilities as capabilitiesTable, modules } from '../db/schema';
 import { findSchemaDrift } from '../db/schema-introspection';
 import { loadControlPlaneSubnet, resolveFirewallNatIp } from '../hooks/capability-loader';
 import type { ModuleManifest } from '../manifest/schema';
@@ -36,9 +39,11 @@ import {
 } from './deployed-systems';
 import { listDnsInternalRecords } from './dns-internal-records';
 import {
+  type LaunchdUnitStatus,
   SUPERVISOR_SCOPES,
   type SupervisorPlatform,
   type SupervisorScope,
+  launchdUnitStatus,
   readInstalledUnit,
   unitMainPid,
 } from './events-daemon';
@@ -193,6 +198,11 @@ export interface DispatcherCheckOptions {
    * no supervision claim rather than guessing.
    */
   unitMainPid?: (scope: SupervisorScope) => number | null;
+  /**
+   * Probe launchd for the unit's health (darwin only — see launchdUnitStatus).
+   * Injected so the crash-loop aspect is testable without launchd.
+   */
+  launchdProbe?: (scope: SupervisorScope) => LaunchdUnitStatus | null;
 }
 
 /**
@@ -225,6 +235,29 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
     remediations.push(
       'start the dispatcher: `systemctl --user enable --now celilo-events.service` (or `celilo events install-daemon` then enable it)',
     );
+    // A unit can be INSTALLED and still dead: the supervisor respawns a
+    // program that cannot start in the unit's environment and it dies again,
+    // forever. On macOS the classic cause is a PATH-less
+    // EnvironmentVariables dict — the global celilo wrapper needs
+    // `command -v bun`, and launchd's default PATH has none (celilo#1373:
+    // 22680 respawns on one Mac). Name the crash loop instead of telling
+    // the operator to "start the dispatcher".
+    if (opts.platform === 'darwin') {
+      const probe = opts.launchdProbe ?? launchdUnitStatus;
+      const st = probe('user');
+      if (st && st.pid === null && st.lastExitStatus != null && st.lastExitStatus !== 0) {
+        const fate =
+          st.lastExitStatus & 0xff
+            ? `signal ${st.lastExitStatus & 0xff}`
+            : `exit code ${st.lastExitStatus >> 8}`;
+        detail.push(
+          `the installed launchd unit is crash-looping (last ${fate}) — launchd respawns it and it dies again, so no dispatcher ever serves the bus`,
+        );
+        remediations.push(
+          "read the unit's stderr (`tail ~/Library/Logs/celilo-events.err.log`) and reinstall with `celilo events install-daemon` so the unit carries an environment that resolves bun (celilo#1373)",
+        );
+      }
+    }
     // Without a heartbeat there's nothing more to assert about it.
     return {
       id: 'dispatcher',
@@ -384,6 +417,172 @@ export function checkDispatcher(bus: Bus, opts: DispatcherCheckOptions = {}): Fl
   };
 }
 
+/**
+ * The build-bus self-update chain, asserted outcome-first like every check
+ * here. Four facts separate the ways `on_upstream_publish` silently does
+ * nothing (celilo#1304: the CLI on a management box aged in place while the
+ * doctor's dispatcher check read green — that check covers the event-bus
+ * dispatcher, which no build-bus hook ever travels through):
+ *
+ *   1. no module declares the hook        → nothing can self-update; not in use
+ *   2. `build-bus.publish` events arrive  → the webhook path works end to end
+ *   3. hook-run rows exist for an event   → the dispatcher ran matching hooks
+ *   4. the newest run exited 0            → the self-update itself succeeded
+ *
+ * Facts 3 and 4 read the `build_bus_hook_runs` ledger the dispatcher writes;
+ * before that ledger existed, a failed self-update existed only as daemon
+ * console output, which no surface reads.
+ */
+export function checkBuildBusPublishing(
+  bus: Bus,
+  db: DbClient,
+  opts: { now?: number } = {},
+): FleetFinding {
+  const now = opts.now ?? Date.now();
+
+  const hookModules = db
+    .select({ id: modules.id })
+    .from(modules)
+    .where(inArray(modules.state, ['INSTALLED', 'VERIFIED']))
+    .all()
+    .filter((m) => hasUpstreamPublishHooks(db, m.id));
+
+  const finding = (
+    status: FleetFindingStatus,
+    summary: string,
+    detail: string[],
+    remediation: string | null,
+  ): FleetFinding => ({
+    id: 'build-bus-publishing',
+    title: 'build-bus webhooks arriving and self-update hooks running',
+    status,
+    summary,
+    detail,
+    remediation,
+    autoFixable: false,
+  });
+
+  if (hookModules.length === 0) {
+    return finding(
+      'ok',
+      'no installed module declares on_upstream_publish — build-bus self-update not in use here',
+      [],
+      null,
+    );
+  }
+
+  const newestPublish = bus.recentEvents({ type: 'build-bus.publish', limit: 1 })[0];
+  if (!newestPublish) {
+    return finding(
+      'warn',
+      `${hookModules.map((m) => m.id).join(', ')} self-update can never fire: no build-bus publish has ever arrived`,
+      [
+        'The webhook receiver (`celilo subscribers serve`) either is not running here or was never sent a verified publish — and nothing in the deploy installs it.',
+      ],
+      "install the receiver under a supervisor unit: `celilo subscribers install-daemon --secret $CELILO_BUS_SECRET` (systemd/launchd; `--print` renders for Ansible), enable it per its next-steps output, and register this host in the publisher's build-bus-subscribers.json with the same secret",
+    );
+  }
+
+  const runs = db
+    .select()
+    .from(buildBusHookRuns)
+    .orderBy(desc(buildBusHookRuns.id))
+    .limit(10)
+    .all();
+
+  const detail: string[] = [
+    `newest publish: ${newestPublish.emittedAt ? `${Math.round((now - newestPublish.emittedAt) / 60000)}min ago` : 'unknown age'}`,
+  ];
+
+  const newest = runs[0];
+  if (!newest) {
+    return finding(
+      'warn',
+      'publish events arrive but no self-update hook has ever run',
+      [
+        ...detail,
+        'The receiver emitted to the bus, yet the in-process hook dispatcher dispatched nothing — it was started with --no-dispatch, no module matched, or the running CLI predates the run ledger.',
+      ],
+      "check how the receiver daemon is started (drop --no-dispatch) and compare the publish events against the installed modules' on_upstream_publish match rules",
+    );
+  }
+
+  const runAgeMin = newest.ranAt ? Math.round((now - newest.ranAt.getTime()) / 60000) : null;
+  const runLine = `newest hook run: ${newest.packageName}@${newest.packageVersion} on ${newest.moduleId}${runAgeMin != null ? `, ${runAgeMin}min ago` : ''}`;
+
+  if (newest.timedOut) {
+    return finding(
+      'warn',
+      `self-update hook timed out after ${newest.durationMs}ms`,
+      [
+        ...detail,
+        runLine,
+        `  stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
+      ],
+      'inspect the ledger: `select * from build_bus_hook_runs order by id desc limit 10` in the celilo DB',
+    );
+  }
+  if (newest.exitCode !== 0) {
+    return finding(
+      'warn',
+      `self-update hook exited ${newest.exitCode ?? 'without an exit code (spawn failure)'}`,
+      [
+        ...detail,
+        runLine,
+        `  stderr tail: ${(newest.stderrTail ?? '(none)').trim().slice(0, 500)}`,
+      ],
+      'fix the failing hook script, then re-publish or re-emit the build-bus.publish event',
+    );
+  }
+
+  // A success that predates the newest publish means the newest publish was
+  // dropped on the dispatch side even though earlier ones ran. The ledger's
+  // ran_at has second resolution (unixepoch), so the event's millisecond
+  // stamp is truncated to the same granularity before comparing — otherwise
+  // a run in the same second as its event reads as older than it.
+  const newestPublishSec = Math.floor((newestPublish.emittedAt ?? 0) / 1000) * 1000;
+  if (newest.ranAt && newest.ranAt.getTime() < newestPublishSec) {
+    return finding(
+      'warn',
+      'newest publish event predates the newest hook run — that publish dispatched nothing',
+      [...detail, runLine],
+      'check the receiver daemon logs for that event and the match rules against what the publisher sent',
+    );
+  }
+
+  return finding(
+    'ok',
+    `last self-update hook succeeded (${newest.packageName}@${newest.packageVersion})`,
+    [...detail, runLine],
+    null,
+  );
+}
+
+/**
+ * Does module `id`'s installed manifest declare an `on_upstream_publish`
+ * hook? Reads the manifest from disk exactly as the hook dispatcher's own
+ * loader does, so the check and the dispatcher cannot disagree about what
+ * "in use" means. A manifest that has vanished or fails to parse reads as
+ * "no hooks" — the dispatcher skips it silently too (Rule 6.2: the skip is
+ * visible here as fact 1 rather than as a crash).
+ */
+function hasUpstreamPublishHooks(db: DbClient, moduleId: string): boolean {
+  const row = db
+    .select({ sourcePath: modules.sourcePath })
+    .from(modules)
+    .where(eq(modules.id, moduleId))
+    .get();
+  if (!row?.sourcePath) return false;
+  try {
+    const parsed = parseYaml(readFileSync(join(row.sourcePath, 'manifest.yml'), 'utf-8')) as {
+      hooks?: { on_upstream_publish?: unknown[] };
+    };
+    return (parsed.hooks?.on_upstream_publish ?? []).length > 0;
+  } catch {
+    return false;
+  }
+}
+
 /** A deployed module + its parsed manifest (INSTALLED/VERIFIED only). */
 interface DeployedModule {
   id: string;
@@ -668,7 +867,10 @@ export function checkCapabilityProviders(db: DbClient): FleetFinding {
   }
   if (traceNeeded.length > 0) {
     detail.push(
-      `${traceNeeded.length} derived value(s) resolve through another capability — verify with \`celilo capability chain <module> <var>\` (ISS-0114): ${traceNeeded.join('; ')}`,
+      // celilo#1308: `celilo capability chain` no longer exists (the ISS-0115
+      // rework removed it), so prescribing it sent the operator to an
+      // "unknown command" error. No CLI command traces a chain; say so.
+      `${traceNeeded.length} derived value(s) resolve through another capability. No CLI command traces a capability chain, so verify these by hand (ISS-0114): ${traceNeeded.join('; ')}`,
     );
   }
 
@@ -1194,6 +1396,7 @@ export async function runFleetChecks(
   return [
     checkSchemaDrift(db),
     checkDispatcher(bus, { now: opts.now, installedCodeMtimeMs: opts.installedCodeMtimeMs }),
+    checkBuildBusPublishing(bus, db, { now: opts.now }),
     checkSubscribers(bus, db),
     checkCapabilityProviders(db),
     checkControlPlaneNetwork(db),

package/src/services/health-runner.test.ts

@@ -1,5 +1,14 @@
-import { describe, expect, test } from 'bun:test';
-import { nextModuleState } from './health-runner';
+import { describe, expect, spyOn, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { type DbClient, getDb } from '../db/client';
+import { modules } from '../db/schema';
+import { resetTestDbPath } from '../test-utils/db-path';
+import * as controlPlaneHealth from './control-plane-health';
+import type { HealthCheckItem } from './health-runner';
+import { deriveHealthStatus, nextModuleState, runModuleHealthCheck } from './health-runner';
 
 /**
  * The D15 rule: a scheduled (monitor-driven) health check observes only and
@@ -9,6 +18,34 @@ import { nextModuleState } from './health-runner';
  * throw, it just makes `celilo module list` flicker between INSTALLED and
  * VERIFIED every monitor interval.
  */
+describe('deriveHealthStatus', () => {
+  const item = (status: HealthCheckItem['status'], name = 'probe'): HealthCheckItem => ({
+    name,
+    status,
+    message: 'msg',
+  });
+
+  test('fail dominates everything', () => {
+    expect(deriveHealthStatus([item('pass'), item('fail'), item('warn')])).toBe('unhealthy');
+  });
+
+  test('warn without fail is degraded', () => {
+    expect(deriveHealthStatus([item('pass'), item('warn')])).toBe('degraded');
+  });
+
+  test('measured passes with a skip are healthy — the audit carries the skip', () => {
+    expect(deriveHealthStatus([item('pass'), item('skip')])).toBe('healthy');
+  });
+
+  test('only skips is no-checks — nothing was measured', () => {
+    expect(deriveHealthStatus([item('skip'), item('skip')])).toBe('no-checks');
+  });
+
+  test('empty checks is healthy (the hook ran and reported nothing)', () => {
+    expect(deriveHealthStatus([])).toBe('healthy');
+  });
+});
+
 describe('nextModuleState', () => {
   describe('operator-invoked run (unattended = false)', () => {
     test('passing check verifies an INSTALLED module', () => {
@@ -48,3 +85,51 @@ describe('nextModuleState', () => {
     });
   });
 });
+
+// celilo#1363: a failed upgrade leaves state ERROR with an errorMessage. The
+// recovery path is an attended health check passing, which moves ERROR →
+// VERIFIED. The message must move with the state, or `module list` keeps
+// printing the old `Error:` line beside a healthy row.
+describe('runModuleHealthCheck — clearing a recorded failure (celilo#1363)', () => {
+  test('an attended healthy check clears errorMessage along with the state', async () => {
+    const tempDir = mkdtempSync(join(tmpdir(), 'celilo-health-'));
+    process.env.CELILO_DB_PATH = join(tempDir, 'test.db');
+
+    try {
+      const db: DbClient = getDb();
+      db.insert(modules)
+        .values({
+          id: 'celilo-mgmt',
+          name: 'Control plane',
+          sourcePath: join(tempDir, 'celilo-mgmt'),
+          version: '1.0.0+5',
+          state: 'ERROR',
+          errorMessage: 'Upgrade to 1.0.0+6 failed to deploy: ssh: connection refused',
+          manifestData: {
+            celilo_contract: '1.0',
+            id: 'celilo-mgmt',
+            name: 'Control plane',
+            version: '1.0.0',
+          },
+        })
+        .run();
+
+      const checks: HealthCheckItem[] = [{ name: 'probe', status: 'pass', message: 'ok' }];
+      const spy = spyOn(controlPlaneHealth, 'controlPlaneHealthChecks').mockResolvedValue(checks);
+      try {
+        const result = await runModuleHealthCheck('celilo-mgmt', db, {});
+        expect(result.status).toBe('healthy');
+      } finally {
+        spy.mockRestore();
+      }
+
+      const row = db.select().from(modules).where(eq(modules.id, 'celilo-mgmt')).get();
+      if (!row) throw new Error('row missing after health check');
+      expect(row.state).toBe('VERIFIED');
+      expect(row.errorMessage).toBeNull();
+    } finally {
+      rmSync(tempDir, { recursive: true, force: true });
+      resetTestDbPath();
+    }
+  });
+});