@celilo/cli 2.2.1 → 3.0.0

package/src/services/health-runner.ts

@@ -15,7 +15,7 @@ import { createHookStores } from '../hooks/hook-store';
 import { loadHookConfigMap } from '../hooks/load-hook-config';
 import { createCapturingLogger, createConsoleLogger, createGaugeLogger } from '../hooks/logger';
 import type { HookLogger, HookResult } from '../hooks/types';
-import type { ModuleManifest } from '../manifest/schema';
+import type { HealthWaiver, ModuleManifest } from '../manifest/schema';
 import { decryptSecret } from '../secrets/encryption';
 import { getOrCreateMasterKey } from '../secrets/master-key';
 import { controlPlaneHealthChecks } from './control-plane-health';
@@ -24,7 +24,14 @@ import { remoteAccessPolicy } from './remote-access';
 
 export interface HealthCheckItem {
   name: string;
-  status: 'pass' | 'warn' | 'fail';
+  /**
+   * `skip` — the check could not perform its real probe (a credential is
+   * not provisioned, a peer is unreachable from the control plane) and
+   * refuses to guess. It is not a pass: a module whose checks all report
+   * skip is unmeasured, and `system audit` renders the fleet UNKNOWN, not
+   * READY. Never claim `pass` for a probe you did not run (celilo#1326).
+   */
+  status: 'pass' | 'warn' | 'fail' | 'skip';
   message: string;
   details?: string;
 }
@@ -44,12 +51,26 @@ export interface HealthCheckResult {
    * watching to re-run the check by hand.
    */
   artifactPaths?: string[];
+  /**
+   * The module's recorded health waiver, if its manifest declares one
+   * (openspec/changes/health-waiver-mechanism). It does not change this
+   * result's severity — an unmeasured module stays unmeasured — it
+   * annotates the finding with who decided and why.
+   */
+  waiver?: HealthWaiver;
 }
 
 export interface HealthCheckOptions {
   debug?: boolean;
   noInteractive?: boolean;
   onProgress?: (msg: string) => void;
+  /**
+   * Presentation-only silence: no gauge and no plain stdout lines, hook
+   * output captured instead. Unlike `unattended` this does NOT touch the
+   * state-transition guard — a `--json` command still runs real checks
+   * and still moves lifecycle state (celilo#1362).
+   */
+  quiet?: boolean;
   /**
    * Scheduled/monitor-driven run: no FuelGauge, no stdout, and — the part
    * that matters beyond cosmetics — the module's lifecycle state is NOT
@@ -98,6 +119,15 @@ function createHealthCheckPresentation(
     return { logger: createCapturingLogger().logger, finish: NO_OP_FINISH };
   }
 
+  // Non-interactive stdout (redirected, piped, or an explicit --json
+  // caller): stdout is a data channel here (celilo#699), so neither the
+  // gauge's frames nor the gauge logger's plain lines may reach it.
+  // Capture everything instead (celilo#1362). State transitions still
+  // happen — only the presentation goes quiet.
+  if (options.quiet || !process.stdout.isTTY) {
+    return { logger: createCapturingLogger().logger, finish: NO_OP_FINISH };
+  }
+
   const gauge = new FuelGauge('Testing app', { skipAnimation: options.noInteractive });
   gauge.start();
   return {
@@ -171,7 +201,7 @@ export async function runModuleHealthCheck(
   const hookDef = manifest.hooks?.health_check;
 
   if (!hookDef) {
-    return { moduleId, status: 'no-checks', checks: [] };
+    return { moduleId, status: 'no-checks', checks: [], waiver: manifest.health_waiver };
   }
 
   // Build config + secrets for hook context. The config map shape
@@ -238,7 +268,31 @@ export async function runModuleHealthCheck(
   };
 
   const checks = outputs.checks || [];
-  return finishHealthCheck(moduleId, module.state, checks, db, options, hookResult.artifactPaths);
+  return finishHealthCheck(
+    moduleId,
+    module.state,
+    checks,
+    db,
+    options,
+    hookResult.artifactPaths,
+    manifest.health_waiver,
+  );
+}
+
+/**
+ * Derive the module-level verdict from the named checks.
+ *
+ * Pure so the folding rule is testable without a database or a hook
+ * subprocess. `skip` checks are folded toward nothing measured: they are
+ * neither pass nor fail, and a module whose checks all skipped is
+ * `no-checks` — nothing ran that deserves the word healthy.
+ */
+export function deriveHealthStatus(checks: HealthCheckItem[]): HealthCheckResult['status'] {
+  if (checks.some((c) => c.status === 'fail')) return 'unhealthy';
+  if (checks.some((c) => c.status === 'warn')) return 'degraded';
+  const measured = checks.filter((c) => c.status !== 'skip');
+  if (checks.length > 0 && measured.length === 0) return 'no-checks';
+  return 'healthy';
 }
 
 /**
@@ -254,22 +308,35 @@ function finishHealthCheck(
   db: DbClient,
   options: HealthCheckOptions,
   artifactPaths?: string[],
+  waiver?: HealthWaiver,
 ): HealthCheckResult {
-  const hasFail = checks.some((c) => c.status === 'fail');
-  const hasWarn = checks.some((c) => c.status === 'warn');
-
-  const status: HealthCheckResult['status'] = hasFail
-    ? 'unhealthy'
-    : hasWarn
-      ? 'degraded'
-      : 'healthy';
+  const status = deriveHealthStatus(checks);
 
-  const nextState = nextModuleState(moduleState, status, options.unattended ?? false);
+  // Only a run that measured something may move lifecycle state. A module
+  // whose checks all skipped (or that declared no hook) stays as it was —
+  // `VERIFIED` asserts that a check was deliberately run and passed, and
+  // nothing was measured here.
+  const nextState =
+    status === 'healthy' || status === 'degraded' || status === 'unhealthy'
+      ? nextModuleState(moduleState, status, options.unattended ?? false)
+      : null;
   if (nextState) {
-    db.update(modules).set({ state: nextState }).where(eq(modules.id, moduleId)).run();
+    // Moving to VERIFIED/INSTALLED means the module just measured healthy (or
+    // was freshly deployed). Clear a recorded failure with the state, so a
+    // healthy row doesn't keep printing its old `Error:` line (celilo#1363).
+    db.update(modules)
+      .set({ state: nextState, errorMessage: null })
+      .where(eq(modules.id, moduleId))
+      .run();
   }
 
-  return { moduleId, status, checks, ...(artifactPaths ? { artifactPaths } : {}) };
+  return {
+    moduleId,
+    status,
+    checks,
+    ...(artifactPaths ? { artifactPaths } : {}),
+    ...(waiver ? { waiver } : {}),
+  };
 }
 
 /**
@@ -314,7 +381,7 @@ async function mapConcurrent<T, R>(
  */
 export async function runAllHealthChecks(
   db: DbClient,
-  options: { debug?: boolean; onProgress?: (msg: string) => void } = {},
+  options: { debug?: boolean; onProgress?: (msg: string) => void; quiet?: boolean } = {},
 ): Promise<HealthCheckResult[]> {
   const eligible = db
     .select()

package/src/services/infrastructure-selector.test.ts

@@ -48,7 +48,7 @@ describe('infrastructure-selector', () => {
 
     // Clear environment variables
     resetTestDbPath();
-    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    delete process.env.CELILO_MASTER_KEY_PATH;
   });
 
   describe('selectInfrastructure', () => {

package/src/services/jail-exemptions.test.ts

@@ -0,0 +1,125 @@
+/**
+ * Tests for the shared jail-exemption collector (per-module-jail-policy
+ * task 3.1-3.3).
+ *
+ * Every reporting surface — `module list`, `system doctor`, `system audit` —
+ * builds its view from this one function, so these tests are what stops the
+ * three from disagreeing about what an exemption is. Isolation mirrors
+ * module-jail.test.ts: CELILO_DB_PATH / CELILO_DATA_DIR are set before the
+ * SUT is imported.
+ */
+
+import { afterAll, beforeEach, describe, expect, test } from 'bun:test';
+import { mkdtempSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+
+const testRoot = mkdtempSync(join(tmpdir(), 'celilo-jail-exemptions-'));
+process.env.CELILO_DB_PATH = join(testRoot, 'celilo.db');
+process.env.CELILO_DATA_DIR = join(testRoot, 'data');
+
+const { getDb, closeDb } = await import('../db/client');
+const { moduleJailPolicies, modules, systemConfig } = await import('../db/schema');
+const { collectJailExemptions } = await import('./jail-exemptions');
+
+function insertModule(id: string): void {
+  getDb()
+    .insert(modules)
+    .values({ id, name: id, sourcePath: `/tmp/${id}`, version: '1.0.0', manifestData: {} })
+    .run();
+}
+
+function setModulePolicy(moduleId: string, policy: string): void {
+  const typed = policy as 'auto' | 'off' | 'required';
+  getDb()
+    .insert(moduleJailPolicies)
+    .values({ moduleId, policy: typed, updatedAt: new Date() })
+    .onConflictDoUpdate({
+      target: moduleJailPolicies.moduleId,
+      set: { policy: typed, updatedAt: new Date() },
+    })
+    .run();
+}
+
+function setSystemPolicy(value: string | undefined): void {
+  const db = getDb();
+  if (value === undefined) {
+    db.delete(systemConfig).where(eq(systemConfig.key, 'hooks.jail_policy')).run();
+  } else {
+    db.insert(systemConfig)
+      .values({ key: 'hooks.jail_policy', value })
+      .onConflictDoUpdate({ target: systemConfig.key, set: { value } })
+      .run();
+  }
+}
+
+afterAll(() => {
+  closeDb();
+  rmSync(testRoot, { recursive: true, force: true });
+});
+
+describe('collectJailExemptions (per-module-jail-policy task 3.4)', () => {
+  beforeEach(() => {
+    const db = getDb();
+    db.delete(moduleJailPolicies).run();
+    db.delete(systemConfig).where(eq(systemConfig.key, 'hooks.jail_policy')).run();
+    db.delete(modules).run();
+    insertModule('strictmod');
+    insertModule('exemptmod');
+  });
+
+  test('no per-module rows produce no exemptions', () => {
+    expect(collectJailExemptions(getDb())).toEqual([]);
+  });
+
+  test("a module policy stronger than the system's is not an exemption", () => {
+    // The fleet default is `off`, so `auto` STRENGTHENS the posture. Reporting
+    // it as an exemption would be a surface reporting an exemption nobody set.
+    setModulePolicy('strictmod', 'required');
+    expect(collectJailExemptions(getDb())).toEqual([]);
+  });
+
+  test("a module policy equal to the system's is not an exemption", () => {
+    setSystemPolicy('auto');
+    setModulePolicy('strictmod', 'auto');
+    expect(collectJailExemptions(getDb())).toEqual([]);
+  });
+
+  test("a module policy weaker than the system's is reported, with both values", () => {
+    setSystemPolicy('auto');
+    setModulePolicy('exemptmod', 'off');
+    expect(collectJailExemptions(getDb())).toEqual([
+      { moduleId: 'exemptmod', policy: 'off', systemPolicy: 'auto' },
+    ]);
+  });
+
+  test('a weaker exemption against the unset-system default (off) is impossible', () => {
+    // off is the floor; nothing can be weaker than it. This is the negative
+    // half of task 3.4: with nobody having set the system key, no module row
+    // — whatever it holds — may read as an exemption.
+    setModulePolicy('exemptmod', 'off');
+    expect(collectJailExemptions(getDb())).toEqual([]);
+  });
+
+  test('two exemptions are both reported', () => {
+    insertModule('exemptmod2');
+    setSystemPolicy('required');
+    setModulePolicy('exemptmod', 'auto');
+    setModulePolicy('exemptmod2', 'off');
+    const exemptions = collectJailExemptions(getDb());
+    expect(exemptions).toHaveLength(2);
+    expect(exemptions.map((e) => e.moduleId).sort()).toEqual(['exemptmod', 'exemptmod2']);
+  });
+
+  test('a stored module policy outside the accepted set throws, not coerces', () => {
+    setModulePolicy('exemptmod', 'sometimes');
+    expect(() => collectJailExemptions(getDb())).toThrow(/not a hook jail policy/);
+  });
+
+  test('a stored system policy outside the accepted set throws, naming the fix', () => {
+    setSystemPolicy('jail-harder');
+    setModulePolicy('exemptmod', 'off');
+    expect(() => collectJailExemptions(getDb())).toThrow(/hooks\.jail_policy='jail-harder'/);
+  });
+});

package/src/services/jail-exemptions.ts

@@ -0,0 +1,81 @@
+/**
+ * The per-module jail exemptions, as one query every reporting surface shares
+ * (per-module-jail-policy task 3.1-3.3).
+ *
+ * An exemption is a recorded module policy weaker than the system's
+ * `hooks.jail_policy`. The three surfaces — `module list`'s marker,
+ * `system doctor`'s count, `system audit`'s category — MUST agree on what
+ * that is, so the comparison lives in `isWeakerJailPolicy` and this file only
+ * fetches rows and applies it. A surface that re-derived the definition here
+ * would eventually report a different set from the other two, and a countable
+ * exemption that two surfaces miss is the failure this slice exists to close.
+ *
+ * Read-time refusal follows the rule the resolvers already follow: a stored
+ * value outside the accepted set THROWS (peba's ruling on ce-8832). The set
+ * path validates, so a bad row can only arrive via a restore or a hand edit —
+ * the exact foreign state the rule refuses to coerce into a silent default.
+ */
+
+import { eq } from 'drizzle-orm';
+import type { getDb as getDbType } from '../db/client';
+import { moduleJailPolicies, systemConfig } from '../db/schema';
+import { type JailPolicy, isWeakerJailPolicy, resolveJailPolicy } from '../hooks/jail';
+
+const JAIL_SYSTEM_KEY = 'hooks.jail_policy';
+
+export interface JailExemption {
+  moduleId: string;
+  /** The module's own recorded policy — the weaker of the two. */
+  policy: JailPolicy;
+  /** The system policy it is weaker than: the stored key or the default. */
+  systemPolicy: JailPolicy;
+}
+
+type Db = ReturnType<typeof getDbType>;
+
+/**
+ * The system-wide policy every module without its own row follows.
+ *
+ * Throws on a stored value outside the accepted set, matching the read-time
+ * refusal rule; the error names the fix, and the same throw will be failing
+ * every hook invocation too (the executor resolves through the same key).
+ */
+function systemJailPolicy(db: Db): JailPolicy {
+  const stored = db
+    .select()
+    .from(systemConfig)
+    .where(eq(systemConfig.key, JAIL_SYSTEM_KEY))
+    .get()?.value;
+  if (stored === undefined) return 'off';
+  try {
+    return resolveJailPolicy(undefined, undefined, stored).policy;
+  } catch {
+    throw new Error(
+      `Stored ${JAIL_SYSTEM_KEY}='${stored}' is not a hook jail policy. Fix it with 'celilo system config set ${JAIL_SYSTEM_KEY} <auto|off|required>'.`,
+    );
+  }
+}
+
+/**
+ * Every module running under a recorded policy weaker than the system's.
+ *
+ * A module row stronger than or equal to the system's is NOT an exemption —
+ * the operator strengthened that module's posture, and reporting it as an
+ * exemption would be a phantom finding (per-module-jail-policy task 3.4).
+ * A bad module row throws rather than being skipped or coerced.
+ */
+export function collectJailExemptions(db: Db): JailExemption[] {
+  const system = systemJailPolicy(db);
+  const rows = db.select().from(moduleJailPolicies).all();
+
+  const exemptions: JailExemption[] = [];
+  for (const row of rows) {
+    // Read-time validation through the same resolver the executor uses, so a
+    // hand-edited row fails with the same message here that a hook would get.
+    const policy = resolveJailPolicy(undefined, row.policy, undefined).policy;
+    if (isWeakerJailPolicy(policy, system)) {
+      exemptions.push({ moduleId: row.moduleId, policy, systemPolicy: system });
+    }
+  }
+  return exemptions;
+}

package/src/services/machine-pool.test.ts

@@ -54,7 +54,7 @@ describe('machine-pool', () => {
 
     // Clear environment variables
     resetTestDbPath();
-    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    delete process.env.CELILO_MASTER_KEY_PATH;
   });
 
   describe('addMachine', () => {

package/src/services/module-deploy-prune.test.ts

@@ -0,0 +1,89 @@
+import { afterEach, beforeEach, describe, expect, test } from 'bun:test';
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { isPreservedGeneratedPath, pruneGeneratedProject } from './module-deploy';
+
+const ROOT = '/tmp/celilo-prune-test';
+const GENERATED = join(ROOT, 'generated');
+
+/**
+ * Mirrors the layout observed on celilo-mgr at 2026-09-16T00:51Z:
+ * `generated/{ansible,deploy.log,terraform}` with state inside terraform/.
+ * Derived from the real tree rather than invented — a hand-written mirror gives
+ * confident results about a world that does not exist.
+ */
+function buildGeneratedTree(): void {
+  mkdirSync(join(GENERATED, 'ansible', 'roles'), { recursive: true });
+  mkdirSync(join(GENERATED, 'terraform', '.terraform', 'providers'), { recursive: true });
+
+  writeFileSync(join(GENERATED, 'deploy.log'), 'deploy output');
+  writeFileSync(join(GENERATED, 'ansible', 'secrets.yml'), 'vault-encrypted secrets');
+  writeFileSync(join(GENERATED, 'ansible', 'roles', 'task.yml'), 'rendered task');
+  writeFileSync(join(GENERATED, 'terraform', 'main.tf'), 'rendered terraform');
+  writeFileSync(join(GENERATED, 'terraform', 'terraform.tfvars'), 'rendered vars');
+  writeFileSync(join(GENERATED, 'terraform', 'terraform.tfstate'), '{"lxc":207}');
+  writeFileSync(join(GENERATED, 'terraform', 'terraform.tfstate.backup'), '{"lxc":207,"old":1}');
+  writeFileSync(join(GENERATED, 'terraform', '.terraform.lock.hcl'), 'provider lock');
+  writeFileSync(join(GENERATED, 'terraform', '.terraform', 'providers', 'bin'), 'huge binary');
+}
+
+beforeEach(() => {
+  rmSync(ROOT, { recursive: true, force: true });
+  buildGeneratedTree();
+});
+
+afterEach(() => {
+  rmSync(ROOT, { recursive: true, force: true });
+});
+
+describe('pruneGeneratedProject', () => {
+  // The recurrence gate for celilo#1374's root cause. Deleting this file is
+  // what armed every container module for destruction on its next deploy.
+  test('keeps terraform state and its lock', () => {
+    pruneGeneratedProject(GENERATED);
+
+    expect(existsSync(join(GENERATED, 'terraform', 'terraform.tfstate'))).toBe(true);
+    expect(existsSync(join(GENERATED, 'terraform', 'terraform.tfstate.backup'))).toBe(true);
+    expect(existsSync(join(GENERATED, 'terraform', '.terraform.lock.hcl'))).toBe(true);
+    // Intact, not merely present.
+    expect(readFileSync(join(GENERATED, 'terraform', 'terraform.tfstate'), 'utf-8')).toBe(
+      '{"lxc":207}',
+    );
+  });
+
+  // D4's actual concern. If this ever passes, the fix has quietly become
+  // "persist the rendered project", which D4 refused for good reasons.
+  test('still deletes every rendered template and secret', () => {
+    pruneGeneratedProject(GENERATED);
+
+    expect(existsSync(join(GENERATED, 'ansible'))).toBe(false);
+    expect(existsSync(join(GENERATED, 'deploy.log'))).toBe(false);
+    expect(existsSync(join(GENERATED, 'terraform', 'main.tf'))).toBe(false);
+    expect(existsSync(join(GENERATED, 'terraform', 'terraform.tfvars'))).toBe(false);
+  });
+
+  // ~19 MB per provider, rebuilt by `terraform init`; excluded from the backup
+  // envelope for the same reason (cross-module-read.ts).
+  test('drops the provider cache', () => {
+    pruneGeneratedProject(GENERATED);
+
+    expect(existsSync(join(GENERATED, 'terraform', '.terraform'))).toBe(false);
+  });
+
+  // A deploy that failed before rendering never made one.
+  test('is a no-op when generated/ is absent', () => {
+    rmSync(GENERATED, { recursive: true, force: true });
+
+    expect(() => pruneGeneratedProject(GENERATED)).not.toThrow();
+  });
+});
+
+describe('isPreservedGeneratedPath', () => {
+  test('preserves only state files directly under terraform/', () => {
+    expect(isPreservedGeneratedPath('terraform/terraform.tfstate')).toBe(true);
+    expect(isPreservedGeneratedPath('terraform/main.tf')).toBe(false);
+    expect(isPreservedGeneratedPath('ansible/terraform.tfstate')).toBe(false);
+    // A nested lookalike must not smuggle a whole tree through.
+    expect(isPreservedGeneratedPath('terraform/nested/terraform.tfstate')).toBe(false);
+  });
+});