@celilo/cli 1.5.0 → 1.6.0

package/src/services/audit/index.ts

@@ -18,9 +18,14 @@ import { type BackupsAuditDeps, auditBackups } from './backups';
 import { type BrowserPinAuditDeps, auditBrowserPin } from './browser-pin';
 import { type CapabilityAbiAuditDeps, auditCapabilityAbi } from './capability-abi';
 import { type CliVersionAuditDeps, auditCliVersion } from './cli-version';
+import {
+  type DetectWithoutConvergeAuditDeps,
+  auditDetectWithoutConverge,
+} from './detect-without-converge';
 import { type HealthAuditDeps, auditHealth } from './health';
 import { type MachinesReachableAuditDeps, auditMachinesReachable } from './machines-reachable';
 import { type ModuleConfigsAuditDeps, auditModuleConfigs } from './module-configs';
+import { type ModuleIntegrityAuditDeps, auditModuleIntegrity } from './module-integrity';
 import { type ModuleVersionsAuditDeps, auditModuleVersions } from './module-versions';
 import { type PublicDnsAuditDeps, auditPublicDns } from './public-dns';
 import { type SchemaAuditDeps, auditSchema } from './schema';
@@ -53,6 +58,8 @@ export interface AuditDeps {
   terraformPlan: TerraformPlanAuditDeps;
   moduleVersions: ModuleVersionsAuditDeps;
   moduleConfigs: ModuleConfigsAuditDeps;
+  moduleIntegrity: ModuleIntegrityAuditDeps;
+  detectWithoutConverge: DetectWithoutConvergeAuditDeps;
   health: HealthAuditDeps;
   backups: BackupsAuditDeps;
   abandonedOperations: AbandonedOperationsAuditDeps;
@@ -107,6 +114,11 @@ export async function runAudit(
     wrap('terraform_plan', auditTerraformPlan(deps.terraformPlan)),
     wrap('module_versions', auditModuleVersions(deps.moduleVersions)),
     wrap('module_configs', auditModuleConfigs(deps.moduleConfigs)),
+    wrap('module_integrity', Promise.resolve(auditModuleIntegrity(deps.moduleIntegrity))),
+    wrap(
+      'detect_without_converge',
+      Promise.resolve(auditDetectWithoutConverge(deps.detectWithoutConverge)),
+    ),
     wrap('health', auditHealth(deps.health)),
     wrap('backups', auditBackups(deps.backups)),
     wrap(

package/src/services/audit/module-integrity.test.ts

@@ -0,0 +1,146 @@
+import { describe, expect, test } from 'bun:test';
+import type { AuditResult } from '../../module/packaging/audit';
+import { auditModuleIntegrity } from './module-integrity';
+import { computeVerdict } from './types';
+
+const clean = (moduleId: string): AuditResult => ({
+  success: true,
+  moduleId,
+  violations: [],
+  moduleVersion: '1.0.0',
+  baselineVersion: '1.0.0',
+});
+
+describe('auditModuleIntegrity', () => {
+  test('a clean module produces nothing', () => {
+    expect(auditModuleIntegrity({ results: [clean('caddy')] })).toEqual([]);
+  });
+
+  test('a check that could not run is unmeasured, and the verdict is UNKNOWN', () => {
+    const findings = auditModuleIntegrity({
+      results: [
+        {
+          success: false,
+          moduleId: 'caddy',
+          violations: [],
+          error: "No integrity data found for module 'caddy'.",
+        },
+      ],
+    });
+    expect(findings.map((f) => f.severity)).toEqual(['unmeasured']);
+    expect(computeVerdict(findings)).toBe('UNKNOWN');
+  });
+
+  test('a stale baseline is unmeasured, and it SUPPRESSES the file findings it explains', () => {
+    // The 72-violation shape. One frozen row made every legitimately-changed
+    // file read as [MODIFIED], and printing all of them is what made `module
+    // verify` unreadable. The baseline is the finding; the files are its
+    // consequence.
+    const findings = auditModuleIntegrity({
+      results: [
+        {
+          ...clean('wireguard'),
+          success: false,
+          baselineVersion: '0.7.0',
+          moduleVersion: '0.8.0',
+          violations: [
+            {
+              type: 'stale-baseline',
+              path: 'checksums.json',
+              message: 'Baseline describes 0.7.0, module records 0.8.0.',
+            },
+            { type: 'modified', path: 'manifest.yml', message: 'Checksum mismatch: manifest.yml' },
+            { type: 'extra', path: 'scripts/new.ts', message: 'Unexpected file: scripts/new.ts' },
+          ],
+        },
+      ],
+    });
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.code).toBe('module_integrity_stale_baseline');
+    expect(findings[0]?.severity).toBe('unmeasured');
+    expect(findings[0]?.remediation).toBe('celilo module update wireguard');
+  });
+
+  test('file drift under a TRUSTWORTHY baseline is drift, and lists every file', () => {
+    const findings = auditModuleIntegrity({
+      results: [
+        {
+          ...clean('wireguard'),
+          success: false,
+          violations: [
+            { type: 'modified', path: 'manifest.yml', message: 'Checksum mismatch: manifest.yml' },
+            { type: 'extra', path: 'scripts/new.ts', message: 'Unexpected file: scripts/new.ts' },
+          ],
+        },
+      ],
+    });
+    expect(findings).toHaveLength(1);
+    expect(findings[0]?.severity).toBe('drift');
+    // Never truncated to the first item. celilo#951's real finding would have
+    // arrived at position 19 of 19 and never been printed.
+    expect(findings[0]?.details).toContain('manifest.yml');
+    expect(findings[0]?.details).toContain('scripts/new.ts');
+  });
+
+  test('a stale generated project is drift, and says a deploy would ship the wrong bytes', () => {
+    const findings = auditModuleIntegrity({
+      results: [
+        {
+          ...clean('wireguard-manager'),
+          success: false,
+          violations: [
+            {
+              type: 'stale-generated',
+              path: 'ansible/roles/vpn/files/bin',
+              message: 'Generated project holds different bytes',
+            },
+          ],
+        },
+      ],
+    });
+    expect(findings.map((f) => f.code)).toEqual(['module_generated_stale']);
+    expect(findings[0]?.severity).toBe('drift');
+    expect(findings[0]?.remediation).toBe('celilo module generate wireguard-manager');
+  });
+
+  test('host-plane results carry through with the right severity', () => {
+    const findings = auditModuleIntegrity({
+      results: [
+        {
+          ...clean('wireguard'),
+          hostPlane: {
+            findings: [
+              { hostname: 'a', state: 'converged' },
+              { hostname: 'b', state: 'drift', detail: 'would change 2 things' },
+              { hostname: 'c', state: 'unmeasured', detail: 'did not answer' },
+            ],
+          },
+        },
+      ],
+    });
+    expect(findings.map((f) => [f.code, f.severity])).toEqual([
+      ['module_host_drifted', 'drift'],
+      ['module_host_unmeasured', 'unmeasured'],
+    ]);
+    // An unreachable host must not let the run read as READY.
+    expect(computeVerdict(findings)).toBe('UNKNOWN');
+  });
+
+  test('every remediation is a runnable celilo command', () => {
+    const findings = auditModuleIntegrity({
+      results: [
+        { success: false, moduleId: 'a', violations: [], error: 'gone' },
+        {
+          ...clean('b'),
+          success: false,
+          violations: [{ type: 'modified', path: 'x', message: 'x' }],
+        },
+      ],
+    });
+    expect(findings.length).toBeGreaterThan(0);
+    for (const f of findings) {
+      expect(`${f.code}: ${f.remediation}`).toMatch(/: celilo /);
+      expect(f.actionable).toBe(true);
+    }
+  });
+});

package/src/services/audit/module-integrity.ts

@@ -0,0 +1,113 @@
+/**
+ * Module integrity as a drift category
+ * (openspec/changes/module-integrity-rigor, D8).
+ *
+ * One implementation, three surfaces: `celilo module verify <id>` for one
+ * module, `celilo system audit` for every module, and `system doctor`'s fleet
+ * section. `system doctor` answers "can this box run celilo, and is the runtime
+ * wired up"; `system audit` answers "has anything drifted from desired state",
+ * and module integrity is a drift category by every structural test.
+ *
+ * Pure of I/O in the same way its siblings are: the caller does the auditing
+ * and passes the results in, so this file is testable without a filesystem.
+ */
+
+import type { AuditResult } from '../../module/packaging/audit';
+import type { DriftFinding } from './types';
+
+export interface ModuleIntegrityAuditDeps {
+  /** One `auditModule` result per installed module. */
+  results: AuditResult[];
+}
+
+/**
+ * A verify failure celilo could not perform at all — no baseline row, module
+ * directory gone — is `unmeasured`, not `drift`. It is the difference between
+ * "the files moved" and "I never looked", which is the whole point of D7.
+ */
+export function auditModuleIntegrity(deps: ModuleIntegrityAuditDeps): DriftFinding[] {
+  const findings: DriftFinding[] = [];
+
+  for (const result of deps.results) {
+    if (result.error) {
+      findings.push({
+        category: 'module_integrity',
+        severity: 'unmeasured',
+        code: 'module_integrity_unmeasured',
+        message: `${result.moduleId}: integrity could not be checked — ${result.error}`,
+        remediation: `celilo module verify ${result.moduleId}`,
+        actionable: true,
+        subject: result.moduleId,
+      });
+      continue;
+    }
+
+    // Ahead of the file findings it explains. When the baseline describes a
+    // different version, every file difference beneath it is a consequence of
+    // that and not independent evidence.
+    const stale = result.violations.filter((v) => v.type === 'stale-baseline');
+    for (const violation of stale) {
+      findings.push({
+        category: 'module_integrity',
+        severity: 'unmeasured',
+        code: 'module_integrity_stale_baseline',
+        message: `${result.moduleId}: ${violation.message}`,
+        remediation: `celilo module update ${result.moduleId}`,
+        actionable: true,
+        subject: result.moduleId,
+      });
+    }
+
+    const generated = result.violations.filter((v) => v.type === 'stale-generated');
+    if (generated.length > 0) {
+      findings.push({
+        category: 'module_integrity',
+        severity: 'drift',
+        code: 'module_generated_stale',
+        message: `${result.moduleId}: ${generated.length} generated asset(s) do not match the installed module — a deploy would ship the wrong bytes`,
+        details: generated.map((v) => `  • ${v.path}`).join('\n'),
+        remediation: `celilo module generate ${result.moduleId}`,
+        actionable: true,
+        subject: result.moduleId,
+      });
+    }
+
+    // File differences are only meaningful once the baseline is trustworthy.
+    // Reporting them under a stale baseline is what made `module verify`
+    // unreadable: 72 violations across two healthy modules, nearly all of them
+    // consequences of one frozen row.
+    const files = result.violations.filter(
+      (v) => v.type === 'modified' || v.type === 'missing' || v.type === 'extra',
+    );
+    if (files.length > 0 && stale.length === 0) {
+      findings.push({
+        category: 'module_integrity',
+        severity: 'drift',
+        code: 'module_files_drifted',
+        message: `${result.moduleId}: ${files.length} installed file(s) do not match the ${result.baselineVersion ?? 'recorded'} baseline`,
+        details: files.map((v) => `  • [${v.type}] ${v.path}`).join('\n'),
+        remediation: `celilo module verify ${result.moduleId} --json`,
+        actionable: true,
+        subject: result.moduleId,
+      });
+    }
+
+    for (const host of result.hostPlane?.findings ?? []) {
+      if (host.state === 'converged') continue;
+      findings.push({
+        category: 'module_integrity',
+        severity: host.state === 'drift' ? 'drift' : 'unmeasured',
+        code: host.state === 'drift' ? 'module_host_drifted' : 'module_host_unmeasured',
+        message: `${result.moduleId} on ${host.hostname}: ${host.detail}`,
+        remediation:
+          host.state === 'drift'
+            ? `celilo module deploy ${result.moduleId}`
+            : `celilo module verify ${result.moduleId} --deep`,
+        actionable: true,
+        subject: result.moduleId,
+      });
+    }
+  }
+
+  return findings;
+}

package/src/services/audit/module-versions.ts

@@ -102,7 +102,10 @@ export async function auditModuleVersions(deps: ModuleVersionsAuditDeps): Promis
     if (r.status === 'rejected') {
       findings.push({
         category: 'module_versions',
-        severity: 'drift',
+        // A registry that did not answer is not a version difference. Reporting
+        // it as `drift` said "this module is behind" when the truth was "I
+        // could not ask" — the ambiguity D7 exists to remove.
+        severity: 'unmeasured',
         code: 'module_version_lookup_failed',
         message: `${installed.id}: registry lookup failed (${String(r.reason).slice(0, 80)})`,
         remediation: 'Check network connectivity to the registry, then press R to re-audit.',

package/src/services/audit/schema.test.ts

@@ -11,13 +11,18 @@ const journalWith = (tags: string[]) => () => ({
 });
 
 describe('auditSchema', () => {
-  test('no finding when journal is missing (folder not on disk)', async () => {
+  test('an unreadable journal is unmeasured, not clean', async () => {
+    // Was `expect(result).toEqual([])`. Silence rendered as READY, and pending
+    // migrations are the difference between a `.deb` that installed and a
+    // `.deb` that works — `apt upgrade` does not run them (celilo#169).
     const result = await auditSchema({
       journal: () => null,
       applied: () => [],
       db: fakeDb,
     });
-    expect(result).toEqual([]);
+    expect(result).toHaveLength(1);
+    expect(result[0]?.severity).toBe('unmeasured');
+    expect(result[0]?.code).toBe('schema_journal_unreadable');
   });
 
   test('no finding when every journal entry is applied', async () => {

package/src/services/audit/schema.ts

@@ -78,7 +78,25 @@ export interface SchemaAuditDeps {
 
 export async function auditSchema(deps: SchemaAuditDeps): Promise<DriftFinding[]> {
   const journal = deps.journal();
-  if (!journal) return []; // No migrations folder → nothing to compare.
+  if (!journal) {
+    // Pending migrations are the difference between a `.deb` that installed and
+    // a `.deb` that works — `apt upgrade` does not run them (celilo#169). With
+    // no journal there is nothing to compare against, and silence used to
+    // render that as READY.
+    return [
+      {
+        category: 'schema',
+        severity: 'unmeasured',
+        code: 'schema_journal_unreadable',
+        message:
+          'No drizzle migration journal could be read, so pending schema migrations are unknown',
+        remediation:
+          'The installed @celilo/cli is missing its `drizzle/meta/_journal.json`. Reinstall it, then re-audit. This records that the comparison did not happen, not that the schema is current.',
+        actionable: false,
+        subject: 'system',
+      },
+    ];
+  }
 
   // drizzle stores SHA-256 hashes (not tag names) in __drizzle_migrations,
   // so we can't intersect tags. The next-best signal is "did all the

package/src/services/audit/terraform-plan.ts

@@ -112,8 +112,23 @@ export async function auditTerraformPlan(deps: TerraformPlanAuditDeps): Promise<
     }
 
     const summary = parsePlanSummary(result.stdout);
-    if (!summary || (summary.add === 0 && summary.change === 0 && summary.destroy === 0)) {
-      // Plan is empty or unparseable-but-non-error — no drift.
+    if (!summary) {
+      // Terraform exited zero and celilo could not read its answer. That is not
+      // "no drift" — it is no measurement, and it used to be indistinguishable
+      // from a clean plan (D7).
+      findings.push({
+        category: 'terraform_plan',
+        severity: 'unmeasured',
+        code: 'terraform_plan_unparseable',
+        message: `${m.id}: terraform plan succeeded but its summary could not be parsed, so infrastructure drift is unknown`,
+        details: result.stdout.slice(0, 500),
+        remediation: `Run terraform plan in ${m.terraformDir} and read it directly. This records that the comparison did not happen, not that the infrastructure matches.`,
+        actionable: false,
+        subject: m.id,
+      });
+      continue;
+    }
+    if (summary.add === 0 && summary.change === 0 && summary.destroy === 0) {
       continue;
     }
 

package/src/services/audit/types.test.ts

@@ -17,6 +17,14 @@ const blocked: DriftFinding = {
   subject: 'lunacycle',
 };
 
+const unmeasured: DriftFinding = {
+  category: 'module_integrity',
+  severity: 'unmeasured',
+  code: 'module_integrity_unmeasured',
+  message: 'wireguard-manager: the host did not answer, so nothing was measured',
+  subject: 'wireguard-manager',
+};
+
 const todo: DriftFinding = {
   category: 'undeployed_modules',
   severity: 'todo',
@@ -59,4 +67,25 @@ describe('computeVerdict', () => {
   test('BLOCKED still wins over todos', () => {
     expect(computeVerdict([todo, drift, blocked])).toBe('BLOCKED');
   });
+
+  // D7. An unmeasured check is not a pass, and it is not the same statement as
+  // a measured difference. Before this, a category that could not reach its
+  // subject contributed nothing, and contributing nothing rendered as READY.
+  test('UNKNOWN when something could not be measured', () => {
+    expect(computeVerdict([unmeasured])).toBe('UNKNOWN');
+  });
+
+  test('an unmeasured finding never lets the verdict return READY', () => {
+    expect(computeVerdict([todo, unmeasured])).not.toBe('READY');
+  });
+
+  test('UNKNOWN outranks DRIFT — you cannot act on a diff you are not sure you have', () => {
+    expect(computeVerdict([drift, unmeasured])).toBe('UNKNOWN');
+    expect(computeVerdict([unmeasured, drift])).toBe('UNKNOWN');
+  });
+
+  test('BLOCKED still outranks UNKNOWN — a hard stop is still a hard stop', () => {
+    expect(computeVerdict([unmeasured, blocked])).toBe('BLOCKED');
+    expect(computeVerdict([blocked, unmeasured, drift, todo])).toBe('BLOCKED');
+  });
 });

package/src/services/audit/types.ts

@@ -9,7 +9,8 @@
  *
  * The overall verdict is computed from the findings:
  * - any `blocked` finding → BLOCKED
- * - any `drift` finding (no blocked) → DRIFT
+ * - any `unmeasured` finding (no blocked) → UNKNOWN
+ * - any `drift` finding (no blocked, no unmeasured) → DRIFT
  * - only `todo` findings (or none) → READY
  *
  * `todo` exists because some categories surface "next-step reminders"
@@ -42,11 +43,35 @@ export type DriftCategory =
   | 'disk_space'
   | 'transport_reads'
   | 'trusted_sources'
-  | 'interface_classification';
+  | 'interface_classification'
+  | 'module_integrity'
+  | 'detect_without_converge';
 
-export type DriftSeverity = 'todo' | 'drift' | 'blocked';
+/**
+ * `unmeasured` is a check that could not reach its subject
+ * (openspec/changes/module-integrity-rigor, D7).
+ *
+ * It exists because there was nowhere to put that. A category that could not
+ * measure either invented a finding of the wrong severity or contributed
+ * nothing, and contributing nothing renders as READY — the same defect as
+ * everything else this change is about, sitting in the framework that is
+ * supposed to catch it.
+ *
+ * Every claim celilo makes about the fleet is either measured against the thing
+ * it describes, or reported as unmeasured. There is no third category, and
+ * unmeasured never renders as green.
+ */
+export type DriftSeverity = 'todo' | 'drift' | 'unmeasured' | 'blocked';
 
-export type AuditVerdict = 'READY' | 'DRIFT' | 'BLOCKED';
+/**
+ * `UNKNOWN` is deliberately distinct from `DRIFT` rather than folded into it.
+ * "I could not tell" and "I can tell, and it is wrong" call for different
+ * operator responses, and collapsing them recreates exactly the ambiguity D1
+ * exists to remove one level down. It ranks below `BLOCKED` — a hard stop is
+ * still a hard stop — and above `DRIFT`, because you cannot act on a diff you
+ * are not sure you have.
+ */
+export type AuditVerdict = 'READY' | 'DRIFT' | 'UNKNOWN' | 'BLOCKED';
 
 /**
  * A single drift finding produced by a category check.
@@ -102,6 +127,7 @@ export interface SystemAuditReport {
  */
 export function computeVerdict(findings: DriftFinding[]): AuditVerdict {
   if (findings.some((f) => f.severity === 'blocked')) return 'BLOCKED';
+  if (findings.some((f) => f.severity === 'unmeasured')) return 'UNKNOWN';
   if (findings.some((f) => f.severity === 'drift')) return 'DRIFT';
   return 'READY';
 }

package/src/services/module-deploy.ts

@@ -559,6 +559,27 @@ async function deployModuleImpl(
 
     log.success('Templates generated');
 
+    // Pre-flight: refuse before contacting any system if the generated project
+    // does not match the module tree it was generated from (D6).
+    //
+    // This is the check that would have turned celilo#925 from a silent success
+    // into a refusal at the point of the fault. A verbatim role asset that
+    // failed to refresh is invisible downstream: `cp` reports no error, Ansible
+    // copies the stale bytes and reports `ok`, unchanged, and the module's
+    // version advances past code that never shipped. Local, milliseconds, no
+    // SSH. It deliberately does NOT verify after the deploy — Ansible's
+    // `changed` is truthful about what Ansible did, and the lie is upstream of
+    // Ansible.
+    const { refuseIfGeneratedIsStale } = await import('../module/packaging/generated-plane');
+    const staleGenerated = await refuseIfGeneratedIsStale(
+      moduleId,
+      module.sourcePath,
+      generatedPath,
+    );
+    if (staleGenerated) {
+      return { success: false, error: staleGenerated, phases };
+    }
+
     // Run validate_config hook if defined (e.g., credential validation via Playwright)
     if (manifest.hooks?.validate_config) {
       const hookDef = manifest.hooks.validate_config;

package/src/services/restore-from-file.ts

@@ -401,8 +401,12 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
     mkdirSync(moduleStorageBase, { recursive: true });
     for (const entry of readdirSync(stagedModuleSrc, { withFileTypes: true })) {
       if (!entry.isDirectory()) continue;
+      // `force` explicit: this merges over an EXISTING module dir and the
+      // restored bytes must win. bun does not honour the documented default
+      // on every cp path (see generator.ts#copyAnsibleRoleFilesDirs).
       cpSync(join(stagedModuleSrc, entry.name), join(moduleStorageBase, entry.name), {
         recursive: true,
+        force: true,
       });
       moduleSourcesApplied += 1;
     }

package/src/services/update/orchestrator.test.ts

@@ -76,6 +76,8 @@ const cleanAudit: AuditDeps = {
   terraformPlan: { modules: [], run: async () => ({ exitCode: 0, stdout: '', stderr: '' }) },
   moduleVersions: { installed: [], fetcher: async () => ({ latest: null }) },
   moduleConfigs: { modules: [] },
+  moduleIntegrity: { results: [] },
+  detectWithoutConverge: { modules: [] },
   health: { results: [] },
   backups: { modules: [] },
   abandonedOperations: { records: [] },