@celilo/cli 1.5.0 → 1.7.0

package/src/module/packaging/classify-module-path.test.ts

@@ -0,0 +1,104 @@
+import { describe, expect, test } from 'bun:test';
+import { readFileSync } from 'node:fs';
+import { join } from 'node:path';
+import { classifyModulePath } from './package-rules';
+
+/**
+ * Real input, not input built by the same helper as the expectation.
+ *
+ * These fixtures are verbatim `celilo module verify` output captured from the
+ * live fleet on 2026-08-19 (see the fixture README). Every path below is a path
+ * that actually exists in an installed module tree on celilo-mgr. celilo#951
+ * shipped 18 false positives because its comparator's test synthesised both
+ * sides of the comparison and so never saw real input.
+ */
+function fixturePaths(name: string): string[] {
+  const raw = readFileSync(
+    join(process.cwd(), 'test-fixtures', 'module-integrity', `${name}-verify.txt`),
+    'utf-8',
+  );
+  const paths: string[] = [];
+  for (const line of raw.split('\n')) {
+    const match = /(?:Checksum mismatch|Unexpected file): (.+)$/.exec(line.trim());
+    if (match?.[1]) paths.push(match[1]);
+  }
+  return paths;
+}
+
+describe('classifyModulePath against the live fleet listings', () => {
+  for (const [moduleName, expectedCount] of [
+    ['wireguard-manager', 47],
+    ['wireguard', 25],
+  ] as const) {
+    describe(moduleName, () => {
+      const paths = fixturePaths(moduleName);
+
+      test(`fixture carries all ${expectedCount} reported paths`, () => {
+        expect(paths.length).toBe(expectedCount);
+      });
+
+      test('no path in a real installed tree classifies unknown', () => {
+        const unknown = paths.filter((p) => classifyModulePath(p) === 'unknown');
+        expect(unknown).toEqual([]);
+      });
+
+      test('celilo-owned paths classify derived', () => {
+        const derived = paths.filter(
+          (p) =>
+            p.startsWith('generated/') ||
+            p.includes('node_modules/') ||
+            p === 'checksums.json' ||
+            p === 'signature.sig',
+        );
+        // Guard the guard: if this is empty the assertion below is vacuous.
+        expect(derived.length).toBeGreaterThan(0);
+        for (const p of derived) {
+          expect(`${p} => ${classifyModulePath(p)}`).toBe(`${p} => derived`);
+        }
+      });
+
+      test("the module's own content classifies package", () => {
+        const own = paths.filter(
+          (p) =>
+            !p.startsWith('generated/') &&
+            !p.includes('node_modules/') &&
+            p !== 'checksums.json' &&
+            p !== 'signature.sig',
+        );
+        expect(own.length).toBeGreaterThan(0);
+        for (const p of own) {
+          expect(`${p} => ${classifyModulePath(p)}`).toBe(`${p} => package`);
+        }
+      });
+    });
+  }
+});
+
+describe('classifyModulePath: source-tree paths that must never be installed', () => {
+  test.each([
+    ['e2e/deploy.test.ts'],
+    ['.git/config'],
+    ['scripts/tsconfig.json'],
+    ['.DS_Store'],
+    ['server/src/api.test.ts'],
+    ['wireguard.netapp'],
+    ['node_modules/tldts/package.json'],
+  ])('%s is unknown', (relPath) => {
+    expect(classifyModulePath(relPath)).toBe('unknown');
+  });
+});
+
+describe('classifyModulePath: composes with includeNodeModulesPath', () => {
+  test('the hook runtime closure is derived, not unknown', () => {
+    expect(classifyModulePath('scripts/node_modules/tldts/index.js')).toBe('derived');
+  });
+
+  test('a .bin shim is still excluded', () => {
+    expect(classifyModulePath('scripts/node_modules/.bin/tsc')).toBe('unknown');
+  });
+
+  test('non-scripts node_modules ships only @celilo/capabilities', () => {
+    expect(classifyModulePath('node_modules/@celilo/capabilities/src/index.ts')).toBe('derived');
+    expect(classifyModulePath('node_modules/@celilo/cli/index.js')).toBe('unknown');
+  });
+});

package/src/module/packaging/extract.ts

@@ -5,13 +5,40 @@ import { extract as tarExtract } from 'tar';
 import { z } from 'zod';
 import { parseJsonWithValidation } from '../../validation/schemas';
 import { computeFileChecksum } from './checksum';
+import { classifyModulePath } from './package-rules';
 import { verifySignature } from './signature';
 
 /**
  * Integrity violation types
  */
 export interface IntegrityViolation {
-  type: 'missing' | 'modified' | 'extra';
+  /**
+   * `stale-baseline` is not a file finding. It says the recorded checksums
+   * describe a DIFFERENT version of the module than the one celilo has
+   * installed, so every file finding beneath it is explained by the baseline
+   * being old rather than by the files having changed. Only `auditModule`
+   * produces it; package verification compares a package to its own manifest
+   * and cannot be stale in this sense.
+   *
+   * `stale-generated` is the second plane (D3): a verbatim role asset in
+   * `generated/` whose bytes are not the installed module's. That is what
+   * celilo#925 was, and what would ship on the next deploy.
+   */
+  type: 'missing' | 'modified' | 'extra' | 'stale-baseline' | 'stale-generated';
+  /**
+   * What the baseline says the file should hash to, and what it actually
+   * hashes to. Both optional because not every violation is about a digest —
+   * a `stale-baseline` finding is about the row, not a file.
+   *
+   * These exist so `module verify --json` can answer "is the installed tree
+   * the 0.3.2 package" in one call. celilo#925 stalled for days on that
+   * question because nothing could read a file on celilo-mgr, and the
+   * tempting fix — a remote read primitive — is a real security surface (every
+   * module's secrets and vault material live under the same tree) for a
+   * question file hashes answer directly (D9).
+   */
+  expectedDigest?: string;
+  actualDigest?: string | null;
   path: string;
   message: string;
 }
@@ -164,8 +191,9 @@ export async function verifyPackageIntegrity(
     const expectedFiles = new Set(Object.keys(expectedChecksums));
 
     for (const file of actualFiles) {
-      // Skip checksums.json and signature.sig
-      if (file === 'checksums.json' || file === 'signature.sig') {
+      // `checksums.json` / `signature.sig` and the rest of the derived set are
+      // never listed by the manifest they accompany.
+      if (classifyModulePath(file) === 'derived') {
         continue;
       }
 

package/src/module/packaging/generated-plane.test.ts

@@ -0,0 +1,79 @@
+import { describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  compareVerbatimRoleAssets,
+  describeVerbatimDifferences,
+  readVerbatimRoleAssets,
+} from './generated-plane';
+
+describe('compareVerbatimRoleAssets', () => {
+  const asset = 'ansible/roles/vpn/files/vpn-linux-x86_64';
+
+  test('identical trees produce no differences', () => {
+    const both = new Map([[asset, 'aaa']]);
+    expect(compareVerbatimRoleAssets(both, new Map(both))).toEqual([]);
+  });
+
+  test('the celilo#925 shape: generated holds the previous version bytes', () => {
+    const differences = compareVerbatimRoleAssets(
+      new Map([[asset, 'new-digest']]),
+      new Map([[asset, 'old-digest']]),
+    );
+    expect(differences).toEqual([
+      {
+        relPath: asset,
+        reason: 'stale',
+        installedDigest: 'new-digest',
+        generatedDigest: 'old-digest',
+      },
+    ]);
+    expect(describeVerbatimDifferences(differences)[0]).toContain(asset);
+  });
+
+  test('an asset generation never produced is missing, not stale', () => {
+    expect(compareVerbatimRoleAssets(new Map([[asset, 'aaa']]), new Map())).toEqual([
+      { relPath: asset, reason: 'missing', installedDigest: 'aaa', generatedDigest: null },
+    ]);
+  });
+
+  test('output for a role the module no longer has is not a difference', () => {
+    // Nothing includes it, so refusing a deploy over it would refuse a correct
+    // module for a file Ansible never reads.
+    expect(
+      compareVerbatimRoleAssets(new Map(), new Map([['ansible/roles/gone/files/x', 'aaa']])),
+    ).toEqual([]);
+  });
+});
+
+describe('readVerbatimRoleAssets', () => {
+  test('digests role files/ assets and nothing else', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'celilo-plane-'));
+    try {
+      mkdirSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'nested'), { recursive: true });
+      mkdirSync(join(root, 'ansible', 'roles', 'vpn', 'templates'), { recursive: true });
+      writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'bin'), 'NEW');
+      writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'files', 'nested', 'blob'), 'B');
+      // Templated, and expected to differ once generated. Must not be compared.
+      writeFileSync(join(root, 'ansible', 'roles', 'vpn', 'templates', 'x.j2'), '{{ v }}');
+
+      const assets = await readVerbatimRoleAssets(root);
+      expect([...assets.keys()].sort()).toEqual([
+        'ansible/roles/vpn/files/bin',
+        'ansible/roles/vpn/files/nested/blob',
+      ]);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test('a module with no ansible/ at all yields nothing rather than throwing', async () => {
+    const root = mkdtempSync(join(tmpdir(), 'celilo-plane-'));
+    try {
+      expect((await readVerbatimRoleAssets(root)).size).toBe(0);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+});

package/src/module/packaging/generated-plane.ts

@@ -0,0 +1,134 @@
+/**
+ * The second plane: is what we would deploy built from what we installed?
+ *
+ * A module exists in four places — the published package, the installed tree,
+ * the generated project, and the host — and celilo reported a version for the
+ * first as though it described the last. Between them sit three copies and two
+ * transformations, and nothing verified any of them against the one before it
+ * (openspec/changes/module-integrity-rigor, D3).
+ *
+ * Only VERBATIM assets have a meaningful expected digest. Ansible templates
+ * most of what it writes, and a `.j2` in `generated/` is supposed to differ
+ * from its source. `ansible/roles/<role>/files/` is the exception: it holds
+ * static assets — built binaries, certs, blobs — copied byte for byte because
+ * they need no variable resolution and may not survive utf-8 round-tripping.
+ *
+ * That is exactly where celilo#925 lived. `copyAnsibleRoleFilesDirs` skipped an
+ * existing destination on bun, so a module's built binary landed in
+ * `generated/` once, at first generate, and no later version replaced it.
+ * `cp` reported no error, Ansible copied that first binary forever and reported
+ * `ok`, and the module's version field advanced past code that was never
+ * shipped. This comparison is local, takes milliseconds, needs no SSH, and is
+ * the check that turns that silence into a refusal.
+ */
+
+import { existsSync } from 'node:fs';
+import { readdir } from 'node:fs/promises';
+import { join, relative } from 'node:path';
+import { computeFileChecksum } from './checksum';
+
+export interface VerbatimAssetDifference {
+  /** Path relative to the module root, identical in both trees. */
+  relPath: string;
+  /** `missing`: generation never produced it. `stale`: it holds other bytes. */
+  reason: 'missing' | 'stale';
+  installedDigest: string;
+  generatedDigest: string | null;
+}
+
+/**
+ * Pure. Both sides are digest maps keyed by the same module-relative path.
+ *
+ * An asset present in `generated` but absent from `installed` is NOT reported.
+ * That is a role the module no longer has, whose old output nothing includes,
+ * and failing a deploy for it would refuse a correct module for a stale file
+ * Ansible never reads.
+ */
+export function compareVerbatimRoleAssets(
+  installed: ReadonlyMap<string, string>,
+  generated: ReadonlyMap<string, string>,
+): VerbatimAssetDifference[] {
+  const differences: VerbatimAssetDifference[] = [];
+  for (const [relPath, installedDigest] of installed) {
+    const generatedDigest = generated.get(relPath) ?? null;
+    if (generatedDigest === null) {
+      differences.push({ relPath, reason: 'missing', installedDigest, generatedDigest: null });
+    } else if (generatedDigest !== installedDigest) {
+      differences.push({ relPath, reason: 'stale', installedDigest, generatedDigest });
+    }
+  }
+  return differences;
+}
+
+/**
+ * Digest every `ansible/roles/<role>/files/**` asset under `root`, keyed by its
+ * path relative to `root`. The generated project mirrors that layout, so the
+ * two maps this produces are directly comparable.
+ */
+export async function readVerbatimRoleAssets(root: string): Promise<Map<string, string>> {
+  const assets = new Map<string, string>();
+  const rolesDir = join(root, 'ansible', 'roles');
+  if (!existsSync(rolesDir)) return assets;
+
+  for (const role of await readdir(rolesDir, { withFileTypes: true })) {
+    if (!role.isDirectory()) continue;
+    const filesDir = join(rolesDir, role.name, 'files');
+    if (!existsSync(filesDir)) continue;
+    for (const filePath of await listFiles(filesDir)) {
+      assets.set(relative(root, filePath), await computeFileChecksum(filePath));
+    }
+  }
+  return assets;
+}
+
+async function listFiles(dir: string): Promise<string[]> {
+  const found: string[] = [];
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      found.push(...(await listFiles(full)));
+    } else if (entry.isFile()) {
+      found.push(full);
+    }
+  }
+  return found;
+}
+
+/**
+ * One line per difference, naming the file. celilo#925 took days partly because
+ * nothing anywhere named the file that had gone stale.
+ */
+export function describeVerbatimDifferences(differences: VerbatimAssetDifference[]): string[] {
+  return differences.map((d) =>
+    d.reason === 'missing'
+      ? `${d.relPath}: generation never produced it (installed ${d.installedDigest})`
+      : `${d.relPath}: generated holds ${d.generatedDigest}, installed is ${d.installedDigest}`,
+  );
+}
+
+/**
+ * The deploy pre-flight (D6). Returns the refusal message, or `null` to proceed.
+ *
+ * Local, milliseconds, no SSH, and it runs before anything contacts a system.
+ * It deliberately does NOT verify after the deploy: Ansible's `changed` is
+ * truthful about what Ansible did, and the lie in celilo#925 was upstream of
+ * Ansible.
+ */
+export async function refuseIfGeneratedIsStale(
+  moduleId: string,
+  modulePath: string,
+  generatedPath: string,
+): Promise<string | null> {
+  const differences = compareVerbatimRoleAssets(
+    await readVerbatimRoleAssets(modulePath),
+    await readVerbatimRoleAssets(generatedPath),
+  );
+  if (differences.length === 0) return null;
+  return [
+    `Refusing to deploy ${moduleId}: the generated project does not match the installed module.`,
+    `  ${differences.length} verbatim asset(s) differ:`,
+    ...describeVerbatimDifferences(differences).map((line) => `    ${line}`),
+    '',
+    `Deploying would ship these bytes and report success. Run 'celilo module generate ${moduleId}' and try again.`,
+  ].join('\n');
+}

package/src/module/packaging/host-plane.test.ts

@@ -0,0 +1,132 @@
+import { describe, expect, test } from 'bun:test';
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import type { ModuleManifest } from '../../manifest/schema';
+import { classifyHostRecap, verifyModuleOnHosts } from './host-plane';
+
+const recap = (over: Partial<Parameters<typeof classifyHostRecap>[0]> = {}) => ({
+  host: 'vpn-manager',
+  changed: 0,
+  unreachable: 0,
+  failed: 0,
+  skipped: 0,
+  ...over,
+});
+
+describe('classifyHostRecap', () => {
+  test('nothing would change → converged', () => {
+    expect(classifyHostRecap(recap(), 'm').state).toBe('converged');
+  });
+
+  test('something would change → drift, and never blocked', () => {
+    const finding = classifyHostRecap(recap({ changed: 3 }), 'wireguard-manager');
+    expect(finding.state).toBe('drift');
+    expect(finding.detail).toContain('celilo module deploy wireguard-manager');
+  });
+
+  test('unreachable → unmeasured, not converged', () => {
+    expect(classifyHostRecap(recap({ unreachable: 1 }), 'm').state).toBe('unmeasured');
+  });
+
+  test('failed → unmeasured', () => {
+    expect(classifyHostRecap(recap({ failed: 1 }), 'm').state).toBe('unmeasured');
+  });
+
+  test('THE case that must not read as converged: check mode skipped tasks', () => {
+    // Ansible does not evaluate a task it cannot support — it SKIPS it. A role
+    // of `command:` / `shell:` tasks can finish with changed=0 having never
+    // been applied to the host at all. Two states would call that converged.
+    const finding = classifyHostRecap(recap({ skipped: 4 }), 'm');
+    expect(finding.state).toBe('unmeasured');
+    expect(finding.detail).toContain('4 task(s)');
+  });
+
+  test('a skip outranks a change, because the run as a whole was not measured', () => {
+    expect(classifyHostRecap(recap({ skipped: 1, changed: 2 }), 'm').state).toBe('unmeasured');
+  });
+});
+
+describe('verifyModuleOnHosts', () => {
+  function generatedTreeWithPlaybook(): string {
+    const root = mkdtempSync(join(tmpdir(), 'celilo-hostplane-'));
+    mkdirSync(join(root, 'ansible'), { recursive: true });
+    writeFileSync(join(root, 'ansible', 'playbook.yml'), '---\n- hosts: all\n');
+    return root;
+  }
+
+  const manifest = { id: 'm', name: 'M', version: '1.0.0' } as unknown as ModuleManifest;
+
+  test('a manifest opt-out is carried back, not silently dropped', async () => {
+    const optedOut = {
+      ...manifest,
+      verify: { deep: false, reason: 'the role is command:-driven' },
+    } as unknown as ModuleManifest;
+
+    const result = await verifyModuleOnHosts({
+      moduleId: 'm',
+      manifest: optedOut,
+      generatedPath: '/nonexistent',
+      execute: async () => {
+        throw new Error('must not run ansible for an opted-out module');
+      },
+    });
+    expect(result.optedOut?.reason).toBe('the role is command:-driven');
+    expect(result.findings).toEqual([]);
+  });
+
+  test('no generated playbook is unmeasured, not clean', async () => {
+    const result = await verifyModuleOnHosts({
+      moduleId: 'm',
+      manifest,
+      generatedPath: '/nonexistent',
+      execute: async () => {
+        throw new Error('must not run ansible without a playbook');
+      },
+    });
+    expect(result.findings.map((f) => f.state)).toEqual(['unmeasured']);
+  });
+
+  test('a run producing no PLAY RECAP is unmeasured, not clean', async () => {
+    // celilo#951's shape in another comparator: nothing measured, rendered as
+    // nothing wrong.
+    const root = generatedTreeWithPlaybook();
+    try {
+      const result = await verifyModuleOnHosts({
+        moduleId: 'm',
+        manifest,
+        generatedPath: root,
+        execute: async () => ({ success: false, output: 'ssh: connect refused', error: 'boom' }),
+      });
+      expect(result.findings.map((f) => f.state)).toEqual(['unmeasured']);
+      expect(result.findings[0]?.detail).toContain('no PLAY RECAP');
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+
+  test('one finding per host in the recap', async () => {
+    const root = generatedTreeWithPlaybook();
+    try {
+      const result = await verifyModuleOnHosts({
+        moduleId: 'm',
+        manifest,
+        generatedPath: root,
+        execute: async () => ({
+          success: true,
+          output: [
+            'PLAY RECAP *********',
+            'vpn-manager : ok=10 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0',
+            'edge : ok=4 changed=2 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0',
+          ].join('\n'),
+        }),
+      });
+      expect(result.findings).toEqual([
+        { hostname: 'vpn-manager', state: 'converged' },
+        expect.objectContaining({ hostname: 'edge', state: 'drift' }),
+      ]);
+    } finally {
+      rmSync(root, { recursive: true, force: true });
+    }
+  });
+});

package/src/module/packaging/host-plane.ts

@@ -0,0 +1,135 @@
+/**
+ * The third plane: is the code running on the host the code we generated?
+ *
+ * Measured by asking Ansible, not by hashing files. celilo does not know where
+ * Ansible puts what it writes — the destination lives in the role's
+ * `tasks/main.yml` `dest:`, which celilo does not parse and should not start
+ * parsing. Ansible already computes exactly this, for every task, including
+ * templated files whose content celilo could not predict
+ * (openspec/changes/module-integrity-rigor, D4).
+ *
+ * This SSHes to every system the module deploys, so it is gated behind
+ * `--deep` and never runs in a default pass.
+ *
+ * The honest caveat, in the spec and not only in the code: check mode does not
+ * evaluate a task it cannot support — it SKIPS it — so a role built from
+ * `command:` / `shell:` tasks can finish a check run with `changed=0` having
+ * never been applied to the host at all. So a skip is `unmeasured`, never a
+ * pass, and a finding here is `drift`, never `blocked`. A check that cries wolf
+ * is the disease; shipping one here would be an unusually stupid way to catch
+ * it.
+ */
+
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+import type { ModuleManifest } from '../../manifest/schema';
+import { executeAnsible, parseAnsibleRecap } from '../../services/deploy-ansible';
+
+/**
+ * `unmeasured` is not a hedge and must never collapse into `converged`.
+ * Absence of a change is not evidence of convergence when nothing was assessed.
+ */
+export type HostPlaneState = 'converged' | 'drift' | 'unmeasured';
+
+export interface HostPlaneFinding {
+  hostname: string;
+  state: HostPlaneState;
+  /** Why, in operator-facing words. Always set for anything but `converged`. */
+  detail?: string;
+}
+
+export interface HostPlaneResult {
+  findings: HostPlaneFinding[];
+  /**
+   * Set when the module declared `verify.deep: false`. Carried so callers can
+   * PRINT the opt-out — an opt-out nobody sees is a check that quietly
+   * disappeared.
+   */
+  optedOut?: { reason: string };
+}
+
+/**
+ * Classify one Ansible `PLAY RECAP` line. Pure, so the interesting decisions
+ * are testable without an SSH round trip.
+ */
+export function classifyHostRecap(
+  recap: { host: string; changed: number; unreachable: number; failed: number; skipped: number },
+  moduleId: string,
+): HostPlaneFinding {
+  if (recap.unreachable > 0 || recap.failed > 0) {
+    return {
+      hostname: recap.host,
+      state: 'unmeasured',
+      detail: 'The host could not be evaluated — it did not answer, or the play errored on it.',
+    };
+  }
+  if (recap.skipped > 0) {
+    return {
+      hostname: recap.host,
+      state: 'unmeasured',
+      detail: `${recap.skipped} task(s) check mode cannot evaluate were skipped, so convergence was not measured. Prefer check-capable Ansible modules (copy, template, lineinfile, file, package, service), or give a command/shell task an honest changed_when:.`,
+    };
+  }
+  if (recap.changed > 0) {
+    return {
+      hostname: recap.host,
+      state: 'drift',
+      detail: `The playbook would change ${recap.changed} thing(s) on this host, so what is running is not what celilo generated. Run 'celilo module deploy ${moduleId}' to converge it.`,
+    };
+  }
+  return { hostname: recap.host, state: 'converged' };
+}
+
+/**
+ * Evaluate the module's generated playbook against its systems in check mode.
+ *
+ * `executeAnsible(..., { check: true })` is the existing plumbing and the only
+ * one — `--check` is one argument, not a second execution path.
+ */
+export async function verifyModuleOnHosts(args: {
+  moduleId: string;
+  manifest: ModuleManifest;
+  generatedPath: string;
+  /** Injected in tests, exactly as `verifyAspectCoverage` does it. */
+  execute?: typeof executeAnsible;
+}): Promise<HostPlaneResult> {
+  const { moduleId, manifest, generatedPath } = args;
+
+  if (manifest.verify?.deep === false) {
+    return { findings: [], optedOut: { reason: manifest.verify.reason } };
+  }
+
+  const playbookPath = join(generatedPath, 'ansible', 'playbook.yml');
+  if (!existsSync(playbookPath)) {
+    return {
+      findings: [
+        {
+          hostname: '(none)',
+          state: 'unmeasured',
+          detail: `No generated playbook at ${playbookPath}. Run 'celilo module generate ${moduleId}' first.`,
+        },
+      ],
+    };
+  }
+
+  const execute = args.execute ?? executeAnsible;
+  const result = await execute(generatedPath, { check: true, noInteractive: true });
+  const recaps = parseAnsibleRecap(result.output ?? '');
+
+  if (recaps.length === 0) {
+    // A run that produced no recap at all must not read as success. This is
+    // the shape celilo#951 got wrong in another comparator: nothing measured,
+    // rendered as nothing wrong.
+    return {
+      findings: [
+        {
+          hostname: '(none)',
+          state: 'unmeasured',
+          detail: `Ansible produced no PLAY RECAP, so nothing was measured.${result.error ? ` ${result.error}` : ''}`,
+        },
+      ],
+    };
+  }
+
+  return { findings: recaps.map((recap) => classifyHostRecap(recap, moduleId)) };
+}