@celilo/cli 1.4.0 → 1.6.0

package/src/cli/commands/system-audit.ts

@@ -211,6 +211,13 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
 
   const healthResults = await runAllHealthChecks(db, { onProgress });
 
+  // Module integrity. Shallow by default: the installed tree against its
+  // baseline, and the generated project against the installed tree. Both are
+  // local and take milliseconds. The host plane is one SSH per system and is
+  // reached through `module verify --deep`, not from here.
+  const { auditModule } = await import('../../module/packaging/audit');
+  const integrityResults = await Promise.all(installed.map((m) => auditModule(m.id, db)));
+
   // Services-credentials: try decrypting each container service's
   // credential envelope. Failures (missing master key, wrong
   // provider shape, corrupt envelope) become BLOCKED audit findings.
@@ -369,6 +376,14 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
       fetcher: makeRegistryFetcher(registryClient),
     },
     moduleConfigs: { modules: installedConfigs },
+    moduleIntegrity: { results: integrityResults },
+    detectWithoutConverge: {
+      modules: deployedModules.map((m) => ({
+        id: m.id,
+        state: m.state,
+        manifest: m.manifestData as ModuleManifest,
+      })),
+    },
     health: { results: healthResults },
     backups: { modules: installedBackupInfo },
     abandonedOperations: { records: loadAbandonedOperations(db) },
@@ -419,11 +434,13 @@ async function buildAuditDeps(onProgress?: (msg: string) => void) {
 
 const VERDICT_ICON: Record<string, string> = {
   READY: '●',
+  UNKNOWN: '?',
   DRIFT: '⚠',
   BLOCKED: '✗',
 };
 
 const SEVERITY_ICON: Record<string, string> = {
+  unmeasured: '?',
   drift: '⚠',
   blocked: '✗',
 };

package/src/cli/commands/system-doctor.ts

@@ -472,7 +472,67 @@ async function renderAspectCoverage(opts: { fix: boolean }): Promise<{
   return { lines, failCount: missing.length, warnCount: unknown.length };
 }
 
-async function renderFleetSection(opts: { forced: boolean; fix: boolean }): Promise<{
+/**
+ * The fleet-section line for module integrity, one row per module.
+ *
+ * `module verify` is the detailed surface and `system audit` is the machine
+ * one; doctor gets the summary, because doctor is where an operator looks
+ * first. Same implementation behind all three.
+ */
+async function renderModuleIntegrity(opts: {
+  db: ReturnType<typeof getDb>;
+  deep: boolean;
+}): Promise<{ lines: string[]; failCount: number; warnCount: number }> {
+  const { auditModule } = await import('../../module/packaging/audit');
+  const { auditModuleIntegrity } = await import('../../services/audit/module-integrity');
+  const { modules: modulesTable } = await import('../../db/schema');
+
+  const installed = opts.db.select().from(modulesTable).all();
+  if (installed.length === 0) return { lines: [], failCount: 0, warnCount: 0 };
+
+  const results = await Promise.all(
+    installed.map((m) => auditModule(m.id, opts.db, { deep: opts.deep })),
+  );
+  const findings = auditModuleIntegrity({ results });
+
+  const drifted = findings.filter((f) => f.severity === 'drift');
+  const unmeasured = findings.filter((f) => f.severity === 'unmeasured');
+
+  if (findings.length === 0) {
+    const scope = opts.deep
+      ? 'including what is running on their hosts'
+      : 'installed and generated';
+    return {
+      lines: [
+        `  ${ANSI.green}✔${ANSI.reset} Module integrity  ${ANSI.dim}— ${installed.length} module(s) match their recorded version (${scope})${ANSI.reset}`,
+      ],
+      failCount: 0,
+      warnCount: 0,
+    };
+  }
+
+  const lines: string[] = [
+    `  ${drifted.length > 0 ? `${ANSI.red}✗${ANSI.reset}` : `${ANSI.yellow}?${ANSI.reset}`} Module integrity  ${ANSI.dim}— ${drifted.length} drifted, ${unmeasured.length} unmeasured, of ${installed.length} module(s)${ANSI.reset}`,
+  ];
+  // Every finding, never just the first. celilo#951 printed `drift[0]` of 19,
+  // so a real finding would have arrived at position 19 and never been seen.
+  for (const f of findings) {
+    lines.push(`      ${ANSI.dim}${f.message}${ANSI.reset}`);
+    if (f.remediation) lines.push(`    ${ANSI.dim}→ ${f.remediation}${ANSI.reset}`);
+  }
+  if (!opts.deep) {
+    lines.push(
+      `    ${ANSI.dim}Run \`celilo system doctor --deep\` to also ask each host what it is running.${ANSI.reset}`,
+    );
+  }
+  return { lines, failCount: drifted.length, warnCount: unmeasured.length };
+}
+
+async function renderFleetSection(opts: {
+  forced: boolean;
+  fix: boolean;
+  deep: boolean;
+}): Promise<{
   lines: string[];
   failCount: number;
   warnCount: number;
@@ -518,6 +578,18 @@ async function renderFleetSection(opts: { forced: boolean; fix: boolean }): Prom
       if (f.status === 'fail') failCount++;
       else if (f.status === 'warn') warnCount++;
     }
+
+    // Module integrity: is the code on each box the code celilo thinks it is?
+    // Shallow here by design — the installed tree against its baseline and the
+    // generated project against the installed tree are both local and take
+    // milliseconds. The host plane is one SSH per system and is reached with
+    // `--deep`, alongside aspect coverage, for the same reason
+    // (openspec/changes/module-integrity-rigor, D8).
+    const integrity = await renderModuleIntegrity({ db, deep: opts.deep });
+    lines.push(...integrity.lines);
+    failCount += integrity.failCount;
+    warnCount += integrity.warnCount;
+
     return { lines, failCount, warnCount };
   } finally {
     bus.close();
@@ -647,7 +719,11 @@ export async function handleSystemDoctor(
 
   // Fleet-runtime section (state-aware; only renders on a management
   // plane with a celilo DB, or when --fleet forces it).
-  const fleet = await renderFleetSection({ forced: flags.fleet === true, fix });
+  const fleet = await renderFleetSection({
+    forced: flags.fleet === true,
+    fix,
+    deep: flags.deep === true,
+  });
   if (fleet.lines.length > 0) {
     lines.push(...fleet.lines);
     lines.push('');

package/src/cli/commands/system-update.ts

@@ -495,6 +495,11 @@ export async function handleSystemUpdate(
   const migrationsFolder = findMigrationsFolderSafe();
   const healthResults = await runAllHealthChecks(db);
 
+  // Module integrity, shallow. Local and milliseconds; the host plane is one
+  // SSH per system and belongs to `module verify --deep`.
+  const { auditModule } = await import('../../module/packaging/audit');
+  const integrityResults = await Promise.all(upgradableModules.map((m) => auditModule(m.id, db)));
+
   const latestBackupByModule = new Map<string, number>();
   try {
     const successfulBackups = db
@@ -563,6 +568,14 @@ export async function handleSystemUpdate(
         configs: configsByModule.get(m.id) ?? {},
       })),
     },
+    moduleIntegrity: { results: integrityResults },
+    detectWithoutConverge: {
+      modules: upgradableModules.map((m) => ({
+        id: m.id,
+        state: m.state,
+        manifest: m.manifestData as ModuleManifest,
+      })),
+    },
     health: { results: healthResults },
     backups: {
       modules: upgradableModules.map((m) => ({
@@ -714,7 +727,7 @@ export async function handleSystemUpdate(
   // orchestrator was reacting to).
   const successfulModuleSteps = result.modules.filter((m) => m.step === 'done');
   if (result.ok && successfulModuleSteps.length > 0) {
-    const refreshedAudit = await runAudit(rebuildAuditDepsForRerun(auditDeps, db));
+    const refreshedAudit = await runAudit(await rebuildAuditDepsForRerun(auditDeps, db));
     result.audit = refreshedAudit;
   }
 
@@ -747,10 +760,10 @@ type AuditDeps = Parameters<typeof runAudit>[0];
  * reflect post-upgrade reality (e.g., a module_versions drift
  * finding for a module we just upgraded is no longer reported).
  */
-export function rebuildAuditDepsForRerun(
+export async function rebuildAuditDepsForRerun(
   original: AuditDeps,
   db: ReturnType<typeof getDb>,
-): AuditDeps {
+): Promise<AuditDeps> {
   const installed = db.select().from(modules).all();
   const upgradeEligibleStates = new Set(['INSTALLED', 'VERIFIED', 'IMPORTED']);
   const upgradable = installed.filter((m) => upgradeEligibleStates.has(m.state));
@@ -763,6 +776,8 @@ export function rebuildAuditDepsForRerun(
     configsByModule.set(c.moduleId, m);
   }
 
+  const { auditModule } = await import('../../module/packaging/audit');
+
   // Backup recency is unchanged across an orchestrator run (only
   // celilo-DB snapshots happen, not per-module backup writes), so
   // we look up each module's prior lastSuccessfulBackupAt by id
@@ -793,6 +808,21 @@ export function rebuildAuditDepsForRerun(
         configs: configsByModule.get(m.id) ?? {},
       })),
     },
+    // Re-MEASURED, not carried over. An upgrade rewrites the installed tree and
+    // the baseline, so the pre-upgrade result describes files that are no
+    // longer on disk. Reusing it would be reporting a stored claim about a
+    // state that has since changed, which is the exact mistake this whole
+    // change exists to remove.
+    moduleIntegrity: {
+      results: await Promise.all(upgradable.map((m) => auditModule(m.id, db))),
+    },
+    detectWithoutConverge: {
+      modules: upgradable.map((m) => ({
+        id: m.id,
+        state: m.state,
+        manifest: m.manifestData as ModuleManifest,
+      })),
+    },
     // Unchanged across an orchestrator run — an upgrade does not reclaim
     // abandoned operations, so re-reading them would be the same rows.
     abandonedOperations: original.abandonedOperations,

package/src/cli/index.ts

@@ -1577,9 +1577,9 @@ export async function runCli(argv: string[]): Promise<CommandResult> {
       case 'changeset':
         return handleModuleChangeset(parsed.args, parsed.flags);
       case 'audit':
-        return moduleAudit(parsed.args);
+        return moduleAudit(parsed.args, parsed.flags);
       case 'verify':
-        return moduleVerify(parsed.args);
+        return moduleVerify(parsed.args, parsed.flags);
       case 'config': {
         // Config requires additional subcommand (set/get)
         const configSubcommand = parsed.args[0];

package/src/cli/tui/audit-state.ts

@@ -56,12 +56,12 @@ export type ModalState =
 /**
  * Per-category progress state — drives the analyzing modal's
  * fuel-gauges. `done` carries the verdict so the row can show the
- * right icon (✓ clean / ▲ drift / × blocked).
+ * right icon (✓ clean / ? unmeasured / ▲ drift / × blocked).
  */
 export type CategoryStatus =
   | 'pending'
   | 'running'
-  | { kind: 'done'; verdict: 'clean' | 'drift' | 'blocked' };
+  | { kind: 'done'; verdict: 'clean' | 'unmeasured' | 'drift' | 'blocked' };
 
 /**
  * The full set of categories the audit emits — used to seed the
@@ -90,9 +90,13 @@ export const ALL_CATEGORIES: readonly DriftCategory[] = [
   'transport_reads',
   'trusted_sources',
   'interface_classification',
+  'module_integrity',
+  'detect_without_converge',
 ];
 
 export const CATEGORY_LABELS: Record<DriftCategory, string> = {
+  module_integrity: 'Module integrity',
+  detect_without_converge: 'Drift without converge',
   interface_classification: 'Firewall interfaces',
   cli_version: 'CLI version',
   schema: 'Schema migrations',
@@ -181,7 +185,11 @@ export type AuditTuiAction =
   | { type: 'select-finding'; index: number }
   // Per-category audit-progress lifecycle.
   | { type: 'category-start'; category: DriftCategory }
-  | { type: 'category-end'; category: DriftCategory; verdict: 'clean' | 'drift' | 'blocked' }
+  | {
+      type: 'category-end';
+      category: DriftCategory;
+      verdict: 'clean' | 'unmeasured' | 'drift' | 'blocked';
+    }
   | { type: 'reset-category-progress' };
 
 const PANE_ORDER: PaneId[] = ['summary', 'categories', 'findings', 'detail', 'log'];

package/src/cli/tui/audit-tui.tsx

@@ -129,11 +129,17 @@ export function AuditTui({ source, theme: themeName }: Props) {
         return;
       }
       const findings = event.findings ?? [];
-      const verdict: 'clean' | 'drift' | 'blocked' = findings.some((f) => f.severity === 'blocked')
+      // Same ranking as `computeVerdict`: an unmeasured category is not clean
+      // and is not the same statement as a measured difference (D7).
+      const verdict: 'clean' | 'unmeasured' | 'drift' | 'blocked' = findings.some(
+        (f) => f.severity === 'blocked',
+      )
         ? 'blocked'
-        : findings.length > 0
-          ? 'drift'
-          : 'clean';
+        : findings.some((f) => f.severity === 'unmeasured')
+          ? 'unmeasured'
+          : findings.length > 0
+            ? 'drift'
+            : 'clean';
       dispatch({ type: 'category-end', category: event.category, verdict });
     };
 

package/src/cli/tui/icons.ts

@@ -8,12 +8,16 @@ export interface SeverityVisual {
 
 export const SEVERITY_VISUALS: Record<DriftSeverity, SeverityVisual> = {
   blocked: { icon: '×', color: 'red', label: 'BLOCKED' },
+  // Deliberately NOT green and NOT gray. An unmeasured check is not a pass and
+  // is not a reminder — it is a hole in what celilo knows.
+  unmeasured: { icon: '?', color: 'yellow', label: 'UNMEASURED' },
   drift: { icon: '▲', color: 'yellow', label: 'DRIFT' },
   todo: { icon: '➤', color: 'gray', label: 'TODO' },
 };
 
 export const VERDICT_VISUALS: Record<AuditVerdict, SeverityVisual> = {
   BLOCKED: { icon: '×', color: 'red', label: 'BLOCKED' },
+  UNKNOWN: { icon: '?', color: 'yellow', label: 'UNKNOWN' },
   DRIFT: { icon: '▲', color: 'yellow', label: 'DRIFT' },
   READY: { icon: '✓', color: 'green', label: 'READY' },
 };
@@ -23,6 +27,9 @@ export const VERDICT_VISUALS: Record<AuditVerdict, SeverityVisual> = {
 // what actually needs attention.
 export function severityRank(s: DriftSeverity): number {
   if (s === 'blocked') return 0;
-  if (s === 'drift') return 1;
-  return 2; // todo
+  // Above drift, for the same reason UNKNOWN outranks DRIFT: you cannot act on
+  // a diff you are not sure you have.
+  if (s === 'unmeasured') return 1;
+  if (s === 'drift') return 2;
+  return 3; // todo
 }

package/src/cli/tui/modals/analyzing.tsx

@@ -36,6 +36,9 @@ function StatusRow({ category, status }: { category: string; status: CategorySta
   if (status.verdict === 'clean') {
     return <Text color="green">✓ {padded}clean</Text>;
   }
+  if (status.verdict === 'unmeasured') {
+    return <Text color="yellow">? {padded}unmeasured</Text>;
+  }
   if (status.verdict === 'drift') {
     return <Text color="yellow">▲ {padded}drift</Text>;
   }

package/src/db/schema.ts

@@ -241,6 +241,11 @@ export const moduleIntegrity = sqliteTable('module_integrity', {
     .unique()
     .references(() => modules.id, { onDelete: 'cascade' }),
   checksums: text('checksums', { mode: 'json' }).$type<Record<string, string>>().notNull(), // { "path/to/file": "xxhash", ... }
+  // Which version these checksums describe. NULL means the row was written
+  // before celilo stamped versions — a fact `module verify` reports rather
+  // than papers over, because a baseline whose version is unknown cannot be
+  // compared to the module's and is not evidence of anything.
+  version: text('version'),
   signature: text('signature'), // Nullable - null for directory imports, populated for .netapp packages
   importedAt: integer('imported_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),
   updatedAt: integer('updated_at', { mode: 'timestamp' }).notNull().default(sql`(unixepoch())`),

package/src/manifest/json-schema-roundtrip.test.ts

@@ -3,10 +3,18 @@
  * under both the Zod schema (ModuleManifestSchema) and the exported JSON
  * Schema at <repo>/schemas/module-manifest.schema.json.
  *
- * This guards two things:
- *   1. The exported JSON Schema is in sync with the Zod source (the same as
- *      'bun run check:schema'). If someone forgets to regenerate after editing
- *      schema.ts, this test fails.
+ * This is the INDIRECT drift detector, not the primary guard. `bun run
+ * check:schema` (a step in the CI validate job) regenerates the schema and
+ * compares it byte-for-byte, so it fails the moment schema.ts changes without a
+ * regenerate. This test only fails once some real manifest in the repo happens
+ * to declare something a stale schema rejects, which can be days later and in
+ * an unrelated PR. Do not mistake one for the other: if you add a field to
+ * schema.ts, `check:schema` is what tells you to run `bun run export:schema`.
+ *
+ * What this test is actually for:
+ *   1. Every manifest the repo ships really does validate under both
+ *      validators, so a stale schema cannot sit unnoticed once it starts
+ *      rejecting live manifests.
  *   2. zod-to-json-schema's translation hasn't lost fidelity in a way that
  *      would let bad manifests pass the editor (red underlines) while still
  *      failing at module import time. If a real manifest in the repo is

package/src/manifest/schema.ts

@@ -677,6 +677,29 @@ export const ModuleManifestSchema = z
 
     hooks: z.object(HOOK_SCHEMAS).strict().optional(),
 
+    /**
+     * How much of `celilo module verify` this module can honestly answer.
+     *
+     * `deep: false` opts the module out of the host plane — the `--deep` pass
+     * that evaluates its generated playbook in check mode. The escape hatch
+     * exists because Ansible's check mode SKIPS a task it cannot evaluate, so a
+     * role driven by `command:` / `shell:` can report `changed=0` having never
+     * been applied. For most roles that lands as `unmeasured`, which is honest.
+     * A role where it would land as a persistent false `drift` should say so
+     * here instead of training operators to ignore the output.
+     *
+     * `reason` is REQUIRED. An opt-out with no stated reason is a check that
+     * disappeared, and celilo prints the reason wherever the module is verified
+     * (openspec/changes/module-integrity-rigor, D4 / task 6.3).
+     */
+    verify: z
+      .object({
+        deep: z.literal(false),
+        reason: z.string().min(1),
+      })
+      .strict()
+      .optional(),
+
     build: z
       .object({
         /** Inline shell command to build the module (run via bash -c). Mutually exclusive with script. */

package/src/module/import.ts

@@ -1,7 +1,7 @@
 import { execSync } from 'node:child_process';
 import { existsSync, statSync, writeFileSync } from 'node:fs';
 import { copyFile, mkdir, readFile, readdir } from 'node:fs/promises';
-import { join } from 'node:path';
+import { join, relative } from 'node:path';
 import { eq } from 'drizzle-orm';
 import { z } from 'zod';
 import { getWellKnownCapability, isWellKnown } from '../capabilities/well-known';
@@ -23,6 +23,7 @@ import {
 } from '../manifest/validate';
 import { parseJsonWithValidation } from '../validation/schemas';
 import { cleanupTempDir, extractPackage, verifyPackageIntegrity } from './packaging/extract';
+import { classifyModulePath } from './packaging/package-rules';
 
 /**
  * Phase-timing helper for `celilo module import`. Set CELILO_IMPORT_DEBUG=1
@@ -232,29 +233,14 @@ export async function copyModuleFiles(sourcePath: string, targetPath: string): P
     for (const entry of entries) {
       const srcPath = join(src, entry.name);
       const destPath = join(dest, entry.name);
-
-      // Skip metadata files
-      if (entry.name === 'checksums.json' || entry.name === 'signature.sig') {
-        continue;
-      }
-
-      // Skip directories that shouldn't be copied into celilo's data store.
-      // node_modules inside scripts/ ARE kept — they contain the module's
-      // runtime deps (@celilo/capabilities etc.). Top-level node_modules
-      // (monorepo workspace deps, build tools) are skipped.
-      if (
-        entry.isDirectory() &&
-        (entry.name === '.git' || entry.name === '.next' || entry.name === '.cache')
-      ) {
-        continue;
-      }
-
-      if (
-        entry.isDirectory() &&
-        entry.name === 'node_modules' &&
-        !src.endsWith('/scripts') &&
-        !src.endsWith('\\scripts')
-      ) {
+      const relPath = relative(sourcePath, srcPath);
+
+      // `unknown` is everything that belongs to the module's SOURCE tree and
+      // not to its install: `.git/`, `e2e/`, tests, `tsconfig.json`, the
+      // node_modules the canonical rule drops. Skipping it here is what keeps
+      // the `unknown` class empty on a healthy install, so any `unknown` that
+      // audit later reports is real. `package` and `derived` both land.
+      if (classifyModulePath(relPath) === 'unknown') {
         continue;
       }
 
@@ -699,17 +685,32 @@ export async function importModule(options: ModuleImportOptions): Promise<Module
     // Execution: Store integrity data from the package's checksums + signature.
     // Directory imports go through the packager too (see top of importModule),
     // so by the time we reach this point we always have these.
-    try {
-      const integrityData: NewModuleIntegrity = {
-        moduleId: manifest.id,
-        checksums: checksums ?? {},
-        signature: signature?.trim() ?? null,
-      };
-      db.insert(moduleIntegrity).values(integrityData).run();
-    } catch (error) {
-      // Non-fatal - module is already imported
-      console.warn('Warning: Failed to store module integrity data', error);
-    }
+    //
+    // UPSERT, not INSERT. `moduleId` is UNIQUE, so a re-import of an already
+    // imported module used to collide, get caught by a warn-and-continue, and
+    // leave the FIRST import's checksums in place forever. Every file that
+    // legitimately changed since then read as [MODIFIED] and the baseline
+    // described a version nobody could name. Failing to record the baseline is
+    // not "non-fatal": it is the state that made `module verify` useless, so
+    // this no longer swallows its own errors.
+    const integrityData: NewModuleIntegrity = {
+      moduleId: manifest.id,
+      checksums: checksums ?? {},
+      version: manifest.version,
+      signature: signature?.trim() ?? null,
+    };
+    db.insert(moduleIntegrity)
+      .values(integrityData)
+      .onConflictDoUpdate({
+        target: moduleIntegrity.moduleId,
+        set: {
+          checksums: integrityData.checksums,
+          version: integrityData.version,
+          signature: integrityData.signature,
+          updatedAt: new Date(),
+        },
+      })
+      .run();
 
     // Record a successful build entry so deploy-time validation sees the
     // artifacts in place and skips rebuild. The packager runs the manifest