@celilo/cli 2.1.0 → 2.2.0

package/src/policy/fixture-capability-coverage.test.ts

@@ -0,0 +1,322 @@
+/**
+ * A module's e2e fixtures must deploy (at least import) a provider for every
+ * capability its manifest requires.
+ *
+ * The failure this exists to catch is silent in the worst way: the manifest
+ * requirement makes stage 1 of the module's own suite fail at
+ * `module import <id>` with "Required capability 'X' not found", and every
+ * later stage cascade-skips on requireStage. Nothing else goes red — the
+ * suite is quarantined, or CI never runs it, so a fixture can sit
+ * structurally incapable of passing for six weeks (celilo#1260:
+ * registry-pipeline required idp@1.2.0 from 7e2d27b9 onward and never
+ * deployed an idp provider).
+ *
+ * Both sides of the comparison are computed sets. The required set comes
+ * from each module's manifest.yml parsed by the production validator; the
+ * provided set comes from the manifests of whatever modules the module's
+ * fixtures actually import (`module import <ref>` statements, scanned in
+ * source). A hand-written list of "modules that need an idp" would rot the
+ * moment the next module declares the requirement.
+ *
+ * The check runs the real `validateCapabilityAccess` against an in-memory
+ * database seeded with the fixtures' providers — the same code path the CLI
+ * executes at import time, including the secret-allowlist gate — so the
+ * assertion is over the actual contract stage 1 exercises, not a re-derivation
+ * of it.
+ *
+ * ## What this gate does NOT prove. Read before trusting green.
+ *
+ * - Import is necessary, not sufficient. A fixture can import the provider
+ *   and still deploy the consumer before it, or never deploy the provider
+ *   at all (the consumer's on_install would then fail against a dead idp).
+ *   Only running the suite proves deploy order.
+ * - The import scan is a source scan. An import built dynamically in a way
+ *   this scanner cannot parse is invisible here (the scanner requires a
+ *   literal module id, a `${CONST}` reference to a string literal in the
+ *   same file, or a trailing path segment).
+ * - Fixtures living outside `modules/<id>/e2e/*.test.ts` (shared harness
+ *   files that deploy on the suite's behalf) are not scanned.
+ */
+
+import { Database, type Database as DatabaseType } from 'bun:sqlite';
+import { describe, expect, test } from 'bun:test';
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  readdirSync,
+  rmSync,
+  writeFileSync,
+} from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { validateCapabilityAccess } from '../capabilities/validation';
+import type { ModuleManifest } from '../manifest/schema';
+import { validateManifest } from '../manifest/validate';
+import { repoRoot } from './capability-shape';
+
+/** Every module manifest under `modulesDir`, parsed by the production validator. */
+function loadModuleManifests(modulesDir: string): Map<string, ModuleManifest> {
+  const manifests = new Map<string, ModuleManifest>();
+  for (const id of readdirSync(modulesDir)) {
+    const path = join(modulesDir, id, 'manifest.yml');
+    if (!existsSync(path)) continue;
+    const result = validateManifest(readFileSync(path, 'utf8'));
+    if (result.success) manifests.set(id, result.data);
+    // Manifests that fail validation are another gate's job (validate.test.ts
+    // and scripts/check-modules.sh); scanning them here would double-report.
+  }
+  return manifests;
+}
+
+/**
+ * Module ids a fixture source imports, via `module import <ref>` statements.
+ *
+ * Handles the three shapes fixtures actually use: a bare literal
+ * (`'module import authentik'`), a path (`'module import /celilo/modules/authentik'`),
+ * and a `${CONST}` reference to a string literal defined in the same file
+ * (`module import ${PRIMARY}`). Trailing CLI flags (`--accept-aspects`) and
+ * timeouts are stripped. Anything that does not resolve to a plausible
+ * module id is skipped rather than guessed.
+ */
+function scanFixtureImports(fixtureSource: string): string[] {
+  const constants = new Map<string, string>();
+  for (const match of fixtureSource.matchAll(/const\s+(\w+)\s*=\s*'([^']+)'/g)) {
+    constants.set(match[1], match[2]);
+  }
+
+  const imports: string[] = [];
+  for (const match of fixtureSource.matchAll(/module import ([^'"\n]+?)['"`]/g)) {
+    let ref = match[1].trim();
+    const constRef = ref.match(/^\$\{(\w+)\}/);
+    if (constRef) ref = constants.get(constRef[1]) ?? '';
+    const id = ref.split(/\s+/)[0]?.split('/').pop() ?? '';
+    if (id && /^[a-z0-9-]+$/.test(id)) imports.push(id);
+  }
+  return imports;
+}
+
+/** The union of modules any `*.test.ts` fixture in `moduleId`'s e2e/ imports. */
+function fixtureImportedModules(modulesDir: string, moduleId: string): string[] {
+  const e2eDir = join(modulesDir, moduleId, 'e2e');
+  if (!existsSync(e2eDir)) return [];
+  const imported = new Set<string>();
+  for (const file of readdirSync(e2eDir)) {
+    if (!file.endsWith('.test.ts')) continue;
+    for (const id of scanFixtureImports(readFileSync(join(e2eDir, file), 'utf8'))) {
+      imported.add(id);
+    }
+  }
+  return [...imported];
+}
+
+/**
+ * An in-memory database shaped like the CLI's module registry, seeded with
+ * every capability `modules` provide — what `getProviderManifest` queries.
+ */
+function providerDatabase(manifests: Map<string, ModuleManifest>): DatabaseType {
+  const db = new Database(':memory:');
+  db.exec('CREATE TABLE modules (id TEXT PRIMARY KEY, manifest_data TEXT)');
+  db.exec('CREATE TABLE capabilities (module_id TEXT, capability_name TEXT)');
+  const insertModule = db.prepare('INSERT INTO modules (id, manifest_data) VALUES (?, ?)');
+  const insertCapability = db.prepare(
+    'INSERT INTO capabilities (module_id, capability_name) VALUES (?, ?)',
+  );
+  for (const [id, manifest] of manifests) {
+    insertModule.run(id, JSON.stringify(manifest));
+    for (const capability of manifest.provides?.capabilities ?? []) {
+      insertCapability.run(id, capability.name);
+    }
+  }
+  return db;
+}
+
+/**
+ * Modules with e2e fixtures whose required capabilities no imported module
+ * provides. The provider union is seeded into a real database and each
+ * consumer is checked with the production `validateCapabilityAccess`.
+ */
+async function capabilityCoverageViolations(modulesDir: string): Promise<string[]> {
+  const manifests = loadModuleManifests(modulesDir);
+  const violations: string[] = [];
+  for (const [id, manifest] of manifests) {
+    if (fixtureImportedModules(modulesDir, id).length === 0) continue;
+    const imported = fixtureImportedModules(modulesDir, id)
+      .map((dep) => manifests.get(dep))
+      .filter((m): m is ModuleManifest => m !== undefined);
+    const db = providerDatabase(new Map(imported.map((m) => [m.id, m])));
+    try {
+      const result = await validateCapabilityAccess(manifest, db);
+      if (!result.success) {
+        violations.push(`${id}: ${result.error ?? 'unknown error'}`);
+      }
+    } finally {
+      db.close();
+    }
+  }
+  return violations;
+}
+
+describe('fixture capability coverage', () => {
+  test('every module with e2e fixtures imports a provider for each required capability', async () => {
+    const violations = await capabilityCoverageViolations(join(repoRoot(), 'modules'));
+    expect(violations).toEqual([]);
+  });
+
+  // Reach measurement (Rule 2 of the lane plan): the real scanner and checker
+  // run against a mirrored module tree with planted markers, so the pass and
+  // fail cases are both proven to fire — not reasoned about. This is the
+  // mutation test for the gate itself: the real-tree test above can only go
+  // green, so the mirror is where the gate is shown to fail.
+  describe('reach: the gate fires on a mirrored tree (synthetic modules)', () => {
+    function mirrorTree(
+      dir: string,
+      modules: Record<string, { manifest: string; fixtures?: string[] }>,
+    ): string {
+      mkdirSync(dir, { recursive: true });
+      for (const [id, spec] of Object.entries(modules)) {
+        const moduleDir = join(dir, id);
+        mkdirSync(join(moduleDir, 'e2e'), { recursive: true });
+        writeFileSync(join(moduleDir, 'manifest.yml'), spec.manifest);
+        for (const [i, fixture] of (spec.fixtures ?? []).entries()) {
+          writeFileSync(join(moduleDir, 'e2e', `fixture-${i}.test.ts`), fixture);
+        }
+      }
+      return dir;
+    }
+
+    const providerManifest = [
+      'celilo_contract: "1.0"',
+      'id: marker-provider',
+      'name: Marker Provider',
+      'version: 1.0.0',
+      'description: synthetic provider',
+      'provides:',
+      '  capabilities:',
+      '    - name: marker_cap',
+      '      version: 1.0.0',
+      '',
+    ].join('\n');
+
+    const consumerFixture = "await net.celilo('module import marker-provider');\n";
+
+    test('a fixture importing the provider satisfies the requirement', async () => {
+      const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-ok-'));
+      try {
+        mirrorTree(dir, {
+          'marker-provider': { manifest: providerManifest },
+          'marker-consumer': {
+            manifest: [
+              'celilo_contract: "1.0"',
+              'id: marker-consumer',
+              'name: Marker Consumer',
+              'version: 1.0.0',
+              'description: synthetic consumer',
+              'requires:',
+              '  capabilities:',
+              '    - name: marker_cap',
+              '      version: 1.0.0',
+              '',
+            ].join('\n'),
+            fixtures: [consumerFixture],
+          },
+        });
+        expect(await capabilityCoverageViolations(dir)).toEqual([]);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    });
+
+    test('a fixture missing the provider fails, naming the module and capability', async () => {
+      const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-fail-'));
+      try {
+        mirrorTree(dir, {
+          'marker-provider': { manifest: providerManifest },
+          'marker-consumer': {
+            manifest: [
+              'celilo_contract: "1.0"',
+              'id: marker-consumer',
+              'name: Marker Consumer',
+              'version: 1.0.0',
+              'description: synthetic consumer',
+              'requires:',
+              '  capabilities:',
+              '    - name: marker_cap',
+              '      version: 1.0.0',
+              '    - name: marker_other',
+              '      version: 1.0.0',
+              '',
+            ].join('\n'),
+            // provider-b covers marker_cap; nothing covers marker_other
+            fixtures: [consumerFixture],
+          },
+          'marker-nonprovider': {
+            manifest: providerManifest.replace('marker_cap', 'marker_other'),
+          },
+        });
+        const violations = await capabilityCoverageViolations(dir);
+        expect(violations).toEqual([
+          expect.stringContaining("marker-consumer: Required capability 'marker_other' not found"),
+        ]);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    });
+
+    test('a privilege the framework grants needs no provider', async () => {
+      const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-priv-'));
+      try {
+        mirrorTree(dir, {
+          'marker-consumer': {
+            manifest: [
+              'celilo_contract: "1.0"',
+              'id: marker-consumer',
+              'name: Marker Consumer',
+              'version: 1.0.0',
+              'description: synthetic consumer',
+              'requires:',
+              '  capabilities:',
+              '    - name: cross_module_read',
+              '      version: 1.0.0',
+              '',
+            ].join('\n'),
+            fixtures: ["await net.celilo('module import nothing-here');\n"],
+          },
+        });
+        expect(await capabilityCoverageViolations(dir)).toEqual([]);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    });
+
+    test('a ${CONST} import resolves through the same-file string literal', async () => {
+      const dir = mkdtempSync(join(tmpdir(), 'cap-coverage-const-'));
+      try {
+        mirrorTree(dir, {
+          'marker-provider': { manifest: providerManifest },
+          'marker-consumer': {
+            manifest: [
+              'celilo_contract: "1.0"',
+              'id: marker-consumer',
+              'name: Marker Consumer',
+              'version: 1.0.0',
+              'description: synthetic consumer',
+              'requires:',
+              '  capabilities:',
+              '    - name: marker_cap',
+              '      version: 1.0.0',
+              '',
+            ].join('\n'),
+            fixtures: [
+              "const PROVIDER = 'marker-provider';\nawait net.celilo(`module import ${PROVIDER} --accept-aspects`);\n",
+            ],
+          },
+        });
+        expect(await capabilityCoverageViolations(dir)).toEqual([]);
+      } finally {
+        rmSync(dir, { recursive: true, force: true });
+      }
+    });
+  });
+});

package/src/policy/module-script-scan.test.ts

@@ -12,7 +12,11 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { buildModule } from '../module/packaging/build';
-import { scanModuleDirectory, scanModuleScriptSource } from './module-script-scan';
+import {
+  JAILED_CLI_SPAWN_RULE,
+  scanModuleDirectory,
+  scanModuleScriptSource,
+} from './module-script-scan';
 
 const rules = (src: string) => scanModuleScriptSource('f.ts', src).map((v) => v.rule);
 
@@ -60,6 +64,43 @@ describe('module script scan — namespace tools (hygiene, not a boundary)', ()
   });
 });
 
+describe('module script scan — jailed CLI spawn rule', () => {
+  // Each spawn spelling asserted separately. An alternation that loses a leg
+  // still passes a test that only exercises the first.
+  it.each([
+    [
+      "execFileSync('celilo', argv)",
+      "execFileSync('celilo', ['module', 'config', 'set', 'x', 'y']);",
+    ],
+    ['execFileSync("celilo", argv)', 'execFileSync("celilo", args);'],
+    ['execSync(`celilo …`)', 'execSync(`celilo module secret set x y`);'],
+    [
+      'exec(`celilo …`) after a shell prefix',
+      'exec(`cd /opt && sudo celilo system config get k`);',
+    ],
+    ["spawn('celilo', argv)", "const child = spawn('celilo', ['events', 'status']);"],
+  ])('catches %s', (_spelling, src) => {
+    expect(rules(src)).toContain(JAILED_CLI_SPAWN_RULE);
+  });
+
+  it('does not fire on operator prose that merely names the CLI', () => {
+    expect(rules('throw new Error("re-run `celilo module deploy x` to regenerate it");')).toEqual(
+      [],
+    );
+  });
+
+  // The `\w` after `celilo ` is what separates a CLI invocation from an
+  // argument that merely begins with the string, so the negative case has to
+  // be a hyphenated name, which would match a bare `\bcelilo\b`.
+  it('does not fire on a container argument named celilo-*', () => {
+    expect(rules('execSync(`docker exec celilo-mgr status`);')).toEqual([]);
+  });
+
+  it('does not fire on the celilo.db file name', () => {
+    expect(rules('execSync(`cp /var/lib/celilo/celilo.db /tmp/x`);')).toEqual([]);
+  });
+});
+
 describe('module script scan — raw-exec escape hatch', () => {
   it('flags a runAppCommand call with no justification', () => {
     expect(rules('const r = runAppCommand(system, "rm -f /tmp/x", run);')).toContain(