@celilo/cli 5.2.0 → 5.2.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "5.2.0",
+  "version": "5.2.2",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {

package/src/module/packaging/artifact-bytes.test.ts

@@ -0,0 +1,91 @@
+import { describe, expect, test } from 'bun:test';
+import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { inspectArtifactFile, verifyBuildArtifacts } from './artifact-bytes';
+
+const ELF = Buffer.from([0x7f, 0x45, 0x4c, 0x46]);
+
+function dir(): string {
+  return mkdtempSync(join(tmpdir(), 'artifact-bytes-'));
+}
+
+function write(base: string, rel: string, body: Buffer, exec = false): string {
+  const full = join(base, rel);
+  mkdirSync(join(full, '..'), { recursive: true });
+  writeFileSync(full, body);
+  if (exec) chmodSync(full, 0o755);
+  return full;
+}
+
+describe('inspectArtifactFile', () => {
+  test('rejects a right-sized file of pure NUL — the celilo#1463 shape', () => {
+    const d = dir();
+    const p = write(d, 'server-linux-x86_64', Buffer.alloc(4096), true);
+    expect(inspectArtifactFile(p)).toMatch(/4096 bytes of pure NUL/);
+  });
+
+  test('rejects an empty file', () => {
+    const d = dir();
+    expect(inspectArtifactFile(write(d, 'bin', Buffer.alloc(0), true))).toMatch(/empty/);
+  });
+
+  test('accepts a real ELF', () => {
+    const d = dir();
+    const p = write(d, 'bin', Buffer.concat([ELF, Buffer.alloc(1024, 0x41)]), true);
+    expect(inspectArtifactFile(p)).toBeNull();
+  });
+
+  test('rejects an executable carrying no executable magic', () => {
+    const d = dir();
+    const p = write(d, 'bin', Buffer.from('error: build failed\n'), true);
+    expect(inspectArtifactFile(p)).toMatch(/no known executable magic/);
+  });
+
+  test('does not apply the magic check to non-executable artifacts', () => {
+    // Modules declare .xz tarballs, .sha256 sidecars and index.html as
+    // artifacts. None is a program and none may be rejected as one.
+    const d = dir();
+    expect(inspectArtifactFile(write(d, 'forgejo.xz.sha256', Buffer.from('abc  f\n')))).toBeNull();
+    expect(inspectArtifactFile(write(d, 'index.html', Buffer.from('<!doctype html>')))).toBeNull();
+  });
+
+  test('a NUL file is rejected even when it is not executable', () => {
+    const d = dir();
+    expect(inspectArtifactFile(write(d, 'forgejo.xz', Buffer.alloc(64)))).toMatch(/pure NUL/);
+  });
+
+  test('a file whose NUL run ends late is still accepted', () => {
+    // Guards the streaming scan: content past the first chunk must be seen.
+    const d = dir();
+    const body = Buffer.alloc((1 << 20) + 16);
+    body[body.length - 1] = 0x41;
+    expect(inspectArtifactFile(write(d, 'blob', body))).toBeNull();
+  });
+});
+
+describe('verifyBuildArtifacts', () => {
+  test('names the module, the artifact and its size', () => {
+    const d = dir();
+    write(d, 'ansible/roles/r/files/srv-linux-x86_64', Buffer.alloc(106129743 % 8192), true);
+    const reason = verifyBuildArtifacts(d, ['ansible/roles/r/files/srv-linux-x86_64'], 'dj-live');
+    expect(reason).toContain('dj-live');
+    expect(reason).toContain('ansible/roles/r/files/srv-linux-x86_64');
+    expect(reason).toMatch(/bytes of pure NUL/);
+  });
+
+  test('passes a sound set', () => {
+    const d = dir();
+    write(d, 'files/a', Buffer.concat([ELF, Buffer.alloc(16, 1)]), true);
+    write(d, 'files/b.sha256', Buffer.from('deadbeef\n'));
+    expect(verifyBuildArtifacts(d, ['files/a', 'files/b.sha256'], 'm')).toBeNull();
+  });
+
+  test('walks a declared directory artifact', () => {
+    const d = dir();
+    write(d, 'spa/index.html', Buffer.from('<!doctype html>'));
+    write(d, 'spa/assets/app.js', Buffer.alloc(32));
+    const reason = verifyBuildArtifacts(d, ['spa'], 'console');
+    expect(reason).toContain('assets/app.js');
+  });
+});

package/src/module/packaging/artifact-bytes.ts

@@ -0,0 +1,135 @@
+import { closeSync, openSync, readSync, readdirSync, statSync } from 'node:fs';
+import { join, relative } from 'node:path';
+
+/**
+ * Executable magics, for a file the build marked executable.
+ *
+ * Only consulted when the artifact carries an exec bit, because
+ * `build.artifacts` is not a list of programs — modules declare `.xz`
+ * tarballs, a `.sha256` sidecar, an `index.html` and a whole `spa/`
+ * directory. A magic check applied to all of them would fail every module in
+ * the fleet; applied to the files the build itself chmod'd +x it is exactly
+ * the "is this runnable" question.
+ */
+const EXEC_MAGICS: readonly (readonly number[])[] = [
+  [0x7f, 0x45, 0x4c, 0x46], // ELF
+  [0xfe, 0xed, 0xfa, 0xce], // Mach-O 32
+  [0xfe, 0xed, 0xfa, 0xcf], // Mach-O 64
+  [0xce, 0xfa, 0xed, 0xfe], // Mach-O 32, byte-swapped
+  [0xcf, 0xfa, 0xed, 0xfe], // Mach-O 64, byte-swapped
+  [0xca, 0xfe, 0xba, 0xbe], // Mach-O universal
+  [0x4d, 0x5a], // PE/COFF "MZ"
+  [0x23, 0x21], // "#!" script
+];
+
+const CHUNK = 1 << 20;
+
+/** Read up to `n` leading bytes without loading the file. */
+function head(path: string, n: number): Buffer {
+  const fd = openSync(path, 'r');
+  try {
+    const buf = Buffer.alloc(n);
+    const read = readSync(fd, buf, 0, n, 0);
+    return buf.subarray(0, read);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * True when every byte of the file is NUL.
+ *
+ * Streams, and stops at the first non-zero byte — so a healthy artifact costs
+ * one 1 MiB read (its first byte is non-zero) and only a genuinely corrupt one
+ * is read through.
+ */
+function isAllNul(path: string, size: number): boolean {
+  const fd = openSync(path, 'r');
+  try {
+    const buf = Buffer.alloc(CHUNK);
+    for (let off = 0; off < size; ) {
+      const read = readSync(fd, buf, 0, CHUNK, off);
+      if (read === 0) break;
+      for (let i = 0; i < read; i++) {
+        if (buf[i] !== 0) return false;
+      }
+      off += read;
+    }
+    return true;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * Inspect one artifact file's own bytes. Returns a reason, or null when sound.
+ *
+ * Takes the verdict from the bytes rather than from the exit code of whatever
+ * wrote them. celilo#1463: two 100 MiB module binaries arrived as right-sized
+ * files of pure NUL, and every stage downstream reported success — because
+ * every one of those stages hashes, signs and verifies the artifact it is
+ * given. A checksum computed over corruption certifies corruption. The first
+ * thing that noticed was systemd, on a production host, with `Exec format
+ * error` and a restart counter.
+ */
+export function inspectArtifactFile(path: string): string | null {
+  const size = statSync(path).size;
+  if (size === 0) return 'file is empty (0 bytes)';
+  if (isAllNul(path, size)) return `file is ${size} bytes of pure NUL — nothing was written`;
+
+  // Only a file the build marked runnable has to look runnable.
+  const mode = statSync(path).mode;
+  if (!(mode & 0o111)) return null;
+
+  const magic = head(path, 4);
+  const known = EXEC_MAGICS.some((m) => m.every((b, i) => magic[i] === b));
+  if (!known) {
+    return `file is executable but carries no known executable magic (starts ${magic.toString('hex')}) — not a runnable program`;
+  }
+  return null;
+}
+
+/**
+ * Gate every declared build artifact on its own bytes.
+ *
+ * Returns a human-readable failure naming module, artifact and size, or null
+ * when all artifacts are sound. A declared directory is walked, because a
+ * module can declare a built tree (`spa/`) and a NUL file inside one is as
+ * invisible as a NUL file beside it.
+ */
+export function verifyBuildArtifacts(
+  buildDir: string,
+  artifacts: readonly string[],
+  moduleId: string,
+): string | null {
+  const failures: string[] = [];
+
+  const check = (fullPath: string, label: string): void => {
+    const reason = inspectArtifactFile(fullPath);
+    if (reason) failures.push(`  ${label}: ${reason}`);
+  };
+
+  // `root` stays the artifact's own directory so a nested file reports its
+  // whole path under the artifact, not just its basename.
+  const walk = (dir: string, root: string, artifact: string): void => {
+    for (const entry of readdirSync(dir, { withFileTypes: true })) {
+      const full = join(dir, entry.name);
+      if (entry.isDirectory()) walk(full, root, artifact);
+      else if (entry.isFile()) check(full, join(artifact, relative(root, full)));
+    }
+  };
+
+  for (const artifact of artifacts) {
+    const full = join(buildDir, artifact);
+    if (statSync(full).isDirectory()) walk(full, full, artifact);
+    else check(full, artifact);
+  }
+
+  if (failures.length === 0) return null;
+  return [
+    `Refusing to package ${moduleId}: the build reported success but produced an unusable artifact.`,
+    ...failures,
+    '',
+    "The verdict is taken from the artifact's own bytes, not from the build command's exit code. Every check downstream of here — checksums, signature, the registry sha256 — would hash this faithfully and certify it (celilo#1463).",
+  ].join('\n');
+}

package/src/module/packaging/build.ts

@@ -9,6 +9,7 @@ import { parse as parseYaml } from 'yaml';
 import { log } from '../../cli/prompts';
 import { formatViolations, scanModuleDirectory } from '../../policy/module-script-scan';
 import { validateModuleDirectory } from '../import';
+import { verifyBuildArtifacts } from './artifact-bytes';
 import { resolveBuildCommandPaths } from './build-paths';
 import { computeFileChecksum } from './checksum';
 import { classifyModulePath, includeNodeModulesPath } from './package-rules';
@@ -385,7 +386,14 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
         return { success: false, error: `Auto-build failed: Build exited with code ${msg}` };
       }
 
-      // Verify artifacts exist
+      // Verify artifacts exist, and that they are actually usable.
+      //
+      // Existence was the whole check, and existence is exactly what a
+      // right-sized file of pure NUL has. celilo#1463: two ~100 MiB module
+      // binaries were packaged, signed, published, downloaded, verified and
+      // installed as all-zero files, and every stage reported success, because
+      // this is the last point in the pipeline that sees the artifact before it
+      // becomes the thing all those checks are computed OVER.
       if (manifest.build.artifacts) {
         const missing = manifest.build.artifacts.filter(
           (a: string) => !existsSync(join(buildDir, a)),
@@ -396,6 +404,10 @@ export async function buildModule(options: ModuleBuildOptions): Promise<ModuleBu
             error: `Build succeeded but artifacts missing: ${missing.join(', ')}`,
           };
         }
+        const unusable = verifyBuildArtifacts(buildDir, manifest.build.artifacts, moduleId);
+        if (unusable) {
+          return { success: false, error: unusable };
+        }
       }
     }
 

package/src/policy/module-business-baseline.ts

@@ -345,12 +345,6 @@ export const CAPABILITY_NAME_BASELINE: readonly CapabilityNameRow[] = [
     count: 1,
     why: 'S13 — ZONE_REQUIREMENTS, a second hand-maintained copy of well-known.ts; Phase 2 (#937)',
   },
-  {
-    file: 'apps/celilo/src/templates/generator.ts',
-    capability: 'dns_internal',
-    count: 1,
-    why: 'X4 — declaration-driven and the model for the rest; only the error string names the capability (#945)',
-  },
   {
     file: 'apps/celilo/src/variables/context.ts',
     capability: 'dns_internal',

package/src/templates/copy-role-files.test.ts

@@ -21,9 +21,9 @@ import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { copyAnsibleRoleFilesDirs } from './generator';
 
-function moduleWithRoleFile(binary: string): string {
+function moduleWithRoleFile(binary: string, layout = 'ansible'): string {
   const root = mkdtempSync(join(tmpdir(), 'celilo-rolefiles-'));
-  const filesDir = join(root, 'ansible', 'roles', 'demo', 'files');
+  const filesDir = join(root, layout, 'roles', 'demo', 'files');
   mkdirSync(filesDir, { recursive: true });
   writeFileSync(join(filesDir, 'demo-linux-x86_64'), binary);
   return root;
@@ -59,6 +59,24 @@ describe('copyAnsibleRoleFilesDirs', () => {
     expect(generatedBinary(outputPath)).toBe('v2');
   });
 
+  /**
+   * celilo#1461. The template pipeline accepts BOTH `ansible/` and
+   * `celilo/ansible/`; this copy accepted one, so a celilo/-layout module got
+   * its role's tasks/ and templates/ and never its files/. `existsSync` was
+   * false, the function returned, and nothing anywhere said so.
+   *
+   * Generated output is the standard layout either way — the source layout is
+   * what varies.
+   */
+  test('finds role files/ under the celilo/ layout too', async () => {
+    const modulePath = moduleWithRoleFile('v1', 'celilo/ansible');
+    const outputPath = mkdtempSync(join(tmpdir(), 'celilo-out-'));
+
+    await copyAnsibleRoleFilesDirs(modulePath, outputPath);
+
+    expect(generatedBinary(outputPath)).toBe('v1');
+  });
+
   test('a module with no role files/ directory is not an error', async () => {
     const root = mkdtempSync(join(tmpdir(), 'celilo-norole-'));
     mkdirSync(join(root, 'ansible', 'roles', 'demo', 'tasks'), { recursive: true });

package/src/templates/generator.ts

@@ -60,9 +60,18 @@ const COPY_AS_IS_EXTENSIONS = ['.yml', '.yaml', '.tf'];
 /**
  * Template directories to process
  */
+// A module may keep its deployment tree at the root or under `celilo/`.
+// This is THE list of supported layouts — everything that has to find a
+// module's ansible/ or terraform/ derives it from here. celilo#1461 was two
+// functions deciding that independently and disagreeing.
+const LAYOUT_ROOTS = ['', 'celilo'];
+
+const layoutDirs = (name: string): string[] =>
+  LAYOUT_ROOTS.map((root) => (root ? `${root}/${name}` : name));
+
 // Directories to scan for template files. Supports both root-level and
 // celilo/ subdirectory layouts (e.g., ansible/ or celilo/ansible/).
-const TEMPLATE_DIRS = ['terraform', 'ansible', 'celilo/terraform', 'celilo/ansible'];
+const TEMPLATE_DIRS = [...layoutDirs('terraform'), ...layoutDirs('ansible')];
 
 /**
  * Check if file is a template that needs variable resolution
@@ -513,8 +522,9 @@ export async function readTemplateFiles(
  * @param files - Generated files to write
  */
 /**
- * Copy each `ansible/roles/<role>/files/` directory from the module source
- * to the generated output, verbatim (preserving binary content + mode bits).
+ * Copy each `<layout>/ansible/roles/<role>/files/` directory from the module
+ * source to the generated output, verbatim (preserving binary content + mode
+ * bits). Both supported layouts are scanned — see LAYOUT_ROOTS.
  *
  * These hold Ansible role-local static assets — they may be binaries and
  * don't need template variable resolution, so the standard template
@@ -524,38 +534,49 @@ export async function copyAnsibleRoleFilesDirs(
   modulePath: string,
   outputPath: string,
 ): Promise<void> {
-  const rolesDir = join(modulePath, 'ansible', 'roles');
-  if (!existsSync(rolesDir)) return;
-  const roles = await readdir(rolesDir, { withFileTypes: true });
-  for (const role of roles) {
-    if (!role.isDirectory()) continue;
-    const srcFilesDir = join(rolesDir, role.name, 'files');
-    if (!existsSync(srcFilesDir)) continue;
-    const destFilesDir = join(outputPath, 'ansible', 'roles', role.name, 'files');
-    await mkdir(dirname(destFilesDir), { recursive: true });
-    // `force: true` is LOAD-BEARING on bun, and its absence was celilo#925.
-    //
-    // Node defaults `force` to true, so this looked correct and is correct
-    // under Node. Bun 1.3.3 does not, on this path specifically — measured,
-    // copying "NEW" over an existing "OLD":
-    //
-    //   recursive only                          -> NEW
-    //   recursive + force                       -> NEW
-    //   recursive + preserveTimestamps          -> OLD   <- what this was
-    //   recursive + force + preserveTimestamps  -> NEW
-    //
-    // celilo runs on bun. So a module's built binary landed in `generated/`
-    // exactly once, at first generate, and no later version ever replaced it —
-    // silently, because `cp` reports no error, so the caller's try/catch has
-    // nothing to catch. Ansible then copies that first binary forever and
-    // reports `ok`, unchanged, while the module's version field advances.
-    //
-    // The sibling call in `storage-set-path.ts:153` already passes `force`.
-    await cp(srcFilesDir, destFilesDir, {
-      recursive: true,
-      force: true,
-      preserveTimestamps: true,
-    });
+  let copied = 0;
+  for (const ansibleDir of layoutDirs('ansible')) {
+    const rolesDir = join(modulePath, ansibleDir, 'roles');
+    if (!existsSync(rolesDir)) continue;
+    const roles = await readdir(rolesDir, { withFileTypes: true });
+    for (const role of roles) {
+      if (!role.isDirectory()) continue;
+      const srcFilesDir = join(rolesDir, role.name, 'files');
+      if (!existsSync(srcFilesDir)) continue;
+      const destFilesDir = join(outputPath, 'ansible', 'roles', role.name, 'files');
+      await mkdir(dirname(destFilesDir), { recursive: true });
+      // `force: true` is LOAD-BEARING on bun, and its absence was celilo#925.
+      //
+      // Node defaults `force` to true, so this looked correct and is correct
+      // under Node. Bun 1.3.3 does not, on this path specifically — measured,
+      // copying "NEW" over an existing "OLD":
+      //
+      //   recursive only                          -> NEW
+      //   recursive + force                       -> NEW
+      //   recursive + preserveTimestamps          -> OLD   <- what this was
+      //   recursive + force + preserveTimestamps  -> NEW
+      //
+      // celilo runs on bun. So a module's built binary landed in `generated/`
+      // exactly once, at first generate, and no later version ever replaced it —
+      // silently, because `cp` reports no error, so the caller's try/catch has
+      // nothing to catch. Ansible then copies that first binary forever and
+      // reports `ok`, unchanged, while the module's version field advances.
+      //
+      // The sibling call in `storage-set-path.ts:153` already passes `force`.
+      await cp(srcFilesDir, destFilesDir, {
+        recursive: true,
+        force: true,
+        preserveTimestamps: true,
+      });
+      copied++;
+    }
+  }
+
+  // "I looked and found nothing" must not read the same as "I copied
+  // everything". A module with no role files/ is legitimate; a module that has
+  // one somewhere this function cannot see is celilo#1461, and it was silent.
+  if (copied === 0) {
+    log.info(`No Ansible role files/ directories found under ${modulePath}`);
   }
 }