@celilo/cli 1.13.0 → 1.14.0

package/src/services/fleet-key.test.ts

@@ -0,0 +1,47 @@
+/**
+ * celilo's fleet SSH keypair, minted by the framework rather than by
+ * celilo-mgmt's on_install (design D9b of
+ * openspec/changes/hook-process-boundary).
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { existsSync, mkdtempSync, readFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { ensureFleetKey, getFleetSshDir } from './fleet-key';
+
+describe('ensureFleetKey', () => {
+  let dataDir: string;
+
+  beforeEach(() => {
+    dataDir = mkdtempSync(join(tmpdir(), 'celilo-fleet-key-'));
+    process.env.CELILO_DB_PATH = join(dataDir, 'celilo.db');
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DB_PATH = undefined;
+    rmSync(dataDir, { recursive: true, force: true });
+  });
+
+  it('mints an ed25519 keypair next to the DB and returns the public half', () => {
+    const key = ensureFleetKey();
+
+    expect(key.created).toBe(true);
+    expect(key.publicKey.startsWith('ssh-ed25519 ')).toBe(true);
+    expect(existsSync(join(dataDir, '.ssh', 'id_ed25519'))).toBe(true);
+    expect(readFileSync(join(dataDir, '.ssh', 'id_ed25519.pub'), 'utf-8').trim()).toBe(
+      key.publicKey,
+    );
+    expect(getFleetSshDir()).toBe(join(dataDir, '.ssh'));
+  });
+
+  it('reuses an existing key instead of re-keying', () => {
+    const first = ensureFleetKey();
+    const second = ensureFleetKey();
+
+    // Re-keying would strand every machine whose authorized_keys holds the
+    // old public half, and a redeploy calls this every time.
+    expect(second.created).toBe(false);
+    expect(second.publicKey).toBe(first.publicKey);
+  });
+});

package/src/services/fleet-key.ts

@@ -0,0 +1,75 @@
+/**
+ * celilo's fleet SSH keypair — the key celilo authenticates to managed
+ * machines with. The DB carries only the public half (`ssh.public_key`);
+ * the private half lives on disk and never leaves the management box.
+ *
+ * It used to be minted by celilo-mgmt's `on_install`, which derived
+ * `dirname(config.db_path)` and wrote into celilo's data directory from
+ * inside a module hook. That is a write into the one directory the hook
+ * jail exists to keep out of the mount set
+ * (openspec/changes/hook-process-boundary, design D9b): staging covers
+ * copies OUT of celilo's state, and nothing covers writes IN.
+ *
+ * So minting moved here. celilo owns the key's lifecycle, and "create it
+ * if absent" wants to be idempotent and tested once rather than in each
+ * module that reaches for it.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { existsSync, mkdirSync, readFileSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { getDbPath } from '../config/paths';
+
+/**
+ * Where the fleet keypair lives.
+ *
+ * Next to the DB, not under `getDataDir()`. Those are the same directory
+ * on a deb install (`CELILO_DATA_DIR=/var/celilo`, db at
+ * `/var/celilo/celilo.db`) and differ only when `CELILO_DB_PATH` points
+ * somewhere custom. Following the DB is what `on_install` did, so it is
+ * where every existing box's key already sits, and restore has followed
+ * the same rule since it was written (`applyStagedSystemFiles`).
+ *
+ * One exported helper rather than two hand-derived joins is the point:
+ * mint and restore can no longer drift to different directories.
+ */
+export function getFleetSshDir(): string {
+  return join(dirname(getDbPath()), '.ssh');
+}
+
+export interface FleetKey {
+  /** The public half, as it goes into `ssh.public_key` and authorized_keys. */
+  publicKey: string;
+  /** True when this call minted the key; false when it was already there. */
+  created: boolean;
+}
+
+/**
+ * Ensure the fleet keypair exists and return its public half.
+ *
+ * Idempotent: an existing key is reused, never regenerated. Re-keying
+ * would silently strand every machine whose authorized_keys holds the old
+ * public half, and a redeploy must not do that.
+ *
+ * Permissions are left to `mkdirSync`'s mode and to ssh-keygen, which
+ * `fchmod`s the private half to 0600 itself. An explicit chmod pass was
+ * written here first and then removed: umask only ever REMOVES mode bits, so
+ * neither the directory nor the key can come out wider than asked for, and no
+ * test could be made to fail without it.
+ */
+export function ensureFleetKey(): FleetKey {
+  const sshDir = getFleetSshDir();
+  const keyPath = join(sshDir, 'id_ed25519');
+  const publicKeyPath = `${keyPath}.pub`;
+
+  if (existsSync(publicKeyPath)) {
+    return { publicKey: readFileSync(publicKeyPath, 'utf-8').trim(), created: false };
+  }
+
+  mkdirSync(sshDir, { recursive: true, mode: 0o700 });
+  execFileSync('ssh-keygen', ['-t', 'ed25519', '-N', '', '-f', keyPath, '-C', 'celilo-fleet'], {
+    stdio: 'pipe',
+  });
+
+  return { publicKey: readFileSync(publicKeyPath, 'utf-8').trim(), created: true };
+}

package/src/services/restore-from-file.ts

@@ -32,7 +32,7 @@ import {
   rmSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
+import { join } from 'node:path';
 import { eq } from 'drizzle-orm';
 import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
 import { closeDb, getDb } from '../db/client';
@@ -47,6 +47,7 @@ import { decryptFileToFile } from './backup-cipher';
 import { assertCompatibleSchema, parseManifest } from './backup-manifest';
 import { applyCrossModuleWriteRoot, moduleHasCrossModuleRead } from './cross-module-read';
 import { getModuleSystems } from './deployed-systems';
+import { getFleetSshDir } from './fleet-key';
 import {
   completeOperation,
   failOperation,
@@ -376,10 +377,10 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
   // managed machines; the DB only carries the public half, so without the
   // private half on disk the restored box can't authenticate to the fleet.
   if (existsSync(stagedSsh)) {
-    // The fleet key lives next to the DB (on_install writes it to
-    // dirname(db_path)/.ssh), so follow the DB's location — not getDataDir()
-    // — in case CELILO_DB_PATH points somewhere custom.
-    const liveSshDir = join(dirname(getDbPath()), '.ssh');
+    // One helper, shared with the minting side, so restore and mint cannot
+    // drift to different directories. See getFleetSshDir for why it follows
+    // the DB rather than getDataDir().
+    const liveSshDir = getFleetSshDir();
     mkdirSync(liveSshDir, { recursive: true, mode: 0o700 });
     chmodSync(liveSshDir, 0o700);
     for (const entry of readdirSync(stagedSsh)) {

package/src/services/system-state-stage.test.ts

@@ -0,0 +1,165 @@
+/**
+ * Staging celilo's own state for a backup hook (design D9b of
+ * openspec/changes/hook-process-boundary).
+ *
+ * These assertions used to live in celilo-mgmt-hooks.test.ts, because the
+ * work used to live in celilo-mgmt's on_backup. They moved here with the
+ * code: the hook no longer reads celilo's data directory, the framework
+ * copies it into a staged directory first.
+ */
+
+import { Database } from 'bun:sqlite';
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { snapshotDatabase, stageSystemState } from './system-state-stage';
+
+describe('stageSystemState', () => {
+  let dataDir: string;
+  let stageRoot: string;
+  let masterKeyPath: string;
+
+  beforeEach(() => {
+    dataDir = mkdtempSync(join(tmpdir(), 'celilo-system-state-'));
+    stageRoot = join(dataDir, 'staged');
+    process.env.CELILO_DATA_DIR = dataDir;
+    process.env.CELILO_DB_PATH = join(dataDir, 'celilo.db');
+
+    const seed = new Database(join(dataDir, 'celilo.db'));
+    seed.run('CREATE TABLE probe (id INTEGER PRIMARY KEY, v TEXT)');
+    seed.run("INSERT INTO probe (v) VALUES ('hello')");
+    seed.close();
+
+    masterKeyPath = join(dataDir, 'master.key');
+    writeFileSync(masterKeyPath, 'fake-master-key-32-bytes-padding!');
+    process.env.CELILO_MASTER_KEY_PATH = masterKeyPath;
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DATA_DIR = undefined;
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    rmSync(dataDir, { recursive: true, force: true });
+  });
+
+  it('stages the DB snapshot and master.key into the root it is given', () => {
+    const staged = stageSystemState(stageRoot);
+
+    expect(existsSync(join(stageRoot, 'celilo.db'))).toBe(true);
+    expect(existsSync(join(stageRoot, 'master.key'))).toBe(true);
+    expect(staged.masterKeyStaged).toBe(true);
+    expect(staged.root).toBe(stageRoot);
+  });
+
+  it('reports a missing master.key rather than throwing', () => {
+    rmSync(masterKeyPath);
+
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.masterKeyStaged).toBe(false);
+    expect(existsSync(join(stageRoot, 'master.key'))).toBe(false);
+    // The DB still travels: a snapshot without the key is degraded, not
+    // useless, and refusing here would block backups on a box whose key
+    // path is overridden.
+    expect(existsSync(join(stageRoot, 'celilo.db'))).toBe(true);
+  });
+
+  it('stages the fleet keypair, private half included', () => {
+    const sshDir = join(dataDir, '.ssh');
+    mkdirSync(sshDir, { recursive: true });
+    writeFileSync(join(sshDir, 'id_ed25519'), 'PRIVATE');
+    writeFileSync(join(sshDir, 'id_ed25519.pub'), 'ssh-ed25519 AAAA celilo-fleet');
+
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.fleetSshStaged).toBe(true);
+    // The DB carries only the public half. Without the private half on disk a
+    // restored box cannot reach machines that already trust the key.
+    expect(readFileSync(join(stageRoot, 'ssh', 'id_ed25519'), 'utf-8')).toBe('PRIVATE');
+  });
+
+  it('reports an absent fleet keypair rather than staging an empty dir', () => {
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.fleetSshStaged).toBe(false);
+    expect(existsSync(join(stageRoot, 'ssh'))).toBe(false);
+  });
+
+  it('captures LEAN module source: no generated/, no node_modules/, no oversized files', () => {
+    const modSrc = join(dataDir, 'modules', 'caddy');
+    mkdirSync(join(modSrc, 'scripts', 'node_modules', '@celilo'), { recursive: true });
+    mkdirSync(join(modSrc, 'generated', 'terraform'), { recursive: true });
+    mkdirSync(join(modSrc, 'ansible', 'files'), { recursive: true });
+    writeFileSync(join(modSrc, 'manifest.yml'), 'id: caddy');
+    writeFileSync(join(modSrc, 'scripts', 'hook.ts'), '// hook');
+    writeFileSync(join(modSrc, 'generated', 'terraform', 'main.tf'), 'resource {}');
+    writeFileSync(join(modSrc, 'scripts', 'node_modules', '@celilo', 'dep.js'), '// vendored');
+    // A >2MB "compiled binary" sitting in source — skipped by size, because
+    // excluding by directory name misses the ones outside a known build dir.
+    writeFileSync(join(modSrc, 'ansible', 'files', 'server-bin'), Buffer.alloc(3 * 1024 * 1024));
+
+    const staged = stageSystemState(stageRoot);
+    const at = (...parts: string[]) => join(stageRoot, 'module_src', 'caddy', ...parts);
+
+    expect(staged.moduleSourceCount).toBe(1);
+    expect(existsSync(at('manifest.yml'))).toBe(true);
+    expect(existsSync(at('scripts', 'hook.ts'))).toBe(true);
+    expect(existsSync(at('generated'))).toBe(false);
+    expect(existsSync(at('scripts', 'node_modules'))).toBe(false);
+    expect(existsSync(at('ansible', 'files', 'server-bin'))).toBe(false);
+  });
+
+  it('names every file the size cap dropped', () => {
+    const modSrc = join(dataDir, 'modules', 'caddy');
+    mkdirSync(join(modSrc, 'ansible'), { recursive: true });
+    writeFileSync(join(modSrc, 'ansible', 'server-bin'), Buffer.alloc(3 * 1024 * 1024));
+
+    const staged = stageSystemState(stageRoot);
+
+    // No silent caps: a backup that quietly dropped a file reads as complete.
+    expect(staged.skippedLarge).toHaveLength(1);
+    expect(staged.skippedLarge[0]).toContain('caddy/ansible/server-bin');
+    expect(staged.skippedLarge[0]).toContain('3.0MB');
+  });
+});
+
+describe('snapshotDatabase', () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-db-snapshot-'));
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('captures rows still sitting in the WAL, uncheckpointed', () => {
+    // The defect this guards: celilo runs the DB in WAL mode, so committed
+    // rows live in celilo.db-wal until a checkpoint folds them into the main
+    // file. A plain copy of the main file produces a snapshot that opens
+    // cleanly and contains NOTHING, and restore then installs it. Swap
+    // serialize() for copyFileSync and this test is the thing that notices.
+    const src = join(dir, 'celilo.db');
+    const live = new Database(src);
+    live.run('PRAGMA journal_mode = WAL');
+    live.run('CREATE TABLE probe (id INTEGER PRIMARY KEY, v TEXT)');
+    for (let i = 0; i < 200; i++) {
+      live.run('INSERT INTO probe (v) VALUES (?)', [`row-${i}`]);
+    }
+
+    const dest = join(dir, 'snapshot.db');
+    snapshotDatabase(src, dest);
+    live.close();
+
+    // Opened read-write, not readonly: the serialized bytes carry WAL journal
+    // mode in their header, so the FIRST open has to be able to create the
+    // -wal/-shm sidecars. Restore opens it read-write too (it copies the file
+    // into place and runs migrations), so this is the real consumer's path.
+    const restored = new Database(dest);
+    const row = restored.query('SELECT COUNT(*) AS n FROM probe').get() as { n: number };
+    restored.close();
+    expect(row.n).toBe(200);
+  });
+});

package/src/services/system-state-stage.ts

@@ -0,0 +1,196 @@
+/**
+ * Staging of celilo's own state for a backup hook
+ * (openspec/changes/hook-process-boundary, design D9b).
+ *
+ * celilo-mgmt's `on_backup` used to reach into celilo's data directory and
+ * copy `master.key`, the DB, the fleet `.ssh` and every other module's
+ * source tree out of it. Measured across the whole hook, it read none of
+ * those bytes: every one was a `copyFileSync` / `cpSync` into `backup_dir`.
+ * It does not need those files in its filesystem view. It needs them to end
+ * up in the backup.
+ *
+ * So the framework copies them into a directory it creates and hands over,
+ * exactly as `materializeCrossModuleRoot` already does for
+ * `cross_module_read`. The hook reads from a staged location it was given,
+ * celilo-mgmt is fully jailed, and "the jail applies to every module" stays
+ * true with no exemption to audit.
+ *
+ * The layout mirrors what `on_backup` puts in the envelope, so the hook's
+ * remaining job is a copy:
+ *
+ *   <root>/celilo.db            WAL-correct snapshot
+ *   <root>/master.key           if present
+ *   <root>/ssh/                 fleet keypair, if present
+ *   <root>/module_src/<id>/     each module's lean source
+ */
+
+import { Database } from 'bun:sqlite';
+import {
+  copyFileSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
+import { getFleetSshDir } from './fleet-key';
+
+/**
+ * Consistent SQLite snapshot via bun:sqlite's serialize().
+ *
+ * celilo runs the DB in WAL mode (apps/celilo/src/db/client.ts), so committed
+ * rows live in `celilo.db-wal` until a checkpoint folds them into the main
+ * file. The main file is routinely a single near-empty page while ALL the
+ * real data (20+ tables, modules, config, secrets) sits in the WAL. A readonly
+ * connection reads THROUGH the WAL, so serialize() captures the full committed
+ * state into one standalone file — exactly what restore needs.
+ *
+ * An earlier implementation shelled out to `sqlite3 ".backup"` and fell back
+ * to a plain copyFileSync when the CLI was absent. On a deb-installed box
+ * there IS no sqlite3 CLI, so the fallback ran — and a plain copy of the main
+ * file alone DROPS the WAL, producing a silently EMPTY backup (restore then
+ * installs an empty DB). bun:sqlite is a Bun built-in and reads the WAL
+ * correctly — no CLI dependency, no data loss.
+ *
+ * This lives in the framework rather than in a module because celilo owns the
+ * schema and `getDbPath()`, and because a bug that silently empties backups
+ * should be fixed once, where it is tested.
+ */
+export function snapshotDatabase(srcPath: string, destPath: string): void {
+  const db = new Database(srcPath, { readonly: true });
+  try {
+    writeFileSync(destPath, db.serialize());
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Directories never worth capturing from a module's source tree. Every one
+ * is rebuilt on deploy (`module build` / `generate`) or re-vendored on
+ * restore (`installScriptDependencies`).
+ */
+const EXCLUDE_DIRS = new Set([
+  'node_modules',
+  'generated',
+  'dist',
+  'coverage',
+  'coverage-raw',
+  '.git',
+  'screenshots',
+  'e2e',
+]);
+
+/**
+ * Size ceiling for a single captured source file.
+ *
+ * celilo module dirs bundle large BUILD artifacts (compiled binaries, built
+ * assets, `*.netapp` packages) — turnip's were ~1.6 GB, which made the
+ * in-memory tar+encrypt segfault. Excluding by directory name misses the ones
+ * that sit outside a known build dir, so a size cap catches them generically.
+ */
+const MAX_SRC_FILE_BYTES = 2 * 1024 * 1024;
+
+export interface StagedSystemState {
+  /** The directory the caller passes to the hook. */
+  root: string;
+  masterKeyStaged: boolean;
+  fleetSshStaged: boolean;
+  moduleSourceCount: number;
+  /** Files the size cap or the `.netapp` rule dropped, named. No silent caps. */
+  skippedLarge: string[];
+}
+
+/**
+ * Populate `rootDir` with celilo's own state and return what landed there.
+ *
+ * Absent pieces are reported rather than thrown on: a box with no fleet key
+ * yet is an ordinary pre-deploy state, and a missing `master.key` is a fact
+ * the caller surfaces to the operator (secrets in the snapshot would be
+ * unreadable on restore) rather than a reason to abort the backup.
+ */
+export function stageSystemState(rootDir: string): StagedSystemState {
+  mkdirSync(rootDir, { recursive: true });
+
+  snapshotDatabase(getDbPath(), join(rootDir, 'celilo.db'));
+
+  // `getMasterKeyPath()` honours CELILO_MASTER_KEY_PATH and otherwise sits
+  // under getDataDir(). The hook used to re-derive it as
+  // `dirname(db_path)/master.key`, which is the same file on a deb install
+  // and a different one whenever CELILO_DB_PATH points elsewhere.
+  const masterKeyPath = getMasterKeyPath();
+  const masterKeyStaged = existsSync(masterKeyPath);
+  if (masterKeyStaged) {
+    copyFileSync(masterKeyPath, join(rootDir, 'master.key'));
+  }
+
+  // The private half too: the DB carries only `ssh.public_key`, so without
+  // it a restored box cannot reach the fleet, and re-keying means
+  // re-authorizing every managed machine.
+  const fleetSshDir = getFleetSshDir();
+  const fleetSshStaged = existsSync(join(fleetSshDir, 'id_ed25519'));
+  if (fleetSshStaged) {
+    cpSync(fleetSshDir, join(rootDir, 'ssh'), { recursive: true });
+  }
+
+  const { moduleSourceCount, skippedLarge } = stageModuleSources(join(rootDir, 'module_src'));
+
+  return { root: rootDir, masterKeyStaged, fleetSshStaged, moduleSourceCount, skippedLarge };
+}
+
+/**
+ * Capture each module's SOURCE (manifest, scripts, ansible, templates) into
+ * `destDir`, minus build artifacts.
+ *
+ * The DB references every module by `source_path`, but the rest of the
+ * envelope carries only DB + state, not module CODE. Restoring onto a fresh
+ * box — especially a different OS, where the source box's absolute
+ * `source_path` does not exist — would otherwise leave it unable to deploy
+ * ANY module, including non-registry ones (e.g. lunacycle) that a
+ * re-import-from-registry cannot recover.
+ *
+ * Reads `getModuleStoragePath()`, which is where `applyStagedSystemFiles`
+ * lays the source back down on restore. The hook derived
+ * `dirname(db_path)/modules` instead — the same directory on a deb install,
+ * and a different one otherwise, so backup and restore could disagree about
+ * where module source lives.
+ */
+function stageModuleSources(destDir: string): {
+  moduleSourceCount: number;
+  skippedLarge: string[];
+} {
+  const modulesSrcDir = getModuleStoragePath();
+  const skippedLarge: string[] = [];
+  let moduleSourceCount = 0;
+
+  if (!existsSync(modulesSrcDir)) {
+    return { moduleSourceCount, skippedLarge };
+  }
+
+  for (const entry of readdirSync(modulesSrcDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const srcModuleDir = join(modulesSrcDir, entry.name);
+    cpSync(srcModuleDir, join(destDir, entry.name), {
+      recursive: true,
+      filter: (src: string) => {
+        const rel = src.slice(srcModuleDir.length).replace(/^\//, '');
+        if (rel === '') return true; // module root
+        if (rel.split('/').some((seg) => EXCLUDE_DIRS.has(seg))) return false;
+        const st = statSync(src);
+        if (st.isDirectory()) return true;
+        if (src.endsWith('.netapp')) return false;
+        if (st.size > MAX_SRC_FILE_BYTES) {
+          skippedLarge.push(`${entry.name}/${rel} (${(st.size / 1048576).toFixed(1)}MB)`);
+          return false;
+        }
+        return true;
+      },
+    });
+    moduleSourceCount += 1;
+  }
+
+  return { moduleSourceCount, skippedLarge };
+}