@celilo/cli 1.13.0 → 2.0.0

package/src/services/static-content-converge.ts

@@ -0,0 +1,299 @@
+/**
+ * The static-content converge (openspec/changes/capability-owned-tables,
+ * stage 4 / design D10).
+ *
+ * `/srv/www` on the public_web provider stops being whatever the last
+ * hand-built ssh upload left there and becomes convergent state: every
+ * declared static route's release directory exists, `/srv/www/<slug>` points
+ * at the current one through an atomic symlink swap, and everything past the
+ * retention count is pruned. One Ansible converge makes the host match.
+ *
+ * Two callers, one code path:
+ *  - the `public_web` capability's publish (`convergeStaticContent` callback
+ *    injected by capability-loader), which runs the whole converge including
+ *    `executeAnsible`, so a publish returns only once the host matches; and
+ *  - the provider's OWN deploy (module-deploy), which writes the same release
+ *    set into its generated inventory and lets the deploy's existing
+ *    `executeAnsible` run the same role tasks — that is how a rebuilt host
+ *    recovers with no consumer involvement (design D10, task 4.5).
+ *
+ * The desired state is the declared `web_routes` release set plus the source
+ * directory core derived from `modules.source_path`. Nothing here writes into
+ * a module tree: reading a module's web root is safe, writing is what breaks
+ * `module verify` (celilo#1018, deliberately left to that thread).
+ */
+
+import { existsSync, readFileSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { and, eq, isNotNull } from 'drizzle-orm';
+import { stringify as stringifyYaml } from 'yaml';
+import type { DbClient } from '../db/client';
+import { moduleConfigs, modules, webRoutes } from '../db/schema';
+import { resolveModuleWebRoot } from '../module/web-root';
+import { parseStoredConfigValue } from './module-config';
+
+/** One content-hashed release the provider host must serve. */
+export interface StaticRelease {
+  /** The `/srv/www/<slug>` key. Independent of hostname on purpose. */
+  slug: string;
+  /** Content hash of the release — the release directory's suffix. */
+  contentHash: string;
+  /** Every FQDN served by this slug's routes (a slug spans hostnames). */
+  hostnames: string[];
+  /** Absolute celilo-mgr path core derived: `<module source_path>/site/dist`. */
+  sourceDir: string;
+}
+
+export interface StaticContentPlan {
+  releases: StaticRelease[];
+  /** How many release directories to keep per slug after a converge. */
+  retention: number;
+}
+
+/**
+ * Retention when the operator has not set one. Mirrors the manifest default
+ * declared on the provider module (`static_release_retention`); the manifest
+ * is the operator-facing source of truth, this only covers a provider whose
+ * manifest predates the variable.
+ */
+const DEFAULT_STATIC_RELEASE_RETENTION = 5;
+
+export interface StaticContentConvergeResult {
+  success: boolean;
+  error?: string;
+  /** The written vars file, repo-absolute — for logs and tests. */
+  varsPath?: string;
+}
+
+/**
+ * Where the converge's desired state lands inside the provider's generated
+ * project. `group_vars/all/` is the existing auto-load directory (it already
+ * holds `secrets.yml`), so no playbook change is needed to read it.
+ */
+export function staticContentVarsPath(generatedPath: string): string {
+  return join(generatedPath, 'ansible', 'inventory', 'group_vars', 'all', 'static_content.yml');
+}
+
+/**
+ * Resolve the retention count for the provider module.
+ *
+ * Operator config first (`static_release_retention` module config), then the
+ * manifest's declared default, then the constant above. ONE resolution — the
+ * role reads the variable bare and never carries a literal.
+ */
+export function resolveStaticContentRetention(db: DbClient, providerModuleId: string): number {
+  const configRow = db
+    .select()
+    .from(moduleConfigs)
+    .where(
+      and(
+        eq(moduleConfigs.moduleId, providerModuleId),
+        eq(moduleConfigs.key, 'static_release_retention'),
+      ),
+    )
+    .get();
+  if (configRow) {
+    const parsed = parseStoredConfigValue(configRow);
+    const count = typeof parsed === 'number' ? parsed : Number.parseInt(String(parsed), 10);
+    if (Number.isInteger(count) && count >= 1) return count;
+    throw new Error(
+      `static_release_retention for ${providerModuleId} must be a positive integer, got: ${String(parsed)}`,
+    );
+  }
+
+  const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
+  if (module) {
+    const manifest = module.manifestData as {
+      variables?: { owns?: Array<{ name?: string; default?: unknown }> };
+    } | null;
+    const declared = manifest?.variables?.owns?.find((v) => v?.name === 'static_release_retention');
+    const fallback = typeof declared?.default === 'number' ? declared.default : undefined;
+    if (typeof fallback === 'number' && Number.isInteger(fallback) && fallback >= 1)
+      return fallback;
+  }
+
+  return DEFAULT_STATIC_RELEASE_RETENTION;
+}
+
+/**
+ * Build the release set from the declared `web_routes` rows.
+ *
+ * Pure: reads the DB, decides nothing on the host (Rule 10.4). Grouping is by
+ * slug, NOT by route — one slug is one release directory, and a slug can span
+ * hostnames (lunacycle's two routes, both at `/`). A group that spans modules
+ * or disagrees with itself about the hash is a contradiction the converge
+ * cannot express, so it fails here with both rows named rather than silently
+ * converging one of them.
+ */
+export function planStaticContent(db: DbClient): StaticRelease[] {
+  const rows = db
+    .select()
+    .from(webRoutes)
+    .where(and(eq(webRoutes.type, 'static'), isNotNull(webRoutes.contentHash)))
+    .all();
+
+  const bySlug = new Map<string, typeof rows>();
+  for (const row of rows) {
+    const group = bySlug.get(row.slug) ?? [];
+    group.push(row);
+    bySlug.set(row.slug, group);
+  }
+
+  const releases: StaticRelease[] = [];
+  for (const [slug, group] of bySlug) {
+    const moduleIds = [...new Set(group.map((r) => r.moduleId))];
+    if (moduleIds.length > 1) {
+      throw new Error(
+        `Static route slug "${slug}" is claimed by ${moduleIds.length} modules (${moduleIds.join(', ')}). The slug is the release key — /srv/www/${slug} cannot serve two release sets. Fix the colliding modules' registered paths.`,
+      );
+    }
+    const moduleId = moduleIds[0];
+    if (!moduleId) continue;
+
+    const hashes = [...new Set(group.map((r) => r.contentHash))];
+    if (hashes.length > 1) {
+      throw new Error(
+        `Static route slug "${slug}" (module ${moduleId}) holds ${hashes.length} content hashes (${hashes.join(', ')}) — rows sharing a slug must share a release. Re-publish the module so every row carries the current hash.`,
+      );
+    }
+    const contentHash = hashes[0];
+    if (!contentHash) continue;
+
+    const sourceDir = resolveModuleWebRoot(moduleId, db);
+    if (!sourceDir) {
+      throw new Error(
+        `Static route slug "${slug}" belongs to module "${moduleId}", which is not installed — its release has no source to converge from.`,
+      );
+    }
+    if (!existsSync(sourceDir)) {
+      throw new Error(
+        `Static route slug "${slug}": no built site at ${sourceDir}. A module that serves a static site ships it at <module root>/site/dist. Redeploy the module.`,
+      );
+    }
+
+    releases.push({
+      slug,
+      contentHash,
+      hostnames: [...new Set(group.map((r) => r.hostname))].sort(),
+      sourceDir,
+    });
+  }
+
+  releases.sort((a, b) => a.slug.localeCompare(b.slug));
+  return releases;
+}
+
+/**
+ * The full desired state: the release set plus the retention count resolved
+ * for THIS provider module. The unit the vars file and the tests speak in.
+ */
+export function buildStaticContentPlan(db: DbClient, providerModuleId: string): StaticContentPlan {
+  return {
+    releases: planStaticContent(db),
+    retention: resolveStaticContentRetention(db, providerModuleId),
+  };
+}
+
+/**
+ * Render the vars file. Presentation function (Rule 10.1) — deterministic
+ * YAML, keyed by slug, carrying what 4.1 names: slug, content_hash, the
+ * hostnames served, and the source directory core derived.
+ */
+export function staticContentVarsYaml(plan: StaticContentPlan): string {
+  const vars = {
+    static_release_retention: plan.retention,
+    static_releases: plan.releases.map((r) => ({
+      slug: r.slug,
+      content_hash: r.contentHash,
+      hostnames: r.hostnames,
+      source_dir: r.sourceDir,
+    })),
+  };
+  const header =
+    '# Desired static-content state, generated by celilo (capability-owned-tables D10).\n' +
+    '# Hand edits are overwritten by the next converge. K = static_release_retention.\n';
+  return `${header}${stringifyYaml(vars, { lineWidth: 0, sortMapEntries: true })}`;
+}
+
+/**
+ * Plan the release set and write it into the provider's generated inventory.
+ *
+ * The shared half of both converge triggers: the capability's callback adds
+ * `executeAnsible` after this, and the provider's own deploy already runs it.
+ * Writing happens AFTER the deploy's staleness check (`refuseIfGeneratedIsStale`
+ * only inspects verbatim role assets), so an injected vars file never reads as
+ * a stale generation.
+ */
+export async function writeStaticContentVars(
+  db: DbClient,
+  providerModuleId: string,
+  generatedPath: string,
+): Promise<StaticContentConvergeResult> {
+  const varsPath = staticContentVarsPath(generatedPath);
+
+  const plan = buildStaticContentPlan(db, providerModuleId);
+
+  await mkdir(join(varsPath, '..'), { recursive: true });
+  await writeFile(varsPath, staticContentVarsYaml(plan), 'utf-8');
+  return { success: true, varsPath };
+}
+
+/**
+ * The full converge: plan, write the desired state, run Ansible against the
+ * provider's generated project.
+ *
+ * `generatedPath` must already exist — it is the project the provider's last
+ * deploy generated. If the project predates the converge role tasks, the
+ * written state would be read by nothing and the transfer would silently
+ * never happen, so the playbook is checked for the converge's own variable
+ * first and a stale project fails loudly with the redeploy that fixes it.
+ */
+export async function convergeStaticContent(
+  db: DbClient,
+  providerModuleId: string,
+): Promise<StaticContentConvergeResult> {
+  const module = db.select().from(modules).where(eq(modules.id, providerModuleId)).get();
+  if (!module) {
+    return { success: false, error: `Provider module '${providerModuleId}' is not installed` };
+  }
+  const generatedPath = join(module.sourcePath, 'generated');
+  if (!existsSync(generatedPath)) {
+    return {
+      success: false,
+      error:
+        `public_web provider '${providerModuleId}' has no generated deploy project at ${generatedPath}. ` +
+        `Run \`celilo module deploy ${providerModuleId}\` once, then retry the publish.`,
+    };
+  }
+
+  const written = await writeStaticContentVars(db, providerModuleId, generatedPath);
+
+  // A generated project that predates the converge role tasks would read the
+  // new vars file and converge nothing. `static_releases` appearing nowhere in
+  // the playbook is the tell. Generic string check — core names no module.
+  const playbookPath = join(generatedPath, 'ansible', 'playbook.yml');
+  if (
+    existsSync(playbookPath) &&
+    !readFileSync(playbookPath, 'utf-8').includes('static_releases')
+  ) {
+    return {
+      success: false,
+      error: `The generated project for '${providerModuleId}' predates the static-content converge (its playbook never reads static_releases). Redeploy the provider (\`celilo module deploy ${providerModuleId}\`) to regenerate it, then retry the publish.`,
+    };
+  }
+
+  const { executeAnsible } = await import('./deploy-ansible');
+  // Tag-scoped: the full playbook re-templates the bootstrap Caddyfile, which
+  // would replace the real config the provider's reconcile wrote and reload
+  // caddy into serving nothing. A publish converge touches /srv/www only.
+  const result = await executeAnsible(generatedPath, { tags: ['static_content'] });
+  if (!result.success) {
+    return {
+      success: false,
+      error: `Static-content converge failed on ${providerModuleId}: ${result.error ?? 'unknown error'}`,
+      varsPath: written.varsPath,
+    };
+  }
+  return { success: true, varsPath: written.varsPath };
+}

package/src/services/system-state-stage.test.ts

@@ -0,0 +1,165 @@
+/**
+ * Staging celilo's own state for a backup hook (design D9b of
+ * openspec/changes/hook-process-boundary).
+ *
+ * These assertions used to live in celilo-mgmt-hooks.test.ts, because the
+ * work used to live in celilo-mgmt's on_backup. They moved here with the
+ * code: the hook no longer reads celilo's data directory, the framework
+ * copies it into a staged directory first.
+ */
+
+import { Database } from 'bun:sqlite';
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { snapshotDatabase, stageSystemState } from './system-state-stage';
+
+describe('stageSystemState', () => {
+  let dataDir: string;
+  let stageRoot: string;
+  let masterKeyPath: string;
+
+  beforeEach(() => {
+    dataDir = mkdtempSync(join(tmpdir(), 'celilo-system-state-'));
+    stageRoot = join(dataDir, 'staged');
+    process.env.CELILO_DATA_DIR = dataDir;
+    process.env.CELILO_DB_PATH = join(dataDir, 'celilo.db');
+
+    const seed = new Database(join(dataDir, 'celilo.db'));
+    seed.run('CREATE TABLE probe (id INTEGER PRIMARY KEY, v TEXT)');
+    seed.run("INSERT INTO probe (v) VALUES ('hello')");
+    seed.close();
+
+    masterKeyPath = join(dataDir, 'master.key');
+    writeFileSync(masterKeyPath, 'fake-master-key-32-bytes-padding!');
+    process.env.CELILO_MASTER_KEY_PATH = masterKeyPath;
+  });
+
+  afterEach(() => {
+    process.env.CELILO_DATA_DIR = undefined;
+    process.env.CELILO_DB_PATH = undefined;
+    process.env.CELILO_MASTER_KEY_PATH = undefined;
+    rmSync(dataDir, { recursive: true, force: true });
+  });
+
+  it('stages the DB snapshot and master.key into the root it is given', () => {
+    const staged = stageSystemState(stageRoot);
+
+    expect(existsSync(join(stageRoot, 'celilo.db'))).toBe(true);
+    expect(existsSync(join(stageRoot, 'master.key'))).toBe(true);
+    expect(staged.masterKeyStaged).toBe(true);
+    expect(staged.root).toBe(stageRoot);
+  });
+
+  it('reports a missing master.key rather than throwing', () => {
+    rmSync(masterKeyPath);
+
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.masterKeyStaged).toBe(false);
+    expect(existsSync(join(stageRoot, 'master.key'))).toBe(false);
+    // The DB still travels: a snapshot without the key is degraded, not
+    // useless, and refusing here would block backups on a box whose key
+    // path is overridden.
+    expect(existsSync(join(stageRoot, 'celilo.db'))).toBe(true);
+  });
+
+  it('stages the fleet keypair, private half included', () => {
+    const sshDir = join(dataDir, '.ssh');
+    mkdirSync(sshDir, { recursive: true });
+    writeFileSync(join(sshDir, 'id_ed25519'), 'PRIVATE');
+    writeFileSync(join(sshDir, 'id_ed25519.pub'), 'ssh-ed25519 AAAA celilo-fleet');
+
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.fleetSshStaged).toBe(true);
+    // The DB carries only the public half. Without the private half on disk a
+    // restored box cannot reach machines that already trust the key.
+    expect(readFileSync(join(stageRoot, 'ssh', 'id_ed25519'), 'utf-8')).toBe('PRIVATE');
+  });
+
+  it('reports an absent fleet keypair rather than staging an empty dir', () => {
+    const staged = stageSystemState(stageRoot);
+
+    expect(staged.fleetSshStaged).toBe(false);
+    expect(existsSync(join(stageRoot, 'ssh'))).toBe(false);
+  });
+
+  it('captures LEAN module source: no generated/, no node_modules/, no oversized files', () => {
+    const modSrc = join(dataDir, 'modules', 'caddy');
+    mkdirSync(join(modSrc, 'scripts', 'node_modules', '@celilo'), { recursive: true });
+    mkdirSync(join(modSrc, 'generated', 'terraform'), { recursive: true });
+    mkdirSync(join(modSrc, 'ansible', 'files'), { recursive: true });
+    writeFileSync(join(modSrc, 'manifest.yml'), 'id: caddy');
+    writeFileSync(join(modSrc, 'scripts', 'hook.ts'), '// hook');
+    writeFileSync(join(modSrc, 'generated', 'terraform', 'main.tf'), 'resource {}');
+    writeFileSync(join(modSrc, 'scripts', 'node_modules', '@celilo', 'dep.js'), '// vendored');
+    // A >2MB "compiled binary" sitting in source — skipped by size, because
+    // excluding by directory name misses the ones outside a known build dir.
+    writeFileSync(join(modSrc, 'ansible', 'files', 'server-bin'), Buffer.alloc(3 * 1024 * 1024));
+
+    const staged = stageSystemState(stageRoot);
+    const at = (...parts: string[]) => join(stageRoot, 'module_src', 'caddy', ...parts);
+
+    expect(staged.moduleSourceCount).toBe(1);
+    expect(existsSync(at('manifest.yml'))).toBe(true);
+    expect(existsSync(at('scripts', 'hook.ts'))).toBe(true);
+    expect(existsSync(at('generated'))).toBe(false);
+    expect(existsSync(at('scripts', 'node_modules'))).toBe(false);
+    expect(existsSync(at('ansible', 'files', 'server-bin'))).toBe(false);
+  });
+
+  it('names every file the size cap dropped', () => {
+    const modSrc = join(dataDir, 'modules', 'caddy');
+    mkdirSync(join(modSrc, 'ansible'), { recursive: true });
+    writeFileSync(join(modSrc, 'ansible', 'server-bin'), Buffer.alloc(3 * 1024 * 1024));
+
+    const staged = stageSystemState(stageRoot);
+
+    // No silent caps: a backup that quietly dropped a file reads as complete.
+    expect(staged.skippedLarge).toHaveLength(1);
+    expect(staged.skippedLarge[0]).toContain('caddy/ansible/server-bin');
+    expect(staged.skippedLarge[0]).toContain('3.0MB');
+  });
+});
+
+describe('snapshotDatabase', () => {
+  let dir: string;
+
+  beforeEach(() => {
+    dir = mkdtempSync(join(tmpdir(), 'celilo-db-snapshot-'));
+  });
+
+  afterEach(() => {
+    rmSync(dir, { recursive: true, force: true });
+  });
+
+  it('captures rows still sitting in the WAL, uncheckpointed', () => {
+    // The defect this guards: celilo runs the DB in WAL mode, so committed
+    // rows live in celilo.db-wal until a checkpoint folds them into the main
+    // file. A plain copy of the main file produces a snapshot that opens
+    // cleanly and contains NOTHING, and restore then installs it. Swap
+    // serialize() for copyFileSync and this test is the thing that notices.
+    const src = join(dir, 'celilo.db');
+    const live = new Database(src);
+    live.run('PRAGMA journal_mode = WAL');
+    live.run('CREATE TABLE probe (id INTEGER PRIMARY KEY, v TEXT)');
+    for (let i = 0; i < 200; i++) {
+      live.run('INSERT INTO probe (v) VALUES (?)', [`row-${i}`]);
+    }
+
+    const dest = join(dir, 'snapshot.db');
+    snapshotDatabase(src, dest);
+    live.close();
+
+    // Opened read-write, not readonly: the serialized bytes carry WAL journal
+    // mode in their header, so the FIRST open has to be able to create the
+    // -wal/-shm sidecars. Restore opens it read-write too (it copies the file
+    // into place and runs migrations), so this is the real consumer's path.
+    const restored = new Database(dest);
+    const row = restored.query('SELECT COUNT(*) AS n FROM probe').get() as { n: number };
+    restored.close();
+    expect(row.n).toBe(200);
+  });
+});

package/src/services/system-state-stage.ts

@@ -0,0 +1,196 @@
+/**
+ * Staging of celilo's own state for a backup hook
+ * (openspec/changes/hook-process-boundary, design D9b).
+ *
+ * celilo-mgmt's `on_backup` used to reach into celilo's data directory and
+ * copy `master.key`, the DB, the fleet `.ssh` and every other module's
+ * source tree out of it. Measured across the whole hook, it read none of
+ * those bytes: every one was a `copyFileSync` / `cpSync` into `backup_dir`.
+ * It does not need those files in its filesystem view. It needs them to end
+ * up in the backup.
+ *
+ * So the framework copies them into a directory it creates and hands over,
+ * exactly as `materializeCrossModuleRoot` already does for
+ * `cross_module_read`. The hook reads from a staged location it was given,
+ * celilo-mgmt is fully jailed, and "the jail applies to every module" stays
+ * true with no exemption to audit.
+ *
+ * The layout mirrors what `on_backup` puts in the envelope, so the hook's
+ * remaining job is a copy:
+ *
+ *   <root>/celilo.db            WAL-correct snapshot
+ *   <root>/master.key           if present
+ *   <root>/ssh/                 fleet keypair, if present
+ *   <root>/module_src/<id>/     each module's lean source
+ */
+
+import { Database } from 'bun:sqlite';
+import {
+  copyFileSync,
+  cpSync,
+  existsSync,
+  mkdirSync,
+  readdirSync,
+  statSync,
+  writeFileSync,
+} from 'node:fs';
+import { join } from 'node:path';
+import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
+import { getFleetSshDir } from './fleet-key';
+
+/**
+ * Consistent SQLite snapshot via bun:sqlite's serialize().
+ *
+ * celilo runs the DB in WAL mode (apps/celilo/src/db/client.ts), so committed
+ * rows live in `celilo.db-wal` until a checkpoint folds them into the main
+ * file. The main file is routinely a single near-empty page while ALL the
+ * real data (20+ tables, modules, config, secrets) sits in the WAL. A readonly
+ * connection reads THROUGH the WAL, so serialize() captures the full committed
+ * state into one standalone file — exactly what restore needs.
+ *
+ * An earlier implementation shelled out to `sqlite3 ".backup"` and fell back
+ * to a plain copyFileSync when the CLI was absent. On a deb-installed box
+ * there IS no sqlite3 CLI, so the fallback ran — and a plain copy of the main
+ * file alone DROPS the WAL, producing a silently EMPTY backup (restore then
+ * installs an empty DB). bun:sqlite is a Bun built-in and reads the WAL
+ * correctly — no CLI dependency, no data loss.
+ *
+ * This lives in the framework rather than in a module because celilo owns the
+ * schema and `getDbPath()`, and because a bug that silently empties backups
+ * should be fixed once, where it is tested.
+ */
+export function snapshotDatabase(srcPath: string, destPath: string): void {
+  const db = new Database(srcPath, { readonly: true });
+  try {
+    writeFileSync(destPath, db.serialize());
+  } finally {
+    db.close();
+  }
+}
+
+/**
+ * Directories never worth capturing from a module's source tree. Every one
+ * is rebuilt on deploy (`module build` / `generate`) or re-vendored on
+ * restore (`installScriptDependencies`).
+ */
+const EXCLUDE_DIRS = new Set([
+  'node_modules',
+  'generated',
+  'dist',
+  'coverage',
+  'coverage-raw',
+  '.git',
+  'screenshots',
+  'e2e',
+]);
+
+/**
+ * Size ceiling for a single captured source file.
+ *
+ * celilo module dirs bundle large BUILD artifacts (compiled binaries, built
+ * assets, `*.netapp` packages) — turnip's were ~1.6 GB, which made the
+ * in-memory tar+encrypt segfault. Excluding by directory name misses the ones
+ * that sit outside a known build dir, so a size cap catches them generically.
+ */
+const MAX_SRC_FILE_BYTES = 2 * 1024 * 1024;
+
+export interface StagedSystemState {
+  /** The directory the caller passes to the hook. */
+  root: string;
+  masterKeyStaged: boolean;
+  fleetSshStaged: boolean;
+  moduleSourceCount: number;
+  /** Files the size cap or the `.netapp` rule dropped, named. No silent caps. */
+  skippedLarge: string[];
+}
+
+/**
+ * Populate `rootDir` with celilo's own state and return what landed there.
+ *
+ * Absent pieces are reported rather than thrown on: a box with no fleet key
+ * yet is an ordinary pre-deploy state, and a missing `master.key` is a fact
+ * the caller surfaces to the operator (secrets in the snapshot would be
+ * unreadable on restore) rather than a reason to abort the backup.
+ */
+export function stageSystemState(rootDir: string): StagedSystemState {
+  mkdirSync(rootDir, { recursive: true });
+
+  snapshotDatabase(getDbPath(), join(rootDir, 'celilo.db'));
+
+  // `getMasterKeyPath()` honours CELILO_MASTER_KEY_PATH and otherwise sits
+  // under getDataDir(). The hook used to re-derive it as
+  // `dirname(db_path)/master.key`, which is the same file on a deb install
+  // and a different one whenever CELILO_DB_PATH points elsewhere.
+  const masterKeyPath = getMasterKeyPath();
+  const masterKeyStaged = existsSync(masterKeyPath);
+  if (masterKeyStaged) {
+    copyFileSync(masterKeyPath, join(rootDir, 'master.key'));
+  }
+
+  // The private half too: the DB carries only `ssh.public_key`, so without
+  // it a restored box cannot reach the fleet, and re-keying means
+  // re-authorizing every managed machine.
+  const fleetSshDir = getFleetSshDir();
+  const fleetSshStaged = existsSync(join(fleetSshDir, 'id_ed25519'));
+  if (fleetSshStaged) {
+    cpSync(fleetSshDir, join(rootDir, 'ssh'), { recursive: true });
+  }
+
+  const { moduleSourceCount, skippedLarge } = stageModuleSources(join(rootDir, 'module_src'));
+
+  return { root: rootDir, masterKeyStaged, fleetSshStaged, moduleSourceCount, skippedLarge };
+}
+
+/**
+ * Capture each module's SOURCE (manifest, scripts, ansible, templates) into
+ * `destDir`, minus build artifacts.
+ *
+ * The DB references every module by `source_path`, but the rest of the
+ * envelope carries only DB + state, not module CODE. Restoring onto a fresh
+ * box — especially a different OS, where the source box's absolute
+ * `source_path` does not exist — would otherwise leave it unable to deploy
+ * ANY module, including non-registry ones (e.g. lunacycle) that a
+ * re-import-from-registry cannot recover.
+ *
+ * Reads `getModuleStoragePath()`, which is where `applyStagedSystemFiles`
+ * lays the source back down on restore. The hook derived
+ * `dirname(db_path)/modules` instead — the same directory on a deb install,
+ * and a different one otherwise, so backup and restore could disagree about
+ * where module source lives.
+ */
+function stageModuleSources(destDir: string): {
+  moduleSourceCount: number;
+  skippedLarge: string[];
+} {
+  const modulesSrcDir = getModuleStoragePath();
+  const skippedLarge: string[] = [];
+  let moduleSourceCount = 0;
+
+  if (!existsSync(modulesSrcDir)) {
+    return { moduleSourceCount, skippedLarge };
+  }
+
+  for (const entry of readdirSync(modulesSrcDir, { withFileTypes: true })) {
+    if (!entry.isDirectory()) continue;
+    const srcModuleDir = join(modulesSrcDir, entry.name);
+    cpSync(srcModuleDir, join(destDir, entry.name), {
+      recursive: true,
+      filter: (src: string) => {
+        const rel = src.slice(srcModuleDir.length).replace(/^\//, '');
+        if (rel === '') return true; // module root
+        if (rel.split('/').some((seg) => EXCLUDE_DIRS.has(seg))) return false;
+        const st = statSync(src);
+        if (st.isDirectory()) return true;
+        if (src.endsWith('.netapp')) return false;
+        if (st.size > MAX_SRC_FILE_BYTES) {
+          skippedLarge.push(`${entry.name}/${rel} (${(st.size / 1048576).toFixed(1)}MB)`);
+          return false;
+        }
+        return true;
+      },
+    });
+    moduleSourceCount += 1;
+  }
+
+  return { moduleSourceCount, skippedLarge };
+}