@celilo/cli 0.4.0 → 0.4.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@celilo/cli",
-  "version": "0.4.0",
+  "version": "0.4.1",
   "description": "Celilo — home lab orchestration CLI",
   "type": "module",
   "bin": {

package/src/cli/commands/restore.ts

@@ -98,6 +98,11 @@ export async function handleRestore(
   if (result.systemDbApplied) lines.push('  • celilo.db swapped into place');
   if (result.masterKeyApplied) lines.push('  • master.key swapped into place');
   if (result.sshKeyApplied) lines.push('  • fleet SSH key restored to <data-dir>/.ssh/');
+  if (result.moduleSourcesApplied && result.moduleSourcesApplied > 0) {
+    lines.push(
+      `  • module source laid down for ${result.moduleSourcesApplied} module(s) (paths reconciled to this box)`,
+    );
+  }
   if (result.crossModuleApplied && result.crossModuleApplied.length > 0) {
     lines.push(
       `  • terraform state restored for ${result.crossModuleApplied.length} module(s): ${result.crossModuleApplied.join(', ')}`,

package/src/module/import.ts

@@ -402,11 +402,16 @@ async function installScriptDependencies(
   targetPath: string,
   manifest: ModuleManifest,
 ): Promise<void> {
-  // Determine where scripts live by looking at hook declarations.
-  // All hooks use paths like `./scripts/setup-admin.ts`, so the
-  // scripts directory is the common parent.
-  if (!manifest.hooks || Object.keys(manifest.hooks).length === 0) {
-    return; // No hooks → no scripts → nothing to install
+  // A module's scripts/ holds hook scripts AND/OR capability-provider
+  // implementations (e.g. dns_registrar's register-host.ts, referenced via
+  // provides.capabilities, not hooks). BOTH import @celilo/capabilities and
+  // need deps. A capability-only provider (hooks: {}) still has scripts to
+  // resolve — keying solely off hooks left its node_modules un-vendored, so
+  // the capability-loader's import() failed with ENOENT @celilo/capabilities.
+  const hasHooks = Boolean(manifest.hooks && Object.keys(manifest.hooks).length > 0);
+  const hasCapabilities = (manifest.provides?.capabilities?.length ?? 0) > 0;
+  if (!hasHooks && !hasCapabilities) {
+    return; // No hook or capability scripts → nothing to install
   }
 
   const scriptsDir = join(targetPath, 'scripts');

package/src/services/celilo-mgmt-hooks.test.ts

@@ -147,11 +147,30 @@ describe('celilo-mgmt on_backup', () => {
     ).toBe(true);
     expect(existsSync(join(backupDir, 'cross_module_state', 'index.json'))).toBe(true);
 
-    expect(result.schema_version).toBe('1.0');
+    expect(result.schema_version).toBe('1.1');
     expect(result.artifact_count).toBeGreaterThan(0);
     expect(result.size_bytes).toBeGreaterThan(0);
   });
 
+  it('captures module SOURCE (excluding generated/) into module_src/', async () => {
+    // stateDir = dirname(db_path) = dir; on_backup reads dir/modules/<id>.
+    const modSrc = join(dir, 'modules', 'caddy');
+    mkdirSync(join(modSrc, 'scripts'), { recursive: true });
+    mkdirSync(join(modSrc, 'generated', 'terraform'), { recursive: true });
+    writeFileSync(join(modSrc, 'manifest.yml'), 'id: caddy');
+    writeFileSync(join(modSrc, 'scripts', 'hook.ts'), '// hook');
+    writeFileSync(join(modSrc, 'generated', 'terraform', 'main.tf'), 'resource {}');
+
+    const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
+    await hook(buildContext({ backup_dir: backupDir, cross_module_root: crossModuleRoot }));
+
+    // Source files captured...
+    expect(existsSync(join(backupDir, 'module_src', 'caddy', 'manifest.yml'))).toBe(true);
+    expect(existsSync(join(backupDir, 'module_src', 'caddy', 'scripts', 'hook.ts'))).toBe(true);
+    // ...but generated/ excluded (TF state travels via cross_module_state).
+    expect(existsSync(join(backupDir, 'module_src', 'caddy', 'generated'))).toBe(false);
+  });
+
   it('machine-pool.json is valid JSON (array)', async () => {
     const { default: hook } = await import(`${HOOK_DIR}/on_backup.ts`);
     await hook(buildContext({ backup_dir: backupDir, cross_module_root: crossModuleRoot }));
@@ -168,7 +187,7 @@ describe('celilo-mgmt on_backup', () => {
 
     expect(existsSync(join(backupDir, 'celilo.db'))).toBe(true);
     expect(existsSync(join(backupDir, 'cross_module_state'))).toBe(false);
-    expect(result.schema_version).toBe('1.0');
+    expect(result.schema_version).toBe('1.1');
   });
 
   it('proceeds (with a warning) when master.key is missing on disk', async () => {

package/src/services/restore-from-file.test.ts

@@ -8,6 +8,7 @@
  * via this service) is an e2e concern, not exercised at the unit level.
  */
 
+import { Database } from 'bun:sqlite';
 import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
 import {
   existsSync,
@@ -38,11 +39,14 @@ describe('applyStagedSystemFiles', () => {
     keyPath = join(dir, 'master.key');
     process.env.CELILO_DB_PATH = livePath;
     process.env.CELILO_MASTER_KEY_PATH = keyPath;
+    process.env.CELILO_DATA_DIR = dir; // getModuleStoragePath() = dir/modules
   });
 
   afterEach(() => {
+    closeDb();
     process.env.CELILO_DB_PATH = undefined;
     process.env.CELILO_MASTER_KEY_PATH = undefined;
+    process.env.CELILO_DATA_DIR = undefined;
     try {
       rmSync(dir, { recursive: true, force: true });
     } catch {
@@ -124,6 +128,48 @@ describe('applyStagedSystemFiles', () => {
     expect(result.dbApplied).toBe(false);
     expect(result.keyApplied).toBe(false);
   });
+
+  it('lays down staged module source dirs at the modules storage path', () => {
+    const stagedSrc = join(stagingDir, 'module_src');
+    mkdirSync(join(stagedSrc, 'caddy', 'scripts'), { recursive: true });
+    writeFileSync(join(stagedSrc, 'caddy', 'manifest.yml'), 'id: caddy');
+    writeFileSync(join(stagedSrc, 'caddy', 'scripts', 'hook.ts'), '// hook');
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.moduleSourcesApplied).toBe(1);
+    // getModuleStoragePath() = <CELILO_DATA_DIR>/modules = dir/modules.
+    const laid = join(dir, 'modules', 'caddy');
+    expect(readFileSync(join(laid, 'manifest.yml'), 'utf-8')).toBe('id: caddy');
+    expect(readFileSync(join(laid, 'scripts', 'hook.ts'), 'utf-8')).toBe('// hook');
+  });
+
+  it("rewrites every module's source_path to this box on the staged DB before swap", () => {
+    // A real staged SQLite DB whose module points at a FOREIGN (macOS) path —
+    // exactly the macOS→Linux cross-host case ISS-0051 broke on. Build a minimal
+    // self-contained DB (no runMigrations, which would hold a connection and
+    // lock the journal-mode switch the rewrite needs).
+    const stagedDbPath = join(stagingDir, 'celilo.db');
+    const seed = new Database(stagedDbPath);
+    seed.run(
+      'CREATE TABLE modules (id TEXT PRIMARY KEY, name TEXT, version TEXT, manifest_data TEXT, source_path TEXT)',
+    );
+    seed.run(
+      "INSERT INTO modules (id, name, version, manifest_data, source_path) VALUES ('caddy', 'caddy', '2.0.0', '{}', '/Users/someone/Library/Application Support/celilo/modules/caddy')",
+    );
+    seed.close();
+
+    const result = applyStagedSystemFiles(stagingDir);
+    expect(result.dbApplied).toBe(true);
+
+    // The swapped-in live DB must now point at THIS box's modules dir, not the
+    // source box's dead macOS path.
+    const live = new Database(livePath);
+    const row = live.query("SELECT source_path AS sp FROM modules WHERE id = 'caddy'").get() as {
+      sp: string;
+    };
+    live.close();
+    expect(row.sp).toBe(join(dir, 'modules', 'caddy'));
+  });
 });
 
 describe('restoreFromArtifactFile error paths', () => {

package/src/services/restore-from-file.ts

@@ -16,13 +16,18 @@
  * of the restore the hook itself cannot do (live DB is open).
  */
 
+import { Database } from 'bun:sqlite';
 import { execSync } from 'node:child_process';
 import {
   chmodSync,
+  closeSync,
   copyFileSync,
+  cpSync,
   existsSync,
   mkdirSync,
+  openSync,
   readFileSync,
+  readSync,
   readdirSync,
   rmSync,
   writeFileSync,
@@ -30,7 +35,7 @@ import {
 import { tmpdir } from 'node:os';
 import { dirname, join } from 'node:path';
 import { eq } from 'drizzle-orm';
-import { getDbPath, getMasterKeyPath } from '../config/paths';
+import { getDbPath, getMasterKeyPath, getModuleStoragePath } from '../config/paths';
 import { closeDb, getDb } from '../db/client';
 import { runMigrations } from '../db/migrate';
 import { modules } from '../db/schema';
@@ -61,6 +66,8 @@ export interface RestoreFromFileResult {
   masterKeyApplied?: boolean;
   /** Was the fleet SSH keypair restored to <dataDir>/.ssh/? */
   sshKeyApplied?: boolean;
+  /** How many module source dirs were laid down at the target's modules dir. */
+  moduleSourcesApplied?: number;
 }
 
 export interface RestoreFromFileOptions {
@@ -258,6 +265,7 @@ export async function restoreFromArtifactFile(
       systemDbApplied: apply.dbApplied,
       masterKeyApplied: apply.keyApplied,
       sshKeyApplied: apply.sshApplied,
+      moduleSourcesApplied: apply.moduleSourcesApplied,
     };
   } finally {
     try {
@@ -272,6 +280,61 @@ export interface StagedSystemApplyResult {
   dbApplied: boolean;
   keyApplied: boolean;
   sshApplied: boolean;
+  /** How many module source dirs were laid down at the target's modules dir. */
+  moduleSourcesApplied: number;
+}
+
+/** Cheap check for the SQLite file magic without reading the whole DB. */
+function looksLikeSqlite(path: string): boolean {
+  let fd: number;
+  try {
+    fd = openSync(path, 'r');
+  } catch {
+    return false;
+  }
+  try {
+    const buf = Buffer.alloc(16);
+    readSync(fd, buf, 0, 16, 0);
+    // Magic header is "SQLite format 3" (15 bytes) + a NUL terminator.
+    return buf.subarray(0, 15).toString('utf-8') === 'SQLite format 3' && buf[15] === 0;
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/**
+ * Recompute every module's source_path to THIS box's modules dir, on the
+ * STAGED DB file (pre-swap). The artifact carries the SOURCE box's absolute
+ * source_path (e.g. macOS `~/Library/Application Support/celilo/modules/<id>`),
+ * which doesn't exist on a different box/OS — so a restored DB references module
+ * code at a dead path and no deploy can generate. By construction (import.ts)
+ * source_path is always `${dataDir}/modules/<id>`, so it's safe to recompute for
+ * the target; a no-op when the paths already match (same-OS restore).
+ *
+ * Done on the staged file (a fresh connection that's about to be swapped in),
+ * never the live DB, so there's no post-swap in-process reopen (ISS-0037). The
+ * PRAGMA forces commits into the main file (no -wal sibling) so the copy below
+ * captures the rewrite; the live DB re-enters WAL when celilo reopens it.
+ */
+function rewriteStagedModuleSourcePaths(stagedDbPath: string): void {
+  // Tolerate non-SQLite staged files: some callers/tests stage placeholder
+  // bytes to exercise the file-copy path. A real artifact DB always opens
+  // with the "SQLite format 3\0" magic header — skip anything that doesn't
+  // (a valid-header-but-corrupt DB still surfaces by throwing below).
+  if (!looksLikeSqlite(stagedDbPath)) return;
+
+  const moduleStorageBase = getModuleStoragePath();
+  const staged = new Database(stagedDbPath);
+  try {
+    staged.query('PRAGMA journal_mode = DELETE').run();
+    const rows = staged.query('SELECT id FROM modules').all() as Array<{ id: string }>;
+    const update = staged.query('UPDATE modules SET source_path = ? WHERE id = ?');
+    for (const { id } of rows) {
+      update.run(join(moduleStorageBase, id), id);
+    }
+  } finally {
+    staged.close();
+  }
 }
 
 /**
@@ -291,14 +354,16 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
   let dbApplied = false;
   let keyApplied = false;
   let sshApplied = false;
+  let moduleSourcesApplied = 0;
 
   if (!existsSync(systemStagingDir)) {
-    return { dbApplied, keyApplied, sshApplied };
+    return { dbApplied, keyApplied, sshApplied, moduleSourcesApplied };
   }
 
   const stagedDb = join(systemStagingDir, 'celilo.db');
   const stagedKey = join(systemStagingDir, 'master.key');
   const stagedSsh = join(systemStagingDir, 'ssh');
+  const stagedModuleSrc = join(systemStagingDir, 'module_src');
 
   // master.key first: it's not load-bearing for the running process
   // (already in memory if the daemon's running). Safe to swap before
@@ -329,9 +394,31 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
     sshApplied = true;
   }
 
+  // Module SOURCE code → lay down at THIS box's modules dir so the restored
+  // box has the code for EVERY module (incl. non-registry ones like lunacycle).
+  // on_backup captured each module's source; without this the restored DB
+  // references modules whose code isn't on disk and no deploy can generate.
+  // The artifact carries no generated/ subtree, so any existing generated/
+  // under a module dir (e.g. restored TF state) is preserved by the merge.
+  if (existsSync(stagedModuleSrc)) {
+    const moduleStorageBase = getModuleStoragePath();
+    mkdirSync(moduleStorageBase, { recursive: true });
+    for (const entry of readdirSync(stagedModuleSrc, { withFileTypes: true })) {
+      if (!entry.isDirectory()) continue;
+      cpSync(join(stagedModuleSrc, entry.name), join(moduleStorageBase, entry.name), {
+        recursive: true,
+      });
+      moduleSourcesApplied += 1;
+    }
+  }
+
   // celilo.db: must close the live DB connection before replacing
   // the file or SQLite gets confused (macOS holds a vnode handle).
   if (existsSync(stagedDb)) {
+    // Reconcile module source_paths to THIS box on the staged file BEFORE the
+    // swap (and before closeDb) — fixes cross-host / cross-OS restores. See
+    // rewriteStagedModuleSourcePaths.
+    rewriteStagedModuleSourcePaths(stagedDb);
     closeDb();
     const livePath = getDbPath();
     mkdirSync(join(livePath, '..'), { recursive: true });
@@ -348,7 +435,7 @@ export function applyStagedSystemFiles(systemStagingDir: string): StagedSystemAp
     dbApplied = true;
   }
 
-  return { dbApplied, keyApplied, sshApplied };
+  return { dbApplied, keyApplied, sshApplied, moduleSourcesApplied };
 }
 
 /**