@celilo/cli 0.1.0

package/src/module/import.ts

@@ -0,0 +1,676 @@
+import { execSync } from 'node:child_process';
+import { existsSync, statSync, writeFileSync } from 'node:fs';
+import { copyFile, mkdir, readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { eq } from 'drizzle-orm';
+import { z } from 'zod';
+import { getWellKnownCapability, isWellKnown } from '../capabilities/well-known';
+import { log } from '../cli/prompts';
+import { getModuleStoragePath } from '../config/paths';
+import { type DbClient, getDb } from '../db/client';
+import { capabilities, moduleIntegrity, modules } from '../db/schema';
+import type { NewModule, NewModuleIntegrity } from '../db/schema';
+import type { ModuleManifest } from '../manifest/schema';
+import {
+  validateCapabilityNames,
+  validateDeriveFromSources,
+  validateHookContract,
+  validateManifest,
+  validateProvidesNoCrossCapabilityRefs,
+  validateVariableSources,
+  validateZoneRequirements,
+} from '../manifest/validate';
+import { parseJsonWithValidation } from '../validation/schemas';
+import { cleanupTempDir, extractPackage, verifyPackageIntegrity } from './packaging/extract';
+
+/**
+ * Module import options
+ */
+export interface ModuleImportOptions {
+  sourcePath: string;
+  targetBasePath?: string;
+  db?: DbClient;
+  flags?: Record<string, string | boolean>;
+}
+
+/**
+ * Module import result
+ */
+export interface ModuleImportSuccess {
+  success: true;
+  moduleId: string;
+  targetPath: string;
+}
+
+export interface ModuleImportError {
+  success: false;
+  error: string;
+  details?: unknown;
+}
+
+export type ModuleImportResult = ModuleImportSuccess | ModuleImportError;
+
+/**
+ * Get default target base path for module storage
+ * Uses platform-specific defaults with environment variable overrides
+ */
+function getDefaultTargetBase(): string {
+  return getModuleStoragePath();
+}
+
+/**
+ * Validate module directory structure
+ *
+ * Policy function (Rule 10.1) - validates input only, no side effects
+ *
+ * @param sourcePath - Path to module directory
+ * @returns Error message if invalid, null if valid
+ */
+export function validateModuleDirectory(sourcePath: string): string | null {
+  // Check directory exists
+  if (!existsSync(sourcePath)) {
+    return `Module directory does not exist: ${sourcePath}`;
+  }
+
+  // Check it's a directory
+  const stats = statSync(sourcePath);
+  if (!stats.isDirectory()) {
+    return `Path is not a directory: ${sourcePath}`;
+  }
+
+  // Check manifest.yml exists
+  const manifestPath = join(sourcePath, 'manifest.yml');
+  if (!existsSync(manifestPath)) {
+    return 'manifest.yml not found in module directory';
+  }
+
+  return null;
+}
+
+/**
+ * Read and validate manifest from module directory
+ *
+ * Policy function - reads and validates, no database access
+ *
+ * @param sourcePath - Path to module directory
+ * @returns Validated manifest or error
+ */
+export async function readModuleManifest(
+  sourcePath: string,
+): Promise<{ success: true; manifest: ModuleManifest } | { success: false; error: string }> {
+  const manifestPath = join(sourcePath, 'manifest.yml');
+
+  let yamlContent: string;
+  try {
+    yamlContent = await readFile(manifestPath, 'utf-8');
+  } catch (error) {
+    return {
+      success: false,
+      error: `Failed to read manifest.yml: ${error instanceof Error ? error.message : 'Unknown error'}`,
+    };
+  }
+
+  const validationResult = validateManifest(yamlContent);
+
+  if (!validationResult.success) {
+    const errorMessages = validationResult.errors.map((e) => `${e.path}: ${e.message}`).join(', ');
+    return {
+      success: false,
+      error: `Manifest validation failed: ${errorMessages}`,
+    };
+  }
+
+  const zoneValidation = validateZoneRequirements(validationResult.data);
+  if (zoneValidation) {
+    const errorMessages = zoneValidation.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
+    return {
+      success: false,
+      error: `Zone validation failed:\n${errorMessages}`,
+    };
+  }
+
+  const deriveCheck = validateDeriveFromSources(validationResult.data);
+  if (deriveCheck) {
+    const errorMessages = deriveCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
+    return {
+      success: false,
+      error: `Variable derive_from validation failed:\n${errorMessages}`,
+    };
+  }
+
+  const hookCheck = validateHookContract(validationResult.data);
+  if (hookCheck) {
+    const errorMessages = hookCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
+    return {
+      success: false,
+      error: `Hook contract validation failed:\n${errorMessages}`,
+    };
+  }
+
+  const crossCapCheck = validateProvidesNoCrossCapabilityRefs(validationResult.data);
+  if (crossCapCheck) {
+    const errorMessages = crossCapCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
+    return {
+      success: false,
+      error: `Capability data validation failed:\n${errorMessages}`,
+    };
+  }
+
+  const capNameCheck = validateCapabilityNames(validationResult.data);
+  if (capNameCheck) {
+    const errorMessages = capNameCheck.errors.map((e) => `${e.path}: ${e.message}`).join('\n');
+    return {
+      success: false,
+      error: `Capability name validation failed:\n${errorMessages}`,
+    };
+  }
+
+  const variableSourceCheck = validateVariableSources(validationResult.data);
+  if (variableSourceCheck) {
+    const errorMessages = variableSourceCheck.errors
+      .map((e) => `${e.path}: ${e.message}`)
+      .join('\n');
+    return {
+      success: false,
+      error: `Variable source validation failed:\n${errorMessages}`,
+    };
+  }
+
+  return {
+    success: true,
+    manifest: validationResult.data,
+  };
+}
+
+/**
+ * Copy module files to target directory
+ *
+ * Execution function (Rule 10.1) - performs file I/O
+ *
+ * @param sourcePath - Source module directory
+ * @param targetPath - Target directory to copy to
+ */
+export async function copyModuleFiles(sourcePath: string, targetPath: string): Promise<void> {
+  // Create target directory
+  await mkdir(targetPath, { recursive: true });
+
+  // Get all files in source directory
+  async function copyRecursive(src: string, dest: string) {
+    const entries = await readdir(src, { withFileTypes: true });
+
+    for (const entry of entries) {
+      const srcPath = join(src, entry.name);
+      const destPath = join(dest, entry.name);
+
+      // Skip metadata files
+      if (entry.name === 'checksums.json' || entry.name === 'signature.sig') {
+        continue;
+      }
+
+      // Skip directories that shouldn't be copied into celilo's data store.
+      // node_modules inside scripts/ ARE kept — they contain the module's
+      // runtime deps (@celilo/capabilities etc.). Top-level node_modules
+      // (monorepo workspace deps, build tools) are skipped.
+      if (
+        entry.isDirectory() &&
+        (entry.name === '.git' || entry.name === '.next' || entry.name === '.cache')
+      ) {
+        continue;
+      }
+
+      if (
+        entry.isDirectory() &&
+        entry.name === 'node_modules' &&
+        !src.endsWith('/scripts') &&
+        !src.endsWith('\\scripts')
+      ) {
+        continue;
+      }
+
+      if (entry.isDirectory()) {
+        await mkdir(destPath, { recursive: true });
+        await copyRecursive(srcPath, destPath);
+      } else if (entry.isFile()) {
+        await copyFile(srcPath, destPath);
+      }
+    }
+  }
+
+  await copyRecursive(sourcePath, targetPath);
+}
+
+/**
+ * Insert module into database
+ *
+ * Execution function - performs database write
+ *
+ * @param manifest - Validated manifest
+ * @param targetPath - Target path where files are stored
+ * @param db - Database client (optional, for testing)
+ * @returns Inserted module record
+ */
+export async function insertModuleToDb(
+  manifest: ModuleManifest,
+  targetPath: string,
+  db = getDb(),
+): Promise<void> {
+  const newModule: NewModule = {
+    id: manifest.id,
+    name: manifest.name,
+    version: manifest.version,
+    description: manifest.description,
+    sourcePath: targetPath,
+    manifestData: manifest as Record<string, unknown>,
+    state: 'IMPORTED',
+  };
+
+  db.insert(modules).values(newModule).run();
+}
+
+/**
+ * Check if module already exists in database
+ *
+ * @param moduleId - Module ID to check
+ * @param db - Database client (optional, for testing)
+ * @returns True if module exists
+ */
+export function moduleExists(moduleId: string, db = getDb()): boolean {
+  const existing = db.select().from(modules).where(eq(modules.id, moduleId)).get();
+  return existing !== undefined;
+}
+
+/**
+ * Validate well-known capabilities
+ *
+ * Policy function - checks if module's well-known capabilities are valid:
+ * 1. No other module provides the same well-known capability (uniqueness)
+ * 2. Module's zone matches capability's required zone (zone enforcement)
+ *
+ * @param manifest - Module manifest
+ * @param db - Database client
+ * @returns Error message if invalid, null if valid
+ */
+export async function validateWellKnownCapabilities(
+  manifest: ModuleManifest,
+  db = getDb(),
+): Promise<string | null> {
+  const providedCapabilities = manifest.provides?.capabilities ?? [];
+
+  for (const capability of providedCapabilities) {
+    // Skip non-well-known capabilities
+    if (!isWellKnown(capability.name)) {
+      continue;
+    }
+
+    const wellKnown = getWellKnownCapability(capability.name);
+
+    // Check 1: Capability uniqueness - only one module can provide this capability
+    const existingCapability = await db
+      .select()
+      .from(capabilities)
+      .where(eq(capabilities.capabilityName, capability.name))
+      .all();
+
+    if (existingCapability.length > 0) {
+      const conflictingModule = existingCapability[0];
+      return `Well-known capability '${capability.name}' is already provided by module '${conflictingModule.moduleId}'. A home lab can only have one module providing this capability. Remove '${conflictingModule.moduleId}' before importing this module.`;
+    }
+
+    // Check 2: Zone enforcement - module must be in the correct zone
+    const moduleZone = manifest.requires?.machine?.zone;
+
+    if (wellKnown.zone_enforced && moduleZone) {
+      if (moduleZone !== wellKnown.required_zone) {
+        return `Capability '${capability.name}' requires zone='${wellKnown.required_zone}' (security requirement). Module manifest specifies zone='${moduleZone}'. Update the module manifest to use the correct zone.`;
+      }
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Import a module from a directory or .netapp package
+ *
+ * Orchestration function (Rule 10.1) - coordinates policy, planning, and execution
+ * This is the main entry point for module import
+ *
+ * @param options - Import options
+ * @returns Import result
+ */
+/**
+ * The minimum package.json auto-generated for modules that have hook
+ * scripts but no package.json. Contains only the framework dep.
+ */
+const DEFAULT_SCRIPTS_PACKAGE_JSON = JSON.stringify(
+  {
+    private: true,
+    dependencies: {
+      '@celilo/capabilities': '^0.1.0',
+    },
+  },
+  null,
+  2,
+);
+
+/**
+ * Install npm dependencies for a module's hook scripts.
+ *
+ * Per NPM_PACKAGE_RESOLUTION design doc (Option B): each module's
+ * scripts/ directory has its own package.json + node_modules for
+ * fully isolated dependency resolution. This function:
+ *
+ * 1. Finds the scripts/ directory (derived from manifest hook paths).
+ * 2. If package.json exists and node_modules/ is missing, runs
+ *    `bun install`.
+ * 3. If no package.json exists but hook scripts do, auto-generates a
+ *    default one with `@celilo/capabilities` and then installs.
+ * 4. If no hooks are declared, does nothing.
+ */
+async function installScriptDependencies(
+  targetPath: string,
+  manifest: ModuleManifest,
+): Promise<void> {
+  // Determine where scripts live by looking at hook declarations.
+  // All hooks use paths like `./scripts/setup-admin.ts`, so the
+  // scripts directory is the common parent.
+  if (!manifest.hooks || Object.keys(manifest.hooks).length === 0) {
+    return; // No hooks → no scripts → nothing to install
+  }
+
+  const scriptsDir = join(targetPath, 'scripts');
+  if (!existsSync(scriptsDir)) {
+    return; // No scripts directory on disk
+  }
+
+  const pkgJsonPath = join(scriptsDir, 'package.json');
+  const nodeModulesPath = join(scriptsDir, 'node_modules');
+
+  // Auto-generate package.json if missing but hooks exist
+  if (!existsSync(pkgJsonPath)) {
+    log.info('Auto-generating scripts/package.json with @celilo/capabilities');
+    writeFileSync(pkgJsonPath, DEFAULT_SCRIPTS_PACKAGE_JSON, 'utf-8');
+  }
+
+  // Skip install if node_modules already exists (pre-bundled via
+  // future --bundle-deps, or a re-import of an already-installed module)
+  if (existsSync(nodeModulesPath)) {
+    return;
+  }
+
+  // Run bun install in the scripts directory
+  log.info('Installing script dependencies...');
+  execSync('bun install', {
+    cwd: scriptsDir,
+    timeout: 120_000,
+    stdio: 'pipe',
+  });
+}
+
+export async function importModule(options: ModuleImportOptions): Promise<ModuleImportResult> {
+  const { sourcePath, targetBasePath = getDefaultTargetBase(), db = getDb(), flags = {} } = options;
+
+  let actualSourcePath = sourcePath;
+  let tempDir: string | null = null;
+  let checksums: Record<string, string> | null = null;
+  let signature: string | null = null;
+
+  try {
+    // Check if source is a .netapp package
+    if (sourcePath.endsWith('.netapp')) {
+      // Extract and verify package
+      const extractResult = await extractPackage(sourcePath);
+      if (!extractResult.success || !extractResult.tempDir) {
+        return { success: false, error: extractResult.error || 'Failed to extract package' };
+      }
+
+      tempDir = extractResult.tempDir;
+
+      // Verify package integrity (unless --skip-verify)
+      const skipVerify = flags['skip-verify'] === true;
+      if (skipVerify) {
+        log.warn('Skipping package signature verification (--skip-verify)');
+      } else {
+        const verifyResult = await verifyPackageIntegrity(tempDir);
+        if (!verifyResult.success) {
+          await cleanupTempDir(tempDir);
+          return {
+            success: false,
+            error: verifyResult.error || 'Package integrity verification failed',
+          };
+        }
+      }
+
+      // Read checksums and signature for database storage
+      const checksumsJson = await readFile(join(tempDir, 'checksums.json'), 'utf-8');
+      const ChecksumsFileSchema = z.object({
+        files: z.record(z.string(), z.string()),
+      });
+      const checksumsData = parseJsonWithValidation(
+        checksumsJson,
+        ChecksumsFileSchema,
+        'package checksums.json',
+      );
+      checksums = checksumsData.files;
+      signature = await readFile(join(tempDir, 'signature.sig'), 'utf-8');
+
+      // Use extracted directory as source
+      actualSourcePath = tempDir;
+    }
+
+    // Policy: Validate directory structure
+    const dirError = validateModuleDirectory(actualSourcePath);
+    if (dirError) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return { success: false, error: dirError };
+    }
+
+    // Policy: Read and validate manifest
+    const manifestResult = await readModuleManifest(actualSourcePath);
+    if (!manifestResult.success) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return { success: false, error: manifestResult.error };
+    }
+
+    const manifest = manifestResult.manifest;
+
+    // Policy: Check if module already exists
+    if (moduleExists(manifest.id, db)) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: `Module '${manifest.id}' already exists. Use update or remove it first.`,
+      };
+    }
+
+    // Policy: Validate well-known capabilities (import-time check)
+    const capabilityError = await validateWellKnownCapabilities(manifest, db);
+    if (capabilityError) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: capabilityError,
+      };
+    }
+
+    // Policy: Validate template variable references (import-time check)
+    const { validateModuleTemplates, formatTemplateValidationErrors } = await import(
+      '../manifest/template-validator'
+    );
+    const templateValidation = await validateModuleTemplates(actualSourcePath, manifest);
+    if (!templateValidation.success) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: formatTemplateValidationErrors(templateValidation.errors),
+      };
+    }
+
+    // Policy: Check for Ansible dependency conflicts and install collections
+    const { installCollectionsForModule } = await import('../ansible/dependencies');
+    const installResult = await installCollectionsForModule(manifest, db);
+
+    if (!installResult.success) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: installResult.error || 'Failed to install Ansible collections',
+        details: installResult.details,
+      };
+    }
+
+    // Execution: Validate capability access if module requires capabilities
+    if (manifest.requires?.capabilities && manifest.requires.capabilities.length > 0) {
+      const { validateCapabilityAccess } = await import('../capabilities/validation');
+      const accessResult = await validateCapabilityAccess(manifest, db.$client);
+
+      if (!accessResult.success) {
+        if (tempDir) await cleanupTempDir(tempDir);
+        return {
+          success: false,
+          error: accessResult.error || 'Capability access denied',
+          details: accessResult.details,
+        };
+      }
+    }
+
+    // Planning: Determine target path
+    const targetPath = join(targetBasePath, manifest.id);
+
+    // Execution: Copy files
+    try {
+      await copyModuleFiles(actualSourcePath, targetPath);
+    } catch (error) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: 'Failed to copy module files',
+        details: error,
+      };
+    }
+
+    // Execution: Install hook script dependencies (NPM_PACKAGE_RESOLUTION Option B).
+    // If the module's scripts/ directory has a package.json, run `bun install`
+    // so the hook scripts can resolve their npm imports (e.g. @celilo/capabilities).
+    // If no package.json exists but hook scripts do, auto-generate one with
+    // just the framework dep — smooths migration for existing modules.
+    try {
+      await installScriptDependencies(targetPath, manifest);
+    } catch (error) {
+      // Non-fatal: the module is importable without deps, but hooks
+      // will fail at runtime. Warn and continue.
+      const msg = error instanceof Error ? error.message : String(error);
+      log.warn(`Failed to install script dependencies: ${msg}`);
+      log.warn('Hook scripts may fail to resolve imports until this is fixed.');
+    }
+
+    // Execution: Insert to database
+    try {
+      await insertModuleToDb(manifest, targetPath, db);
+    } catch (error) {
+      if (tempDir) await cleanupTempDir(tempDir);
+      return {
+        success: false,
+        error: 'Failed to insert module to database',
+        details: error,
+      };
+    }
+
+    // Execution: Register capabilities if module provides them
+    if (manifest.provides?.capabilities && manifest.provides.capabilities.length > 0) {
+      const { registerModuleCapabilities } = await import('../capabilities/registration');
+      const capResult = await registerModuleCapabilities(manifest.id, manifest, db.$client, flags);
+
+      if (!capResult.success) {
+        if (tempDir) await cleanupTempDir(tempDir);
+        return {
+          success: false,
+          error: capResult.error || 'Failed to register module capabilities',
+          details: capResult.details,
+        };
+      }
+    }
+
+    // Execution: Store integrity data
+    // For .netapp packages: store checksums + signature
+    // For directory imports: calculate and store checksums (no signature)
+    try {
+      let finalChecksums: Record<string, string>;
+      let finalSignature: string | null = null;
+
+      if (checksums && signature) {
+        // From .netapp package - already verified
+        finalChecksums = checksums;
+        finalSignature = signature.trim();
+      } else {
+        // From directory - calculate checksums now
+        const { computeChecksums } = await import('./packaging/build');
+        const checksumsData = await computeChecksums(targetPath);
+        finalChecksums = checksumsData.files;
+        // No signature for directory imports
+      }
+
+      const integrityData: NewModuleIntegrity = {
+        moduleId: manifest.id,
+        checksums: finalChecksums,
+        signature: finalSignature,
+      };
+      db.insert(moduleIntegrity).values(integrityData).run();
+    } catch (error) {
+      // Non-fatal - module is already imported
+      console.warn('Warning: Failed to store module integrity data', error);
+    }
+
+    // For .netapp packages with build artifacts: record a successful build
+    // so that `module generate` doesn't require a rebuild
+    if (sourcePath.endsWith('.netapp') && manifest.build?.artifacts) {
+      const { existsSync: fileExists } = await import('node:fs');
+      const artifactPaths = manifest.build.artifacts
+        .map((a: string) => join(targetPath, a))
+        .filter((p: string) => fileExists(p));
+
+      if (artifactPaths.length > 0) {
+        const { moduleBuilds } = await import('../db/schema');
+        db.insert(moduleBuilds)
+          .values({
+            moduleId: manifest.id,
+            version: manifest.version,
+            artifacts: artifactPaths,
+            status: 'success',
+            buildLog: 'Pre-built artifacts from .netapp package',
+          })
+          .run();
+      }
+    }
+
+    // Update dependency cache
+    try {
+      const { updateDependencyCache } = await import('../ansible/dependencies');
+      await updateDependencyCache(db);
+    } catch (error) {
+      // Non-fatal - cache is optional
+      console.warn('Warning: Failed to update dependency cache', error);
+    }
+
+    // Cleanup temp directory if used
+    if (tempDir) {
+      await cleanupTempDir(tempDir);
+    }
+
+    return {
+      success: true,
+      moduleId: manifest.id,
+      targetPath,
+    };
+  } catch (error) {
+    // Cleanup on unexpected error
+    if (tempDir) {
+      await cleanupTempDir(tempDir);
+    }
+    return {
+      success: false,
+      error: 'Unexpected error during import',
+      details: error,
+    };
+  }
+}