@underpostnet/underpost 2.97.0 → 2.97.1

package/src/cli/baremetal.js

@@ -82,9 +82,12 @@ class UnderpostBaremetal {
      * @param {string} [options.isoUrl=''] - Uses a custom ISO URL for baremetal machine commissioning.
      * @param {boolean} [options.ubuntuToolsBuild=false] - Builds ubuntu tools for chroot environment.
      * @param {boolean} [options.ubuntuToolsTest=false] - Tests ubuntu tools in chroot environment.
+     * @param {boolean} [options.rockyToolsBuild=false] - Builds rocky linux tools for chroot environment.
+     * @param {boolean} [options.rockyToolsTest=false] - Tests rocky linux tools in chroot environment.
      * @param {string} [options.bootcmd=''] - Comma-separated list of boot commands to execute.
      * @param {string} [options.runcmd=''] - Comma-separated list of run commands to execute.
      * @param {boolean} [options.nfsBuild=false] - Flag to build the NFS root filesystem.
+     * @param {boolean} [options.nfsBuildServer=false] - Flag to build the NFS server components.
      * @param {boolean} [options.nfsMount=false] - Flag to mount the NFS root filesystem.
      * @param {boolean} [options.nfsUnmount=false] - Flag to unmount the NFS root filesystem.
      * @param {boolean} [options.nfsSh=false] - Flag to chroot into the NFS environment for shell access.
@@ -127,9 +130,12 @@ class UnderpostBaremetal {
         isoUrl: '',
         ubuntuToolsBuild: false,
         ubuntuToolsTest: false,
+        rockyToolsBuild: false,
+        rockyToolsTest: false,
         bootcmd: '',
         runcmd: '',
         nfsBuild: false,
+        nfsBuildServer: false,
         nfsMount: false,
         nfsUnmount: false,
         nfsSh: false,
@@ -168,13 +174,16 @@ class UnderpostBaremetal {
       }
 
       const tftpPrefix = workflowsConfig[workflowId].tftpPrefix || 'rpi4mb';
-      // Define the debootstrap architecture.
-      let debootstrapArch;
+      // Define the bootstrap architecture.
+      let bootstrapArch;
 
-      // Set debootstrap architecture.
-      if (workflowsConfig[workflowId].type === 'chroot') {
+      // Set bootstrap architecture.
+      if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
         const { architecture } = workflowsConfig[workflowId].debootstrap.image;
-        debootstrapArch = architecture;
+        bootstrapArch = architecture;
+      } else if (workflowsConfig[workflowId].type === 'chroot-container') {
+        const { architecture } = workflowsConfig[workflowId].container;
+        bootstrapArch = architecture;
       }
 
       // Define the database provider ID.
@@ -186,6 +195,9 @@ class UnderpostBaremetal {
       // Define the TFTP root prefix path based
       const tftpRootPath = `${process.env.TFTP_ROOT}/${tftpPrefix}`;
 
+      // Define the iPXE cache directory to preserve builds across tftproot cleanups
+      const ipxeCacheDir = `/tmp/ipxe-cache/${tftpPrefix}`;
+
       // Define the bootstrap HTTP server path.
       const bootstrapHttpServerPath = options.bootstrapHttpServerPath
         ? options.bootstrapHttpServerPath
@@ -479,14 +491,9 @@ rm -rf ${artifacts.join(' ')}`);
 
       // Handle NFS shell access option.
       if (options.nfsSh === true) {
-        const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
-        if (!workflowsConfig[workflowId]) {
-          throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
-        }
-        const { debootstrap } = workflowsConfig[workflowId];
         // Copy the chroot command to the clipboard for easy execution.
-        if (debootstrap.image.architecture !== callbackMetaData.runnerHost.architecture)
-          switch (debootstrap.image.architecture) {
+        if (bootstrapArch && bootstrapArch !== callbackMetaData.runnerHost.architecture)
+          switch (bootstrapArch) {
             case 'arm64':
               pbcopy(`sudo chroot ${nfsHostPath} /usr/bin/qemu-aarch64-static /bin/bash`);
               break;
@@ -515,9 +522,9 @@ rm -rf ${artifacts.join(' ')}`);
       // Handle control server uninstallation.
       if (options.controlServerUninstall === true) {
         // Stop and remove MAAS services, handling potential errors gracefully.
-        shellExec(`sudo snap stop maas.pebble || true`);
+        shellExec(`sudo snap stop maas.pebble`);
         shellExec(`sudo snap stop maas`);
-        shellExec(`sudo snap remove maas --purge || true`);
+        shellExec(`sudo snap remove maas --purge`);
 
         // Remove residual snap data to ensure a clean uninstall.
         shellExec(`sudo rm -rf /var/snap/maas`);
@@ -555,7 +562,7 @@ rm -rf ${artifacts.join(' ')}`);
 
       // Handle NFS mount operation.
       if (options.nfsMount === true) {
-        const { isMounted } = UnderpostBaremetal.API.nfsMountCallback({
+        await UnderpostBaremetal.API.nfsMountCallback({
           hostname,
           nfsHostPath,
           workflowId,
@@ -566,7 +573,7 @@ rm -rf ${artifacts.join(' ')}`);
 
       // Handle NFS unmount operation.
       if (options.nfsUnmount === true) {
-        const { isMounted } = UnderpostBaremetal.API.nfsMountCallback({
+        await UnderpostBaremetal.API.nfsMountCallback({
           hostname,
           nfsHostPath,
           workflowId,
@@ -577,21 +584,19 @@ rm -rf ${artifacts.join(' ')}`);
 
       // Handle NFS root filesystem build operation.
       if (options.nfsBuild === true) {
-        const { isMounted } = UnderpostBaremetal.API.nfsMountCallback({
+        await UnderpostBaremetal.API.nfsMountCallback({
           hostname,
           nfsHostPath,
           workflowId,
           unmount: true,
         });
 
-        if (isMounted) throw new Error(`NFS path ${nfsHostPath} is currently mounted. Please unmount before building.`);
-
         // Clean and create the NFS host path.
         shellExec(`sudo rm -rf ${nfsHostPath}/*`);
         shellExec(`mkdir -p ${nfsHostPath}`);
 
         // Perform the first stage of debootstrap.
-        {
+        if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
           const { architecture, name } = workflowsConfig[workflowId].debootstrap.image;
           shellExec(
             [
@@ -604,6 +609,12 @@ rm -rf ${artifacts.join(' ')}`);
               `http://ports.ubuntu.com/ubuntu-ports/`,
             ].join(' '),
           );
+        } else if (workflowsConfig[workflowId].type === 'chroot-container') {
+          const { image } = workflowsConfig[workflowId].container;
+          shellExec(`sudo podman pull --arch=${bootstrapArch} ${image}`);
+          shellExec(`sudo podman create --arch=${bootstrapArch} --name chroot-source ${image}`);
+          shellExec(`sudo podman export chroot-source | sudo tar -x -C ${nfsHostPath}`);
+          shellExec(`sudo podman rm chroot-source`);
         }
 
         // Create a podman container to extract QEMU static binaries.
@@ -611,10 +622,10 @@ rm -rf ${artifacts.join(' ')}`);
         shellExec(`podman ps -a`); // List all podman containers for verification.
 
         // If cross-architecture, copy the QEMU static binary into the chroot.
-        if (debootstrapArch !== callbackMetaData.runnerHost.architecture)
+        if (bootstrapArch !== callbackMetaData.runnerHost.architecture)
           UnderpostBaremetal.API.crossArchBinFactory({
             nfsHostPath,
-            debootstrapArch,
+            bootstrapArch,
           });
 
         // Clean up the temporary podman container.
@@ -623,7 +634,7 @@ rm -rf ${artifacts.join(' ')}`);
         shellExec(`file ${nfsHostPath}/bin/bash`); // Verify the bash executable in the chroot.
 
         // Mount necessary filesystems and register binfmt for the second stage.
-        UnderpostBaremetal.API.nfsMountCallback({
+        await UnderpostBaremetal.API.nfsMountCallback({
           hostname,
           nfsHostPath,
           workflowId,
@@ -631,13 +642,96 @@ rm -rf ${artifacts.join(' ')}`);
         });
 
         // Perform the second stage of debootstrap within the chroot environment.
-        UnderpostBaremetal.API.crossArchRunner({
-          nfsHostPath,
-          debootstrapArch,
-          callbackMetaData,
-          steps: [`/debootstrap/debootstrap --second-stage`],
-        });
-        return;
+        if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [`/debootstrap/debootstrap --second-stage`],
+          });
+        } else if (
+          workflowsConfig[workflowId].type === 'chroot-container' &&
+          workflowsConfig[workflowId].osIdLike.match('rhel')
+        ) {
+          // Copy resolv.conf to allow network access inside chroot
+          shellExec(`sudo cp /etc/resolv.conf ${nfsHostPath}/etc/resolv.conf`);
+
+          // Consolidate all package installations into one step to avoid redundancy
+          const { packages } = workflowsConfig[workflowId].container;
+          const basePackages = [
+            'findutils',
+            'systemd',
+            'sudo',
+            'dracut',
+            'dracut-network',
+            'dracut-config-generic',
+            'nfs-utils',
+            'file',
+            'binutils',
+            'kernel-modules-core',
+            'NetworkManager',
+            'dhclient',
+            'iputils',
+          ];
+          const allPackages = packages && packages.length > 0 ? [...basePackages, ...packages] : basePackages;
+
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [
+              `dnf install -y --allowerasing ${allPackages.join(' ')} 2>/dev/null || yum install -y --allowerasing ${allPackages.join(' ')} 2>/dev/null || echo "Package install completed"`,
+              `dnf clean all`,
+              `echo "=== Installed packages verification ==="`,
+              `rpm -qa | grep -E "dracut|kernel|nfs" | sort`,
+              `echo "=== Boot directory contents ==="`,
+              `ls -la /boot /lib/modules/*/`,
+              // Search for bootable kernel in order of preference:
+              // 1. Raw ARM64 Image file (preferred for GRUB)
+              // 2. vmlinuz or vmlinux (may be PE32+ on Rocky Linux)
+              `echo "Searching for bootable kernel..."`,
+              `KERNEL_FILE=""`,
+              // First try to find raw Image file
+              `if [ -f /boot/Image ]; then KERNEL_FILE=/boot/Image; echo "Found raw ARM64 Image: $KERNEL_FILE"; fi`,
+              `if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "Image" -o -name "Image.gz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found kernel Image in modules: $KERNEL_FILE"; fi`,
+              // Fallback to vmlinuz
+              `if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /boot -name "vmlinuz-*" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz: $KERNEL_FILE"; fi`,
+              `if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinuz" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinuz in modules: $KERNEL_FILE"; fi`,
+              // Last resort: any vmlinux
+              `if [ -z "$KERNEL_FILE" ]; then KERNEL_FILE=$(find /lib/modules -name "vmlinux" 2>/dev/null | head -n 1); test -n "$KERNEL_FILE" && echo "Found vmlinux: $KERNEL_FILE"; fi`,
+              `if [ -z "$KERNEL_FILE" ]; then echo "ERROR: No kernel found!"; exit 1; fi`,
+              // Copy and check kernel type
+              `cp "$KERNEL_FILE" /boot/vmlinuz-efi.tmp`,
+              // Decompress if gzipped
+              `if file /boot/vmlinuz-efi.tmp | grep -q gzip; then echo "Decompressing gzipped kernel..."; gunzip -c /boot/vmlinuz-efi.tmp > /boot/vmlinuz-efi && rm /boot/vmlinuz-efi.tmp; else mv /boot/vmlinuz-efi.tmp /boot/vmlinuz-efi; fi`,
+              `KERNEL_TYPE=$(file /boot/vmlinuz-efi 2>/dev/null)`,
+              `echo "Final kernel file type: $KERNEL_TYPE"`,
+              // Handle PE32+ if still present - use kernel directly without extraction since iPXE can boot it
+              `case "$KERNEL_TYPE" in *PE32+*|*EFI*application*) echo "WARNING: Kernel is PE32+ EFI executable"; echo "GRUB may fail to boot this - recommend using iPXE chainload or installing kernel-core package"; echo "Keeping PE32+ kernel as-is for now..."; ;; *ARM64*|*aarch64*|*Image*|*data*) echo "Kernel appears to be raw ARM64 format - suitable for GRUB"; ;; *) echo "Unknown kernel format - attempting to use anyway"; ;; esac`,
+              // Get kernel version for initramfs rebuild
+              `KVER=$(basename $(dirname "$KERNEL_FILE"))`,
+              `echo "Kernel version: $KVER"`,
+              // Rebuild initramfs with NFS and network support
+              `echo "Rebuilding initramfs with NFS and network support..."`,
+              `echo "Available dracut modules:"`,
+              `dracut --list-modules 2>/dev/null | grep -E "network|nfs" || echo "No network modules listed"`,
+              // Use network-manager module (it's available in Rocky 9) for better compatibility
+              `dracut --force --add "nfs network base" --add-drivers "nfs sunrpc" --kver "$KVER" /boot/initrd.img "$KVER" 2>&1 || echo "Initramfs rebuild failed"`,
+              // Fallback: if rebuild fails, use existing initramfs
+              `if [ ! -f /boot/initrd.img ]; then echo "Initramfs rebuild failed, using existing..."; INITRD=$(find /boot -name "initramfs-$KVER.img" 2>/dev/null | head -n 1); if [ -z "$INITRD" ]; then INITRD=$(find /boot -name "initramfs*.img" 2>/dev/null | grep -v kdump | head -n 1); fi; if [ -n "$INITRD" ]; then cp "$INITRD" /boot/initrd.img; echo "Copied existing initramfs: $INITRD"; else echo "ERROR: No initramfs found!"; fi; fi`,
+              `echo "=== Final boot files ==="`,
+              `ls -lh /boot/vmlinuz-efi /boot/initrd.img`,
+              `file /boot/vmlinuz-efi`,
+              `file /boot/initrd.img`,
+              `echo "=== Setting root password ==="`,
+              `echo "root:root" | chpasswd`,
+            ],
+          });
+        } else {
+          throw new Error(
+            `Unsupported workflow type for NFS build: ${workflowsConfig[workflowId].type} and like os ID ${workflowsConfig[workflowId].osIdLike}`,
+          );
+        }
       }
 
       // Fetch boot resources and machines if commissioning or listing.
@@ -679,13 +773,158 @@ rm -rf ${artifacts.join(' ')}`);
           ignore: machine ? [machine.system_id] : [],
         });
 
-      // Handle commissioning tasks (placeholder for future implementation).
+      if (workflowsConfig[workflowId].type === 'chroot-debootstrap') {
+        if (options.ubuntuToolsBuild) {
+          UnderpostCloudInit.API.buildTools({
+            workflowId,
+            nfsHostPath,
+            hostname,
+            callbackMetaData,
+            dev: options.dev,
+          });
+
+          const { chronyc, keyboard } = workflowsConfig[workflowId];
+          const { timezone, chronyConfPath } = chronyc;
+          const systemProvisioning = 'ubuntu';
 
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].base(),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].user(),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].timezone({
+                timezone,
+                chronyConfPath,
+              }),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].keyboard(keyboard.layout),
+            ],
+          });
+        }
+
+        if (options.ubuntuToolsTest)
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [
+              `chmod +x /underpost/date.sh`,
+              `chmod +x /underpost/keyboard.sh`,
+              `chmod +x /underpost/dns.sh`,
+              `chmod +x /underpost/help.sh`,
+              `chmod +x /underpost/host.sh`,
+              `chmod +x /underpost/test.sh`,
+              `chmod +x /underpost/start.sh`,
+              `chmod +x /underpost/reset.sh`,
+              `chmod +x /underpost/shutdown.sh`,
+              `chmod +x /underpost/device_scan.sh`,
+              `chmod +x /underpost/mac.sh`,
+              `chmod +x /underpost/enlistment.sh`,
+              `sudo chmod 700 ~/.ssh/`, // Set secure permissions for .ssh directory.
+              `sudo chmod 600 ~/.ssh/authorized_keys`, // Set secure permissions for authorized_keys.
+              `sudo chmod 644 ~/.ssh/known_hosts`, // Set permissions for known_hosts.
+              `sudo chmod 600 ~/.ssh/id_rsa`, // Set secure permissions for private key.
+              `sudo chmod 600 /etc/ssh/ssh_host_ed25519_key`, // Set secure permissions for host key.
+              `chown -R root:root ~/.ssh`, // Ensure root owns the .ssh directory.
+              `/underpost/test.sh`,
+            ],
+          });
+      }
+
+      if (
+        workflowsConfig[workflowId].type === 'chroot-container' &&
+        workflowsConfig[workflowId].osIdLike.match('rhel')
+      ) {
+        if (options.rockyToolsBuild) {
+          const { chronyc, keyboard } = workflowsConfig[workflowId];
+          const { timezone } = chronyc;
+          const systemProvisioning = 'rocky';
+
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].base(),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].user(),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].timezone({
+                timezone,
+                chronyConfPath: chronyc.chronyConfPath,
+              }),
+              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].keyboard(keyboard.layout),
+            ],
+          });
+        }
+
+        if (options.rockyToolsTest)
+          UnderpostBaremetal.API.crossArchRunner({
+            nfsHostPath,
+            bootstrapArch,
+            callbackMetaData,
+            steps: [
+              `node --version`,
+              `npm --version`,
+              `underpost --version`,
+              `timedatectl status`,
+              `localectl status`,
+              `id root`,
+              `ls -la /home/root/.ssh/`,
+              `cat /home/root/.ssh/authorized_keys`,
+              'underpost test',
+            ],
+          });
+      }
+
+      if (options.cloudInit || options.cloudInitUpdate) {
+        const { chronyc, networkInterfaceName } = workflowsConfig[workflowId];
+        const { timezone, chronyConfPath } = chronyc;
+        const authCredentials = UnderpostCloudInit.API.authCredentialsFactory();
+        const { cloudConfigSrc } = UnderpostCloudInit.API.configFactory(
+          {
+            controlServerIp: callbackMetaData.runnerHost.ip,
+            hostname,
+            commissioningDeviceIp: ipAddress,
+            gatewayip: callbackMetaData.runnerHost.ip,
+            mac: macAddress,
+            timezone,
+            chronyConfPath,
+            networkInterfaceName,
+            ubuntuToolsBuild: options.ubuntuToolsBuild,
+            bootcmd: options.bootcmd,
+            runcmd: options.runcmd,
+          },
+          authCredentials,
+        );
+
+        UnderpostBaremetal.API.httpBootstrapServerStaticFactory({
+          bootstrapHttpServerPath,
+          hostname,
+          cloudConfigSrc,
+        });
+      }
+
+      // Rebuild NFS server configuration.
+      if (
+        (options.nfsBuildServer === true || options.commission === true) &&
+        (workflowsConfig[workflowId].type === 'iso-nfs' ||
+          workflowsConfig[workflowId].type === 'chroot-debootstrap' ||
+          workflowsConfig[workflowId].type === 'chroot-container')
+      ) {
+        shellExec(`${underpostRoot}/scripts/nat-iptables.sh`, { silent: true });
+        UnderpostBaremetal.API.rebuildNfsServer({
+          nfsHostPath,
+        });
+      }
+      // Handle commissioning tasks
       if (options.commission === true) {
         let { firmwares, networkInterfaceName, maas, menuentryStr, type } = workflowsConfig[workflowId];
 
         // Use commissioning config (Ubuntu ephemeral) for PXE boot resources
-        const commissioningImage = maas.commissioning;
+        const commissioningImage = maas?.commissioning || {
+          architecture: 'arm64/generic',
+          name: 'ubuntu/noble',
+        };
         const resource = resources.find(
           (o) => o.architecture === commissioningImage.architecture && o.name === commissioningImage.name,
         );
@@ -701,6 +940,11 @@ rm -rf ${artifacts.join(' ')}`);
         shellExec(`sudo rm -rf ${tftpRootPath}`);
         shellExec(`mkdir -p ${tftpRootPath}/pxe`);
 
+        // Restore iPXE build from cache if available and not forcing rebuild
+        if (fs.existsSync(`${ipxeCacheDir}/ipxe.efi`) && !options.ipxeRebuild) {
+          shellExec(`cp ${ipxeCacheDir}/ipxe.efi ${tftpRootPath}/ipxe.efi`);
+        }
+
         // Process firmwares for TFTP.
         for (const firmware of firmwares) {
           const { url, gateway, subnet } = firmware;
@@ -717,7 +961,7 @@ rm -rf ${artifacts.join(' ')}`);
               const bootConfSrc = UnderpostBaremetal.API.bootConfFactory({
                 workflowId,
                 tftpIp: callbackMetaData.runnerHost.ip,
-                tftpPrefixStr: hostname,
+                tftpPrefixStr: tftpPrefix,
                 macAddress,
                 clientIp: ipAddress,
                 subnet,
@@ -732,12 +976,25 @@ rm -rf ${artifacts.join(' ')}`);
         {
           // Fetch kernel and initrd paths from MAAS boot resource.
           // Both NFS and disk-based commissioning use MAAS boot resources.
-          const { kernelFilesPaths, resourcesPath } = UnderpostBaremetal.API.kernelFactory({
-            resource,
-            type,
-            nfsHostPath,
-            isoUrl: options.isoUrl || workflowsConfig[workflowId].isoUrl,
-          });
+          let kernelFilesPaths, resourcesPath;
+          if (workflowsConfig[workflowId].type === 'chroot-container') {
+            const arch = commissioningImage.architecture.split('/')[0];
+            resourcesPath = `/var/snap/maas/common/maas/image-storage/bootloaders/uefi/${arch}`;
+            kernelFilesPaths = {
+              'vmlinuz-efi': `${nfsHostPath}/boot/vmlinuz-efi`,
+              'initrd.img': `${nfsHostPath}/boot/initrd.img`,
+            };
+          } else {
+            const kf = UnderpostBaremetal.API.kernelFactory({
+              resource,
+              type,
+              nfsHostPath,
+              isoUrl: options.isoUrl || workflowsConfig[workflowId].isoUrl,
+              workflowId,
+            });
+            kernelFilesPaths = kf.kernelFilesPaths;
+            resourcesPath = kf.resourcesPath;
+          }
 
           const { cmd } = UnderpostBaremetal.API.kernelCmdBootParamsFactory({
             ipClient: ipAddress,
@@ -755,6 +1012,8 @@ rm -rf ${artifacts.join(' ')}`);
             macAddress,
             cloudInit: options.cloudInit,
             machine,
+            dev: options.dev,
+            osIdLike: workflowsConfig[workflowId].osIdLike || '',
           });
 
           // Check if iPXE mode is enabled AND the iPXE EFI binary exists
@@ -783,47 +1042,15 @@ rm -rf ${artifacts.join(' ')}`);
               path: `${tftpRootPath}/stable-id.ipxe`,
               embeddedPath: `${tftpRootPath}/boot.ipxe`,
             });
-            if (macAddress === null) {
-              logger.info('ℹ Hardware MAC mode - device will use actual hardware MAC address');
-              logger.info('ℹ MAAS will identify the machine by its hardware MAC after discovery');
-            } else {
-              logger.info('ℹ Machine registered in MAAS with MAC:', macAddress);
-              if (macAddress !== '00:00:00:00:00:00') {
-                logger.info('ℹ MAAS will identify the machine by MAC:', macAddress);
-              } else {
-                logger.info('ℹ Device will boot with its actual hardware MAC address');
-                logger.info('ℹ MAAS will identify the machine by its hardware MAC');
-              }
-            }
-
-            // Rebuild iPXE with embedded boot script if requested or if binary doesn't exist
-            const embeddedScriptPath = `${tftpRootPath}/boot.ipxe`;
-            const shouldRebuild = options.ipxeRebuild || !fs.existsSync(`${tftpRootPath}/ipxe.efi`);
-
-            if (shouldRebuild && fs.existsSync(embeddedScriptPath)) {
-              logger.info('Rebuilding iPXE with embedded boot script...', {
-                embeddedScriptPath,
-                forced: options.ipxeRebuild,
-              });
-              shellExec(
-                `${underpostRoot}/scripts/ipxe-setup.sh ${tftpRootPath} --target-arch ${arch} --embed-script ${embeddedScriptPath} --rebuild`,
-              );
-            } else if (shouldRebuild) {
-              logger.warn('⚠ Embedded script not found, building without embedded script');
-              shellExec(`${underpostRoot}/scripts/ipxe-setup.sh ${tftpRootPath} --target-arch ${arch}`);
-            } else {
-              logger.info('ℹ Using existing iPXE binary (use --ipxe-rebuild to force rebuild)');
-            }
 
-            // Only use iPXE chainloading if the binary exists
-            if (fs.existsSync(`${tftpRootPath}/ipxe.efi`)) {
-              logger.info('✓ iPXE EFI binary found');
-            } else {
-              useIpxe = false;
-              logger.warn(`⚠ iPXE EFI binary not found at ${tftpRootPath}/ipxe.efi - falling back to direct GRUB boot`);
-              logger.warn('  The iPXE script was generated but cannot be used without the iPXE EFI binary.');
-              logger.warn('  Consider booting without --ipxe flag or providing the iPXE EFI binary.');
-            }
+            UnderpostBaremetal.API.ipxeEfiFactory({
+              tftpRootPath,
+              ipxeCacheDir,
+              arch,
+              underpostRoot,
+              embeddedScriptPath: `${tftpRootPath}/boot.ipxe`,
+              forceRebuild: options.ipxeRebuild,
+            });
           }
 
           const { grubCfgSrc } = UnderpostBaremetal.API.grubFactory({
@@ -850,7 +1077,7 @@ rm -rf ${artifacts.join(' ')}`);
         }
 
         // Pass architecture from commissioning or deployment config
-        const grubArch = maas.commissioning.architecture;
+        const grubArch = commissioningImage.architecture;
         UnderpostBaremetal.API.efiGrubModulesFactory({ image: { architecture: grubArch } });
 
         // Set ownership and permissions for TFTP root.
@@ -863,130 +1090,24 @@ rm -rf ${artifacts.join(' ')}`);
           bootstrapHttpServerPort:
             options.bootstrapHttpServerPort || workflowsConfig[workflowId].bootstrapHttpServerPort,
         });
-      }
-
-      if (options.cloudInit || options.cloudInitUpdate) {
-        const { chronyc, networkInterfaceName } = workflowsConfig[workflowId];
-        const { timezone, chronyConfPath } = chronyc;
-        const authCredentials = UnderpostCloudInit.API.authCredentialsFactory();
-        const { cloudConfigSrc } = UnderpostCloudInit.API.configFactory(
-          {
-            controlServerIp: callbackMetaData.runnerHost.ip,
-            hostname,
-            commissioningDeviceIp: ipAddress,
-            gatewayip: callbackMetaData.runnerHost.ip,
-            mac: macAddress,
-            timezone,
-            chronyConfPath,
-            networkInterfaceName,
-            ubuntuToolsBuild: options.ubuntuToolsBuild,
-            bootcmd: options.bootcmd,
-            runcmd: options.runcmd,
-          },
-          authCredentials,
-        );
-
-        shellExec(`mkdir -p ${bootstrapHttpServerPath}`);
-        fs.writeFileSync(
-          `${bootstrapHttpServerPath}/${hostname}/cloud-init/user-data`,
-          `#cloud-config\n${cloudConfigSrc}`,
-          'utf8',
-        );
-        fs.writeFileSync(
-          `${bootstrapHttpServerPath}/${hostname}/cloud-init/meta-data`,
-          `instance-id: ${hostname}\nlocal-hostname: ${hostname}`,
-          'utf8',
-        );
-        fs.writeFileSync(`${bootstrapHttpServerPath}/${hostname}/cloud-init/vendor-data`, ``, 'utf8');
-
-        logger.info(`Cloud-init files written to ${bootstrapHttpServerPath}`);
-        if (options.cloudInitUpdate) return;
-      }
-
-      if (workflowsConfig[workflowId].type === 'chroot') {
-        if (options.ubuntuToolsBuild) {
-          UnderpostCloudInit.API.buildTools({
-            workflowId,
-            nfsHostPath,
-            hostname,
-            callbackMetaData,
-            dev: options.dev,
-          });
-
-          const { chronyc, keyboard } = workflowsConfig[workflowId];
-          const { timezone, chronyConfPath } = chronyc;
-          const systemProvisioning = 'ubuntu';
-
-          UnderpostBaremetal.API.crossArchRunner({
-            nfsHostPath,
-            debootstrapArch,
-            callbackMetaData,
-            steps: [
-              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].base(),
-              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].user(),
-              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].timezone({
-                timezone,
-                chronyConfPath,
-              }),
-              ...UnderpostBaremetal.API.systemProvisioningFactory[systemProvisioning].keyboard(keyboard.layout),
-            ],
-          });
-        }
-
-        if (options.ubuntuToolsTest)
-          UnderpostBaremetal.API.crossArchRunner({
-            nfsHostPath,
-            debootstrapArch,
-            callbackMetaData,
-            steps: [
-              `chmod +x /underpost/date.sh`,
-              `chmod +x /underpost/keyboard.sh`,
-              `chmod +x /underpost/dns.sh`,
-              `chmod +x /underpost/help.sh`,
-              `chmod +x /underpost/host.sh`,
-              `chmod +x /underpost/test.sh`,
-              `chmod +x /underpost/start.sh`,
-              `chmod +x /underpost/reset.sh`,
-              `chmod +x /underpost/shutdown.sh`,
-              `chmod +x /underpost/device_scan.sh`,
-              `chmod +x /underpost/mac.sh`,
-              `chmod +x /underpost/enlistment.sh`,
-              `sudo chmod 700 ~/.ssh/`, // Set secure permissions for .ssh directory.
-              `sudo chmod 600 ~/.ssh/authorized_keys`, // Set secure permissions for authorized_keys.
-              `sudo chmod 644 ~/.ssh/known_hosts`, // Set permissions for known_hosts.
-              `sudo chmod 600 ~/.ssh/id_rsa`, // Set secure permissions for private key.
-              `sudo chmod 600 /etc/ssh/ssh_host_ed25519_key`, // Set secure permissions for host key.
-              `chown -R root:root ~/.ssh`, // Ensure root owns the .ssh directory.
-              `/underpost/test.sh`,
-            ],
-          });
-      }
-
-      shellExec(`${underpostRoot}/scripts/nat-iptables.sh`, { silent: true });
-      // Rebuild NFS server configuration.
-      if (workflowsConfig[workflowId].type === 'iso-nfs' || workflowsConfig[workflowId].type === 'chroot')
-        UnderpostBaremetal.API.rebuildNfsServer({
-          nfsHostPath,
-        });
 
-      // Final commissioning steps.
-      if (options.commission === true) {
-        const { type } = workflowsConfig[workflowId];
-
-        if (type === 'chroot') {
-          const { isMounted } = UnderpostBaremetal.API.nfsMountCallback({
+        if (type === 'chroot-debootstrap' || type === 'chroot-container')
+          await UnderpostBaremetal.API.nfsMountCallback({
             hostname,
             nfsHostPath,
             workflowId,
             mount: true,
           });
-          if (!isMounted) throw new Error('NFS root filesystem is not mounted');
-        }
+
         const commissionMonitorPayload = {
           macAddress,
           ipAddress,
           hostname,
-          maas: workflowsConfig[workflowId].maas,
+          architecture:
+            workflowsConfig[workflowId].maas?.commissioning?.architecture ||
+            workflowsConfig[workflowId].container?.architecture ||
+            workflowsConfig[workflowId].debootstrap?.image?.architecture ||
+            'arm64/generic',
           machine,
         };
         logger.info('Waiting for commissioning...', {
@@ -996,7 +1117,7 @@ rm -rf ${artifacts.join(' ')}`);
 
         const { discovery } = await UnderpostBaremetal.API.commissionMonitor(commissionMonitorPayload);
 
-        if (type === 'chroot' && options.cloudInit === true) {
+        if ((type === 'chroot-debootstrap' || type === 'chroot-container') && options.cloudInit === true) {
           openTerminal(`node ${underpostRoot}/bin baremetal ${workflowId} ${ipAddress} ${hostname} --logs cloud-init`);
           openTerminal(
             `node ${underpostRoot}/bin baremetal ${workflowId} ${ipAddress} ${hostname} --logs cloud-init-machine`,
@@ -1036,60 +1157,44 @@ rm -rf ${artifacts.join(' ')}`);
     },
 
     /**
-     * @method downloadUbuntuLiveISO
-     * @description Downloads Ubuntu live ISO and extracts casper boot files for live boot.
+     * @method downloadISO
+     * @description Downloads a generic ISO and extracts kernel boot files.
      * @param {object} params - Parameters for the method.
      * @param {object} params.resource - The MAAS boot resource object.
      * @param {string} params.architecture - The architecture (arm64 or amd64).
      * @param {string} params.nfsHostPath - The NFS host path to store the ISO and extracted files.
-     * @returns {object} An object containing paths to the extracted kernel, initrd, and squashfs.
+     * @param {string} params.isoUrl - The full URL to the ISO file to download.
+     * @param {string} params.osIdLike - OS family identifier (e.g., 'debian ubuntu' or 'rhel centos fedora').
+     * @returns {object} An object containing paths to the extracted kernel, initrd, and optionally squashfs.
      * @memberof UnderpostBaremetal
      */
-    downloadUbuntuLiveISO({ resource, architecture, nfsHostPath, isoUrl }) {
+    downloadISO({ resource, architecture, nfsHostPath, isoUrl, osIdLike }) {
       const arch = architecture || resource.architecture.split('/')[0];
-      const osName = resource.name.split('/')[1]; // e.g., "focal", "jammy", "noble"
-
-      // Map Ubuntu codenames to versions - different versions available for different architectures
-      // ARM64 ISOs are hosted on cdimage.ubuntu.com, AMD64 on releases.ubuntu.com
-      const versionMap = {
-        arm64: {
-          focal: '20.04.5', // ARM64 focal only up to 20.04.5 on cdimage
-          jammy: '22.04.5',
-          noble: '24.04.3', // ubuntu-24.04.3-live-server-arm64+largemem.iso
-          bionic: '18.04.6',
-        },
-        amd64: {
-          focal: '20.04.6',
-          jammy: '22.04.5',
-          noble: '24.04.1',
-          bionic: '18.04.6',
-        },
-      };
 
-      shellExec(`mkdir -p ${nfsHostPath}/casper`);
+      // Validate that isoUrl is provided
+      if (!isoUrl) {
+        throw new Error('isoUrl parameter is required. Please specify the full ISO URL in the workflow configuration.');
+      }
+
+      // Extract ISO filename from URL
+      const isoFilename = isoUrl.split('/').pop();
 
-      const version = (versionMap[arch] && versionMap[arch][osName]) || '20.04.5';
-      const majorVersion = version.split('.').slice(0, 2).join('.');
+      // Determine OS family from osIdLike
+      const isDebianBased = osIdLike && osIdLike.match(/debian|ubuntu/i);
+      const isRhelBased = osIdLike && osIdLike.match(/rhel|centos|fedora|alma|rocky/i);
 
-      // Determine ISO filename and URL based on architecture
-      // ARM64 ISOs are on cdimage.ubuntu.com, AMD64 on releases.ubuntu.com
-      let isoFilename;
-      if (arch === 'arm64') {
-        isoFilename = `ubuntu-${version}-live-server-arm64${osName === 'noble' ? '+largemem' : ''}.iso`;
-      } else {
-        isoFilename = `ubuntu-${version}-live-server-amd64.iso`;
-      }
-      if (!isoUrl) isoUrl = `https://cdimage.ubuntu.com/releases/${majorVersion}/release/${isoFilename}`;
-      else isoFilename = isoUrl.split('/').pop();
+      // Set extraction directory based on OS family
+      const extractDirName = isDebianBased ? 'casper' : 'iso-extract';
+      shellExec(`mkdir -p ${nfsHostPath}/${extractDirName}`);
 
-      const isoPath = `/var/tmp/ubuntu-live-iso/${isoFilename}`;
-      const extractDir = `${nfsHostPath}/casper`;
+      const isoPath = `/var/tmp/live-iso/${isoFilename}`;
+      const extractDir = `${nfsHostPath}/${extractDirName}`;
 
       if (!fs.existsSync(isoPath)) {
-        logger.info(`Downloading Ubuntu ${version} live ISO for ${arch}...`);
+        logger.info(`Downloading ISO for ${arch}...`);
         logger.info(`URL: ${isoUrl}`);
-        logger.info(`This may take a while (typically 1-2 GB)...`);
-        shellExec(`wget --progress=bar:force -O ${isoPath} "${isoUrl}"`, { silent: false });
+        shellExec(`mkdir -p /var/tmp/live-iso`);
+        shellExec(`wget --progress=bar:force -O ${isoPath} "${isoUrl}"`);
         // Verify download by checking file existence and size (not exit code, which can be unreliable)
         if (!fs.existsSync(isoPath)) {
           throw new Error(`Failed to download ISO from ${isoUrl} - file not created`);
@@ -1102,46 +1207,32 @@ rm -rf ${artifacts.join(' ')}`);
         logger.info(`Downloaded ISO to ${isoPath} (${(stats.size / 1024 / 1024 / 1024).toFixed(2)} GB)`);
       }
 
-      // Mount ISO and extract casper files
-      const mountPoint = `${nfsHostPath}/mnt-${osName}-${arch}`;
+      // Mount ISO and extract boot files
+      const mountPoint = `${nfsHostPath}/mnt-iso-${arch}`;
       shellExec(`mkdir -p ${mountPoint}`);
 
       // Ensure mount point is not already mounted
-      shellExec(`sudo umount ${mountPoint} 2>/dev/null || true`, { silent: true });
+      shellExec(`sudo umount ${mountPoint} 2>/dev/null`, { silent: true });
 
       try {
         // Mount the ISO
         shellExec(`sudo mount -o loop,ro ${isoPath} ${mountPoint}`, { silent: false });
-        // Verify mount succeeded by checking if casper directory exists
-        if (!fs.existsSync(`${mountPoint}/casper`)) {
-          throw new Error(`Failed to mount ISO or casper directory not found: ${isoPath}`);
-        }
         logger.info(`Mounted ISO at ${mountPoint}`);
 
-        // List casper directory to see what's available
-        logger.info(`Checking casper directory contents...`);
-        shellExec(`ls -la ${mountPoint}/casper/ 2>/dev/null || echo "casper directory not found"`, { silent: false });
-
-        // Extract casper files
-        shellExec(`sudo cp -a ${mountPoint}/casper/* ${extractDir}/`);
-        shellExec(`sudo chown -R $(whoami):$(whoami) ${extractDir}`);
-        logger.info(`Extracted casper files from ISO`);
-
-        // Rename kernel and initrd to standard names if needed
-        if (!fs.existsSync(`${extractDir}/vmlinuz`)) {
-          const vmlinuz = shellExec(`ls ${extractDir}/vmlinuz* | head -1`, {
-            silent: true,
-            stdout: true,
-          }).stdout.trim();
-          if (vmlinuz) shellExec(`mv ${vmlinuz} ${extractDir}/vmlinuz`);
-        }
-        if (!fs.existsSync(`${extractDir}/initrd`)) {
-          const initrd = shellExec(`ls ${extractDir}/initrd* | head -1`, { silent: true, stdout: true }).stdout.trim();
-          if (initrd) shellExec(`mv ${initrd} ${extractDir}/initrd`);
+        // Distribution-specific extraction logic
+        if (isDebianBased) {
+          // Ubuntu/Debian: Extract from casper directory
+          if (!fs.existsSync(`${mountPoint}/casper`)) {
+            throw new Error(`Failed to mount ISO or casper directory not found: ${isoPath}`);
+          }
+          logger.info(`Checking casper directory contents...`);
+          shellExec(`ls -la ${mountPoint}/casper/ 2>/dev/null || echo "casper directory not found"`);
+          shellExec(`sudo cp -a ${mountPoint}/casper/* ${extractDir}/`);
         }
       } finally {
         shellExec(`ls -la ${mountPoint}/`);
 
+        shellExec(`sudo chown -R $(whoami):$(whoami) ${extractDir}`);
         // Unmount ISO
         shellExec(`sudo umount ${mountPoint}`, { silent: true });
         logger.info(`Unmounted ISO`);
@@ -1174,15 +1265,12 @@ rm -rf ${artifacts.join(' ')}`);
         hostname: '',
         ipAddress: '',
         powerType: 'manual',
-        maas: {},
+        architecture: 'arm64/generic',
       },
     ) {
       if (!options.powerType) options.powerType = 'manual';
-      const maas = options.maas || {};
       const payload = {
-        architecture: (maas.commissioning?.architecture || 'arm64/generic').match('arm')
-          ? 'arm64/generic'
-          : 'amd64/generic',
+        architecture: (options.architecture || 'arm64/generic').match('arm') ? 'arm64/generic' : 'amd64/generic',
         mac_address: options.macAddress,
         mac_addresses: options.macAddress,
         hostname: options.hostname,
@@ -1214,21 +1302,25 @@ rm -rf ${artifacts.join(' ')}`);
      * @description Retrieves kernel, initrd, and root filesystem paths from a MAAS boot resource.
      * @param {object} params - Parameters for the method.
      * @param {object} params.resource - The MAAS boot resource object.
-     * @param {boolean} params.useLiveIso - Whether to use Ubuntu live ISO instead of MAAS boot resources.
-     * @param {string} params.nfsHostPath - The NFS host path for storing extracted files.
+     * @param {string} params.type - The type of boot (e.g., 'iso-ram', 'iso-nfs', etc.).
+     * @param {string} params.nfsHostPath - The NFS host path (used for ISO types).
+     * @param {string} params.isoUrl - The ISO URL (used for ISO types).
+     * @param {string} params.workflowId - The workflow identifier.
      * @returns {object} An object containing paths to the kernel, initrd, and root filesystem.
      * @memberof UnderpostBaremetal
      */
-    kernelFactory({ resource, type, nfsHostPath, isoUrl }) {
-      // For disk-based commissioning (casper), use Ubuntu live ISO files
+    kernelFactory({ resource, type, nfsHostPath, isoUrl, workflowId }) {
+      // For disk-based commissioning (casper/iso), use live ISO files
       if (type === 'iso-ram' || type === 'iso-nfs') {
-        logger.info('Using Ubuntu live ISO for casper boot (disk-based commissioning)');
+        logger.info('Using live ISO for boot (disk-based commissioning)');
         const arch = resource.architecture.split('/')[0];
-        const kernelFilesPaths = UnderpostBaremetal.API.downloadUbuntuLiveISO({
+        const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
+        const kernelFilesPaths = UnderpostBaremetal.API.downloadISO({
           resource,
           architecture: arch,
           nfsHostPath,
           isoUrl,
+          osIdLike: workflowsConfig[workflowId].osIdLike || '',
         });
         const resourcesPath = `/var/snap/maas/common/maas/image-storage/bootloaders/uefi/${arch}`;
         return { kernelFilesPaths, resourcesPath };
@@ -1665,6 +1757,42 @@ shell
 `;
     },
 
+    /**
+     * @method ipxeEfiFactory
+     * @description Manages iPXE EFI binary build with cache support.
+     * Checks cache, builds only if needed, saves to cache after build.
+     * @param {object} params - The parameters for iPXE build.
+     * @param {string} params.tftpRootPath - TFTP root directory path.
+     * @param {string} params.ipxeCacheDir - iPXE cache directory path.
+     * @param {string} params.arch - Target architecture (arm64/amd64).
+     * @param {string} params.underpostRoot - Underpost root directory.
+     * @param {string} [params.embeddedScriptPath] - Path to embedded boot script.
+     * @param {boolean} [params.forceRebuild=false] - Force rebuild regardless of cache.
+     * @returns {void}
+     * @memberof UnderpostBaremetal
+     */
+    ipxeEfiFactory({ tftpRootPath, ipxeCacheDir, arch, underpostRoot, embeddedScriptPath, forceRebuild = false }) {
+      const shouldRebuild =
+        forceRebuild || (!fs.existsSync(`${tftpRootPath}/ipxe.efi`) && !fs.existsSync(`${ipxeCacheDir}/ipxe.efi`));
+
+      if (!shouldRebuild) return;
+
+      if (embeddedScriptPath && fs.existsSync(embeddedScriptPath)) {
+        logger.info('Rebuilding iPXE with embedded boot script...', {
+          embeddedScriptPath,
+          forced: forceRebuild,
+        });
+        shellExec(
+          `${underpostRoot}/scripts/ipxe-setup.sh ${tftpRootPath} --target-arch ${arch} --embed-script ${embeddedScriptPath} --rebuild`,
+        );
+      } else if (shouldRebuild) {
+        shellExec(`${underpostRoot}/scripts/ipxe-setup.sh ${tftpRootPath} --target-arch ${arch}`);
+      }
+
+      shellExec(`mkdir -p ${ipxeCacheDir}`);
+      shellExec(`cp ${tftpRootPath}/ipxe.efi ${ipxeCacheDir}/ipxe.efi`);
+    },
+
     /**
      * @method grubFactory
      * @description Generates the GRUB configuration file content.
@@ -1729,6 +1857,45 @@ shell
       };
     },
 
+    /**
+     * @method httpBootstrapServerStaticFactory
+     * @description Creates static files for the bootstrap HTTP server including cloud-init configuration.
+     * @param {object} params - Parameters for creating static files.
+     * @param {string} params.bootstrapHttpServerPath - The path where static files will be created.
+     * @param {string} params.hostname - The hostname of the client machine.
+     * @param {string} params.cloudConfigSrc - The cloud-init configuration YAML source.
+     * @param {object} [params.metadata] - Optional metadata to include in meta-data file.
+     * @param {string} [params.vendorData] - Optional vendor-data content (default: empty string).
+     * @memberof UnderpostBaremetal
+     * @returns {void}
+     */
+    httpBootstrapServerStaticFactory({
+      bootstrapHttpServerPath,
+      hostname,
+      cloudConfigSrc,
+      metadata = {},
+      vendorData = '',
+    }) {
+      // Create directory structure
+      shellExec(`mkdir -p ${bootstrapHttpServerPath}/${hostname}/cloud-init`);
+
+      // Write user-data file
+      fs.writeFileSync(
+        `${bootstrapHttpServerPath}/${hostname}/cloud-init/user-data`,
+        `#cloud-config\n${cloudConfigSrc}`,
+        'utf8',
+      );
+
+      // Write meta-data file
+      const metaDataContent = `instance-id: ${metadata.instanceId || hostname}\nlocal-hostname: ${metadata.localHostname || hostname}`;
+      fs.writeFileSync(`${bootstrapHttpServerPath}/${hostname}/cloud-init/meta-data`, metaDataContent, 'utf8');
+
+      // Write vendor-data file
+      fs.writeFileSync(`${bootstrapHttpServerPath}/${hostname}/cloud-init/vendor-data`, vendorData, 'utf8');
+
+      logger.info(`Cloud-init files written to ${bootstrapHttpServerPath}/${hostname}/cloud-init`);
+    },
+
     /**
      * @method httpBootstrapServerRunnerFactory
      * @description Starts a simple HTTP server to serve boot files for network booting.
@@ -1749,7 +1916,7 @@ shell
       shellExec(`mkdir -p ${bootstrapHttpServerPath}/${hostname}/cloud-init`);
 
       // Kill any existing HTTP server
-      shellExec(`sudo pkill -f 'python3 -m http.server ${port}' || true`, { silent: true });
+      shellExec(`sudo pkill -f 'python3 -m http.server ${port}'`, { silent: true });
 
       shellExec(
         `cd ${bootstrapHttpServerPath} && nohup python3 -m http.server ${port} --bind 0.0.0.0 > /tmp/http-boot-server.log 2>&1 &`,
@@ -1771,7 +1938,8 @@ shell
     /**
      * @method updateKernelFiles
      * @description Copies EFI bootloaders, kernel, and initrd images to the TFTP root path.
-     * It also handles decompression of the kernel if necessary for ARM64 compatibility.
+     * It also handles decompression of the kernel if necessary for ARM64 compatibility,
+     * and extracts raw kernel images from PE32+ EFI wrappers (common in Rocky Linux ARM64).
      * @param {object} params - The parameters for the function.
      * @param {object} params.commissioningImage - The commissioning image configuration.
      * @param {string} params.resourcesPath - The path where resources are located.
@@ -1797,11 +1965,21 @@ shell
         if (file === 'vmlinuz-efi') {
           const kernelDest = `${tftpRootPath}/pxe/${file}`;
           const fileType = shellExec(`file ${kernelDest}`, { silent: true }).stdout;
+
+          // Handle gzip compressed kernels
           if (fileType.includes('gzip compressed data')) {
             logger.info(`Decompressing kernel ${file} for ARM64 UEFI compatibility...`);
             shellExec(`sudo mv ${kernelDest} ${kernelDest}.gz`);
             shellExec(`sudo gunzip ${kernelDest}.gz`);
           }
+
+          // Handle PE32+ EFI wrapped kernels (common in Rocky Linux ARM64)
+          // Rocky Linux ARM64 kernels are distributed as PE32+ EFI executables, which
+          // are bootable directly via UEFI firmware. However, GRUB's 'linux' command
+          // expects a raw ARM64 Linux kernel Image format, not a PE32+ wrapper.
+          if (fileType.includes('PE32+') || fileType.includes('EFI application')) {
+            logger.warn('Detected PE32+ EFI wrapped kernel. Need to extract raw kernel image for GRUB.');
+          }
         }
       }
     },
@@ -1820,10 +1998,12 @@ shell
      * @param {string} options.networkInterfaceName - The name of the network interface.
      * @param {string} options.fileSystemUrl - The URL of the root filesystem.
      * @param {number} options.bootstrapHttpServerPort - The port of the bootstrap HTTP server.
-     * @param {string} options.type - The type of boot ('iso-ram', 'chroot', 'iso-nfs', etc.).
+     * @param {string} options.type - The type of boot ('iso-ram', 'chroot-debootstrap', 'chroot-container', 'iso-nfs', etc.).
      * @param {string} options.macAddress - The MAC address of the client.
      * @param {boolean} options.cloudInit - Whether to include cloud-init parameters.
      * @param {object} options.machine - The machine object containing system_id.
+     * @param {boolean} [options.dev=false] - Whether to enable dev mode with dracut debugging parameters.
+     * @param {string} [options.osIdLike=''] - OS family identifier (e.g., 'rhel centos fedora' or 'debian ubuntu').
      * @returns {object} An object containing the constructed command line string.
      * @memberof UnderpostBaremetal
      */
@@ -1843,6 +2023,8 @@ shell
         macAddress: '',
         cloudInit: false,
         machine: { system_id: '' },
+        dev: false,
+        osIdLike: '',
       },
     ) {
       // Construct kernel command line arguments for NFS boot.
@@ -1860,6 +2042,7 @@ shell
         type,
         macAddress,
         cloudInit,
+        osIdLike,
       } = options;
 
       const ipParam = true
@@ -1868,12 +2051,11 @@ shell
         : 'ip=dhcp';
 
       const nfsOptions = `${
-        type === 'chroot'
+        type === 'chroot-debootstrap' || type === 'chroot-container'
           ? [
               'tcp',
               'nfsvers=3',
               'nolock',
-              'vers=3',
               // 'protocol=tcp',
               // 'hard=true',
               'port=2049',
@@ -1953,19 +2135,36 @@ shell
         'overlayroot_cfgdisk=disabled', // Ignore external overlay configurations
       ];
 
-      const baseNfsParams = [`netboot=nfs`];
-
       let cmd = [];
       if (type === 'iso-ram') {
         const netBootParams = [`netboot=url`];
         if (fileSystemUrl) netBootParams.push(`url=${fileSystemUrl.replace('https', 'http')}`);
         cmd = [ipParam, `boot=casper`, ...netBootParams, ...kernelParams];
-      } else if (type === 'chroot') {
-        const qemuNfsRootParams = [`root=/dev/nfs`, `rootfstype=nfs`, `initrd=initrd.img`, `init=/sbin/init`];
-        cmd = [ipParam, ...baseNfsParams, ...qemuNfsRootParams, nfsRootParam, ...kernelParams];
+      } else if (type === 'chroot-debootstrap' || type === 'chroot-container') {
+        let qemuNfsRootParams = [`root=/dev/nfs`, `rootfstype=nfs`];
+
+        // Determine OS family from osIdLike configuration
+        const isRhelBased = osIdLike && osIdLike.match(/rhel|centos|fedora|alma|rocky/i);
+        const isDebianBased = osIdLike && osIdLike.match(/debian|ubuntu/i);
+
+        // Add RHEL/Rocky/Fedora based images specific parameters
+        if (isRhelBased) {
+          qemuNfsRootParams = qemuNfsRootParams.concat([`rd.neednet=1`, `rd.timeout=180`, `selinux=0`, `enforcing=0`]);
+        }
+        // Add Debian/Ubuntu based images specific parameters
+        else if (isDebianBased) {
+          qemuNfsRootParams = qemuNfsRootParams.concat([`initrd=initrd.img`, `init=/sbin/init`]);
+        }
+
+        // Add debugging parameters in dev mode for dracut troubleshooting
+        if (options.dev) {
+          // qemuNfsRootParams = qemuNfsRootParams.concat([`rd.shell`, `rd.debug`]);
+        }
+
+        cmd = [ipParam, ...qemuNfsRootParams, nfsRootParam, ...kernelParams];
       } else {
         // 'iso-nfs'
-        cmd = [ipParam, ...baseNfsParams, nfsRootParam, ...kernelParams, ...performanceParams];
+        cmd = [ipParam, `netboot=nfs`, nfsRootParam, ...kernelParams, ...performanceParams];
 
         cmd.push(`ifname=${networkInterfaceName}:${macAddress}`);
 
@@ -1998,12 +2197,12 @@ shell
      * @param {string} params.macAddress - The MAC address to monitor for.
      * @param {string} params.ipAddress - The IP address of the machine (used if MAC is all zeros).
      * @param {string} [params.hostname] - The hostname for the machine (optional).
-     * @param {object} [params.maas] - Additional MAAS-specific options (optional).
+     * @param {string} [params.architecture] - The architecture of the machine (optional).
      * @param {object} [params.machine] - Existing machine payload to use (optional).
-     * @returns {Promise<void>} A promise that resolves when commissioning is initiated or after a delay.
+     * @returns {Promise<void>} A promise object with machine and discovery details.
      * @memberof UnderpostBaremetal
      */
-    async commissionMonitor({ macAddress, ipAddress, hostname, maas, machine }) {
+    async commissionMonitor({ macAddress, ipAddress, hostname, architecture, machine }) {
       {
         // Query observed discoveries from MAAS.
         const discoveries = JSON.parse(
@@ -2038,7 +2237,7 @@ shell
                 ipAddress,
                 macAddress: discovery.mac_address,
                 hostname,
-                maas,
+                architecture,
               }).machine;
               console.log('New machine system id:', machine.system_id.bgYellow.bold.black);
               UnderpostBaremetal.API.writeGrubConfigToFile({
@@ -2079,14 +2278,6 @@ shell
                 logger.info('✓ Machine interface MAC address updated successfully');
 
                 // commissioning_scripts=90-verify-user.sh
-                machine = JSON.parse(
-                  shellExec(
-                    `maas ${process.env.MAAS_ADMIN_USERNAME} machine commission --debug --insecure ${systemId} enable_ssh=1 skip_bmc_config=1 skip_networking=1 skip_storage=1`,
-                    {
-                      silent: true,
-                    },
-                  ),
-                );
               }
               logger.info('Machine resource uri', machine.resource_uri);
               for (const iface of machine.interface_set)
@@ -2101,7 +2292,13 @@ shell
           }
         }
         await timer(1000);
-        return await UnderpostBaremetal.API.commissionMonitor({ macAddress, ipAddress, hostname, maas, machine });
+        return await UnderpostBaremetal.API.commissionMonitor({
+          macAddress,
+          ipAddress,
+          hostname,
+          architecture,
+          machine,
+        });
       }
     },
 
@@ -2189,8 +2386,8 @@ shell
      * @param {'arm64'|'amd64'} params.debootstrapArch - The target architecture of the debootstrap environment.
      * @returns {void}
      */
-    crossArchBinFactory({ nfsHostPath, debootstrapArch }) {
-      switch (debootstrapArch) {
+    crossArchBinFactory({ nfsHostPath, bootstrapArch }) {
+      switch (bootstrapArch) {
         case 'arm64':
           // Copy QEMU static binary for ARM64.
           shellExec(`sudo podman cp extract:/usr/bin/qemu-aarch64-static ${nfsHostPath}/usr/bin/`);
@@ -2201,7 +2398,7 @@ shell
           break;
         default:
           // Log a warning or throw an error for unsupported architectures.
-          logger.warn(`Unsupported debootstrap architecture: ${debootstrapArch}`);
+          logger.warn(`Unsupported bootstrap architecture: ${bootstrapArch}`);
           break;
       }
       // Install GRUB EFI modules for both architectures to ensure compatibility.
@@ -2221,14 +2418,14 @@ shell
      * @param {string[]} params.steps - An array of shell commands to execute.
      * @returns {void}
      */
-    crossArchRunner({ nfsHostPath, debootstrapArch, callbackMetaData, steps }) {
+    crossArchRunner({ nfsHostPath, bootstrapArch, callbackMetaData, steps }) {
       // Render the steps with logging for better visibility during execution.
       steps = UnderpostBaremetal.API.stepsRender(steps, false);
 
       let qemuCrossArchBash = '';
       // Determine if QEMU is needed for cross-architecture execution.
-      if (debootstrapArch !== callbackMetaData.runnerHost.architecture)
-        switch (debootstrapArch) {
+      if (bootstrapArch !== callbackMetaData.runnerHost.architecture)
+        switch (bootstrapArch) {
           case 'arm64':
             qemuCrossArchBash = '/usr/bin/qemu-aarch64-static ';
             break;
@@ -2287,20 +2484,25 @@ EOF`);
      * @param {string} params.workflowId - The identifier for the workflow configuration.
      * @param {boolean} [params.mount] - If true, attempts to mount the NFS paths.
      * @param {boolean} [params.unmount] - If true, attempts to unmount the NFS paths.
+     * @param {number} [currentRecall=0] - The current recall attempt count for retries.
+     * @param {number} [maxRecalls=5] - The maximum number of recall attempts allowed.
      * @memberof UnderpostBaremetal
-     * @returns {{isMounted: boolean}} An object indicating whether any NFS path is currently mounted.
+     * @returns {Promise<void>} A promise that resolves when the mount/unmount operations are complete.
      */
-    nfsMountCallback({ hostname, nfsHostPath, workflowId, mount, unmount }) {
+    async nfsMountCallback({ hostname, nfsHostPath, workflowId, mount, unmount }, currentRecall = 0, maxRecalls = 5) {
       // Mount binfmt_misc filesystem.
       if (mount) UnderpostBaremetal.API.mountBinfmtMisc();
-      let isMounted = false;
+      const unMountCmds = [];
       const mountCmds = [];
-      const currentMounts = [];
       const workflowsConfig = UnderpostBaremetal.API.loadWorkflowsConfig();
+      let recall = false;
       if (!workflowsConfig[workflowId]) {
         throw new Error(`Workflow configuration not found for ID: ${workflowId}`);
       }
-      if (workflowsConfig[workflowId].type === 'chroot') {
+      if (
+        workflowsConfig[workflowId].type === 'chroot-debootstrap' ||
+        workflowsConfig[workflowId].type === 'chroot-container'
+      ) {
         const mounts = {
           bind: ['/proc', '/sys', '/run'],
           rbind: ['/dev'],
@@ -2315,12 +2517,11 @@ EOF`);
             );
 
             if (isPathMounted) {
-              currentMounts.push(mountPath);
-              if (!isMounted) isMounted = true; // Set overall mounted status.
               logger.warn('Nfs path already mounted', mountPath);
               if (unmount === true) {
                 // Unmount if requested.
-                mountCmds.push(`sudo umount ${hostMountPath}`);
+                unMountCmds.push(`sudo umount -Rfl ${hostMountPath}`);
+                if (!recall) recall = true;
               }
             } else {
               if (mount === true) {
@@ -2332,17 +2533,27 @@ EOF`);
             }
           }
         }
-
-        if (!isMounted) {
-          // if all path unmounted, set ownership and permissions for the NFS host path.
+        for (const unMountCmd of unMountCmds) shellExec(unMountCmd);
+        if (recall) {
+          if (currentRecall >= maxRecalls) {
+            throw new Error(
+              `Maximum recall attempts (${maxRecalls}) reached for nfsMountCallback. Hostname: ${hostname}`,
+            );
+          }
+          logger.info(`nfsMountCallback recall attempt ${currentRecall + 1}/${maxRecalls} for hostname: ${hostname}`);
+          await timer(1000);
+          return await UnderpostBaremetal.API.nfsMountCallback(
+            { hostname, nfsHostPath, workflowId, mount, unmount },
+            currentRecall + 1,
+            maxRecalls,
+          );
+        }
+        if (mountCmds.length > 0) {
           shellExec(`sudo chown -R $(whoami):$(whoami) ${nfsHostPath}`);
           shellExec(`sudo chmod -R 755 ${nfsHostPath}`);
+          for (const mountCmd of mountCmds) shellExec(mountCmd);
         }
-        for (const mountCmd of mountCmds) shellExec(mountCmd);
-        if (mount) isMounted = true;
-        logger.info('Current mounts', currentMounts);
       }
-      return { isMounted, currentMounts };
     },
 
     /**
@@ -2387,27 +2598,25 @@ EOF`);
          * @returns {string[]} An array of shell commands.
          */
         base: () => [
-          // Configure APT sources for Ubuntu ports.
+          // Configure APT sources for Ubuntu ports
           `cat <<SOURCES | tee /etc/apt/sources.list
 deb http://ports.ubuntu.com/ubuntu-ports noble main restricted universe multiverse
 deb http://ports.ubuntu.com/ubuntu-ports noble-updates main restricted universe multiverse
 deb http://ports.ubuntu.com/ubuntu-ports noble-security main restricted universe multiverse
 SOURCES`,
 
-          // Update package lists and perform a full system upgrade.
+          // Update package lists and perform a full system upgrade
           `apt update -qq`,
           `apt -y full-upgrade`,
-          // Install essential development and system utilities.
-          `apt install -y build-essential xinput x11-xkb-utils usbutils uuid-runtime`,
-          'apt install -y linux-image-generic',
 
-          // Install cloud-init, systemd, SSH, sudo, locales, udev, and networking tools.
-          `apt install -y systemd-sysv openssh-server sudo locales udev util-linux systemd-sysv iproute2 netplan.io ca-certificates curl wget chrony`,
-          `ln -sf /lib/systemd/systemd /sbin/init`, // Ensure systemd is the init system.
+          // Install all essential packages in one consolidated step
+          `DEBIAN_FRONTEND=noninteractive apt install -y build-essential xinput x11-xkb-utils usbutils uuid-runtime linux-image-generic systemd-sysv openssh-server sudo locales udev util-linux iproute2 netplan.io ca-certificates curl wget chrony apt-utils tzdata kmod keyboard-configuration console-setup iputils-ping`,
+
+          // Ensure systemd is the init system
+          `ln -sf /lib/systemd/systemd /sbin/init`,
 
-          `apt-get update`,
-          `DEBIAN_FRONTEND=noninteractive apt-get install -y apt-utils`, // Install apt-utils non-interactively.
-          `DEBIAN_FRONTEND=noninteractive apt-get install -y tzdata kmod keyboard-configuration console-setup iputils-ping`, // Install timezone data, kernel modules, and network tools.
+          // Clean up
+          `apt-get clean`,
         ],
         /**
          * @method user
@@ -2524,6 +2733,157 @@ logdir /var/log/chrony
           `sudo systemctl restart keyboard-setup.service`,
         ],
       },
+      /**
+       * @property {object} rocky
+       * @description Provisioning steps for Rocky Linux-based systems.
+       * @memberof UnderpostBaremetal.systemProvisioningFactory
+       * @namespace UnderpostBaremetal.systemProvisioningFactory.rocky
+       */
+      rocky: {
+        /**
+         * @method base
+         * @description Generates shell commands for basic Rocky Linux system provisioning.
+         * This includes installing Node.js, npm, and underpost CLI tools.
+         * @param {object} params - The parameters for the function.
+         * @memberof UnderpostBaremetal.systemProvisioningFactory.rocky
+         * @returns {string[]} An array of shell commands.
+         */
+        base: () => [
+          // Update system and install EPEL repository
+          `dnf -y update`,
+          `dnf -y install epel-release`,
+
+          // Install essential system tools (avoiding duplicates from container packages)
+          `dnf -y install --allowerasing bzip2 openssh-server nano vim-enhanced less openssl-devel git gnupg2 libnsl perl`,
+          `dnf clean all`,
+
+          // Install Node.js
+          `curl -fsSL https://rpm.nodesource.com/setup_24.x | bash -`,
+          `dnf install -y nodejs`,
+          `dnf clean all`,
+
+          // Verify Node.js and npm versions
+          `node --version`,
+          `npm --version`,
+
+          // Install underpost ci/cd cli
+          `npm install -g underpost`,
+          `underpost --version`,
+        ],
+        /**
+         * @method user
+         * @description Generates shell commands for creating a root user and configuring SSH access on Rocky Linux.
+         * This is a critical security step for initial access to the provisioned system.
+         * @memberof UnderpostBaremetal.systemProvisioningFactory.rocky
+         * @returns {string[]} An array of shell commands.
+         */
+        user: () => [
+          `useradd -m -s /bin/bash -G wheel root`, // Create a root user with bash shell and wheel group (sudo on RHEL)
+          `echo 'root:root' | chpasswd`, // Set a default password for the root user
+          `mkdir -p /home/root/.ssh`, // Create .ssh directory for authorized keys
+          // Add the public SSH key to authorized_keys for passwordless login
+          `echo '${fs.readFileSync(
+            `/home/dd/engine/engine-private/deploy/id_rsa.pub`,
+            'utf8',
+          )}' > /home/root/.ssh/authorized_keys`,
+          `chown -R root:root /home/root/.ssh`, // Set ownership for security
+          `chmod 700 /home/root/.ssh`, // Set permissions for the .ssh directory
+          `chmod 600 /home/root/.ssh/authorized_keys`, // Set permissions for authorized_keys
+        ],
+        /**
+         * @method timezone
+         * @description Generates shell commands for configuring the system timezone on Rocky Linux.
+         * @param {object} params - The parameters for the function.
+         * @param {string} params.timezone - The timezone string (e.g., 'America/Santiago').
+         * @param {string} params.chronyConfPath - The path to the Chrony configuration file (optional).
+         * @memberof UnderpostBaremetal.systemProvisioningFactory.rocky
+         * @returns {string[]} An array of shell commands.
+         */
+        timezone: ({ timezone, chronyConfPath = '/etc/chrony.conf' }) => [
+          // Set system timezone using both methods (for chroot and running system)
+          `ln -sf /usr/share/zoneinfo/${timezone} /etc/localtime`,
+          `echo '${timezone}' > /etc/timezone`,
+          `timedatectl set-timezone ${timezone} 2>/dev/null`,
+
+          // Configure chrony with local NTP server and common NTP pools
+          `echo '# Local NTP server' > ${chronyConfPath}`,
+          `echo 'server 192.168.1.1 iburst prefer' >> ${chronyConfPath}`,
+          `echo '' >> ${chronyConfPath}`,
+          `echo '# Fallback public NTP servers' >> ${chronyConfPath}`,
+          `echo 'server 0.pool.ntp.org iburst' >> ${chronyConfPath}`,
+          `echo 'server 1.pool.ntp.org iburst' >> ${chronyConfPath}`,
+          `echo 'server 2.pool.ntp.org iburst' >> ${chronyConfPath}`,
+          `echo 'server 3.pool.ntp.org iburst' >> ${chronyConfPath}`,
+          `echo '' >> ${chronyConfPath}`,
+          `echo '# Configuration' >> ${chronyConfPath}`,
+          `echo 'driftfile /var/lib/chrony/drift' >> ${chronyConfPath}`,
+          `echo 'makestep 1.0 3' >> ${chronyConfPath}`,
+          `echo 'rtcsync' >> ${chronyConfPath}`,
+          `echo 'logdir /var/log/chrony' >> ${chronyConfPath}`,
+
+          // Enable chronyd to start on boot
+          `systemctl enable chronyd 2>/dev/null`,
+
+          // Create systemd link for boot (works in chroot)
+          `mkdir -p /etc/systemd/system/multi-user.target.wants`,
+          `ln -sf /usr/lib/systemd/system/chronyd.service /etc/systemd/system/multi-user.target.wants/chronyd.service 2>/dev/null`,
+
+          // Start chronyd if systemd is running
+          `systemctl start chronyd 2>/dev/null`,
+
+          // Restart chronyd to apply configuration
+          `systemctl restart chronyd 2>/dev/null`,
+
+          // Force immediate time synchronization (only if chronyd is running)
+          `chronyc makestep 2>/dev/null`,
+
+          // Verify timezone configuration
+          `ls -l /etc/localtime`,
+          `cat /etc/timezone || echo 'No /etc/timezone file'`,
+          `timedatectl status 2>/dev/null || echo 'Timezone set to ${timezone} (timedatectl not available in chroot)'`,
+          `chronyc tracking 2>/dev/null || echo 'Chrony configured but not running (will start on boot)'`,
+        ],
+        /**
+         * @method keyboard
+         * @description Generates shell commands for configuring the keyboard layout on Rocky Linux.
+         * This uses localectl to set the keyboard layout for both console and X11.
+         * @param {string} [keyCode='us'] - The keyboard layout code (e.g., 'us', 'es').
+         * @memberof UnderpostBaremetal.systemProvisioningFactory.rocky
+         * @returns {string[]} An array of shell commands.
+         */
+        keyboard: (keyCode = 'us') => [
+          // Configure vconsole.conf for console keyboard layout (persistent)
+          `echo 'KEYMAP=${keyCode}' > /etc/vconsole.conf`,
+          `echo 'FONT=latarcyrheb-sun16' >> /etc/vconsole.conf`,
+
+          // Configure locale.conf for system locale
+          `echo 'LANG=en_US.UTF-8' > /etc/locale.conf`,
+          `echo 'LC_ALL=en_US.UTF-8' >> /etc/locale.conf`,
+
+          // Set keyboard layout using localectl (works if systemd is running)
+          `localectl set-locale LANG=en_US.UTF-8 2>/dev/null`,
+          `localectl set-keymap ${keyCode} 2>/dev/null`,
+          `localectl set-x11-keymap ${keyCode} 2>/dev/null`,
+
+          // Configure X11 keyboard layout file directly
+          `mkdir -p /etc/X11/xorg.conf.d`,
+          `echo 'Section "InputClass"' > /etc/X11/xorg.conf.d/00-keyboard.conf`,
+          `echo '    Identifier "system-keyboard"' >> /etc/X11/xorg.conf.d/00-keyboard.conf`,
+          `echo '    MatchIsKeyboard "on"' >> /etc/X11/xorg.conf.d/00-keyboard.conf`,
+          `echo '    Option "XkbLayout" "${keyCode}"' >> /etc/X11/xorg.conf.d/00-keyboard.conf`,
+          `echo 'EndSection' >> /etc/X11/xorg.conf.d/00-keyboard.conf`,
+
+          // Load the keymap immediately (if not in chroot)
+          `loadkeys ${keyCode} 2>/dev/null || echo 'Keymap ${keyCode} configured (loadkeys not available in chroot)'`,
+
+          // Verify configuration
+          `echo 'Keyboard configuration files:'`,
+          `cat /etc/vconsole.conf`,
+          `cat /etc/locale.conf`,
+          `cat /etc/X11/xorg.conf.d/00-keyboard.conf 2>/dev/null || echo 'X11 config created'`,
+          `localectl status 2>/dev/null || echo 'Keyboard layout set to ${keyCode} (localectl not available in chroot)'`,
+        ],
+      },
     },
 
     /**