kameleon-builder 2.0.0 → 2.1.0

data/templates/steps/bootstrap/fedora/liveos_bootstrap.yaml

@@ -0,0 +1,123 @@
+# # Bootstrap
+- include: >
+    python yum rpm initscripts passwd rsyslog vim-minimal dhclient chkconfig
+    rootfiles policycoreutils fedora-release openssh-server net-tools nc
+    traceroute kernel syslinux-extlinux systemd procps-ng console-setup
+    bash grep coreutils findutils sed cpio cyrus-sasl file-libs gawk xz
+
+- mirrorlist_server: mirrors.kernel.org
+- mirror_distrib_path: fedora/releases/$${release}/Fedora/$$arch/os
+- mirrorlist_url_rsync: $$mirrorlist_server::$$mirror_distrib_path
+
+- download_live_os:
+  - check_cmd_out: rsync
+  - exec_out: |
+      if [ ! -d "$$rootfs_download_path" ]; then
+        if [ ! -f "./LiveOS/squashfs.img" ]; then
+          echo "Downloading LiveOS squashfs file system from $$mirrorlist_url_rsync ..."
+          rsync -avh --progress $$mirrorlist_url_rsync/LiveOS .
+          echo "Download of squashfs image complete."
+        else
+          echo "Using cached LiveOS squashfs file system."
+        fi
+      fi
+
+- copy_ro_to_rw:
+  - exec_out: |
+      if [ ! -d "$$rootfs_download_path" ]; then
+        mkdir -p squashfs readonlyfs partial
+        mount -o loop ./LiveOS/squashfs.img squashfs ||  \
+          fail Mount of LiveOS squashfs image failed! You mush have squashfs support available to mount image.
+        mount -o loop squashfs/LiveOS/rootfs.img readonlyfs
+        rsync -aAHS readonlyfs/ partial/
+      fi
+  - umount_out: readonlyfs
+  - umount_out: squashfs
+
+- bootstrap_partial:
+  - exec_out: |
+      if [ ! -d "$$rootfs_download_path" ]; then
+        mount -o bind /dev  partial/dev
+        mount -o bind /dev/pts partial/dev/pts
+        mount -t proc /proc  partial/proc
+        mount -t sysfs /sys  partial/sys
+        test -f partial/etc/mtab || cat /proc/mounts > partial/etc/mtab
+        cp /etc/resolv.conf partial/etc/
+        rsync -av $$mirrorlist_url_rsync/Packages/r/rpm-[0-9]* $$mirrorlist_url_rsync//Packages/y/yum-[0-9]* partial
+        chroot partial rpm -ivh --nodeps --replacepkgs rpm-* yum-* > /dev/null
+        rsync -av $$mirrorlist_url_rsync/Packages/f/fedora-release-$${release}* partial
+        mkdir -p partial/run/install
+        chroot partial  rpm --nodeps -ivh fedora-release-*
+        sed -i "s|\$basearch|$$arch|" partial/etc/yum.repos.d/*
+        chroot partial  rpm --root /run/install --nodeps -ivh fedora-release-*
+        sed -i "s|\$basearch|$$arch|" partial/run/install/etc/yum.repos.d/*
+        chroot partial yum --installroot /run/install -y --nogpgcheck install $$include
+      fi
+  - umount_out: partial/dev/pts
+  - umount_out: partial/dev
+  - umount_out: partial/proc
+  - umount_out: partial/sys
+
+- create_rootfs:
+  - exec_out: |
+      if [ ! -d "$$rootfs_download_path" ]; then
+        mkdir -p $$rootfs_download_path
+        rsync -aAX --delete --one-file-system \
+          --exclude '/.kameleon_timestamp' \
+          --exclude '/dev/*' \
+          --exclude '/proc/*' \
+          --exclude '/sys/*' \
+          partial/run/install/* $$rootfs_download_path
+      fi
+
+- download_pxeboot_initrd:
+  - exec_out: |
+      wget -c -O $$rootfs_download_path/boot/initrd.img \
+        http://$$mirrorlist_server/$$mirror_distrib_path/images/pxeboot/initrd.img \
+           2>&1 | tail -f
+
+- download_pxeboot_vmlinuz:
+  - exec_out: |
+      wget -c -O $$rootfs_download_path/boot/vmlinuz \
+        http://$$mirrorlist_server/$$mirror_distrib_path/images/pxeboot/vmlinuz \
+          2>&1 | tail -f
+
+- enable_sshd_service:
+  - exec_out: |
+      chroot $$rootfs_download_path \
+        ln -sf \
+          '/usr/lib/systemd/system/sshd.service' \
+          '/etc/systemd/system/multi-user.target.wants/sshd.service'
+
+- enable_dhcp_service:
+  - exec_out: |
+      echo -n 'root:root' | chroot $$rootfs_download_path  chpasswd
+      chroot $$rootfs_download_path \
+        ln -sf \
+          '/usr/lib/systemd/system/network.service' \
+          '/etc/systemd/system/multi-user.target.wants/network.service'
+
+- enable_networking:
+  - write_out:
+    - $$rootfs_download_path/etc/sysconfig/network-scripts/ifcfg-eth0
+    - |
+      DEVICE=eth0
+      BOOTPROTO=dhcp
+      ONBOOT=yes
+      HOSTNAME=fedora-$${release}
+      NM_CONTROLLED=no
+      TYPE=Ethernet
+  - write_out:
+    - $$rootfs_download_path/etc/sysconfig/network
+    - |
+      NETWORKING=yes
+      HOSTNAME=fedora-$${release}
+
+- clean:
+  - on_bootstrap_clean:
+    - umount_out: partial/dev/pts
+    - umount_out: partial/dev
+    - umount_out: partial/proc
+    - umount_out: partial/sys
+    - umount_out: readonlyfs
+    - umount_out: squashfs

data/templates/steps/bootstrap/g5k_reserv.yaml

@@ -0,0 +1,70 @@
+- create_local_ssh_config_base:
+  - write_local:
+    - $$ssh_config_file
+    - |
+      Host *
+        UserKnownHostsFile /dev/null
+        StrictHostKeyChecking no
+        ConnectTimeout 2
+        LogLevel FATAL
+        ForwardAgent yes
+        Protocol 2
+        ControlPath /tmp/$${kameleon_short_uuid}%r@%h:%p
+        ControlMaster auto
+        ControlPersist yes
+        Compression yes
+      Host g5kaccess
+        User $$g5k_user
+        Hostname access.grid5000.fr
+      Host $$g5k_site
+        User $$g5k_user
+        ProxyCommand ssh -F $$ssh_config_file g5kaccess "nc -q 1 $$g5k_site %p"
+
+- deploy_image:
+  - exec_out: echo "Deploying image $$kenv with kadeploy"
+  - exec_out: sleep 1
+  - exec_out: echo "Submitting a job for deployment"
+    # We wait long to keep the job
+    # We submit a job only if there is no job already
+  - exec_out : |
+      oarstat | grep -q "$$kameleon_recipe_name" \
+          || oarsub -n "$$kameleon_recipe_name" -l walltime=$$walltime -t deploy "sleep 100000"
+  - exec_out : |
+      echo -n "Waiting for the reservation to be ready"
+      until $(oarstat -fu $$g5k_user | grep -q "state = Running")
+      do
+        echo -n .
+        sleep 2
+      done
+      sleep 1
+      echo " ~> OK"
+  - exec_out: echo "Getting the machine name"
+  - exec_out: export machine=`oarstat -fu $$g5k_user | grep assigned_hostnames | cut -d ' ' -f 7`
+  - exec_out: echo "Deploying environment $$kenv on $machine"
+  - exec_out: kadeploy3 -e $$kenv -m $machine -k
+  - pipe:
+      - exec_out: echo $machine
+      - exec_local: cat >$$kameleon_cwd/g5k_machine
+  - exec_out: sleep 1
+
+- create_out_ssh_config:
+  - write_out:
+    - $$out_cwd/ssh_config
+    - |
+      Host $$kameleon_recipe_name
+        User root
+        Hostname $machine
+        UserKnownHostsFile /dev/null
+        StrictHostKeyChecking no
+        ConnectTimeout 2
+        LogLevel FATAL
+        ForwardAgent yes
+        Protocol 2
+
+- append_ssh_node_config:
+  - append_local:
+    - $$ssh_config_file
+    - |
+      Host $$kameleon_recipe_name
+        User root
+        ProxyCommand ssh -F $$ssh_config_file $$g5k_site "nc -q 1 `cat $$kameleon_cwd/g5k_machine | awk '{print $NF}'` %p"

data/templates/steps/bootstrap/initialize_disk_chroot.yaml

@@ -0,0 +1,84 @@
+# Initialize disk
+
+# this step create the appliance disk and initialize the
+# checkpoint mechanism with Qemu Network Block Device (NBD)
+
+- create_initial_image:
+  - check_cmd_out: qemu-img
+  - exec_out: mkdir -p $$kameleon_cwd/checkpoints
+  - exec_out: |
+      if [ ! -e "$$image_disk" ] ; then
+        qemu-img create -f qcow2 -o preallocation=metadata $$kameleon_cwd/checkpoints/base.qcow2 $$image_size
+        # keep a link to the last checkpoint disk
+        ln -sf $$kameleon_cwd/checkpoints/base.qcow2 $$image_disk
+      fi
+
+- load_nbd_module:
+  - on_checkpoint: redo
+  - on_bootstrap_init:
+    - exec_out: |
+        lsmod | grep nbd >/dev/null \
+        || modprobe nbd max_part=63 \
+        || fail failed to load nbd module into kernel
+
+- attach_nbd_device:
+  # this microstep must be skipped because it is already done by the
+  # checkpoint restore function
+  - on_checkpoint: redo
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo Connecting $$image_disk to nbd device $$nbd_device
+  - exec_out: echo "sync" ; sync
+  - exec_out: qemu-nbd -d $$nbd_device
+  - exec_out: qemu-nbd -c $$nbd_device $(readlink $$image_disk) -n || fail nbd device $$nbd_device is unavailable
+
+- detach_nbd_device:
+  # this miscrostep must be done again ensure that the nbd device is
+  # detached
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: echo "sync" ; sync
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: "pgrep qemu-nbd | xargs -I {} kill -9 {} || true"
+
+- partition_disk:
+  - check_cmd_out: parted
+  - exec_out: |
+      echo "Partitioning disk..."
+      parted $${nbd_device} mklabel msdos
+      parted $${nbd_device} mkpart primary 0% 100%
+      parted $${nbd_device} set 1 boot on
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p1 || fail cannot create / ext4
+
+- mount_mountdir:
+  - on_checkpoint: redo
+  - exec_out: mkdir -p $$rootfs
+  - exec_out:  "echo Mounting root partition... ;  mount $${nbd_device}p1 $$rootfs || fail cannot mount /"
+  - on_export_clean:
+    - exec_out: "echo try umount $$rootfs... ; mountpoint -q $$rootfs && umount -f -l $$rootfs || true"
+
+- copy_rootfs:
+  - check_cmd_out: rsync
+  - exec_out: echo "Copying rootfs to $$rootfs..."
+  - exec_out: |
+      rsync -aAX --delete --one-file-system\
+        --exclude '/.kameleon_timestamp' \
+        --exclude '/dev/*' \
+        --exclude '/proc/*' \
+        --exclude '/sys/*' \
+        $$rootfs_download_path/* $$rootfs
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$rootfs/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # nbd_device; this may be used with UUID= as a more robust way to name nbd_devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${nbd_device}p1` /               $$filesystem_type    errors=remount-ro  0       1

data/templates/steps/bootstrap/initialize_disk_qemu.yaml

@@ -0,0 +1,72 @@
+# Initialize disk
+
+# this step create the appliance disk and initialize the
+# checkpoint mechanism with Qemu Network Block Device (NBD)
+
+- create_initial_image:
+  - check_cmd_out: qemu-img
+  - exec_out: |
+      rm -f $$image_disk
+      qemu-img create -f qcow2 -o preallocation=metadata $$image_disk $$image_size
+
+- load_nbd_module:
+  - on_checkpoint: redo
+  - on_bootstrap_init:
+    - exec_out: |
+        lsmod | grep nbd >/dev/null \
+        || modprobe nbd max_part=63 \
+        || fail failed to load nbd module into kernel
+
+- attach_nbd_device:
+  # this microstep must be skipped because it is already done by the
+  # checkpoint restore function
+  - on_checkpoint: redo
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo Connecting $$image_disk to nbd device $$nbd_device
+  - exec_out: qemu-nbd -c $$nbd_device $$image_disk -n || fail nbd device $$nbd_device is unavailable
+
+- detach_nbd_device:
+  # this miscrostep must be done again ensure that the nbd device is
+  # detached
+  - on_checkpoint: redo
+  - on_bootstrap_clean:
+    - exec_out: echo "sync" ; sync
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: "pgrep qemu-nbd | xargs -I {} kill -9 {} || true"
+
+- partition_disk:
+  - check_cmd_out: parted
+  - exec_out: |
+      echo "Partitioning disk..."
+      parted $${nbd_device} mklabel msdos
+      parted $${nbd_device} mkpart primary 0% 100%
+      parted $${nbd_device} set 1 boot on
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p1 || fail cannot create / ext4
+
+- mount_mountdir:
+  - on_checkpoint: redo
+  - exec_out: mkdir -p $$rootfs
+  - exec_out:  "echo Mounting root partition... ;  mount $${nbd_device}p1 $$rootfs || fail cannot mount /"
+  - on_bootstrap_clean:
+    - exec_out: "echo try umount $$rootfs... ; mountpoint -q $$rootfs && umount -f -l $$rootfs || true"
+
+- copy_rootfs:
+  - check_cmd_out: rsync
+  - exec_out: echo "Copying rootfs to $$rootfs..."
+  - exec_out: rsync -aAX --exclude '/.kameleon_timestamp' $$rootfs_download_path/* $$rootfs
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$rootfs/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # nbd_device; this may be used with UUID= as a more robust way to name nbd_devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${nbd_device}p1` /               $$filesystem_type    errors=remount-ro  0       1

data/templates/steps/bootstrap/install_bootloader.yaml

@@ -0,0 +1,42 @@
+# Install extlinux bootloader
+
+- install_bootloader:
+  - rescue:
+    - exec_out: LC_ALL=POSIX chroot $$rootfs bash -c "command -V extlinux 2> /dev/null"
+    - breakpoint: "extlinux is missing"
+  - exec_out: LC_ALL=POSIX chroot $$rootfs extlinux --install /boot/extlinux 2>&1
+  - exec_out: |
+      MBR_PATH=
+      PATHS=("$$rootfs/usr/share/syslinux/mbr.bin"
+             "$$rootfs/usr/lib/bios/syslinux/mbr.bin"
+             "$$rootfs/usr/lib/syslinux/bios/mbr.bin"
+             "$$rootfs/usr/lib/extlinux/mbr.bin"
+             "$$rootfs/usr/lib/syslinux/mbr.bin")
+      for element in "${PATHS[@]}"
+      do
+        if [ -f "$element" ]; then
+          MBR_PATH="$element"
+          break
+        fi
+      done
+      if [ "$MBR_PATH" == "" ]; then
+        fail "unable to locate the extlinux mbr"
+      else
+        dd if="$MBR_PATH" of="$$nbd_device" bs=440  2>&1
+      fi
+  - exec_out: echo " sync..." ; sync
+  - write_out:
+    - $$rootfs/boot/extlinux/extlinux.conf
+    - |
+      default linux
+      timeout 1
+      
+      label linux
+      kernel ../`basename $$rootfs/boot/vmlinuz*`
+      append initrd=../`basename $$rootfs/boot/init*` root=UUID=`blkid -s UUID -o value $${nbd_device}p1` ro
+  - exec_out: echo " sync..." ; sync
+  - on_bootstrap_clean:
+    - exec_out: |
+        dir=$$rootfs/var/lib/os-prober/mount
+        test ! -d "$dir" || (umount -f -l "$dir" && rmdir "$dir")
+

data/templates/steps/bootstrap/prepare_chroot.yaml

@@ -0,0 +1,126 @@
+# create the appliance disk and initialize the checkpoint mechanism with Qemu
+# Network Block Device (NBD)
+
+- create_initial_image:
+  - check_cmd_out: qemu-img
+  - exec_out: mkdir -p checkpoints
+  - exec_out: |
+      if [ $$consider_checkpoint = true ]; then
+        if [ ! -e "$$image_disk" ] ; then
+          qemu-img create -f qcow2 checkpoints/base_$$image_disk $$image_size
+          # keep a link to the last checkpoint disk
+          ln -sf checkpoints/base_$$image_disk $$image_disk
+        fi
+      else
+        qemu-img create -f qcow2 $$image_disk $$image_size
+      fi
+
+- load_nbd_module:
+  - on_checkpoint: redo
+  - on_bootstrap_init:
+    - exec_out: |
+        lsmod | grep nbd >/dev/null \
+        || modprobe nbd max_part=63 \
+        || fail failed to load nbd module into kernel
+
+- attach_nbd_device:
+  # this microstep must be skipped because it is already done by the
+  # checkpoint restore function
+  - on_checkpoint: redo
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo Connecting $$image_disk to nbd device $$nbd_device
+  - exec_out: echo "sync" ; sync
+  - exec_out: qemu-nbd -d $$nbd_device
+  - exec_out: |
+      if [ $$consider_checkpoint = true ]; then
+        REAL_PATH="$(readlink $$image_disk)"
+      else
+        REAL_PATH="$$image_disk"
+      fi
+      qemu-nbd -c $$nbd_device $REAL_PATH -n || fail nbd device $$nbd_device is unavailable
+
+- detach_nbd_device:
+  # this miscrostep must be done again ensure that the nbd device is
+  # detached
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: echo "sync" ; sync
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: "pgrep qemu-nbd | xargs -I {} kill -9 {} || true"
+
+- partition_disk:
+  - check_cmd_out: parted
+  - exec_out: |
+      echo "Partitioning disk..."
+      parted $${nbd_device} mklabel msdos
+      parted $${nbd_device} mkpart primary 0% 100%
+      parted $${nbd_device} set 1 boot on
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p1 || fail cannot create / ext4
+
+- mount_rootfs:
+  - exec_out: mkdir -p $$rootfs
+  - exec_out:  "echo Mounting root partition... ;  mount $${nbd_device}p1 $$rootfs || fail cannot mount /"
+  - on_clean:
+    - exec_out: "echo try umount $$rootfs... ; mountpoint -q $$rootfs && umount -f -l $$rootfs || true"
+
+- copy_rootfs:
+  - check_cmd_out: rsync
+  - exec_out: echo "Copying rootfs to $$rootfs..."
+  - exec_out: rsync -aAX --exclude '/.kameleon_timestamp' $$rootfs_partial/* $$rootfs
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$rootfs/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # device; this may be used with UUID= as a more robust way to name devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${nbd_device}p1` /               $$filesystem_type    errors=remount-ro  0       1
+
+- mount_chroot:
+  - check_cmd_out: chroot
+  - exec_out: mount -o bind /dev  $$rootfs/dev
+  - exec_out: mount -o bind /dev/pts $$rootfs/dev/pts
+  - exec_out: mount -t proc /proc  $$rootfs/proc
+  - exec_out: mount -t sysfs /sys  $$rootfs/sys
+  - exec_out: test -f $$rootfs/etc/mtab || cat /proc/mounts > $$rootfs/etc/mtab
+
+- umount_chroot:
+  - on_clean:
+    - exec_out: echo try umount $$rootfs/sys... ; mountpoint -q $$rootfs/sys && umount -f -l $$rootfs/sys  || true
+    - exec_out: echo try umount $$rootfs/proc... ; mountpoint -q $$rootfs/proc && umount  -f -l $$rootfs/proc  || true
+    - exec_out: echo try umount $$rootfs/dev/pts... ; mountpoint -q $$rootfs/dev/pts && umount -f -l $$rootfs/dev/pts  || true
+    - exec_out: echo try umount $$rootfs/dev... ; mountpoint -q $$rootfs/dev && umount -f -l $$rootfs/dev  || true
+
+- configure_sshd:
+  - exec_out: echo -e  'y\n' | ssh-keygen -q -t dsa -f $$insecure_ssh_key -N ''
+  - exec_out: cat $$insecure_ssh_key
+  - exec_out: chroot $$rootfs mkdir -p /root/.ssh
+  - exec_out: |
+        cat $${insecure_ssh_key}.pub | tee -a $$rootfs/root/.ssh/authorized_keys
+  - write_out:
+    - $$ssh_config_file
+    - |
+      Host $$kameleon_recipe_name
+      HostName 127.0.0.1
+      Port $${qemu_ssh_port}
+      User root
+      IdentityFile $$insecure_ssh_key
+      UserKnownHostsFile /dev/null
+      StrictHostKeyChecking no
+      PasswordAuthentication no
+      IdentitiesOnly yes
+      LogLevel FATAL
+      ForwardAgent yes
+      ControlPath $(pwd)/master-%l-%r@%h:%p
+      ControlMaster auto
+      ControlPersist yes
+      Compression yes
+      Protocol 2