kameleon-builder 2.0.0 → 2.1.0

data/templates/steps/bootstrap/prepare_docker.yaml

@@ -1,4 +1,14 @@
 
+- check_docker:
+  - on_bootstrap_init:
+    - exec_out: DOCKER=${DOCKER:-docker}
+    - rescue:
+      - exec_out: command -V $DOCKER
+      - breakpoint: >
+          error: Docker executable no found. Make sure Docker
+          is installed and/or use the DOCKER variable to set Docker
+          executable.
+
 - clean_containers:
   - on_checkpoint: redo
   - on_export_clean:
@@ -19,15 +29,16 @@
 
 - configure_sshd:
   - on_checkpoint: redo
-  - exec_out: echo -e  'y\n' | ssh-keygen -q -t dsa -f $$insecure_ssh_key -N ''
-  - exec_out: chmod 600 $$insecure_ssh_key*
+  - exec_out: echo -e  'y\n' | ssh-keygen -q -t rsa -f $$insecure_ssh_key -N ''
+  - exec_out: cat $$insecure_ssh_key
   - exec_out: |
-      CID=$(docker run --dns $$dns -d -v "$(pwd):/tmp" $$image:latest \
-            /bin/bash -c "rm -fr /root/.ssh ;  \
-                          mkdir -p /root/.ssh ;  \
-                          ssh-keygen -q -t dsa -f /root/.ssh/id_dsa -N '' ; \
-                          cat /tmp/$${insecure_ssh_key}.pub > /root/.ssh/authorized_keys")
-  - exec_out: bash -c "exit $(docker wait $CID)"
+      CID="kameleon_container_$[($RANDOM % ($[50000 - 50] + 1)) + 50]"
+      cat $${insecure_ssh_key}.pub | \
+        docker run -i -a stdin --dns $$dns --name $CID $$image:latest \
+          /bin/bash -c "mkdir -p /root/.ssh ; \
+                        tee -a /root/.ssh/authorized_keys"
+  - on_export_init:
+    - exec_in: rm -rf /root/.ssh
   - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
   - exec_out: |
       docker images | grep $$image \

data/templates/steps/bootstrap/prepare_qemu.yaml

@@ -0,0 +1,47 @@
+- insecure_ssh_key: $$kameleon_cwd/insecure_ssh_key
+
+- mount_chroot:
+  - check_cmd_out: chroot
+  - exec_out: mount -o bind /dev  $$rootfs/dev
+  - exec_out: mount -o bind /dev/pts $$rootfs/dev/pts
+  - exec_out: mount -t proc /proc  $$rootfs/proc
+  - exec_out: mount -t sysfs /sys  $$rootfs/sys
+  - exec_out: test -f $$rootfs/etc/mtab || cat /proc/mounts > $$rootfs/etc/mtab
+
+- umount_chroot:
+  - on_bootstrap_clean:
+    - exec_out: echo try umount $$rootfs/sys... ; mountpoint -q $$rootfs/sys && umount -f -l $$rootfs/sys  || true
+    - exec_out: echo try umount $$rootfs/proc... ; mountpoint -q $$rootfs/proc && umount  -f -l $$rootfs/proc  || true
+    - exec_out: echo try umount $$rootfs/dev/pts... ; mountpoint -q $$rootfs/dev/pts && umount -f -l $$rootfs/dev/pts  || true
+    - exec_out: echo try umount $$rootfs/dev... ; mountpoint -q $$rootfs/dev && umount -f -l $$rootfs/dev  || true
+
+- configure_ssh_access:
+  - exec_out: echo -e  'y\n' | ssh-keygen -q -t dsa -f $$insecure_ssh_key -N ''
+  - exec_out: cat $$insecure_ssh_key
+  - exec_out: chroot $$rootfs mkdir -p /root/.ssh
+  - exec_out: |
+        cat $${insecure_ssh_key}.pub | tee -a $$rootfs/root/.ssh/authorized_keys
+        cat $${insecure_ssh_key}.pub > $$rootfs/root/.ssh/kameleon_insecure_public_key
+        chmod 600 $$rootfs/root/.ssh/authorized_keys
+        chmod 755 $$rootfs/root/.ssh
+
+- create_ssh_config:
+  - write_out:
+    - $$ssh_config_file
+    - |
+      Host $$kameleon_recipe_name
+      HostName 127.0.0.1
+      Port $${qemu_ssh_port}
+      User root
+      IdentityFile $$insecure_ssh_key
+      UserKnownHostsFile /dev/null
+      StrictHostKeyChecking no
+      PasswordAuthentication no
+      IdentitiesOnly yes
+      LogLevel FATAL
+      ForwardAgent yes
+      ControlPath $(pwd)/master-%l-%r@%h:%p
+      ControlMaster auto
+      ControlPersist yes
+      Compression yes
+      Protocol 2

data/templates/steps/bootstrap/start_chroot.yaml

@@ -7,7 +7,7 @@
   - exec_out: mount -t proc /proc  $$rootfs/proc
   - exec_out: mount -t sysfs /sys  $$rootfs/sys
   - exec_out: test -f $$rootfs/etc/mtab || cat /proc/mounts > $$rootfs/etc/mtab
-  - on_export_clean:
+  - on_setup_clean:
     - exec_out: echo try umount $$rootfs/sys... ; mountpoint -q $$rootfs/sys && umount -f -l $$rootfs/sys  || true
     - exec_out: echo try umount $$rootfs/proc... ; mountpoint -q $$rootfs/proc && umount  -f -l $$rootfs/proc  || true
     - exec_out: echo try umount $$rootfs/dev/pts... ; mountpoint -q $$rootfs/dev/pts && umount -f -l $$rootfs/dev/pts  || true
@@ -33,7 +33,7 @@
 
 - clean_cpuset:
   - on_checkpoint: redo
-  - on_export_clean:
+  - on_setup_clean:
     - exec_out: CPUSET_DIR="$$rootfs/$$cpuset_base_name"
     - exec_out: |
         if [ -f $CPUSET_DIR/$$cpuset_name/tasks ]; then
@@ -51,3 +51,12 @@
         fi
     - exec_out: echo try umount $CPUSET_DIR... ; mountpoint -q $CPUSET_DIR && umount -f -l $CPUSET_DIR || true
 
+- enable_checkpoint:
+  - on_checkpoint: redo
+  - on_setup_init:
+    - exec_out: touch $$kameleon_cwd/checkpoint_enabled
+
+- disable_checkpoint:
+  - on_checkpoint: redo
+  - on_setup_clean:
+    - exec_out: rm -f $$kameleon_cwd/checkpoint_enabled

data/templates/steps/bootstrap/start_docker.yaml

@@ -1,8 +1,8 @@
 - start_sshd:
   - on_checkpoint: redo
   - exec_out: |
-      CID=$(docker run -d -p 22 -i --dns $$dns -h $$hostname -privileged "$$image:latest" \
-            /bin/bash -c "mkdir /var/run/sshd ; /usr/sbin/sshd -D")
+      CID=$(docker run -d -p 127.0.0.1::22 -i --dns $$dns -h $$hostname --privileged "$$image:latest" \
+            /bin/bash -c "service ssh restart ; tail -f /dev/null")
   - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
   - exec_out: echo $CID > MAIN_CONTAINER_ID
   - on_export_clean:

data/templates/steps/bootstrap/start_qemu.yaml

@@ -0,0 +1,75 @@
+- start_qemu_vm:
+  - on_checkpoint: redo
+  - check_cmd_out: qemu-system-$$qemu_arch
+  - exec_out: echo "sync" ; sync
+  - exec_out: echo "+++ Starting kvm..."
+  - exec_out: |
+      if [ $$qemu_enable_kvm = true ]; then
+        ENABLE_KVM="-enable-kvm"
+      fi
+  - exec_out: |
+        if [ -f "vm_state_to_load.txt" ]
+        then
+            SAVED_STATE="$(cat vm_state_to_load.txt)"
+            LOADVM="-loadvm $SAVED_STATE"
+            rm -f vm_state_to_load.txt
+        fi
+  - exec_out: |
+        nohup qemu-system-$$qemu_arch  \
+          $ENABLE_KVM -no-reboot \
+          -drive file="$$image_disk",cache=unsafe,media=disk,if=virtio,id=drive0 \
+          -smp $${qemu_cpu} \
+          -m $$qemu_memory_size \
+          -rtc base=localtime \
+          -net nic,model=virtio -net user \
+          -redir tcp:$${qemu_ssh_port}::22 \
+          -monitor tcp::$$qemu_monitor_port,server,nowait \
+          -vnc :1 $LOADVM \
+        1>qemu.log 2>&1 & disown
+  - exec_out: |
+        echo -n "Waiting for SSH to become available"
+        until ssh-keyscan -4 -p $$qemu_ssh_port localhost  2>&1 | grep -e ssh-rsa -e ssh-dsa &> /dev/null
+        do
+          echo -n "."
+          sleep 1
+        done
+        echo " ~> OK"
+
+- shutdown_qemu_vm:
+  - on_checkpoint: redo
+  - on_setup_clean:
+    - exec_in: echo "Removing insecure ssh key..."
+    - exec_in: |
+        grep -f /root/.ssh/kameleon_insecure_public_key -v /root/.ssh/authorized_keys \
+            | cat > /root/.ssh/authorized_keys
+        rm -f /root/.ssh/kameleon_insecure_public_key
+    - exec_out: echo -n "Shutting down qemu virtual machine"
+    - exec_in: |
+        sleep 2 && shutdown -h now &
+    - exec_out: |
+        while nc -w 0 -z localhost $$qemu_monitor_port
+        do
+          sleep 1
+          echo -n "."
+        done
+        echo " ~> OK"
+
+- force_shutdown_qemu_vm:
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: |
+        if nc -w 0 -z localhost $$qemu_monitor_port 2>/dev/null
+        then
+          echo "Shutting down qemu virtual machine..."
+          echo "system_reset" | nc localhost $$qemu_monitor_port 1>/dev/null 2>&1
+        fi
+
+- enable_checkpoint:
+  - on_checkpoint: redo
+  - on_setup_init:
+    - exec_out: touch $$kameleon_cwd/checkpoint_enabled
+
+- disable_checkpoint:
+  - on_checkpoint: redo
+  - on_setup_clean:
+    - exec_out: rm -f $$kameleon_cwd/checkpoint_enabled

data/templates/steps/bootstrap/ubuntu/debootstrap.yaml

@@ -0,0 +1,27 @@
+# # Bootstrap
+- enable_cache: true
+
+- debootstrap:
+  - check_cmd_out: debootstrap
+  - exec_out: mkdir -p $$rootfs_download_path
+  - exec_out: |
+        if [ $$enable_cache = true ]; then
+            [[ ! -f "$$rootfs_download_path/.kameleon_timestamp"  ]] || CACHE_AVAILABLE=1
+        fi
+        [[ -n $CACHE_AVAILABLE  ]] || \
+          debootstrap --no-check-gpg --arch=$$arch --include="$$include_pkg" $$release $$rootfs_download_path $$repository \
+          && date +%s > $$rootfs_download_path/.kameleon_timestamp
+  - write_out:
+    - $$rootfs_download_path/etc/apt/sources.list
+    - |
+      deb $$repository $$release main restricted universe multiverse
+  - exec_out: chroot $$rootfs_download_path apt-get update
+  - exec_out: chroot $$rootfs_download_path apt-get install -y extlinux
+  - exec_out: cat /etc/resolv.conf > $$rootfs_download_path/etc/resolv.conf
+  - write_out:
+    - $$rootfs_download_path/etc/network/interfaces
+    - |
+      auto lo
+      iface lo inet loopback
+      auto eth0
+      iface eth0 inet dhcp

data/templates/steps/breakpoint.yaml

@@ -0,0 +1,2 @@
+- breakpoint:
+  - breakpoint: "Kameleon breakpoint"

data/templates/steps/checkpoints/qcow2.yaml

@@ -0,0 +1,38 @@
+create: |
+    if [ -f "$$kameleon_cwd/checkpoint_enabled" ] ; then
+    if [ -e "$$image_disk" ] ; then
+      parent_of_@microstep_id="$(readlink $$image_disk)"
+      pushd $$kameleon_cwd/checkpoints
+      rm -f @microstep_id_checkpoint.qcow2
+      qemu-img create -f qcow2 "@microstep_id_checkpoint.qcow2" \
+                      -o backing_file=$(basename "$parent_of_@microstep_id")
+      popd
+      ln -sf $$kameleon_cwd/checkpoints/@microstep_id_checkpoint.qcow2 $$image_disk
+
+      lsmod | grep nbd >/dev/null || modprobe nbd max_part=63
+
+      echo "sync" ; sync
+      qemu-nbd -d $$nbd_device
+      qemu-nbd -c $$nbd_device "$(readlink $$image_disk)"
+    fi
+    fi
+
+apply: |
+    # apply command use the backing file that represent the real state to restore
+    previous_id=$(qemu-img info "checkpoints/@microstep_id_checkpoint.qcow2" \
+                  | grep backing \
+                  | sed -e 's/.*checkpoints\/\(.*\)_checkpoint.qcow2\(.*\)/\1/p' \
+                  | uniq)
+    ln -sf "$$kameleon_cwd/checkpoints/${previous_id}_checkpoint.qcow2" "$$image_disk"
+
+clear: |
+    if [ -d "$$kameleon_cwd/checkpoints" ] ; then
+      ls $$kameleon_cwd/checkpoints/ | \
+        xargs -I {} bash -c "echo Removing $$kameleon_cwd/checkpoints/{} ; rm $$kameleon_cwd/checkpoints/{}"
+      rm -f $$image_disk
+    fi
+
+list: |
+    if [ -d "$$kameleon_cwd/checkpoints" ] ; then
+      ls $$kameleon_cwd/checkpoints/ | sed -e 's/\(.*\)_checkpoint.qcow2/\1/p' | uniq
+    fi

data/templates/steps/checkpoints/qemu.yaml

@@ -0,0 +1,39 @@
+create: |
+    if [ -f "$$kameleon_cwd/checkpoint_enabled" ] ; then
+      if nc -w 0 -z localhost $$qemu_monitor_port 2>/dev/null
+      then
+        echo "savevm @microstep_id" | nc localhost $$qemu_monitor_port
+      fi
+    fi
+
+apply: |
+    [[ ! -f "$$kameleon_cwd/vm_state_to_load.txt" ]] || rm -f $$kameleon_cwd/vm_state_to_load.txt
+    if [ -f "$$image_disk" ] ; then
+      _checkpoints=$(qemu-img snapshot -l $$image_disk | tail -n +3 | awk '{print $2}')
+      _checkpoints=($_checkpoints)
+      for checkpoint in "${_checkpoints[@]}"; do
+        if [ -f "$$kameleon_cwd/vm_state_to_load.txt" ]; then
+          echo "Removing old checkpoint $checkpoint"
+          qemu-img snapshot -d $checkpoint $$image_disk
+        fi
+        if [[ "$checkpoint" == "@microstep_id" ]]; then
+            echo "@microstep_id" > $$kameleon_cwd/vm_state_to_load.txt
+            qemu-img snapshot -a $checkpoint $$image_disk
+        fi
+    done
+    fi
+
+clear: |
+    if [ -f "$$image_disk" ] ; then
+      _checkpoints=$(qemu-img snapshot -l $$image_disk | tail -n +3 | awk '{print $2}')
+      _checkpoints=($_checkpoints)
+      for checkpoint in "${_checkpoints[@]}"; do
+        echo "Removing old checkpoint $checkpoint"
+        qemu-img snapshot -d $checkpoint $$image_disk
+    done
+    fi
+
+list: |
+    if [ -f "$$image_disk" ] ; then
+        qemu-img snapshot -l $$image_disk | tail -n +3 | awk '{print $2}'
+    fi

data/templates/steps/export/clean_appliance.yaml

@@ -1,3 +1,9 @@
 # # Clean Appliance
 - clean_udev:
-  - exec_in: rm -f /etc/udev/rules.d/*persistent-net.rules*
+  - on_setup_clean:
+    - exec_in: rm -f /etc/udev/rules.d/*persistent-net.rules*
+
+- clean_tmp:
+  - on_setup_clean:
+    - exec_in: rm -rf /tmp/* || true
+    - exec_in: rm -rf /var/tmp/* || true

data/templates/steps/export/compact_qcow_img.yaml

@@ -0,0 +1,12 @@
+
+
+- nullify_freespace:
+  - exec_out: |
+      echo "Nullify freespace..."
+      dd if=/dev/zero of=$$mountdir/bigemptyfile bs=1M 2>&1 >/dev/null || true
+      rm -f $$mountdir/bigemptyfile
+      echo "sync..." ; sync
+
+- compact_qcow_img:
+  - exec_out: qemu-img convert -c -O qcow2 $$filename compressed_$$filename
+  - exec_out: mv compressed_$$filename $$filename

data/templates/steps/export/save_appliance.yaml

@@ -0,0 +1,58 @@
+# Save Appliance
+
+- save_as_tgz:
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo "Exporting appliance to tar.gz..."
+  - exec_out: qemu-nbd -c $$nbd_device $$input -n || fail nbd device $$nbd_device is unavailable
+  - exec_out: mkdir -p $$rootfs
+  - exec_out: mount $${nbd_device}p1 $$rootfs || fail cannot mount /
+  - exec_out: cp $$rootfs/etc/fstab $$kameleon_cwd/fstab.orig
+  - write_out:
+    - $$rootfs/etc/fstab
+    - |
+      # UNCONFIGURED FSTAB FOR BASE SYSTEM
+  - exec_out: |
+      tar -zcf $$output.tar.gz --numeric-owner \
+      --exclude=tmp/* \
+      --exclude=dev/* \
+      --exclude=proc/* \
+      --exclude=sys/* \
+      --exclude=run/* \
+      --exclude=mnt/* \
+      --exclude=media/* \
+      --exclude=lost+found/* \
+      -C $$rootfs .
+  - exec_out: cat $$kameleon_cwd/fstab.orig > $$rootfs/etc/fstab
+  - exec_out: sync
+  - exec_out: echo "Saved tar.gz appliance to $$output.tar.gz"
+  - on_export_clean:
+    - exec_out: sync
+    - exec_out: "echo try umount $$rootfs... ; mountpoint -q $$rootfs && umount -f -l $$rootfs || true"
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: pgrep qemu-nbd | xargs -I {} kill -9 {} || true
+
+- save_as_raw:
+  - exec_out: qemu-img convert -O raw $$input $$output.raw
+  - exec_out: echo "Saved raw appliance to $$output.raw"
+
+- save_as_qcow2:
+  - exec_out: qemu-img convert -O qcow2 $$input $$output.qcow2
+  - exec_out: echo "Saved qcow2 appliance to $$output.qcow2"
+
+- save_as_qed:
+  - exec_out: qemu-img convert -O qed $$input $$output.qed
+  - exec_out: echo "Saved qed appliance to $$output.qed"
+
+- save_as_vmdk:
+  - exec_out: qemu-img convert -O vmdk $$input $$output.vmdk
+  - exec_out: echo "Saved vmdk appliance to $$output.vmdk"
+
+- save_as_vdi:
+  - exec_out: qemu-img convert -O vdi $$input $$output.vdi
+  - rescue:
+    - exec_out: |
+        echo "Compact the vdi disk"
+        VBoxManage modifyhd $$output.vdi --compact 2>&1
+    - exec_out: |
+        echo "Cannot compact the vdi disk : VBoxManage is missing."
+  - exec_out: echo "Saved vdi appliance to $$output.vdi"

data/templates/steps/export/save_appliance_from_g5k.yaml

@@ -0,0 +1,47 @@
+# # Save G5k Appliance
+
+- save_as_g5k:
+  - exec_out: echo "Using tgz g5k to Export the machine"
+  - exec_in: tgz-g5k $$filename.tar.gz
+  - exec_out: |
+      rsync -avz -e "ssh -F $$out_cwd/ssh_config" \
+        $$kameleon_recipe_name:/root/$$filename.tar.gz \
+        $$out_cwd/$$filename.tar.gz
+  - exec_out: echo "Saved tar.gz appliance to $$out_cwd/$$filename.tar.gz"
+  - exec_in: rm -f $$in_cwd/$$filename.tar.gz
+  - exec_out: echo "Getting the appropiate post install"
+  - pipe:
+    - exec_out: kaenv3 -p $$kenv | grep archive
+    - exec_in: cat > /tmp/postinstall
+
+  - write_in:
+    - $$in_cwd/$${kameleon_recipe_name}_img.yaml
+    - |
+      #
+      # Kameleon generated based on kadeploy description file
+      #
+      ---
+      name: $${kameleon_recipe_name}
+
+      version: 1
+
+      os: linux
+
+      image:
+        file: $$filename.tar.gz
+        kind: tar
+        compression: gzip
+
+      postinstalls:
+      `cat /tmp/postinstall`
+        compression: gzip
+        script: traitement.ash /rambin
+
+      boot:
+        kernel: /boot/`basename /boot/vmlinu*`
+        initrd: /boot/`basename /boot/init*`
+
+      filesystem: $$filesystem
+  - in2out:
+    - $$in_cwd/$${kameleon_recipe_name}_img.yaml
+    - $$out_cwd/$${kameleon_recipe_name}_img.yaml