kameleon-builder 2.0.0.dev

data/templates/steps/bootstrap/debian/debootstrap.yaml

@@ -0,0 +1,13 @@
+# # Bootstrap
+- include: >
+    ifupdown locales libui-dialog-perl dialog isc-dhcp-client netbase
+    net-tools iproute acpid openssh-server pciutils
+- debootstrap:
+  - check_cmd_out: debootstrap
+  - exec_out: mkdir -p $(dirname "$$rootfs_archive")
+  - exec_out: ROOTFS=$(dirname "$$rootfs_archive")/rootfs
+  - exec_out: mkdir -p $ROOTFS
+  - exec_out: test -f "$$rootfs_archive" || debootstrap --no-check-gpg --arch=$$arch --include="$$include" $$release $ROOTFS $$repository
+  - exec_out: test -f "$$rootfs_archive" || cat /etc/resolv.conf > $ROOTFS/etc/resolv.conf
+  - exec_out: test -f "$$rootfs_archive" || tar zcf "$$rootfs_archive" -C "$ROOTFS" --numeric-owner --one-file-system .
+  - exec_out: test -f "$$rootfs_archive" || rm -rf $ROOTFS

data/templates/steps/bootstrap/fedora/docker_bootstrap.yaml

@@ -0,0 +1,25 @@
+# # Bootstrap
+- repository: mattdm/fedora
+- rootfs: rootfs
+- include: >
+    yum initscripts passwd rsyslog vim-minimal dhclient chkconfig rootfiles openssh-server
+    policycoreutils fedora-release openssh-server net-tools nc traceroute
+- create_rootfs:
+  - exec_out: test -f "$$rootfs_archive" || mkdir -p $(dirname "$$rootfs_archive")
+  - exec_out: test -f "$$rootfs_archive" || echo "Pulling image from '$$repository'"
+  - exec_out: test -f "$$rootfs_archive" || docker pull -t f$$release $$repository  > /dev/null
+  - exec_out: |
+      echo "Installing core packages : $$include"
+      if [ ! -f "$$rootfs_archive" ]
+      then
+          BASE_CID=$(docker run --dns $$dns -d $$repository:f$${release} \
+                     bash -c "yum -y --nogpgcheck update ; \
+                              yum -y --nogpgcheck install $$include ; \
+                              /usr/bin/ssh-keygen -A")
+      fi
+  - exec_out: test -f "$$rootfs_archive" || bash -c "exit $(docker wait $BASE_CID)"
+  - exec_out: test -f "$$rootfs_archive" || docker export $BASE_CID > "$$rootfs_archive"
+  - on_bootstrap_clean:
+    - exec_out: test -f "$$rootfs_archive" || docker kill "$BASE_CID"
+    - exec_out: test -f "$$rootfs_archive" || docker rm "$BASE_CID"
+    - exec_out: test -f "$$rootfs_archive" || docker rmi $$repository

data/templates/steps/bootstrap/fedora/yum_bootstrap.yaml

@@ -0,0 +1,22 @@
+# # Bootstrap
+- include: >
+    yum initscripts passwd rsyslog vim-minimal dhclient chkconfig rootfiles
+    policycoreutils fedora-release openssh-server net-tools nc traceroute
+- bootstrap_fedora:
+  - check_cmd_out: yum
+  - check_cmd_out: curl
+  - exec_out: test -f "$$rootfs_archive" || mkdir -p $(dirname "$$rootfs_archive")
+  - exec_out: test -f "$$rootfs_archive" || ROOTFS=$(dirname "$$rootfs_archive")/rootfs
+  - exec_out: test -f "$$rootfs_archive" || mkdir -p $ROOTFS
+  - exec_out: test -f "$$rootfs_archive" || echo "Fetching release mirror"
+  - exec_out: test -f "$$rootfs_archive" || MIRROR_URL=$(curl -s -S -f "$$mirrorlist_url" | head -n2 | tail -n1)
+  - exec_out: test -f "$$rootfs_archive" || RELEASE_URL="$MIRROR_URL/Packages/f/fedora-release-$${release}-1.noarch.rpm"
+  - exec_out: test -f "$$rootfs_archive" || echo "Fetching from $RELEASE_URL"
+  - exec_out: test -f "$$rootfs_archive" || curl -sf "$RELEASE_URL" > "$(basename $RELEASE_URL)" 2> /dev/null
+  - exec_out: test -f "$$rootfs_archive" || mkdir -p $ROOTFS/var/lib/rpm
+  - exec_out: test -f "$$rootfs_archive" || rpm --root $ROOTFS --initdb
+  - exec_out: test -f "$$rootfs_archive" || rpm --root $ROOTFS -ivh "$(basename $RELEASE_URL)"
+  - exec_out: test -f "$$rootfs_archive" || yum --installroot $ROOTFS -y --nogpgcheck install $$include
+  - exec_out: test -f "$$rootfs_archive" || cat /etc/resolv.conf > $ROOTFS/etc/resolv.conf
+  - exec_out: test -f "$$rootfs_archive" || tar zcf "$$rootfs_archive" -C "$ROOTFS" --numeric-owner --one-file-system .
+  - exec_out: test -f "$$rootfs_archive" || rm -fr $ROOTFS

data/templates/steps/bootstrap/prepare_appliance_with_nbd.yaml

@@ -0,0 +1,93 @@
+# Prepare Appliance
+- nbd_device: /dev/nbd0
+
+- create_raw_image:
+  - check_cmd_out: qemu-img
+  - exec_out: mkdir -p checkpoints
+  - exec_out: |
+      if [ ! -e "$$container" ] ; then
+        qemu-img create -f qcow2 checkpoints/base_$$container $$image_size
+        ln -sf checkpoints/base_$$container $$container
+      fi
+
+- load_nbd_module:
+  - on_bootstrap_init:
+    - exec_out: |
+        lsmod | grep nbd >/dev/null \
+        || modprobe nbd max_part=63 \
+        || fail failed to load nbd module into kernel
+
+- attach_nbd_device:
+  - on_checkpoint: skip
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo Connecting $$container to nbd device $$nbd_device
+  - exec_out: qemu-nbd -c $$nbd_device "$(readlink $$container)" -n || fail nbd device $$container is unavailable
+
+
+- detach_nbd_device:
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: echo "sync" ; sync
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: "pgrep qemu-nbd | xargs -I {} kill -9 {} || true"
+
+- partition_disk:
+  - check_cmd_out: sfdisk
+  - exec_out: |
+      echo "Partitioning disk..."
+      sfdisk $${nbd_device} -q -D -uM --force --no-reread <<< '
+      ,200,83,*
+      ;
+      ' 2>&1 || fail cannot partition $$container
+  - exec_out: |
+      echo Creating boot partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p1 || fail cannot create /boot ext4
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p2 || fail cannot create / ext4
+
+- mount_mountdir:
+  - on_checkpoint: redo
+  - exec_out: mkdir -p $$mountdir
+  - exec_out:  "echo Mounting root partition... ;  mount $${nbd_device}p2 $$mountdir || fail cannot mount /"
+  - on_export_clean:
+    - exec_out: "echo try umount $$mountdir... ; mountpoint -q $$mountdir && umount -f -l $$mountdir || true"
+    - exec_out: "test -d $$mountdir && rmdir $$mountdir || true"
+  - exec_out: mkdir -p $$mountdir/boot
+  - exec_out:  "echo Mounting boot partition... ;  mount $${nbd_device}p1 $$mountdir/boot || fail cannot mount /boot"
+  - on_export_clean:
+    - exec_out: "echo try umount $$mountdir/boot... ; mountpoint -q $$mountdir/boot && umount -f -l $$mountdir/boot || true"
+    - exec_out: "test -d $$mountdir/boot && rmdir $$mountdir/boot || true"
+
+- copy_rootfs:
+  - exec_out: tar -xf $$rootfs_archive -C $$mountdir
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$mountdir/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # device; this may be used with UUID= as a more robust way to name devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${nbd_device}p2` /               $$filesystem_type    errors=remount-ro  0       1
+      UUID=`blkid -s UUID -o value $${nbd_device}p1` /boot           $$filesystem_type    sync               0       2
+
+
+- install_bootloader:
+  - on_export_init:
+    - exec_in: extlinux-install $$nbd_device 2>&1
+    - write_in:
+      - /boot/extlinux/extlinux.conf
+      - |
+        default linux
+        timeout 1
+        
+        label linux
+        kernel ../`basename /boot/vmlinuz*`
+        append initrd=../`basename /boot/init*` root=UUID=`blkid -s UUID -o value $${nbd_device}p2` ro
+    - exec_out: echo " sync..." ; sync

data/templates/steps/bootstrap/prepare_docker.yaml

@@ -0,0 +1,38 @@
+
+- clean_containers:
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: echo "Stopping trailing containers"
+    - exec_out: touch CONTAINERS_TO_CLEAN
+    - exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker kill {}
+    - exec_out: echo "Removing trailing containers"
+    - exec_out: cat CONTAINERS_TO_CLEAN | xargs -I {} docker rm {}
+    - exec_out: rm -f CONTAINERS_TO_CLEAN
+
+- import_rootfs:
+  - exec_out: |
+      docker images | grep -q $$image \
+      || (echo "Importing $$image to docker..." && cat "$$rootfs_archive"\
+          | docker import - $$image \
+          | xargs -I {} docker tag {} $$image:base)
+  - exec_out: docker tag $$image:base $$image:latest
+
+- configure_sshd:
+  - on_checkpoint: redo
+  - exec_out: echo -e  'y\n' | ssh-keygen -q -t dsa -f $$insecure_ssh_key -N ''
+  - exec_out: chmod 600 $$insecure_ssh_key*
+  - exec_out: |
+      CID=$(docker run --dns $$dns -d -v "$(pwd):/tmp" $$image:latest \
+            /bin/bash -c "rm -fr /root/.ssh ;  \
+                          mkdir -p /root/.ssh ;  \
+                          ssh-keygen -q -t dsa -f /root/.ssh/id_dsa -N '' ; \
+                          cat /tmp/$${insecure_ssh_key}.pub > /root/.ssh/authorized_keys")
+  - exec_out: bash -c "exit $(docker wait $CID)"
+  - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
+  - exec_out: |
+      docker images | grep $$image \
+                    | grep sshd \
+                    | awk '{print $3}' \
+                    | xargs -I {} docker rmi {}
+  - exec_out: docker commit $CID $$image:sshd > /dev/null
+  - exec_out: docker tag $$image:sshd $$image:latest

data/templates/steps/bootstrap/start_chroot.yaml

@@ -0,0 +1,53 @@
+# # Bootstrap
+- mount_chroot:
+  - on_checkpoint: redo
+  - check_cmd_out: chroot
+  - exec_out: mount -o bind /dev  $$rootfs/dev
+  - exec_out: mount -o bind /dev/pts $$rootfs/dev/pts
+  - exec_out: mount -t proc /proc  $$rootfs/proc
+  - exec_out: mount -t sysfs /sys  $$rootfs/sys
+  - exec_out: test -f $$rootfs/etc/mtab || cat /proc/mounts > $$rootfs/etc/mtab
+  - on_export_clean:
+    - exec_out: echo try umount $$rootfs/sys... ; mountpoint -q $$rootfs/sys && umount -f -l $$rootfs/sys  || true
+    - exec_out: echo try umount $$rootfs/proc... ; mountpoint -q $$rootfs/proc && umount  -f -l $$rootfs/proc  || true
+    - exec_out: echo try umount $$rootfs/dev/pts... ; mountpoint -q $$rootfs/dev/pts && umount -f -l $$rootfs/dev/pts  || true
+    - exec_out: echo try umount $$rootfs/dev... ; mountpoint -q $$rootfs/dev && umount -f -l $$rootfs/dev  || true
+
+- cpuset_base_name: /dev/cpuset
+- cpuset_name: kameleon
+
+- init_cpuset:
+  - on_checkpoint: redo
+  - on_setup_init:
+    - exec_in: |
+        if [ ! -f $$cpuset_base_name/cpus ]; then
+          mkdir -p /dev/cpuset
+          mount -t cgroup -o cpuset none /dev/cpuset
+        fi
+        if [ ! -d $$cpuset_base_name/$$cpuset_name ]; then
+          mkdir $$cpuset_base_name/$$cpuset_name
+        fi
+    - exec_in: echo 0 > $$cpuset_base_name/$$cpuset_name/cpuset.cpus
+    - exec_in: echo 0 > $$cpuset_base_name/$$cpuset_name/cpuset.mems
+    - exec_in: echo $$ > $$cpuset_base_name/$$cpuset_name/tasks
+
+- clean_cpuset:
+  - on_checkpoint: redo
+  - on_export_clean:
+    - exec_out: CPUSET_DIR="$$rootfs/$$cpuset_base_name"
+    - exec_out: |
+        if [ -f $CPUSET_DIR/$$cpuset_name/tasks ]; then
+          for pid in `cat $CPUSET_DIR/$$cpuset_name/tasks`; do
+            kill -9 $pid
+          done
+          echo "Waiting for all processes of the in context to terminate..."
+          while [ ! -z "$(cat $CPUSET_DIR/$$cpuset_name/tasks)" ]; do
+            sleep .2
+          done
+          sleep .2
+          echo > $CPUSET_DIR/$$cpuset_name/cpuset.cpus
+          echo > $CPUSET_DIR/$$cpuset_name/cpuset.mems
+          rmdir $CPUSET_DIR/$$cpuset_name
+        fi
+    - exec_out: echo try umount $CPUSET_DIR... ; mountpoint -q $CPUSET_DIR && umount -f -l $CPUSET_DIR || true
+

data/templates/steps/bootstrap/start_docker.yaml

@@ -0,0 +1,12 @@
+- start_sshd:
+  - on_checkpoint: redo
+  - exec_out: |
+      CID=$(docker run -d -p 22 -i --dns $$dns -h $$hostname -privileged "$$image:latest" \
+            /bin/bash -c "mkdir /var/run/sshd ; /usr/sbin/sshd -D")
+  - exec_out: echo "$CID" >> CONTAINERS_TO_CLEAN
+  - exec_out: echo $CID > MAIN_CONTAINER_ID
+  - on_export_clean:
+    - exec_out: rm -f MAIN_CONTAINER_ID
+  - exec_out: echo $(docker port $CID 22) | cut -d':' -f2 > MAIN_CONTAINER_PORT
+  - on_export_clean:
+    - exec_out: rm -f MAIN_CONTAINER_PORT

data/templates/steps/export/build_appliance_from_docker.yaml

@@ -0,0 +1,105 @@
+- nbd_device: /dev/nbd1
+
+- export_targz:
+  - on_export_init:
+    - exec_out: echo "Exporting the rootfs..."
+    - exec_out: docker export $(cat MAIN_CONTAINER_ID) > $$filename.tar.gz
+
+- create_raw_image:
+  - check_cmd_out: qemu-img
+  - exec_out: bash -c "qemu-img create -f $$format $$filename.$$format $$image_size"
+
+- load_nbd_module:
+  - on_bootstrap_init:
+    - exec_out: |
+        lsmod | grep nbd >/dev/null \
+        || modprobe nbd max_part=63 \
+        || fail failed to load nbd module into kernel
+
+- attach_nbd_device:
+  - on_checkpoint: redo
+  - check_cmd_out: qemu-nbd
+  - exec_out: echo Connecting $$filename.$$format to nbd device $$nbd_device
+  - exec_out: qemu-nbd -c $$nbd_device $$filename.$$format || fail nbd device $$filename.$$format is unavailable
+  - on_export_clean:
+    - exec_out: echo "sync" ; sync
+    - exec_out: qemu-nbd -d $$nbd_device
+    - exec_out: "pgrep qemu-nbd | xargs -I {} kill -9 {} || true"
+
+- partition_disk:
+  - check_cmd_out: sfdisk
+  - exec_out: |
+      echo "Partitioning disk..."
+      sfdisk $${nbd_device} -q -D -uM --force --no-reread <<< '
+      ,200,83,*
+      ;
+      ' 2>&1 || fail cannot partition $$filename.$$format
+  - exec_out: |
+      echo Creating boot partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p1 || fail cannot create /boot ext4
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${nbd_device}p2 || fail cannot create / ext4
+
+- mount_mountdir:
+  - on_checkpoint: redo
+  - exec_out: mkdir -p $$mountdir
+  - exec_out:  "echo Mounting root partition... ;  mount $${nbd_device}p2 $$mountdir || fail cannot mount /"
+  - on_export_clean:
+    - exec_out: "echo try umount $$mountdir... ; mountpoint -q $$mountdir && umount -f -l $$mountdir || true"
+    - exec_out: "test -d $$mountdir && rmdir $$mountdir || true"
+  - exec_out: mkdir -p $$mountdir/boot
+  - exec_out:  "echo Mounting boot partition... ;  mount $${nbd_device}p1 $$mountdir/boot || fail cannot mount /boot"
+  - on_export_clean:
+    - exec_out: "echo try umount $$mountdir/boot... ; mountpoint -q $$mountdir/boot && umount -f -l $$mountdir/boot || true"
+    - exec_out: "test -d $$mountdir/boot && rmdir $$mountdir/boot || true"
+
+- copy_rootfs:
+  - exec_out: echo "Copying the rootfs to $$mountdir..."
+  - exec_out: tar -xf $$filename.tar.gz -C $$mountdir
+  - exec_out: rm -f $$mountdir/.dockerinit
+  - exec_out: rm -f $$mountdir/.dockerenv
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$mountdir/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # device; this may be used with UUID= as a more robust way to name devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${nbd_device}p2` /               $$filesystem_type    errors=remount-ro  0       1
+      UUID=`blkid -s UUID -o value $${nbd_device}p1` /boot           $$filesystem_type    sync               0       2
+  - exec_out: echo "sync..." ; sync
+
+- mount_chroot:
+  - on_checkpoint: redo
+  - check_cmd_out: chroot
+  - exec_out: mount -o bind /dev  $$mountdir/dev
+  - exec_out: mount -o bind /dev/pts $$mountdir/dev/pts
+  - exec_out: mount -t proc /proc  $$mountdir/proc
+  - exec_out: mount -t sysfs /sys  $$mountdir/sys
+  - exec_out: test -f $$mountdir/etc/mtab || cat /proc/mounts > $$mountdir/etc/mtab
+  - on_export_clean:
+    - exec_out: echo try umount $$mountdir/sys... ; mountpoint -q $$mountdir/sys && umount -f -l $$mountdir/sys  || true
+    - exec_out: echo try umount $$mountdir/proc... ; mountpoint -q $$mountdir/proc && umount  -f -l $$mountdir/proc  || true
+    - exec_out: echo try umount $$mountdir/dev/pts... ; mountpoint -q $$mountdir/dev/pts && umount -f -l $$mountdir/dev/pts  || true
+    - exec_out: echo try umount $$mountdir/dev... ; mountpoint -q $$mountdir/dev && umount -f -l $$mountdir/dev  || true
+
+- install_bootloader:
+  - check_cmd_out: extlinux-install
+  - exec_out: chroot $$mountdir extlinux-install $$nbd_device 2>&1
+  - write_out:
+    - $$mountdir/boot/extlinux/extlinux.conf
+    - |
+      default linux
+      timeout 1
+      
+      label linux
+      kernel ../`basename $$mountdir/boot/vmlinuz*`
+      append initrd=../`basename $$mountdir/boot/init*` root=UUID=`blkid -s UUID -o value $${nbd_device}p2` ro
+  - exec_out: echo "sync..." ; sync

data/templates/steps/export/clean_appliance.yaml

@@ -0,0 +1,3 @@
+# # Clean Appliance
+- clean_udev:
+  - exec_in: rm -f /etc/udev/rules.d/*persistent-net.rules*

data/templates/steps/export/save_appliance_from_nbd.yaml

@@ -0,0 +1,54 @@
+# # Save Chroot Appliance
+- save_as_tgz:
+  - exec_out: echo "sync..." ; sync
+  - in2out:
+    - /etc/fstab
+    - ./fstab.bak
+  - on_clean:
+    - exec_out: rm -f ./fstab.bak
+  - write_in:
+    - /etc/fstab
+    - |
+      # UNCONFIGURED FSTAB FOR BASE SYSTEM
+  - pipe:
+    - exec_in: |
+        tar -zcf - --numeric-owner \
+        --exclude=tmp/* \
+        --exclude=dev/* \
+        --exclude=proc/* \
+        --exclude=sys/* \
+        --exclude=run/* \
+        --exclude=mnt/* \
+        --exclude=media/* \
+        --exclude=lost+found/* \
+        -C / .
+    - exec_out: cat > ./$$filename.tar.gz
+  - out2in:
+    - ./fstab.bak
+    - /etc/fstab
+  - exec_out: |
+      echo "Saved tar.gz appliance to $(pwd)/$$filename.tar.gz"
+
+- save_as_raw:
+  - exec_out: echo "sync..." ; sync
+  - exec_out: |
+      qemu-img convert -O raw $(readlink $$container) $$filename.raw
+  - exec_out: echo "Saved raw appliance to $(pwd)/$$filename.raw"
+
+- save_as_qcow2:
+  - exec_out: echo "sync..." ; sync
+  - exec_out: |
+      qemu-img convert -O qcow2 $(readlink $$container) $$filename.qcow2
+  - exec_out: echo "Saved qcow2 appliance to $(pwd)/$$filename.qcow2"
+
+- save_as_vmdk:
+  - exec_out: echo "sync..." ; sync
+  - exec_out: |
+      qemu-img convert -O vmdk $(readlink $$container) $$filename.vmdk
+  - exec_out: echo "Saved vmdk appliance to $(pwd)/$$filename.vmdk"
+
+- save_as_vdi:
+  - exec_out: echo "sync..." ; sync
+  - exec_out: |
+      qemu-img convert -O vdi $(readlink $$container) $$filename.vdi
+  - exec_out: echo "Saved vdi appliance to $(pwd)/$$filename.vdi"

data/templates/steps/setup/create_user.yaml

@@ -0,0 +1,12 @@
+# # Create User
+- create_group:
+  - exec_in: groupadd $$group
+- add_user:
+  # use the create_passwd.py script to generate the crypted password
+  - exec_in: useradd -m -g $$group $$name -s /bin/bash
+  - exec_in: echo -n '$$name:$$password' | chpasswd
+- add_group_to_sudoers:
+  - append_in:
+    - /etc/sudoers
+    - |
+      %admin ALL=(ALL:ALL) ALL