kameleon-builder 2.0.0 → 2.1.0

data/templates/{aliases → steps/aliases}/defaults.yaml

@@ -1,31 +1,44 @@
 write_local:
   - exec_local: |
       mkdir -p $(dirname @1);
-      cat >@1 <<<"@2"
+      cat >@1 <<EOF
+      @2
+      EOF
 
 write_in:
   - exec_in: |
       mkdir -p $(dirname @1);
-      cat >@1 <<<"@2"
+      cat >@1 <<EOF
+      @2
+      EOF
 
 write_out:
   - exec_out: |
       mkdir -p $(dirname @1);
-      cat >@1 <<<"@2"
+      cat >@1 <<EOF
+      @2
+      EOF
 
 append_local:
   - exec_local: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<<"@2"
+      cat >>@1 <<EOF
+      @2
+      EOF
+
 append_in:
   - exec_in: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<<"@2"
+      cat >>@1 <<EOF
+      @2
+      EOF
 
 append_out:
   - exec_out: |
       mkdir -p $(dirname @1);
-      cat >>@1 <<<"@2"
+      cat >@1 <<EOF
+      @2
+      EOF
 
 local2out:
   - exec_out: |
@@ -66,18 +79,30 @@ in2out:
 
 check_cmd_out:
   - on_bootstrap_init:
-    - rescue: 
+    - rescue:
       - exec_out: command -V @1 2> /dev/null
-      - breakpoint: "@1 is missing"
+      - breakpoint: "@1 is missing from out_context"
 
 check_cmd_local:
   - on_bootstrap_init:
-    - rescue: 
+    - rescue:
       - exec_local: command -V @1 2> /dev/null
-      - breakpoint: "@1 is missing"
+      - breakpoint: "@1 is missing from local_context"
 
 check_cmd_in:
   - on_bootstrap_init:
-    - rescue: 
+    - rescue:
       - exec_in: command -V @1 2> /dev/null
-      - breakpoint: "@1 is missing"
+      - breakpoint: "@1 is missing from in_context"
+
+umount_out:
+  - exec_out: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
+
+umount_local:
+  - exec_local: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true
+
+umount_in:
+  - exec_in: |
+      echo "try umount @1..." ; mountpoint -q "@1" && umount -f -l "@1" || true

data/templates/steps/bootstrap/archlinux/arch_bootstrap.yaml

@@ -0,0 +1,219 @@
+# Arch-bootstrap
+#
+# Call the arch-bootstrap.sh script to bootstrap an archlinux distro from
+# any GNU system
+
+# You can choose any group from https://www.archlinux.org/groups/
+# It's recommended to use 'base' (default) or 'base-devel' (with dev tools)
+
+- pkg_group: base
+
+- enable_cache: true
+
+- import_script:
+  - exec_out: |
+      cat > $$kameleon_cwd/arch-bootstrap.sh <<'EOF'
+      #!/bin/bash
+      #
+      # arch-bootstrap: Bootstrap a base Arch Linux system using any GNU distribution.
+      #
+      # Dependencies: bash >= 4, coreutils, wget, sed, gawk, tar, gzip, chroot, xz.
+      # Project: https://github.com/tokland/arch-bootstrap
+      #
+      # Install:
+      #
+      #   # install -m 755 arch-bootstrap.sh /usr/local/bin/arch-bootstrap
+      #
+      # Some examples:
+      #
+      #   # arch-bootstrap destination
+      #   # arch-bootstrap -a x86_64 -r ftp://ftp.archlinux.org destination-x86_64 
+      #
+      # And then you can chroot to the destination directory (root/root):
+      #
+      #   # chroot destination
+      
+      set -e -u -o pipefail
+      
+      # Packages needed by pacman (see get-pacman-dependencies.sh)
+      PACMAN_PACKAGES=(
+        acl archlinux-keyring attr bzip2 curl expat glibc gpgme libarchive
+        libassuan libgpg-error libssh2 lzo2 openssl pacman pacman-mirrorlist xz
+        zlib krb5 e2fsprogs keyutils openssh
+      )
+      BASIC_PACKAGES=(${PACMAN_PACKAGES[*]} filesystem)
+      EXTRA_PACKAGES=(
+        $$pkg_group haveged syslinux
+      )
+      DEFAULT_REPO_URL="http://mirrors.kernel.org/archlinux"
+      DEFAULT_ARCH=`uname -m`
+      
+      # Output to standard error
+      stderr() { echo "$@" >&2; }
+      
+      # Output debug message to standard error
+      debug() { echo "--- $@"; }
+      
+      # Extract href attribute from HTML link
+      extract_href() { sed -n '/<a / s/^.*<a [^>]*href="\([^\"]*\)".*$/\1/p'; }
+      
+      # Simple wrapper around wget
+      fetch() { wget -c --passive-ftp --quiet "$@"; }
+      
+      # Extract FILEPATH gz/xz archive to DEST directory
+      uncompress() {
+        local FILEPATH=$1 DEST=$2
+        
+        case "$FILEPATH" in
+          *.gz) tar xzf "$FILEPATH" -C "$DEST";;
+          *.xz) xz -dc "$FILEPATH" | tar x -C "$DEST";;
+          *) debug "Error: unknown package format: $FILEPATH"
+             return 1;;
+        esac
+      }  
+      
+      ###
+      
+      configure_pacman() {
+        local DEST=$1 ARCH=$2
+        debug "configure DNS and pacman"
+        cp "/etc/resolv.conf" "$DEST/etc/resolv.conf"
+        echo "Server = $REPO_URL/\$repo/os/$ARCH" >> "$DEST/etc/pacman.d/mirrorlist"
+      }
+      
+      configure_minimal_system() {
+        local DEST=$1
+        
+        mkdir -p "$DEST/dev"
+        echo "root:x:0:0:root:/root:/bin/bash" > "$DEST/etc/passwd" 
+        echo 'root:$1$GT9AUpJe$oXANVIjIzcnmOpY07iaGi/:14657::::::' > "$DEST/etc/shadow"
+        touch "$DEST/etc/group"
+        echo "bootstrap" > "$DEST/etc/hostname"
+        
+        test -e "$DEST/etc/mtab" || echo "rootfs / rootfs rw 0 0" > "$DEST/etc/mtab"
+        test -e "$DEST/dev/null" || mknod "$DEST/dev/null" c 1 3
+        test -e "$DEST/dev/random" || mknod -m 0644 "$DEST/dev/random" c 1 8
+        test -e "$DEST/dev/urandom" || mknod -m 0644 "$DEST/dev/urandom" c 1 9
+        
+        sed -i "s/^[[:space:]]*\(CheckSpace\)/# \1/" "$DEST/etc/pacman.conf"
+        sed -i "s/^[[:space:]]*SigLevel[[:space:]]*=.*$/SigLevel = Never/" "$DEST/etc/pacman.conf"
+      }
+      
+      fetch_packages_list() {
+        local REPO=$1 
+        
+        debug "fetch packages list: $REPO/"
+        # Force trailing '/' needed by FTP servers.
+        fetch -O - "$REPO/" | extract_href | awk -F"/" '{print $NF}' | sort -rn ||
+          { debug "Error: cannot fetch packages list: $REPO"; return 1; }
+      }
+      
+      install_pacman_packages() {
+        local BASIC_PACKAGES=$1 DEST=$2 LIST=$3 PACKDIR=$4
+        debug "pacman package and dependencies: $BASIC_PACKAGES"
+        
+        for PACKAGE in $BASIC_PACKAGES; do
+          local FILE=$(echo "$LIST" | grep -m1 "^$PACKAGE-[[:digit:]].*\(\.gz\|\.xz\)$")
+          test "$FILE" || { debug "Error: cannot find package: $PACKAGE"; return 1; }
+          local FILEPATH="$PACKDIR/$FILE"
+          
+          debug "download package: $REPO/$FILE"
+          fetch -O "$FILEPATH" "$REPO/$FILE"
+          debug "uncompress package: $FILEPATH"
+          uncompress "$FILEPATH" "$DEST"
+        done
+      }
+      
+      install_packages() {
+        local ARCH=$1 DEST=$2 PACKAGES=$3
+        debug "install packages: $PACKAGES"
+        mount -t proc /proc $DEST/proc
+        mount /dev $DEST/dev -o bind
+        LC_ALL=C chroot "$DEST" /usr/bin/pacman \
+          --noconfirm --arch $ARCH -Sy --force $PACKAGES
+        sync
+        umount $DEST/{proc,dev}
+      }
+      
+      show_usage() {
+        stderr "show_usage: $(basename "$0") [-a i686|x86_64] [-r REPO_URL] DIRECTORY"
+      }
+      
+      main() {
+        # Process arguments and options
+        test $# -eq 0 && set -- "-h"
+        local ARCH=$DEFAULT_ARCH
+        local REPO_URL=$DEFAULT_REPO_URL
+        
+        while getopts "a:r:h" ARG; do
+          case "$ARG" in
+            a) ARCH=$OPTARG;;
+            r) REPO_URL=$OPTARG;;
+            *) show_usage; return 1;;
+          esac
+        done
+        shift $(($OPTIND-1))
+        test $# -eq 1 || { show_usage; return 1; }
+        
+        local DEST=$1
+        local REPO="${REPO_URL%/}/core/os/$ARCH"
+        local PACKDIR=$(mktemp -d)
+        trap "rm -rf '$PACKDIR'" KILL TERM EXIT
+        debug "destination directory: $DEST"
+        debug "core repository: $REPO"
+        debug "temporary directory: $PACKDIR"
+        
+        # Fetch packages, install and do a minimal system configuration
+        mkdir -p "$DEST"
+        local LIST=$(fetch_packages_list $REPO)
+        install_pacman_packages "${BASIC_PACKAGES[*]}" "$DEST" "$LIST" "$PACKDIR"
+        configure_pacman "$DEST" "$ARCH"
+        configure_minimal_system "$DEST"
+        install_packages "$ARCH" "$DEST" "${BASIC_PACKAGES[*]} ${EXTRA_PACKAGES[*]}"
+        configure_pacman "$DEST" "$ARCH" # Pacman must be re-configured
+        rm -rf "$PACKDIR"
+        
+        debug "done"
+      }
+      
+      main "$@"
+      EOF
+
+- call_arch_boostrap:
+  - exec_out: |
+        if [ $$enable_cache = true ]; then
+            [[ ! -f "$$rootfs_download_path/.kameleon_timestamp"  ]] || CACHE_AVAILABLE=1
+        fi
+        [[ -n $CACHE_AVAILABLE  ]] || \
+          bash $$kameleon_cwd/arch-bootstrap.sh -a $$arch -r $$mirror $$rootfs_download_path \
+            && date +%s > $$rootfs_download_path/.kameleon_timestamp
+  - on_clean:
+    - umount_out: $$rootfs_download_path/proc
+    - umount_out: $$rootfs_download_path/dev
+
+- enable_sshd_service:
+  - exec_out: |
+      chroot $$rootfs_download_path \
+        ln -sf \
+          '/usr/lib/systemd/system/sshd.service' \
+          '/etc/systemd/system/multi-user.target.wants/sshd.service'
+
+- enable_dhcp_service:
+  - exec_out: |
+      chroot $$rootfs_download_path \
+        ln -sf \
+          '/usr/lib/systemd/system/dhcpcd.service' \
+          '/etc/systemd/system/multi-user.target.wants/dhcpcd.service'
+
+- add_nobody_user:  ## To allow sshd to run correctly
+  - exec_out: echo "nobody:x:99:99:nobody:/:/bin/false" >> $$rootfs_download_path/etc/passwd
+  - exec_out: echo "nobody:x:99:" >> $$rootfs_download_path/etc/group
+  - exec_out: echo "nobody:x:14871::::::" >> $$rootfs_download_path/etc/shadow
+
+- configure_pacman:
+  - on_setup_init:
+    # init pacman keyring
+    - exec_in: haveged -w 1024
+    - exec_in: pacman-key --init
+    - exec_in: pkill haveged
+    - exec_in: pacman-key --populate archlinux

data/templates/steps/bootstrap/archlinux/install_bootloader.yaml

@@ -0,0 +1,46 @@
+# Install extlinux bootloader
+
+- install_bootloader:
+  - rescue:
+    - exec_out: LC_ALL=POSIX chroot $$rootfs bash -c "command -V extlinux 2> /dev/null"
+    - breakpoint: "extlinux is missing"
+  - exec_out: cp -r $$rootfs/usr/lib/syslinux/bios/*.c32 $$rootfs/boot/syslinux/
+  - exec_out: LC_ALL=POSIX chroot $$rootfs extlinux --install /boot/syslinux 2>&1
+  - exec_out: |
+      MBR_PATH=
+      PATHS=("$$rootfs/usr/share/syslinux/mbr.bin"
+             "$$rootfs/usr/lib/bios/syslinux/mbr.bin"
+             "$$rootfs/usr/lib/syslinux/bios/mbr.bin"
+             "$$rootfs/usr/lib/extlinux/mbr.bin"
+             "$$rootfs/usr/lib/syslinux/mbr.bin")
+      for element in "${PATHS[@]}"
+      do
+        if [ -f "$element" ]; then
+          MBR_PATH="$element"
+          break
+        fi
+      done
+      if [ "$MBR_PATH" == "" ]; then
+        fail "unable to locate the extlinux mbr"
+      else
+        dd if="$MBR_PATH" of="$$nbd_device" bs=440  2>&1
+      fi
+  - exec_out: echo " sync..." ; sync
+  - write_out:
+    - $$rootfs/boot/syslinux/syslinux.cfg
+    - |
+      default kameleon
+      timeout 5
+      
+      label kameleon
+      kernel ../`basename $$rootfs/boot/vmlinuz*`
+      initrd ../`basename $$rootfs/boot/init*`
+      # the net.ifnames option is disable to get default net interface
+      # names according to this page:
+      # https://wiki.archlinux.org/index.php/Network_Configuration#Device_names
+      append root=UUID=`blkid -s UUID -o value $${nbd_device}p1` rw net.ifnames=0
+  - exec_out: echo " sync..." ; sync
+  - on_bootstrap_clean:
+    - exec_out: |
+        dir=$$rootfs/var/lib/os-prober/mount
+        test ! -d "$dir" || (umount -f -l "$dir" && rmdir "$dir")

data/templates/steps/bootstrap/archlinux/populate_disk.yaml

@@ -0,0 +1,39 @@
+# Populate disk
+#
+# This step is partionning and populating the disk
+
+- partition_disk:
+  - check_cmd_out: parted
+  - exec_out: |
+      echo "Partitioning disk..."
+      parted $${device} mklabel msdos
+      parted $${device} mkpart primary 0% 100%
+      parted $${device} set 1 boot on
+  - exec_out: |
+      echo Creating root partition...
+      mkfs.$$filesystem_type -q $${device}p1 || fail cannot create / ext4
+
+- mount_mountdir:
+  - on_checkpoint: redo
+  - exec_out: mkdir -p $$mountdir
+  - exec_out:  "echo Mounting root partition... ;  mount $${device}p1 $$mountdir || fail cannot mount /"
+  - on_export_clean:
+    - exec_out: "echo try umount $$mountdir... ; mountpoint -q $$mountdir && umount -f -l $$mountdir || true"
+
+- copy_rootfs:
+  - check_cmd_out: rsync
+  - exec_out: rsync -aAX $$rootfs_cache/* $$rootfs
+  - exec_out: echo "sync..." ; sync
+
+- create_fstab:
+  - write_out:
+    - $$mountdir/etc/fstab
+    - |
+      # /etc/fstab: static file system information.
+      #
+      # Use 'blkid' to print the universally unique identifier for a
+      # device; this may be used with UUID= as a more robust way to name devices
+      # that works even if disks are added and removed. See fstab(5).
+      #
+      # <file system> <mount point>   <type>  <options>       <dump>  <pass>
+      UUID=`blkid -s UUID -o value $${device}p1` /               $$filesystem_type    errors=remount-ro  0       1

data/templates/steps/bootstrap/debian/debootstrap.yaml

@@ -1,13 +1,21 @@
 # # Bootstrap
-- include: >
-    ifupdown locales libui-dialog-perl dialog isc-dhcp-client netbase
-    net-tools iproute acpid openssh-server pciutils
+- enable_cache: true
+
 - debootstrap:
   - check_cmd_out: debootstrap
-  - exec_out: mkdir -p $(dirname "$$rootfs_archive")
-  - exec_out: ROOTFS=$(dirname "$$rootfs_archive")/rootfs
-  - exec_out: mkdir -p $ROOTFS
-  - exec_out: test -f "$$rootfs_archive" || debootstrap --no-check-gpg --arch=$$arch --include="$$include" $$release $ROOTFS $$repository
-  - exec_out: test -f "$$rootfs_archive" || cat /etc/resolv.conf > $ROOTFS/etc/resolv.conf
-  - exec_out: test -f "$$rootfs_archive" || tar zcf "$$rootfs_archive" -C "$ROOTFS" --numeric-owner --one-file-system .
-  - exec_out: test -f "$$rootfs_archive" || rm -rf $ROOTFS
+  - exec_out: mkdir -p $$rootfs_download_path
+  - exec_out: |
+        if [ $$enable_cache = true ]; then
+            [[ ! -f "$$rootfs_download_path/.kameleon_timestamp"  ]] || CACHE_AVAILABLE=1
+        fi
+        [[ -n $CACHE_AVAILABLE  ]] || \
+          debootstrap --no-check-gpg --arch=$$arch --include="$$include_pkg" $$release $$rootfs_download_path $$repository \
+          && date +%s > $$rootfs_download_path/.kameleon_timestamp
+  - exec_out: cat /etc/resolv.conf > $$rootfs_download_path/etc/resolv.conf
+  - write_out:
+    - $$rootfs_download_path/etc/network/interfaces
+    - |
+      auto lo
+      iface lo inet loopback
+      auto eth0
+      iface eth0 inet dhcp

data/templates/steps/bootstrap/debian/debootstrap_arm.yaml

@@ -0,0 +1,31 @@
+# # Bootstrap
+- enable_cache: true
+
+- debootstrap:
+  - check_cmd_out: debootstrap
+  - check_cmd_out: qemu-arm-static
+  - exec_out: |
+        if [ $$enable_cache = true ]; then
+            [[ ! -f "$$rootfs_download_path/.kameleon_timestamp"  ]] || CACHE_AVAILABLE=1
+        fi
+        if [ -n $CACHE_AVAILABLE  ]; then
+          mkdir -p $$rootfs_download_path
+          debootstrap --no-check-gpg --foreign --variant=minbase --arch=$$arch --include="$$include_pkg" $$release $$rootfs_download_path $$repository
+          QEMU_USER_PATH=$(which qemu-arm-static)
+          cp $QEMU_USER_PATH $$rootfs_download_path/usr/bin/qemu-arm-static
+          chroot $$rootfs_download_path /usr/bin/qemu-arm-static -cpu cortex-a9 /bin/sh /debootstrap/debootstrap --second-stage
+          echo "deb $$repository wheezy main contrib non-free" > $$rootfs_download_path/etc/apt/sources.list
+          chroot $$rootfs_download_path /usr/bin/qemu-arm-static -cpu cortex-a9 /bin/sh apt-get update
+          chroot $$rootfs_download_path /usr/bin/qemu-arm-static -cpu cortex-a9 /bin/sh apt-get install -y --force-yes vim
+          date +%s > $$rootfs_download_path/.kameleon_timestamp
+        fi
+  - exec_out: false
+  - exec_out: date +%s > $$rootfs_download_path/.kameleon_timestamp
+  - exec_out: cat /etc/resolv.conf > $$rootfs_download_path/etc/resolv.conf
+  - write_out:
+    - $$rootfs_download_path/etc/network/interfaces
+    - |
+      auto lo
+      iface lo inet loopback
+      auto eth0
+      iface eth0 inet dhcp