kameleon-builder 2.0.0 → 2.1.0

data/templates/steps/setup/debian/upgrade_system.yaml

@@ -0,0 +1,15 @@
+
+- install_recommends: false
+- dist_upgrade: true
+
+- exec_in: |
+    if [ $$install_recommends = false ]; then
+      INSTALL_RECOMMENDS="--no-install-recommends"
+    fi
+
+- upgrade_system:
+  - exec_in: apt-get -y --force-yes $INSTALL_RECOMMENDS upgrade 2>&1
+  - exec_in: |
+      if [ $$dist_upgrade = false ]; then
+          apt-get -y --force-yes $INSTALL_RECOMMENDS dist-upgrade 2>&1
+      fi

data/templates/steps/setup/fedora/configure_network.yaml

@@ -0,0 +1,30 @@
+# Network config for Archlinux
+#
+# This should work for every systemd powered system
+- mtu: "1492"
+
+- set_hostname:
+  - exec_in: echo $$hostname > /etc/hostname
+
+- enable_network:
+  - exec_in: chkconfig network on
+
+- set_host:
+  - exec_in: echo "127.0.0.1 localhost $$hostname" > /etc/hosts
+
+- set_interface:
+  - write_in:
+    - /etc/sysconfig/network-scripts/ifcfg-eth0
+    - |
+      DEVICE=eth0
+      BOOTPROTO=dhcp
+      ONBOOT=yes
+      HOSTNAME=$$hostname
+      NM_CONTROLLED=no
+      TYPE=Ethernet
+      MTU=$$mtu
+  - write_in:
+    - /etc/sysconfig/network
+    - |
+      NETWORKING=yes
+      HOSTNAME=$$hostname

data/templates/steps/setup/fedora/configure_system.yaml

@@ -0,0 +1,59 @@
+# System configuration for Archlinux
+#
+# This will set the locals and the timezone
+
+- set_locales:
+  # uncomment the locales
+  - exec_in: |
+      echo $$locales | tr ' ' '\n' | xargs -I {} localedef -v -c -i {} -i {} -f UTF-8 $$lang 2>&1 || :
+
+- set_timezone:
+  - exec_in: ln -sf /usr/share/zoneinfo/$$timezone /etc/localtime
+
+- set_wheel_sudo:
+  - exec_in: echo "%wheel      ALL=(ALL) ALL" >> /etc/sudoers
+
+- reinstall_extlinux:
+  - exec_in: VMLINUZ=$(echo /boot/vmlinuz-* | tr ' ' '\n' | head -n 1)
+  - exec_in: VMLINUZ_VERSION=$(echo $VMLINUZ | sed 's/\/boot\/vmlinuz-//g')
+  - exec_in: INITRAMFS="$(echo $VMLINUZ | sed 's/vmlinuz/initramfs/g').img"
+  - exec_in: depmod -a $VMLINUZ_VERSION
+  - exec_in: dracut -f $INITRAMFS $VMLINUZ_VERSION
+  - exec_in: extlinux --install /boot/extlinux 2>&1
+  - exec_in: |
+      MBR_PATH=
+      PATHS=("/usr/share/syslinux/mbr.bin"
+             "/usr/lib/bios/syslinux/mbr.bin"
+             "/usr/lib/syslinux/bios/mbr.bin"
+             "/usr/lib/extlinux/mbr.bin"
+             "/usr/lib/syslinux/mbr.bin")
+      for element in "${PATHS[@]}"
+      do
+        if [ -f "$element" ]; then
+          MBR_PATH="$element"
+          break
+        fi
+      done
+      if [ "$MBR_PATH" == "" ]; then
+        fail "unable to locate the extlinux mbr"
+      else
+        dd if="$MBR_PATH" of="/dev/vda" bs=440  2>&1
+      fi
+  - write_in:
+    - /boot/extlinux/extlinux.conf
+    - |
+      menu autoboot Welcome to Fedora $$release $${arch} . Automatic boot in # second{,s}. Press a key for options.
+      menu title Fedora-$${release}-$${arch} Boot Options.
+      default Fedora-$${release}-$${arch}
+      timeout 10
+      totaltimeout 600
+      
+      label Fedora-$${release}-$${arch} ($VMLINUZ_VERSION)
+      kernel ../`basename $VMLINUZ`
+      append ro root=UUID=`blkid -s UUID -o value /dev/vda1` console=tty1 console=ttyS0,115200n8
+      initrd ../`basename $INITRAMFS`
+      
+      label Fedora-$${release}-$${arch}-kameleon ($(uname -r))
+      kernel ../vmlinuz
+      append ro root=UUID=`blkid -s UUID -o value /dev/vda1` console=tty1 console=ttyS0,115200n8
+      initrd ../initrd.img

data/templates/steps/setup/fedora/install_software.yaml

@@ -0,0 +1,3 @@
+# Software Install
+- install_packages:
+  - exec_in: yum install -y $$packages 2>&1

data/templates/steps/setup/fedora/update_system.yaml

@@ -0,0 +1,10 @@
+- update_system:
+  - exec_in: yum update yum -y
+  - exec_in: yum groupupdate 'Minimal Install' -y
+  - exec_in: yum clean all
+  - exec_in: yum update -y
+  - exec_in: yum --releasever=$$release distro-sync
+
+- clean:
+  - on_setup_clean:
+    - exec_in: yum clean all

data/templates/steps/setup/oar/oar_config_devel.yaml

@@ -0,0 +1,21 @@
+- update_hostfile:
+  - append_in:
+    - /etc/hosts
+    - |
+      127.0.0.2 node1 node2
+
+- create_resources:
+  - exec_in: /usr/local/sbin/oarproperty -a core
+  - exec_in: /usr/local/sbin/oarproperty -a cpu
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node1 -p cpu=0 -p core=0 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node1 -p cpu=0 -p core=1 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node1 -p cpu=1 -p core=2 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node1 -p cpu=1 -p core=3 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node2 -p cpu=2 -p core=4 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node2 -p cpu=2 -p core=5 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node2 -p cpu=3 -p core=6 -p cpuset=0
+  - exec_in: /usr/local/sbin/oarnodesetting -a -h node2 -p cpu=3 -p core=7 -p cpuset=0
+
+- modify_job_resource_manager:
+  - exec_in: sed -e 's/#exit/exit/' -i /etc/oar/job_resource_manager.pl
+  - exec_in: sed -i 's/job_resource_manager_cgroups\.pl/job_resource_manager\.pl/g' /etc/oar/oar.conf

data/templates/steps/setup/oar/oar_config_frontend.yaml

@@ -0,0 +1,38 @@
+- configure_monika:
+  - exec_in: sed -e "s/^\(username.*\)oar.*/\1oar_ro/" -i /etc/oar/monika.conf
+  - exec_in: sed -e "s/^\(password.*\)oar.*/\1oar_ro/" -i /etc/oar/monika.conf
+  - exec_in: sed -e "s/^\(dbtype.*\)mysql.*/\1psql/" -i /etc/oar/monika.conf
+  - exec_in: sed -e "s/^\(dbport.*\)3306.*/\15432/" -i /etc/oar/monika.conf
+
+- configure_drawgantt_svg:
+  - exec_in: |
+      sed -i "s/\$CONF\['db_type'\]=\"mysql\"/\$CONF\['db_type'\]=\"pg\"/g" /etc/oar/drawgantt-config.inc.php
+      sed -i "s/\$CONF\['db_port'\]=\"3306\"/\$CONF\['db_port'\]=\"5432\"/g" /etc/oar/drawgantt-config.inc.php
+
+- configure_open_api:
+  - exec_in: perl -pi -e "s/Deny from all/#Deny from all/" /etc/oar/apache2/oar-restful-api.conf
+
+- configure_basic_auth_api_priv:
+  - write_in:
+    - /etc/oar/apache2/oar-restful-api-priv.conf
+    - |
+      ScriptAlias /oarapi-priv $$oar_prefix_install/lib/cgi-bin/oarapi/oarapi.cgi
+      ScriptAlias /oarapi-priv-debug $$oar_prefix_install/lib/cgi-bin/oarapi/oarapi.cgi
+      <Location /oarapi-priv>
+        Options ExecCGI -MultiViews FollowSymLinks
+        AuthType      basic
+        AuthUserfile  /etc/oar/api-users
+        AuthName      "OAR API authentication"
+        Require valid-user
+        #RequestHeader set X_REMOTE_IDENT %{REMOTE_USER}e
+        RewriteEngine On
+        RewriteCond %{REMOTE_USER} (.*)
+        RewriteRule .* - [E=MY_REMOTE_IDENT:%1]
+        RequestHeader add X-REMOTE_IDENT %{MY_REMOTE_IDENT}e
+      </Location>
+  - exec_in: htpasswd -b -c /etc/oar/api-users $$user_name $$user_name
+  - exec_in: htpasswd -b /etc/oar/api-users oar $$user_name
+
+- install_chandler:
+  - exec_in: gem install naturalsort --no-ri --no-rdoc
+  - exec_in: cp /home/$$user_name/oar/sources/api/examples/chandler.rb /usr/local/bin/chandler

data/templates/steps/setup/oar/oar_config_node.yaml

@@ -0,0 +1,4 @@
+- configure_nothing:
+  - exec_in: echo "There is nothing to do to configure the node"
+
+

data/templates/steps/setup/oar/oar_config_server.yaml

@@ -0,0 +1,25 @@
+- configure_oar_log_level:
+  - exec_in: sed -e 's/^LOG_LEVEL\=\"2\"/LOG_LEVEL\=\"3\"/' -i /etc/oar/oar.conf
+
+- configure_taktuk:
+  - exec_in: sed -e 's/^#\(TAKTUK_CMD\=\"\/usr\/bin\/taktuk \-t 30 \-s\".*\)/\1/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^#\(PINGCHECKER_TAKTUK_ARG_COMMAND\=\"broadcast exec timeout 5 kill 9 \[ true \]\".*\)/\1/' -i /etc/oar/oar.conf
+
+- configure_db_type:
+  - exec_in: sed -e 's/^\(DB_TYPE\)=.*/\1="Pg"/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^\(DB_PORT\)=.*/\1="5432"/' -i /etc/oar/oar.conf
+
+- configure_cpuset:
+  - exec_in: sed -e 's/^#\(JOB_RESOURCE_MANAGER_PROPERTY_DB_FIELD\=\"cpuset\".*\)/\1/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^#\(JOB_RESOURCE_MANAGER_FILE\=\"\/etc\/oar\/job_resource_manager\.pl\".*\)/\1/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^#\(CPUSET_PATH\=\"\/oar\".*\)/\1/' -i /etc/oar/oar.conf
+
+- configure_database:
+  - exec_in: su postgres -c "psql -c 'DROP DATABASE IF EXISTS oar'"
+  - exec_in: su postgres -c "psql -c 'DROP USER IF EXISTS oar'"
+  - exec_in: su postgres -c "psql -c 'DROP USER IF EXISTS oar_ro'"
+  - exec_in: sed -e 's/^\(DB_BASE_PASSWD\)=.*/\1="oar"/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^\(DB_BASE_LOGIN\)=.*/\1="oar"/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^\(DB_BASE_PASSWD_RO\)=.*/\1="oar_ro"/' -i /etc/oar/oar.conf
+  - exec_in: sed -e 's/^\(DB_BASE_LOGIN_RO\)=.*/\1="oar_ro"/' -i /etc/oar/oar.conf
+  - exec_in: /usr/local/sbin/oar-database --create --db-is-local --db-admin-user root

data/templates/steps/setup/oar/oar_config_system.yaml

@@ -0,0 +1,34 @@
+- setup_fuse:
+  - exec_in: usermod -a -G fuse $$user_name
+  - append_in:
+    - /etc/fuse.conf
+    - |
+      user_allow_other
+- motd_config:
+  - write_in:
+    - /etc/motd
+    - |
+      
+      ************************** WELCOME TO THE OAR APPLIANCE ************************
+      We created 2 fake nodes pointing to localhost (node1 and node2)
+      * You can, for example, submit an interactive job:
+          $ oarsub -I
+      * Or check the API:
+          $ wget -O - http://localhost/oarapi/resources.yaml
+      * Check the API more deeply, submitting a job as the "$$user_name" user:
+          $ curl -i -X POST http://$$user_name:$$user_name@localhost/oarapi-priv/jobs.json \
+            -H'Content-Type: application/json' \
+            -d '{"resource":"/nodes=1,walltime=00:10:00", "command":"sleep 600"}'
+      * Launch the tests: 
+          $ cd oar/tests/rspec && make
+      * Mount your local git repository for coding:
+          $ mkdir git
+          $ sshfs -o allow_root your_login@10.0.2.2:git git
+
+      Notes:
+        - if you want to change the keyboard mapping, use: 
+        'loadkeys <2_letters_country_code>'
+        - root password is: $$user_name
+      
+      ********************************************************************************
+

data/templates/steps/setup/oar/oar_devel_prereq_install.yaml

@@ -0,0 +1,5 @@
+# OAR devel prereq install
+- packages: "sshfs git ruby-dev libapache2-mod-passenger rubygems"
+- req_packages:
+  - exec_in: apt-get -y --force-yes install $$packages
+

data/templates/steps/setup/oar/oar_git_install.yaml

@@ -0,0 +1,21 @@
+- fetch_repository:
+  - exec_in: mkdir -p $$oar_git_directory
+  - exec_in: git init $$oar_git_directory
+  - exec_in: git --git-dir $$oar_git_directory/.git --work-tree $$oar_git_directory fetch --depth 0 $$oar_git_repository $$oar_git_treeish
+  - exec_in: git --git-dir $$oar_git_directory/.git --work-tree $$oar_git_directory checkout FETCH_HEAD
+  - exec_in: make -C $$oar_git_directory clean
+
+- node:
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install node-build
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install node-install
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install node-setup
+
+- frontend:
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install user-build tools-build
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install user-install drawgantt-svg-install monika-install www-conf-install api-install tools-install
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install user-setup drawgantt-svg-setup monika-setup www-conf-setup api-setup tools-setup
+
+- server:
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install server-build
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install server-install
+  - exec_in: make -C $$oar_git_directory PREFIX=$$oar_prefix_install server-setup

data/templates/steps/setup/ubuntu/configure_apt.yaml

@@ -0,0 +1,67 @@
+# Software Install
+- enable_restricted: true
+- install_recommends: false
+
+- configure_apt_to_use_less_disk_space:
+  - write_in:
+    - /etc/apt/apt.conf.d/02compress_indexes
+    - |
+      Acquire::GzipIndexes "true";
+      Acquire::CompressionTypes::Order:: "gz";
+  - exec_in: |
+      if [ $$install_recommends = false ]; then
+        echo 'APT::Install-Recommends "0";' > /etc/apt/apt.conf.d/01norecommend
+        echo 'APT::Install-Suggests "0";' >> /etc/apt/apt.conf.d/01norecommend
+      fi
+  - write_in:
+    - /etc/apt/apt.conf.d/02nocache
+    - |
+      Dir::Cache {
+        srcpkgcache "";
+        pkgcache "";
+      }
+  - write_in:
+    - /etc/dpkg/dpkg.cfg.d/01_nodoc
+    - |
+      path-exclude /usr/share/doc/*
+      # we need to keep copyright files for legal reasons
+      path-include /usr/share/doc/*/copyright
+      path-exclude /usr/share/man/*
+      path-exclude /usr/share/groff/*
+      path-exclude /usr/share/info/*
+      # lintian stuff is small, but really unnecessary
+      path-exclude /usr/share/lintian/*
+      path-exclude /usr/share/linda/*
+      path-exclude /usr/share/locale/*
+      path-include /usr/share/locale/en*
+  - write_in:
+    - /etc/locale.nopurge
+    - |
+      MANDELETE
+      DONTBOTHERNEWLOCALE
+      SHOWFREEDSPACE
+      #QUICKNDIRTYCALC
+      #VERBOSE
+
+- enable_universe_and_multiverse:
+  - write_out:
+    - $$rootfs_download_path/etc/apt/sources.list
+    - |
+      deb $$repository $$release main restricted universe multiverse
+
+- enable_restricted:
+  - exec_in: |
+      if [ $$enable_restricted = true ]; then
+        REPO_RESTRICTED="restricted"
+      fi
+  - exec_in: perl -pi -e "s/multiverse$/multiverse $REPO_RESTRICTED/" /etc/apt/sources.list
+
+- update_repositories:
+  - exec_in: apt-get -y --force-yes update
+  - exec_in: apt-get -y --force-yes install localepurge
+  - on_setup_clean:
+    - exec_in: apt-get -y --force-yes autoclean
+    - exec_in: apt-get -y --force-yes clean
+    - exec_in: apt-get -y --force-yes autoremove
+    - exec_in: localepurge
+

data/templates/ubuntu-12.04-desktop.yaml

@@ -0,0 +1,25 @@
+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Ubuntu 12.04 LTS (Precise Pangolin) Desktop edition.
+#
+#==============================================================================
+
+---
+extend: ubuntu-12.04
+
+global:
+  # Apt options
+  apt_install_recommends: true
+
+bootstrap:
+  - @base
+
+setup:
+  - @base
+  - install_software:
+    - packages: ubuntu-desktop
+
+export:
+  - @base

data/templates/ubuntu-12.04.yaml

@@ -0,0 +1,128 @@
+#==============================================================================
+# vim: softtabstop=2 shiftwidth=2 expandtab fenc=utf-8 cc=81 tw=80
+#==============================================================================
+#
+# DESCRIPTION: Ubuntu 12.04 LTS (Precise Pangolin) base system.
+#
+#==============================================================================
+
+---
+# Loads some helpful aliases
+aliases: defaults.yaml
+# Enables qemu checkpoint
+checkpoint: qemu.yaml
+#== Global variables use by Kameleon engine and the steps
+global:
+  ## User varibales : used by the recipe
+  user_name: kameleon
+  user_password: $$user_name
+
+  # Distribution
+  distrib: ubuntu
+  release: precise
+  arch: amd64
+
+  ## QEMU options
+  qemu_enable_kvm: true
+  qemu_cpu: 2
+  qemu_memory_size: 512
+  qemu_monitor_port: 10023
+  qemu_ssh_port: 55423
+  qemu_arch: x86_64
+
+  ## Disk options
+  nbd_device: /dev/nbd1
+  image_disk: $$kameleon_cwd/base_$$kameleon_recipe_name.qcow2
+  image_size: 10G
+  filesystem_type: ext4
+
+  # rootfs options
+  rootfs: $$kameleon_cwd/rootfs
+  rootfs_download_path: /var/cache/kameleon/$$distrib/$$release/$$arch/rootfs
+
+  # Apt options
+  apt_repository: http://archive.ubuntu.com/ubuntu/
+  apt_enable_restricted: true
+  apt_install_recommends: false
+
+  ## System variables. Required by kameleon engine
+  # Include specific steps
+  include_steps:
+    - $$distrib/$$release
+    - $$distrib
+    - debian
+
+  # Shell session from where we launch exec_out commands. There is often a
+  # local bash session, but it can be a remote shell on other machines or on
+  # any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
+  out_context:
+    cmd: bash
+    workdir: $$kameleon_cwd
+
+  # Shell session that allows us to connect to the building machine in order to
+  # configure it and setup additional programs
+  ssh_config_file: $$kameleon_cwd/ssh_config
+  in_context:
+    cmd: ssh -F $$ssh_config_file $$kameleon_recipe_name -t /bin/bash
+    workdir: /root/kameleon_workdir
+
+#== Bootstrap the new system and create the 'in_context'
+bootstrap:
+  - debootstrap:
+    - include_pkg: >
+        acpi-support kbd udev linux-image-virtual locales openssh-server
+        bzip2 linux-image-generic
+    - release: $$release
+    - arch: $$arch
+    - repository: $$apt_repository
+    - enable_cache: true
+  - initialize_disk_qemu
+  - prepare_qemu
+  - install_bootloader
+  - start_qemu
+
+#== Install and configuration steps
+# WARNING: this part should be independante from the out context (whenever
+# possible...)
+setup:
+  # Install
+  - configure_apt:
+    - repository: $$apt_repository
+    - enable_restricted: $$apt_enable_restricted
+    - install_recommends: $$apt_install_recommends
+  - upgrade_system:
+    - dist_upgrade: true
+  - install_software:
+    - packages: >
+        apt-transport-https ca-certificates rsync sudo less bash-completion
+        curl nano vim psmisc language-pack-en
+  - configure_kernel:
+    - arch: $$arch
+  # Configuration
+  - configure_system:
+    - locales: POSIX C en_US fr_FR de_DE
+    - lang: en_US.UTF-8
+    - timezone: UTC
+  - configure_keyboard:
+    - layout: "us,fr,de"
+  - configure_network:
+    - hostname: kameleon-$$distrib
+  - create_group:
+    - name: admin
+  - create_user:
+    - name: $$user_name
+    - groups: sudo admin
+    - password: $$user_password
+
+#== Export the generated appliance in the format of your choice
+export:
+  - save_appliance:
+    - input: $$image_disk
+    - output: $$kameleon_cwd/$$kameleon_recipe_name
+    - save_as_qcow2
+    # - save_as_qed
+    # - save_as_tgz
+    # - save_as_raw
+    # - save_as_vmdk
+    # - save_as_vdi
+