kameleon-builder 2.2.3 → 2.2.4

data/CHANGELOG.rst

@@ -1,6 +1,19 @@
 Kameleon CHANGELOG
 ==================
 
+version 2.2.4
+-------------
+Released on Sep 2nd 2014
+
+- [template] Added the ``-cpu host`` option to qemu to improve performances
+- [template] Made centos image as close as possible as default Centos installation
+- [template] Configured SELinux for Fedora and Centos
+- [template] Fixed Grid'5000 export step
+- [template] Extended vagrant recipes from virtualbox recipes
+- [core] Adding the step elapsed time to the Kamelon output
+- [proxy cache] Restructured persistent cache and improved caching of pipes
+- [proxy cache] Added ``ProxyAdrres`` paramter to polipo to take into account all the host network interfaces
+
 version 2.2.3
 -------------
 Released on Aug 19th 2014

data/contrib/kameleon_bashrc.sh

@@ -139,14 +139,14 @@ fi
 
 function __download {
     echo "Downloading: $1..."
-    if which curl >/dev/null; then
-        curl -# -L --retry 999 --retry-max-time 0 "$1" -o "$2" 2>&1
+    if which wget >/dev/null; then
+        wget --progress=bar:force "$1" -O "$2" 2>&1
     else
-        fail "curl is missing, trying with wget..."
-        if which wget >/dev/null; then
-            wget --progress=bar:force "$1" -O "$2" 2>&1
+        fail "wget is missing, trying with curl..."
+        if which curl >/dev/null; then
+            curl -# -L --retry 999 --retry-max-time 0 "$1" -o "$2" 2>&1
         else
-            fail "wget is missing, trying with python..."
+            fail "curl is missing, trying with python..."
             if which python >/dev/null; then
                 python -c "
 import sys

data/lib/kameleon/engine.rb

@@ -1,6 +1,7 @@
 require 'kameleon/recipe'
 require 'kameleon/context'
 require 'kameleon/persistent_cache'
+# require 'pry'
 module Kameleon
 
   class Engine
@@ -38,7 +39,7 @@ module Kameleon
         @cache.name = @recipe.name
         @cache.mode = @options[:cache] ? :build : :from
         @cache.cache_path = @options[:from_cache]
-        @cache.recipe_files = @recipe.files # I'm passing the Pathname objects
+        @cache.recipe_files = @recipe.files + @recipe.base_recipes_files# I'm passing the Pathname objects
         @cache.recipe_path = @recipe.path
 
         if @recipe.global["in_context"]["proxy_cache"].nil? then
@@ -151,6 +152,7 @@ module Kameleon
     def do_steps(section_name)
       section = @recipe.sections.fetch(section_name)
       section.sequence do |macrostep|
+        macrostep_time = Time.now.to_i
         if @cache then
           Kameleon.ui.info("Starting proxy cache server for macrostep '#{macrostep.name}'...")
           # the following function start a polipo web proxy and stops a previous run
@@ -186,6 +188,7 @@ module Kameleon
             end
           end
         end
+        Kameleon.ui.info("Step #{macrostep.name} took: #{Time.now.to_i-macrostep_time} secs")
       end
       @cleaned_sections.push(section.name)
     end
@@ -235,7 +238,7 @@ module Kameleon
         end
         first_context = map[first_cmd.key]
         second_context = map[second_cmd.key]
-        @cache.cache_cmd_id(cmd.identifier) if @cache
+        @cache.cache_cmd_raw(cmd.raw_cmd_id) if @cache
         first_context.pipe(first_cmd.value, second_cmd.value, second_context)
       when "rescue"
         first_cmd, second_cmd = cmd.value

data/lib/kameleon/persistent_cache.rb

@@ -1,6 +1,7 @@
 require 'childprocess'
 require 'singleton'
 require 'socket'
+
 module Kameleon
   #This ruby class will control the execution of Polipo web proxy
   class Persistent_cache
@@ -36,6 +37,7 @@ module Kameleon
                              :proxyPort => @polipo_port,
                              :relaxTransparency =>"true",
                              :daemonise => false,
+                             :proxyAddress => "0.0.0.0",
                              :logFile => File.join(Kameleon.env.build_path, 'polipo.log')
                             }
 
@@ -45,10 +47,9 @@ module Kameleon
       @cache_dir = Kameleon.env.cache_path
       @polipo_path = nil
       @cwd = ""
-      #structure {:cmd => "cmd", :stdout_filename => "file_name"}
       @cmd_cached = []
       @cache_path = ""
-      @current_cmd_id = nil
+      @current_raw_cmd = nil
       @current_step_dir = nil
       @recipe_file = nil
       @steps_files = []
@@ -93,7 +94,7 @@ module Kameleon
 
     def create_cache_directory(step_name)
       Kameleon.ui.debug("Creating  cache directory #{step_name} for Polipo")
-      directory_name = File.join(@cache_dir,"#{step_name}")
+      directory_name = File.join(@cache_dir,"DATA","#{step_name}")
       FileUtils.mkdir_p directory_name
       directory_name
     end
@@ -135,10 +136,9 @@ module Kameleon
     end
 
 
-    # This function caches the command with its respective stdout
-    # a command id is associate to a file
-    def cache_cmd_id(cmd_identifier)
-      @current_cmd_id = cmd_identifier
+    # This function caches the raw command specified in the recipe
+    def cache_cmd_raw(raw_cmd_id)
+      @current_raw_cmd = raw_cmd_id
       return true
     end
 
@@ -147,16 +147,18 @@ module Kameleon
       Kameleon.ui.debug("command: cp #{file_path} #{@cwd}/cache/files/")
       FileUtils.mkdir_p @current_step_dir + "/data/"
       FileUtils.cp file_path, @current_step_dir + "/data/"
-      @cmd_cached.push({:cmd_id => @current_cmd_id,
-                        :cmd => cmd ,
+      @cmd_cached.push({:raw_cmd_id => @current_raw_cmd,
                         :stdout_filename => File.basename(file_path)})
     end
 
     def get_cache_cmd(cmd)
       return false if @mode == :build
-      cache_line = @cmd_cached.select{ |reg|
-        (reg[:cmd_id] == @current_cmd_id && reg[:cmd] == cmd) }.first
-
+      cache_line = @cmd_cached.select{ |reg| reg[:raw_cmd_id] == @current_raw_cmd  }.first
+      if cache_line.nil? then
+        # This error can be due to the improper format of the file cache_cmd_index
+        Kameleon.ui.error("Persistent cache missing file")
+        raise BuildError, "Failed to use persistent cache"
+      end
       return File.new("#{@current_step_dir}/data/#{cache_line[:stdout_filename]}","r")
     end
 
@@ -164,22 +166,34 @@ module Kameleon
       @polipo_process.stop
       Kameleon.ui.info("Stopping web proxy polipo")
       Kameleon.ui.info("Finishing persistent cache with last files")
-
+      cache_metadata_dir = File.join(@cache_dir,"metadata")
       if @mode == :build then
-        File.open("#{@cache_dir}/cache_cmd_index",'w+') do |f|
+        File.open("#{cache_metadata_dir}/cache_cmd_index",'w+') do |f|
           f.puts(@cmd_cached.to_yaml)
         end
 
         unless @recipe_files.empty?
-          recipe_dir = Pathname.new(common_prefix(@recipe_files))
           all_files = @recipe_files.push(@recipe_path)
-          Kameleon::Utils.copy_files(recipe_dir, @cache_dir, all_files)
+          recipe_dir = Pathname.new(common_prefix(all_files))
+          cached_recipe_dir = Pathname.new(File.join(@cache_dir,"recipe"))
+       #   binding.pry
+          Kameleon::Utils.copy_files(recipe_dir, cached_recipe_dir, all_files)
         end
         ## Saving metadata information
         Kameleon.ui.info("Caching recipe")
-        File.open("#{@cached_recipe_dir}/header",'w+') do |f|
-          f.puts({:recipe_path => @recipe_path.to_s}.to_yaml)
+
+        File.open("#{cache_metadata_dir}/header",'w+') do |f|
+          f.puts({:recipe_path => @recipe_path.basename.to_s}.to_yaml)
+          f.puts({:date => Time.now.to_i}.to_yaml)
+        end
+
+        #Removing empty directories
+        cache_data_dir = File.join(@cache_dir,"DATA")
+        Dir.foreach(cache_data_dir) do |item|
+          dir_temp = File.join(cache_data_dir,item)
+          Dir.delete(dir_temp) if File.stat(dir_temp).nlink == 2
         end
+
         pack
       end
     end
@@ -193,20 +207,24 @@ module Kameleon
           raise BuildError, "Failed to untar the persistent cache file"
         end
         ## We have to load the file
-        @cmd_cached = YAML.load(File.read("#{@cache_dir}/cache_cmd_index"))
+        metadata_dir = File.join(@cache_dir,"metadata")
+        @cmd_cached = YAML.load(File.read("#{metadata_dir}/cache_cmd_index"))
       end
       @activated = true
-      @cached_recipe_dir = @cache_dir
-      FileUtils.mkdir_p @cached_recipe_dir
+      #@cached_recipe_dir = @cache_dir
+      FileUtils.mkdir_p @cache_dir
+      # Creating sctructure of the cache
+      FileUtils.mkdir_p File.join(@cache_dir,"recipe")
+      FileUtils.mkdir_p File.join(@cache_dir,"DATA")
+      FileUtils.mkdir_p File.join(@cache_dir,"metadata")
     end
 
     def get_recipe()
       cached_recipe=Dir.mktmpdir("cache")
-      puts "cache path : #{@cache_path}"
-      execute("tar","-xf #{@cache_path} -C #{cached_recipe} .")
+      execute("tar","-xf #{@cache_path} -C #{cached_recipe} ./recipe ./metadata")
       Kameleon.ui.info("Getting cached recipe")
-      recipe_header = YAML::load(File.read("#{cached_recipe}/header"))
-      recipe_file = recipe_header[:recipe_path]
+      recipe_header = YAML::load(File.read(File.join(cached_recipe,"metadata","header")))
+      recipe_file = File.join(cached_recipe,"recipe",recipe_header[:recipe_path])
       return recipe_file
     end
 

data/lib/kameleon/recipe.rb

@@ -327,13 +327,6 @@ module Kameleon
       consistency_check
       resolve_checkpoint unless @checkpoint.nil?
 
-      Kameleon.ui.info("Resolving variables")
-      @sections.values.each do |section|
-        section.macrosteps.each do |macrostep|
-          macrostep.resolve_variables!(@global)
-        end
-      end
-
       @sections.values.each do |section|
         section.macrosteps.each do |macrostep|
           # First pass : resolve aliases
@@ -347,6 +340,18 @@ module Kameleon
           # flatten for multiple-command alias + variables
           Kameleon.ui.debug("Resolving check statements for macrostep '#{macrostep.name}'")
           macrostep.microsteps.each { |microstep| microstep.commands.flatten! }
+        end
+      end
+
+      Kameleon.ui.info("Resolving variables")
+      @sections.values.each do |section|
+        section.macrosteps.each do |macrostep|
+          macrostep.resolve_variables!(@global)
+        end
+      end
+
+      @sections.values.each do |section|
+        section.macrosteps.each do |macrostep|
           # Second pass : resolve variables + clean/init hooks
           macrostep.microsteps.each do |microstep|
             microstep.commands.map! do |cmd|

data/lib/kameleon/step.rb

@@ -3,11 +3,13 @@ module Kameleon
   class Command
 
     attr_accessor :string_cmd
+    attr_accessor :raw_cmd_id
     attr_accessor :microstep_name
     attr_accessor :identifier
 
     def initialize(yaml_cmd, microstep_name)
       @string_cmd = YAML.dump(yaml_cmd).gsub("---", "").strip
+      @raw_cmd_id = Digest::SHA1.hexdigest(YAML.dump(yaml_cmd).gsub("---", "").strip)
       @microstep_name = microstep_name
       @identifier = nil
     end

data/templates/qemu/centos6.5-x86_64.yaml

@@ -50,7 +50,10 @@ global:
     - $$distrib
     - "fedora"
 
-  disable_selinux: true
+  # SELinux configuration
+  selinux: permissive  ## Can take one of these three values: enforcing, permissive or disabled
+  selinuxtype: targeted  # Possible values are: strict, targeted
+
   ssh_config_file: $$kameleon_cwd/ssh_config
   out_context:
     cmd: ssh -F $$ssh_config_file $${kameleon_recipe_name} -t /bin/bash
@@ -58,7 +61,7 @@ global:
     proxy_cache: 10.0.2.2
 
   in_context:
-    cmd: ssh -F $$ssh_config_file $${kameleon_recipe_name} -t chroot $$rootfs /bin/bash
+    cmd: ssh -F $$ssh_config_file $${kameleon_recipe_name} -t /bin/bash
     workdir: /root/kameleon_workdir
     proxy_cache: 10.0.2.2
 
@@ -67,12 +70,14 @@ bootstrap:
   - prepare_qemu
   - start_qemu
   - install_requirements:
-    - packages: parted e2fsprogs yum rpm lynx
-  - initialize_disk:
+    - packages: parted e2fsprogs yum rpm lynx extlinux
+  - initialize_disk
   - yum_bootstrap:
     - mirror_packages_url: http://mirrors.kernel.org/$$distrib/$$release/os/$$arch/Packages/
-    - include_pkgs: findutils util-linux
-  - start_chroot
+    - include_pkgs: >
+        findutils yum util-linux dhclient vim-minimal net-tools openssh-server
+        kernel kernel-devel acpid
+  - switch_context_qemu
 
 #== Install and configuration steps
 setup:
@@ -81,9 +86,13 @@ setup:
   - minimal_install
   - install_software:
     - packages: >
-        kernel kernel-devel syslinux-extlinux bash-completion kbd dhclient 
-        sudo openssh-server openssh-clients ntp ntpdate dhclient
-        net-tools NetworkManager
+        syslinux-extlinux kbd sudo openssh-clients ntp ntpdate
+        rsync bridge-utils bzip2 cronie cronie-anacron crontabs
+        dash dhclient dhcp-common dracut dracut-kernel file fuse gnupg2
+        iptables-ipv6 libuser logrotate m4 openssh-server passwd
+        pciutils-libs rsyslog dbus system-config-firewall-base which
+        findutils yum util-linux vim-minimal net-tools openssh-server
+        kernel kernel-devel acpid
   - install_bootloader
   - configure_system:
     - locales: POSIX C en_US fr_FR de_DE

data/templates/qemu/centos7-x86_64.yaml

@@ -50,7 +50,10 @@ global:
     - $$distrib
     - "fedora"
 
-  disable_selinux: true
+  # SELinux configuration
+  selinux: permissive  ## Can take one of these three values: enforcing, permissive or disabled
+  selinuxtype: targeted  # Possible values are: strict, targeted
+
   ssh_config_file: $$kameleon_cwd/ssh_config
   out_context:
     cmd: ssh -F $$ssh_config_file $${kameleon_recipe_name} -t /bin/bash

data/templates/qemu/fedora20-x86_64.yaml

@@ -49,7 +49,10 @@ global:
     - $$distrib/$$release
     - $$distrib
 
-  disable_selinux: true
+  # SELinux configuration
+  selinux: permissive  ## Can take one of these three values: enforcing, permissive or disabled
+  selinuxtype: targeted  # Possible values are: strict, targeted
+
   ssh_config_file: $$kameleon_cwd/ssh_config
   out_context:
     cmd: ssh -F $$ssh_config_file $${kameleon_recipe_name} -t /bin/bash

data/templates/steps/bootstrap/centos/6.5/yum_bootstrap.yaml

@@ -0,0 +1,65 @@
+- fix_yum:
+  - check_cmd_out: yum
+  - exec_out: sed -i "s/opts.ssl_verify_host/2/g" /usr/lib/pymodules/python2.7/urlgrabber/grabber.py
+
+- init_rpm_db:
+  - check_cmd_out: rpm
+  - exec_out: mkdir -p $$rootfs/var/lib/rpm
+  - exec_out: rpm --root $$rootfs --initdb
+
+- fetch_release_package:
+  - check_cmd_out: lynx
+  - exec_out: RELEASE_PACKAGE_URL=$(lynx $$mirror_packages_url -dump -listonly -nonumbers | grep $${distrib}-release | head -1)
+  - exec_out: |
+      if [ -z "${RELEASE_PACKAGE_URL-unset}" ]; then
+          fail "$${distrib}-release package not found!"
+      fi
+
+- install_distrib_release:
+  - download_file_out:
+    - $RELEASE_PACKAGE_URL
+    - $KAMELEON_WORKDIR/$${distrib}-release.rpm
+  - exec_out: rpm --root $$rootfs -ivh --force-debian --nodeps $KAMELEON_WORKDIR/$${distrib}-release.rpm
+
+- install_yum:
+  - exec_out: ln -sf $$rootfs/etc/pki/ /etc/pki
+  - exec_out: yum --installroot $$rootfs -y install yum
+  - exec_out: echo $${release} > $$rootfs/etc/yum/vars/releasever
+
+- mount_chroot:
+  - exec_out: mount -o bind /dev  $$rootfs/dev
+  - exec_out: mount -o bind /dev/pts $$rootfs/dev/pts
+  - exec_out: mount -t proc /proc  $$rootfs/proc
+  - exec_out: mount -t sysfs /sys  $$rootfs/sys
+  - exec_out: cp /etc/resolv.conf $$rootfs/etc/resolv.conf
+
+- install_packages:
+  - exec_out: chroot $$rootfs yum install --releasever=$$release -y $$include_pkgs
+
+- enable_services:
+  - exec_out: chroot $$rootfs chkconfig network on
+  - exec_out: chroot $$rootfs chkconfig sshd on
+
+- set_interface:
+  - write_out:
+    - $$rootfs/etc/sysconfig/network-scripts/ifcfg-eth0
+    - |
+      DEVICE=eth0
+      BOOTPROTO=dhcp
+      ONBOOT=yes
+      HOSTNAME=$$hostname
+      NM_CONTROLLED=no
+      TYPE=Ethernet
+  - write_out:
+    - $$rootfs/etc/sysconfig/network
+    - |
+      NETWORKING=yes
+      HOSTNAME=$$hostname
+
+- umount_chroot:
+  - on_clean:
+    - umount_out: $$rootfs/sys
+    - umount_out: $$rootfs/proc
+    - umount_out: $$rootfs/dev/pts
+    - umount_out: $$rootfs/dev
+    - exec_out: rm -f $$rootfs/etc/resolv.conf

data/templates/steps/bootstrap/start_qemu.yaml

@@ -22,6 +22,7 @@
           -cdrom $$kameleon_cwd/boot2kameleon.iso \
           -drive file="$$qemu_image_disk",cache=unsafe,media=disk,if=virtio,id=drive0 \
           -smp $${qemu_cpu} \
+          -cpu host \
           -m $$qemu_memory_size \
           -rtc base=localtime \
           -net nic,model=virtio -net user \
@@ -31,13 +32,14 @@
           -daemonize -vnc :1 $LOADVM
   - exec_local: |
         NEXT_WAIT_TIME=0
-        until ssh-keyscan -T 1 -4 -p $$qemu_ssh_port localhost  2>&1 | grep -e ssh-rsa -e ssh-dsa &> /dev/null || [ $NEXT_WAIT_TIME -eq $$boot_timeout ];
+        SSH_AVAILABLE=0
+        until ssh-keyscan -T 1 -4 -p $$qemu_ssh_port localhost  2>&1 | grep -e ssh-rsa -e ssh-dsa &> /dev/null && SSH_AVAILABLE=1 || [ $NEXT_WAIT_TIME -eq $$boot_timeout ];
         do
           echo -en "\rWaiting for SSH to become available for out_context...($(( $$boot_timeout - 1 - NEXT_WAIT_TIME++ ))s)"
         done
         echo ""
   - rescue:
-    - exec_local: ssh-keyscan -T 1 -4 -p $$qemu_ssh_port localhost  2>&1 | grep -e ssh-rsa -e ssh-dsa &> /dev/null
+    - exec_local: test $SSH_AVAILABLE -eq 1
     - breakpoint: Failed to connect to VM via SSH. Please verify the VM successfully booted with a vnc client.
 
 - force_shutdown_qemu_vm: