kameleon-builder 2.1.3 → 2.2.0

data/lib/kameleon/shell.rb

@@ -12,18 +12,17 @@ module Kameleon
     attr :process
     attr :shell_cmd
 
-    def initialize(context_name, cmd, shell_workdir, local_workdir, kwargs = {})
-      @logger = Log4r::Logger.new("kameleon::[kameleon]")
+    def initialize(context_name, cmd, shell_workdir, local_workdir, proxy_cache)
       @cmd = cmd.chomp
       @context_name = context_name
       @local_workdir = local_workdir
       @shell_workdir = shell_workdir
-      @bashrc_file = "kameleon_#{@context_name}_bash_rc"
-      @bash_history_file = "kameleon_#{@context_name}_bash_history"
-      @bash_env_file = "kameleon_#{@context_name}_bash_env"
+      @proxy_cache = proxy_cache
+      @bashrc_file = ".kameleon_#{@context_name}_bash_rc"
+      @bash_history_file = ".kameleon_#{@context_name}_bash_history"
+      @bash_env_file = ".kameleon_#{@context_name}_bash_env"
       @default_bashrc_file = File.join(Kameleon.source_root,
-                                       "contrib",
-                                       "kameleon_bashrc.sh")
+                                       "contrib", "kameleon_bashrc.sh")
       if @shell_workdir
         @bashrc_file = File.join(@shell_workdir, @bashrc_file)
         @bash_history_file = File.join(@shell_workdir, @bash_history_file)
@@ -35,10 +34,10 @@ module Kameleon
 
       @shell_cmd = "source #{@default_bashrc_file} 2> /dev/null; "\
                    "#{@cmd} --rcfile #{@bashrc_file}"
-      @logger.debug("Initialize shell (#{self})")
+      Kameleon.ui.debug("Initialize shell (#{self})")
       # Injecting all variables of the options and assign the variables
       instance_variables.each do |v|
-        @logger.debug("  #{v} = #{instance_variable_get(v)}")
+        Kameleon.ui.debug("  #{v} = #{instance_variable_get(v)}")
       end
     end
 
@@ -87,17 +86,29 @@ module Kameleon
       ## log shell error message
       error = read_io(@stderr)
       init_stdout = read_io(@stdout)
-      @logger.error(error) unless error.empty?
-      @logger.info(init_stdout) unless init_stdout.empty?
+      Kameleon.ui.error(error) unless error.empty?
+      Kameleon.ui.info(init_stdout) unless init_stdout.empty?
       bashrc_content = ""
       if File.file?(@default_bashrc_file)
         tpl = ERB.new(File.read(@default_bashrc_file))
         bashrc_content = tpl.result(binding)
         if @cache.activated? then
-          tpl = ERB.new(File.read(@cache.polipo_env))
-          bashrc_content << "\n" + ERB.new(File.read(@cache.polipo_env)).result(binding)
+          if @proxy_cache.nil? then
+            Kameleon.ui.warn("Variable 'proxy_cache' not defined for this context, persistent cache will not be generated")
+          else
+            tpl = ERB.new(File.read(@cache.polipo_env))
+            bashrc_content << "\n" + ERB.new(File.read(@cache.polipo_env)).result(binding)
+          end
         end
       end
+      # Inject sigint handler
+      bashrc_content << <<-SCRIPT
+      function save_state_handler {
+        echo "Saved ENV in #{@bash_env_file} file"
+        (comm -3 <(declare | sort) <(declare -f | sort)) > #{@bash_env_file}
+      }
+      trap save_state_handler EXIT
+      SCRIPT
       bashrc = Shellwords.escape(bashrc_content)
       if @shell_workdir
         unless @shell_workdir.eql? "/"
@@ -114,6 +125,7 @@ module Kameleon
       end
       shell_cmd << "source #{@bashrc_file}\n"
       shell_cmd << "export KAMELEON_WORKDIR=$PWD\n"
+      shell_cmd << "(comm -3 <(declare | sort) <(declare -f | sort)) > #{@bash_env_file}\n"
       shell_cmd
     end
 
@@ -124,11 +136,11 @@ module Kameleon
         shell_cmd << init_shell_cmd
         @sent_first_cmd = true
       end
+      shell_cmd << "source #{@bash_env_file} 2> /dev/null || true\n"
       shell_cmd << "KAMELEON_LAST_COMMAND=#{Shellwords.escape(cmd.value)}\n"
-      shell_cmd << "( set -o posix ; set ) > #{@bash_env_file}\n"
-      shell_cmd << "env | xargs -I {} echo export {} >> #{@bash_env_file}\n"
       shell_cmd << "#{ cmd.value }\nexport __exit_status__=$?\n"
       shell_cmd << "#{ ECHO_CMD } $KAMELEON_LAST_COMMAND >> \"$HISTFILE\"\n"
+      shell_cmd << "(comm -3 <(declare | sort) <(declare -f | sort)) > #{@bash_env_file}\n"
       shell_cmd << "#{ ECHO_CMD } -n #{ cmd.end_err } 1>&2\n"
       shell_cmd << "#{ ECHO_CMD } -n #{ cmd.end_out }\n"
       @process.io.stdin.puts shell_cmd
@@ -156,6 +168,9 @@ module Kameleon
                              :yield     => lambda{|buf| yield(buf, nil)} }
                 }
       while true
+        if @process.exited?
+          raise ShellError, "Process '#{@cmd}' exited..."
+        end
         iodata.each do |_, iodat|
           if iodat[:end] and not iodat[:begin]
             raise ShellError, "Cannot read #{iodat[:begin]} from shell"
@@ -246,8 +261,8 @@ module Kameleon
 
     def fork(io)
       command = ["bash", "-c", @shell_cmd]
-      @logger.notice("Starting process: #{@cmd.inspect}")
-      @logger.debug("Starting shell process: #{ command.inspect}")
+      Kameleon.ui.info("Starting process: #{@cmd.inspect}")
+      Kameleon.ui.debug("Starting shell process: #{ command.inspect}")
       ChildProcess.posix_spawn = true
       process = ChildProcess.build(*command)
       # Create the pipes so we can read the output in real time as

data/lib/kameleon/step.rb

@@ -1,12 +1,15 @@
 module Kameleon
 
   class Command
+
     attr_accessor :string_cmd
     attr_accessor :microstep_name
+    attr_accessor :identifier
 
     def initialize(yaml_cmd, microstep_name)
       @string_cmd = YAML.dump(yaml_cmd).gsub("---", "").strip
       @microstep_name = microstep_name
+      @identifier = nil
     end
 
     def resolve!
@@ -16,7 +19,12 @@ module Kameleon
 
     def key
       if @key.nil?
-        @key = YAML.load(@string_cmd).keys.first
+        object = YAML.load(@string_cmd)
+        if object.kind_of? String
+          @key = object
+        else
+          @key = object.keys.first
+        end
       end
       @key
     rescue
@@ -30,6 +38,8 @@ module Kameleon
         object = YAML.load(@string_cmd)
         if object.kind_of? Command
           @value = object
+        elsif object.kind_of? String
+          @value = nil
         else
           raise RecipeError unless object.kind_of? Hash
           raise RecipeError unless object.keys.count == 1
@@ -68,6 +78,7 @@ module Kameleon
         @value.gsub!(arg1, arg2)
       end
       @string_cmd = YAML.dump(to_array).gsub("---", "").strip
+      return self
     end
 
   end
@@ -122,6 +133,11 @@ module Kameleon
       commands_str = @commands.map { |cmd| cmd.string_cmd.to_s }
       content_id = commands_str.join(' ') + salt
       @identifier = "#{ Digest::SHA1.hexdigest content_id }"[0..11]
+      @commands.each do |cmd|
+        map_id = cmd.string_cmd.to_s + @identifier
+        cmd.identifier = "#{ Digest::SHA1.hexdigest map_id }"[0..11]
+      end
+      @identifier
     end
 
     def to_array
@@ -216,6 +232,7 @@ module Kameleon
             "identifier" => microstep.identifier.to_s,
             "cmds" => microstep.to_array
           }
+
           section_array.push({ microstep.slug => hash })
         end
       end

data/lib/kameleon/ui.rb

@@ -0,0 +1,141 @@
+module Kameleon
+  module UI
+
+    class Silent
+      def info(message, newline = nil)
+      end
+
+      def confirm(message, newline = nil)
+      end
+
+      def warn(message, newline = nil)
+      end
+
+      def error(message, newline = nil)
+      end
+
+      def debug(message, newline = nil)
+      end
+
+      def debug?
+        false
+      end
+
+      def quiet?
+        false
+      end
+
+      def ask(message)
+      end
+
+      def level=(name)
+      end
+
+      def level(name = nil)
+      end
+
+      def trace(message, newline = nil)
+      end
+
+      def silence
+        yield
+      end
+    end
+
+    class Shell
+      LEVELS = %w(silent error warn confirm info debug)
+
+      attr_accessor :shell
+
+      def initialize(options = {})
+        @shell = Thor::Base.shell.new
+        @level = ENV['KAMELEON_DEBUG'] ? "debug" : "info"
+      end
+
+      def info(msg, newline = nil)
+        tell_me(msg, nil, newline) if level("info")
+      end
+
+      def confirm(msg, newline = nil)
+        tell_me(msg, :green, newline) if level("confirm")
+      end
+
+      def warn(msg, newline = nil)
+        tell_me(msg, :yellow, newline) if level("warn")
+      end
+
+      def error(msg, newline = nil)
+        tell_me(msg, :red, newline) if level("error")
+      end
+
+      def debug(msg, newline = nil)
+        tell_me("[debug] #{msg}", nil, newline) if level("debug")
+      end
+
+      def debug?
+        # needs to be false instead of nil to be newline param to other methods
+        level("debug")
+      end
+
+      def quiet?
+        LEVELS.index(@level) <= LEVELS.index("warn")
+      end
+
+      def ask(msg)
+        @shell.ask(msg)
+      end
+
+      def level=(level)
+        raise ArgumentError unless LEVELS.include?(level.to_s)
+        @level = level
+      end
+
+      def level(name = nil)
+        name ? LEVELS.index(name) <= LEVELS.index(@level) : @level
+      end
+
+      def trace(e, newline = nil)
+        msg = ["#{e.class}: #{e.message}", *e.backtrace].join("\n")
+        if debug?
+          tell_me(msg, nil, newline)
+        elsif @trace
+          STDERR.puts "#{msg}#{newline}"
+        end
+      end
+
+      def silence
+        old_level, @level = @level, "silent"
+        yield
+      ensure
+        @level = old_level
+      end
+
+    private
+
+      # valimism
+      def tell_me(msg, color = nil, newline = nil)
+        msg = word_wrap(msg) if newline.is_a?(Hash) && newline[:wrap]
+        if newline.nil?
+          if Kameleon.log_on_progress
+            Kameleon.log_on_progress = false
+            msg = "\n" + msg
+          end
+          @shell.say(msg, color)
+        else
+          @shell.say(msg, color, newline)
+        end
+      end
+
+      def strip_leading_spaces(text)
+        spaces = text[/\A\s+/, 0]
+        spaces ? text.gsub(/#{spaces}/, '') : text
+      end
+
+      def word_wrap(text, line_width = @shell.terminal_width)
+        strip_leading_spaces(text).split("\n").collect do |line|
+          line.length > line_width ? line.gsub(/(.{1,#{line_width}})(\s+|$)/, "\\1\n").strip : line
+        end * "\n"
+      end
+    end
+  end
+end

data/lib/kameleon/utils.rb

@@ -43,5 +43,14 @@ module Kameleon
     rescue
     end
 
+    def self.copy_files(relative_dir, dest_dir, files2copy)
+      files2copy.each do |path|
+        relative_path = path.relative_path_from(relative_dir)
+        dst = File.join(dest_dir,relative_path)
+        FileUtils.mkdir_p File.dirname(dst)
+        FileUtils.copy_file(path, dst)
+      end
+    end
+
   end
 end

data/templates/{debian7-chroot.yaml → chroot/debian7-amd64.yaml}

@@ -21,6 +21,8 @@ global:
   distrib: debian
   release: wheezy
   arch: amd64
+  kernel_arch: $$arch
+  hostname: kameleon-$$distrib
 
   ## Disk options
   nbd_device: /dev/nbd1
@@ -30,7 +32,6 @@ global:
 
   # rootfs options
   rootfs: $$kameleon_cwd/rootfs
-  rootfs_download_path: /var/cache/kameleon/$$distrib/$$release/$$arch/rootfs
 
   ## System variables. Required by kameleon engine
   # Include specific steps
@@ -38,33 +39,36 @@ global:
     - $$distrib/$$release
     - $$distrib
 
+  # Apt options
+  apt_repository: http://ftp.debian.org/debian/
+  apt_enable_contrib: true
+  apt_enable_nonfree: true
+
   # Shell session from where we launch exec_out commands. There is often a
   # local bash session, but it can be a remote shell on other machines or on
   # any shell. (eg. bash, chroot, fakechroot, ssh, tmux, lxc...)
   out_context:
     cmd: bash
     workdir: $$kameleon_cwd
-
+    proxy_cache: 127.0.0.1
   # Shell session that allows us to connect to the building machine in order to
   # configure it and setup additional programs
   in_context:
     cmd: USER=root HOME=/root LC_ALL=POSIX chroot $$kameleon_cwd/rootfs bash
     workdir: /root/kameleon_workdir
+    proxy_cache: 127.0.0.1
 
 #== Bootstrap the new system and create the 'in_context'
 bootstrap:
+  - initialize_disk_chroot
+  - enable_checkpoint
   - debootstrap:
-    - include_pkg: >
-        ifupdown locales libui-dialog-perl dialog isc-dhcp-client netbase
-        net-tools iproute acpid openssh-server pciutils extlinux
-        linux-image-$$arch
+    - include_pkg: apt-utils ca-certificates locales less
     - release: $$release
     - arch: $$arch
-    - repository: http://ftp.fr.debian.org/debian/
-    - enable_cache: true
-  - initialize_disk_chroot
+    - repository: http://ftp.debian.org/debian/
+    - variant: minbase
   - start_chroot
-  - install_bootloader
 
 #== Install and configuration steps
 # WARNING: this part should be independante from the out context (whenever
@@ -72,16 +76,18 @@ bootstrap:
 setup:
   # Install
   - configure_apt:
-    - enable_contrib_repo: true
-    - enable_nonfree_repo: true
-    - install_recommends: false
+    - repository: $$apt_repository
+    - enable_contrib_repo: $$apt_enable_contrib
+    - enable_nonfree_repo: $$apt_enable_nonfree
   - upgrade_system:
     - dist_upgrade: true
   - install_software:
     - packages: >
-        debian-keyring ntp zip unzip rsync sudo less vim bash-completion
-  - configure_kernel:
-    - arch: $$arch
+        debian-keyring ntp sudo less vim bash-completion curl less acpid
+        linux-image-$$kernel_arch isc-dhcp-client isc-dhcp-common ifupdown
+        iptables iputils-ping iproute pciutils psmisc resolvconf extlinux
+  - configure_kernel
+  - install_bootloader
   # Configuration
   - configure_system:
     - locales: POSIX C en_US fr_FR de_DE
@@ -89,8 +95,7 @@ setup:
     - timezone: UTC
   - configure_keyboard:
     - layout: "us,fr,de"
-  - configure_network:
-    - hostname: kameleon-$$distrib
+  - configure_network
   - create_group:
     - name: admin
   - create_user:
@@ -100,12 +105,15 @@ setup:
 
 #== Export the generated appliance in the format of your choice
 export:
-  - save_appliance:
-    - input: $(readlink $$image_disk)
+  - disable_checkpoint
+  - qemu_save_appliance:
+    - input: $$image_disk
     - output: $$kameleon_cwd/$$kameleon_recipe_name
     - save_as_qcow2
     # - save_as_qed
-    # - save_as_tgz
+    # - save_as_tar_gz
+    # - save_as_tar_xz
+    # - save_as_tar_bz2
     # - save_as_raw
     # - save_as_vmdk
     # - save_as_vdi