kameleon-builder 2.0.0 → 2.1.0

data/lib/kameleon/compat.rb

@@ -0,0 +1,39 @@
+require 'securerandom'
+require 'yaml'
+
+YAML::ENGINE.yamler = 'syck' if (defined?(Syck) || defined?(YAML::Syck)) && defined?(YAML::ENGINE)
+
+if RUBY_VERSION < "1.9.3"
+  # Backport of missing SecureRandom methods from 1.9
+  # Snippet from http://softover.com/UUID_in_Ruby_1.8
+  module SecureRandom
+    class << self
+      def method_missing(method_sym, *arguments, &block)
+        case method_sym
+        when :urlsafe_base64
+          r19_urlsafe_base64(*arguments)
+        when :uuid
+          r19_uuid(*arguments)
+        else
+          super
+        end
+      end
+
+      private
+      def r19_urlsafe_base64(n=nil, padding=false)
+        s = [random_bytes(n)].pack("m*")
+        s.delete!("\n")
+        s.tr!("+/", "-_")
+        s.delete!("=") if !padding
+        s
+      end
+
+      def r19_uuid
+        ary = random_bytes(16).unpack("NnnnnN")
+        ary[2] = (ary[2] & 0x0fff) | 0x4000
+        ary[3] = (ary[3] & 0x3fff) | 0x8000
+        "%08x-%04x-%04x-%04x-%04x%08x" % ary
+      end
+    end
+  end
+end

data/lib/kameleon/context.rb

@@ -14,13 +14,27 @@ module Kameleon
       @local_workdir = local_workdir
       @shell = Kameleon::Shell.new(@name, @cmd, @workdir, @local_workdir)
       @logger.debug("Initialize new ctx (#{name})")
+      @log_on_progress = false
 
       instance_variables.each do |v|
         @logger.debug("  #{v} = #{instance_variable_get(v)}")
       end
-      # Start the shell process
-      @shell.start
-      execute("echo The '#{name}_context' has been initialized", :log_level => "info")
+
+      @cache = Kameleon::Persistent_cache.instance
+    end
+
+    def do_log(out, log_level)
+      if @log_on_progress
+        log_progress(log_level, out)
+        @log_on_progress = false if out.match(/\n$/)
+      else
+        if out.match(/\n$/)
+          out.split( /\r?\n/ ).each {|m| log(log_level, m) }
+        else
+          log_progress(log_level, out)
+          @log_on_progress = true
+        end
+      end
     end
 
     def log(log_level, msg)
@@ -29,42 +43,58 @@ module Kameleon
       @logger.debug msg if log_level == "debug"
     end
 
+    def log_progress(log_level, msg)
+      @logger.progress_info msg if log_level == "info"
+      @logger.progress_error msg if log_level == "error"
+      @logger.debug msg if log_level == "debug"
+    end
+
     def execute(cmd, kwargs = {})
+      lazyload_shell
       cmd_with_prefix = "#{@exec_prefix} #{cmd}"
       cmd_with_prefix.split( /\r?\n/ ).each {|m| @logger.debug "+ #{m}" }
       log_level = kwargs.fetch(:log_level, "info")
       exit_status = @shell.execute(cmd_with_prefix, kwargs) do |out, err|
-        out.split( /\r?\n/ ).each {|m| log(log_level, m) } unless out.nil?
-        err.split( /\r?\n/ ).each {|m| log("error", m) } unless err.nil?
+        do_log(out, log_level) unless out.nil?
+        do_log(err, "error") unless err.nil?
       end
       @logger.debug("Exit status : #{exit_status}")
       fail ExecError unless exit_status.eql? 0
-    rescue ShellError => e
-      @logger.error(e.message)
-      fail ExecError
+    rescue ShellError, Errno::EPIPE  => e
+      @logger.debug("Shell cmd failed to launch: #{@shell.shell_cmd}")
+      raise ShellError, e.message + ". Check the cmd argument of the '#{@name}_context'."
     end
 
     def pipe(cmd, other_cmd, other_ctx)
       tmp = Tempfile.new("pipe-#{ Kameleon::Utils.generate_slug(cmd)[0..20] }")
-      @logger.info("Running piped commands")
-      @logger.info("Saving STDOUT from #{@name}_ctx to local file #{tmp.path}")
+      @logger.debug("Running piped commands")
+      @logger.debug("Saving STDOUT from #{@name}_ctx to local file #{tmp.path}")
       execute(cmd, :stdout => tmp)
       tmp.close
-      @logger.info("Forwarding #{tmp.path} to STDIN of #{other_ctx.name}_ctx")
-      dest_pipe_path = "./pipe-#{ Kameleon::Utils.generate_slug(other_cmd)[0..20] }"
+      @logger.debug("Forwarding #{tmp.path} to STDIN of #{other_ctx.name}_ctx")
+      dest_pipe_path = "${KAMELEON_WORKDIR}/pipe-#{ Kameleon::Utils.generate_slug(other_cmd)[0..20] }"
       other_ctx.send_file(tmp.path, dest_pipe_path)
       other_cmd_with_pipe = "cat #{dest_pipe_path} | #{other_cmd} && rm #{dest_pipe_path}"
       other_ctx.execute(other_cmd_with_pipe)
     end
 
+    def lazyload_shell()
+      unless @shell.started?
+        # Start the shell process
+        @shell.start
+        execute("echo The '#{name}_context' has been initialized", :log_level => "info")
+      end
+    end
+
     def start_shell
       #TODO: Load env and history
+      lazyload_shell
       @logger.info("Starting interactive shell")
       @shell.fork_and_wait
     end
 
     def closed?
-      @shell.exited?
+      return !@shell.started? || @shell.exited?
     end
 
     def close!

data/lib/kameleon/engine.rb

@@ -1,6 +1,6 @@
 require 'kameleon/recipe'
 require 'kameleon/context'
-
+require 'kameleon/persistent_cache'
 
 module Kameleon
 
@@ -9,7 +9,7 @@ module Kameleon
 
     def initialize(recipe, options)
       @options = options
-      @logger = Log4r::Logger.new("kameleon::[engine]")
+      @logger = Log4r::Logger.new("kameleon::[kameleon]")
       @recipe = recipe
       @cleaned_sections = []
       @cwd = @recipe.global["kameleon_cwd"]
@@ -29,6 +29,17 @@ module Kameleon
       @enable_checkpoint = !@recipe.checkpoint.nil? if @enable_checkpoint
 
       @recipe.resolve!
+
+      if @options[:cache] || @options[:from_cache] then
+        @cache = Kameleon::Persistent_cache.instance
+        @cache.activated = true
+        @cache.cwd = @cwd
+        @cache.polipo_path = @options[:proxy_path]
+        @cache.check_polipo_binary
+        @cache.name = @recipe.name
+        #saving_steps_files
+      end
+
       @in_context = nil
       begin
         @logger.notice("Creating kameleon working directory : #{@cwd}")
@@ -44,6 +55,34 @@ module Kameleon
                                  @recipe.global["out_context"]["workdir"],
                                  @recipe.global["out_context"]["exec_prefix"],
                                  @cwd)
+      @logger.notice("Building internal context [in]")
+      @in_context = Context.new("in",
+                                @recipe.global["in_context"]["cmd"],
+                                @recipe.global["in_context"]["workdir"],
+                                @recipe.global["in_context"]["exec_prefix"],
+                                @cwd)
+      if @options[:from_cache] then
+        begin
+          @cache.unpack(@options[:from_cache])
+        rescue
+          raise BuildError, "Failed to untar the persistent cache file"
+        end
+      end
+    end
+
+    def saving_steps_files
+      @recipe.files.each do |file|
+        @logger.notice("File #{file} loaded from the recipe")
+        sleep 1
+      end
+
+    end
+
+    def create_cache_directory(step_name)
+      @logger.notice("Creating directory for cache #{step_name}")
+      directory_name = @cache.cache_dir + "/#{step_name}"
+      FileUtils.mkdir_p directory_name
+      directory_name
     end
 
     def create_checkpoint(microstep_id)
@@ -82,16 +121,22 @@ module Kameleon
     def do_steps(section_name)
       section = @recipe.sections.fetch(section_name)
       section.sequence do |macrostep|
+
+        if @cache then
+          # the following function start a polipo web proxy and stops a previous run
+          dir_cache = @cache.create_cache_directory(macrostep.name) #unless @options[:from_cache]
+          @cache.start_web_proxy_in(dir_cache)
+        end
+
         macrostep.sequence do |microstep|
           @logger.notice("Step #{ microstep.order } : #{ microstep.slug }")
-          @logger.notice(" ---> #{ microstep.identifier }")
           if @enable_checkpoint
             if microstep.on_checkpoint == "skip"
               @logger.notice(" ---> Skipped")
               next
             end
             if microstep.in_cache && microstep.on_checkpoint == "use_cache"
-              @logger.notice(" ---> Using cache")
+              @logger.notice(" ---> Using cache this time")
             else
               @logger.notice(" ---> Running step")
               microstep.commands.each do |cmd|
@@ -111,6 +156,12 @@ module Kameleon
         end
       end
       @cleaned_sections.push(section.name)
+
+      if @cache then
+        @cache.stop_web_proxy
+        @cache.pack unless @options[:from_cache]
+      end
+
     end
 
     def safe_exec_cmd(cmd, kwargs = {})
@@ -124,41 +175,32 @@ module Kameleon
     end
 
     def exec_cmd(cmd, kwargs = {})
-      def skip_alert(cmd)
-        @logger.warn("Skipping cmd '#{cmd.string_cmd}'. The in_context is" \
-                     " not ready yet")
-      end
       case cmd.key
       when "breakpoint"
-        breakpoint(cmd.value, )
+        breakpoint(cmd.value)
       when "exec_in"
         skip_alert(cmd) if @in_context.nil?
-        @in_context.execute(cmd.value, kwargs) unless @in_context.nil?
+        @in_context.execute(cmd.value, kwargs)
       when "exec_out"
         @out_context.execute(cmd.value, kwargs)
       when "exec_local"
         @local_context.execute(cmd.value, kwargs)
       when "pipe"
         first_cmd, second_cmd = cmd.value
-        if ((first_cmd.key == "exec_in" || second_cmd.key == "exec_in")\
-             && @in_context.nil?)
-          skip_alert(cmd)
-        else
-          expected_cmds = ["exec_in", "exec_out", "exec_local"]
-          [first_cmd.key, second_cmd.key].each do |key|
-            unless expected_cmds.include?(key)
-              @logger.error("Invalid pipe arguments. Expected "\
-                            "#{expected_cmds} commands")
-              fail ExecError
-            end
+        expected_cmds = ["exec_in", "exec_out", "exec_local"]
+        [first_cmd.key, second_cmd.key].each do |key|
+          unless expected_cmds.include?(key)
+            @logger.error("Invalid pipe arguments. Expected "\
+                          "#{expected_cmds} commands")
+            fail ExecError
           end
-          map = {"exec_in" => @in_context,
-                 "exec_out" => @out_context,
-                 "exec_local" => @local_context,}
-          first_context = map[first_cmd.key]
-          second_context = map[second_cmd.key]
-          first_context.pipe(first_cmd.value, second_cmd.value, second_context)
         end
+        map = {"exec_in" => @in_context,
+               "exec_out" => @out_context,
+               "exec_local" => @local_context,}
+        first_context = map[first_cmd.key]
+        second_context = map[second_cmd.key]
+        first_context.pipe(first_cmd.value, second_cmd.value, second_context)
       when "rescue"
         first_cmd, second_cmd = cmd.value
         begin
@@ -179,17 +221,17 @@ module Kameleon
       msg << "Press [r] to retry\n" if enable_retry
       msg << "Press [c] to continue with execution"
       msg << "\nPress [a] to abort execution"
-      msg << "\nPress [l] to switch to local_context shell" unless @local_context.nil?
-      msg << "\nPress [o] to switch to out_context shell" unless @out_context.nil?
-      msg << "\nPress [i] to switch to in_context shell" unless @in_context.nil?
+      msg << "\nPress [l] to switch to local_context shell"
+      msg << "\nPress [o] to switch to out_context shell"
+      msg << "\nPress [i] to switch to in_context shell"
       responses = {"c" => "continue", "a" => "abort"}
       responses["r"] = "retry" if enable_retry
-      responses.merge!({"l" => "launch local_context"}) unless @out_context.nil?
-      responses.merge!({"o" => "launch out_context"}) unless @out_context.nil?
-      responses.merge!({"i" => "launch in_context"}) unless @in_context.nil?
+      responses.merge!({"l" => "launch local_context"})
+      responses.merge!({"o" => "launch out_context"})
+      responses.merge!({"i" => "launch in_context"})
       while true
         msg.split( /\r?\n/ ).each {|m| @logger.notice "#{m}" }
-        @logger.progress "answer ? [" + responses.keys().join("/") + "]: "
+        @logger.progress_notice "answer ? [" + responses.keys().join("/") + "]: "
         answer = $stdin.gets
         raise AbortError, "Execution aborted..." if answer.nil?
         answer.chomp!
@@ -203,13 +245,14 @@ module Kameleon
             else
               @in_context.start_shell
             end
-            @logger.notice("Getting back to Kameleon ...")
+            @logger.notice("Getting back to Kameleon...")
           elsif answer.eql? "a"
             raise AbortError, "Execution aborted..."
           elsif answer.eql? "c"
             ## resetting the exit status
-            @in_context.execute("true") unless @in_context.nil?
-            @out_context.execute("true") unless @out_context.nil?
+            @in_context.execute("true")
+            @out_context.execute("true")
+            @local_context.execute("true")
             return true
           elsif answer.eql? "r"
             @logger.notice("Retrying the previous command...")
@@ -225,39 +268,32 @@ module Kameleon
       return breakpoint(message, :enable_retry => true)
     end
 
-    def finish_clean()
+    def clean()
       @recipe.sections.values.each do |section|
         next if @cleaned_sections.include?(section.name)
-        begin
-          @logger.notice("Cleaning #{section.name} section")
-          section.clean_macrostep.sequence do |microstep|
-            microstep.commands.each do |cmd|
-              begin
-                exec_cmd(cmd)
-              rescue
-                @logger.warn("An error occurred while executing : #{cmd.value}")
-              end
-            end
-          end
-        end
-      end
-    end
-
-    def clear
-      @recipe.sections.values.each do |section|
+        map = {"exec_in" => @in_context,
+               "exec_out" => @out_context,
+               "exec_local" => @local_context}
         @logger.notice("Cleaning #{section.name} section")
         section.clean_macrostep.sequence do |microstep|
           microstep.commands.each do |cmd|
-            if (cmd.key == "exec_out" || cmd.key == "exec_local")
-              begin
-                exec_cmd(cmd)
-              rescue
-                @logger.warn("An error occurred while executing : #{cmd.value}")
+            if map.keys.include? cmd.key
+              unless map[cmd.key].closed?
+                begin
+                  exec_cmd(cmd)
+                rescue
+                  @logger.warn("An error occurred while executing : #{cmd.value}")
+                end
               end
             end
           end
         end
       end
+      @cache.stop_web_proxy if @options[:cache] ## stopping polipo
+    end
+
+    def clear
+      clean
       unless @recipe.checkpoint.nil?
         @logger.notice("Removing all old checkpoints")
         cmd = @recipe.checkpoint["clear"]
@@ -291,24 +327,27 @@ module Kameleon
       end
       dump_build_recipe
       begin
-        do_steps("bootstrap")
-        @logger.notice("Building internal context [in]")
-        @in_context = Context.new("in",
-                                  @recipe.global["in_context"]["cmd"],
-                                  @recipe.global["in_context"]["workdir"],
-                                  @recipe.global["in_context"]["exec_prefix"],
-                                  @cwd)
-        do_steps("setup")
-        do_steps("export")
-      rescue Exception => e
+        ["bootstrap", "setup", "export"].each do |section|
+          do_steps(section)
+        end
+        clean
+      rescue AbortError => e
+        @logger.error("Aborted...")
         @logger.warn("Waiting for cleanup before exiting...")
-        @out_context.reopen if !@out_context.nil?
-        @in_context.reopen if !@in_context.nil?
-        @local_context.reopen if !@local_context.nil?
-        finish_clean
-        @out_context.close! unless @out_context.nil?
-        @in_context.close! unless @in_context.nil?
-        @local_context.close! unless @local_context.nil?
+        clean
+        raise e
+      rescue SystemExit, Interrupt => e
+        @logger.error("Interrupted...")
+        @out_context.reopen
+        @in_context.reopen
+        @local_context.reopen
+        @logger.warn("Waiting for cleanup before exiting...")
+        clean
+        raise e
+      rescue Exception => e
+        @out_context.close!
+        @in_context.close!
+        @local_context.close!
         raise e
       end
     end
@@ -328,6 +367,8 @@ module Kameleon
     end
 
     def pretty_checkpoints_list
+
+
       def find_microstep_slug_by_id(id)
         @recipe.microsteps.each do |m|
           return m.slug if m.identifier == id