kameleon-builder 2.0.0.dev

data/lib/kameleon/cli.rb

@@ -0,0 +1,176 @@
+require 'kameleon/engine'
+require 'kameleon/recipe'
+require 'kameleon/utils'
+
+module Kameleon
+  class CLI < Thor
+
+    class_option :no_color, :type => :boolean, :default => false,
+                 :desc => "Disable colorization in output"
+    class_option :debug, :type => :boolean, :default => false,
+                 :desc => "Enable debug output"
+    class_option :workspace, :aliases => '-w', :type => :string,
+                 :desc => 'Change the kameleon current work directory. ' \
+                          '(The folder containing your recipes folder).' \
+                          ' Default : ./'
+    no_commands do
+      def logger
+        @logger ||= Log4r::Logger.new("kameleon::[cli]")
+      end
+    end
+
+    method_option :template, :aliases => "-t",
+                  :desc => "Starting from a template", :required => true,
+                  :enum => Kameleon.templates_names
+    method_option :force,:type => :boolean,
+                  :default => false, :aliases => "-f",
+                  :desc => "overwrite the recipe"
+    desc "new [RECIPE_NAME]", "Creates a new recipe"
+    def new(recipe_name)
+      logger.notice("Cloning template '#{options[:template]}'")
+      templates_path = Kameleon.env.templates_path
+      recipes_path = Kameleon.env.recipes_path
+
+      template_path = File.join(templates_path, options[:template]) + '.yaml'
+      template_recipe = RecipeTemplate.new(template_path)
+      template_recipe.copy_template(recipes_path,
+                                    recipe_name,
+                                    options[:force])
+      recipe_path = File.join(recipes_path, recipe_name + ".yaml")
+      logger.notice("New recipe \"#{recipe_name}\" "\
+                    "as been created in #{recipe_path}")
+    end
+
+    desc "templates", "Lists all defined templates"
+    def templates
+      Log4r::Outputter['console'].level = Log4r::ERROR unless Kameleon.env.debug
+      puts "The following templates are available in " \
+                 "#{ Kameleon.templates_path }:"
+      templates_hash = []
+      Kameleon.templates_files.each do |f|
+        begin
+        recipe = RecipeTemplate.new(f)
+        templates_hash.push({
+          "name" => recipe.name,
+          "description" => recipe.metainfo['description'],
+        })
+        rescue => e
+          raise e if Kameleon.env.debug
+        end
+      end
+      tp templates_hash, {"name" => {:width => 30}}, { "description" => {:width => 60}}
+    end
+
+    desc "version", "Prints the Kameleon's version information"
+    def version
+      Log4r::Outputter['console'].level = Log4r::OFF unless Kameleon.env.debug
+      puts "Kameleon version #{Kameleon::VERSION}"
+    end
+    map %w(-v --version) => :version
+
+
+    desc "build [RECIPE_NAME]", "Builds the appliance from the recipe"
+    method_option :build_path, :type => :string ,
+                  :default => nil, :aliases => "-b",
+                  :desc => "Set the build directory path"
+    method_option :from_checkpoint, :type => :string ,
+                  :default => nil,
+                  :desc => "Using specific checkpoint to build the image. " \
+                           "Default value is the last checkpoint."
+    method_option :no_checkpoint, :type => :boolean ,
+                  :default => false,
+                  :desc => "Do not use checkpoints"
+    def build(recipe_name)
+      logger.notice("Starting build recipe '#{recipe_name}'")
+      start_time = Time.now.to_i
+      recipe_path = File.join(Kameleon.env.recipes_path, recipe_name) + '.yaml'
+      engine = Kameleon::Engine.new(Recipe.new(recipe_path), options)
+      engine.build
+      total_time = Time.now.to_i - start_time
+      logger.notice("")
+      logger.notice("Build recipe '#{recipe_name}' is completed !")
+      logger.notice("Build total duration : #{total_time} secs")
+      logger.notice("Build directory : #{engine.cwd}")
+      logger.notice("Build recipe file : #{engine.build_recipe_path}")
+      logger.notice("Log file : #{Kameleon.env.log_file}")
+    end
+
+    desc "checkpoints [RECIPE_NAME]", "Lists all availables checkpoints"
+    method_option :build_path, :type => :string ,
+                  :default => nil, :aliases => "-b",
+                  :desc => "Set the build directory path"
+    def checkpoints(recipe_name)
+      Log4r::Outputter['console'].level = Log4r::ERROR unless Kameleon.env.debug
+      recipe_path = File.join(Kameleon.env.recipes_path, recipe_name) + '.yaml'
+      engine = Kameleon::Engine.new(Recipe.new(recipe_path), options)
+      engine.pretty_checkpoints_list
+    end
+
+    desc "clear [RECIPE_NAME]", "Cleaning out context and removing all checkpoints"
+    method_option :build_path, :type => :string ,
+                  :default => nil, :aliases => "-b",
+                  :desc => "Set the build directory path"
+    def clear(recipe_name)
+      Log4r::Outputter['console'].level = Log4r::INFO
+      recipe_path = File.join(Kameleon.env.recipes_path, recipe_name) + '.yaml'
+      engine = Kameleon::Engine.new(Recipe.new(recipe_path), options)
+      engine.clear
+    end
+
+    # Hack Thor to init Kameleon env soon
+    def self.init(base_config)
+      options = base_config[:shell].base.options
+      workspace ||= options[:workspace] || ENV['KAMELEON_WORKSPACE'] || Dir.pwd
+      env_options = options.merge({:workspace => workspace})
+      FileUtils.mkdir_p workspace
+      # configure logger
+      env_options["debug"] = true if ENV["KAMELEON_LOG"] == "debug"
+      ENV["KAMELEON_LOG"] = "debug" if env_options["debug"]
+      if ENV["KAMELEON_LOG"] && ENV["KAMELEON_LOG"] != ""
+        level_name = ENV["KAMELEON_LOG"]
+      else
+        level_name = "info"
+      end
+      # Require Log4r and define the levels we'll be using
+      require 'log4r-color/config'
+      Log4r.define_levels(*Log4r::Log4rConfig::LogLevels)
+
+      begin
+        level = Log4r.const_get(level_name.upcase)
+      rescue NameError
+        fail KameleonError, "Invalid KAMELEON_LOG level is set: #{level_name}.\n" \
+                            "Please use one of the standard log levels: debug," \
+                            " info, warn, or error"
+      end
+      format = ConsoleFormatter.new
+      # format = Log4r::PatternFormatter.new(:pattern => '%11c: %M')
+      if !$stdout.tty? or env_options["no_color"]
+        console_output = Log4r::StdoutOutputter.new('console',
+                                                    :formatter => format)
+      else
+        console_output = Log4r::ColorOutputter.new 'console', {
+          :colors => { :debug  => :light_black,
+                       :info   => :green,
+                       :notice => :light_blue,
+                       :progress => :light_blue,
+                       :warn   => :yellow,
+                       :error  => :red,
+                       :fatal  => :red,
+                     },
+          :formatter => format,
+        }
+      end
+      logger = Log4r::Logger.new('kameleon')
+      logger.outputters << console_output
+      logger.level = level
+      logger = nil
+      Kameleon.logger.debug("`kameleon` invoked: #{ARGV.inspect}")
+      Kameleon.env = Kameleon::Environment.new(env_options)
+    end
+
+    def self.start(given_args=ARGV, config={})
+        config[:shell] ||= Thor::Base.shell.new
+        dispatch(nil, given_args.dup, nil, config) { init(config) }
+    end
+  end
+end

data/lib/kameleon/context.rb

@@ -0,0 +1,83 @@
+require 'kameleon/shell'
+
+module Kameleon
+  class Context
+
+    attr_accessor :shell, :name
+
+    def initialize(name, cmd, workdir, exec_prefix, local_workdir)
+      @name = name.downcase
+      @logger = Log4r::Logger.new("kameleon::[#{@name}_ctx]")
+      @cmd = cmd
+      @workdir = workdir
+      @exec_prefix = exec_prefix
+      @local_workdir = local_workdir
+      @shell = Kameleon::Shell.new(@name, @cmd, @workdir, @local_workdir)
+      @logger.debug("Initialize new ctx (#{name})")
+
+      instance_variables.each do |v|
+        @logger.debug("  #{v} = #{instance_variable_get(v)}")
+      end
+      # Start the shell process
+      @shell.start
+      execute("echo The '#{name}_context' has been initialized", :log_level => "info")
+    end
+
+    def log(log_level, msg)
+      @logger.info msg if log_level == "info"
+      @logger.error msg if log_level == "error"
+      @logger.debug msg if log_level == "debug"
+    end
+
+    def execute(cmd, kwargs = {})
+      cmd_with_prefix = "#{@exec_prefix} #{cmd}"
+      cmd_with_prefix.split( /\r?\n/ ).each {|m| @logger.debug "+ #{m}" }
+      log_level = kwargs.fetch(:log_level, "info")
+      exit_status = @shell.execute(cmd_with_prefix, kwargs) do |out, err|
+        out.split( /\r?\n/ ).each {|m| log(log_level, m) } unless out.nil?
+        err.split( /\r?\n/ ).each {|m| log("error", m) } unless err.nil?
+      end
+      @logger.debug("Exit status : #{exit_status}")
+      fail ExecError unless exit_status.eql? 0
+    rescue ShellError => e
+      @logger.error(e.message)
+      fail ExecError
+    end
+
+    def pipe(cmd, other_cmd, other_ctx)
+      tmp = Tempfile.new("pipe-#{ Kameleon::Utils.generate_slug(cmd)[0..20] }")
+      @logger.info("Running piped commands")
+      @logger.info("Saving STDOUT from #{@name}_ctx to local file #{tmp.path}")
+      execute(cmd, :stdout => tmp)
+      tmp.close
+      @logger.info("Forwarding #{tmp.path} to STDIN of #{other_ctx.name}_ctx")
+      dest_pipe_path = "./pipe-#{ Kameleon::Utils.generate_slug(other_cmd)[0..20] }"
+      other_ctx.send_file(tmp.path, dest_pipe_path)
+      other_cmd_with_pipe = "cat #{dest_pipe_path} | #{other_cmd} && rm #{dest_pipe_path}"
+      other_ctx.execute(other_cmd_with_pipe)
+    end
+
+    def start_shell
+      #TODO: Load env and history
+      @logger.info("Starting interactive shell")
+      @shell.fork_and_wait
+    end
+
+    def closed?
+      @shell.exited?
+    end
+
+    def close!
+      @shell.stop
+    end
+
+    def reopen
+      @shell.restart
+    end
+
+    def send_file(source_path, dest_path)
+      @shell.send_file(source_path, dest_path)
+    end
+  end
+
+end

data/lib/kameleon/engine.rb

@@ -0,0 +1,357 @@
+require 'kameleon/recipe'
+require 'kameleon/context'
+
+
+module Kameleon
+
+  class Engine
+    attr_accessor :recipe, :cwd, :build_recipe_path, :pretty_list_checkpoints
+
+    def initialize(recipe, options)
+      @options = options
+      @logger = Log4r::Logger.new("kameleon::[engine]")
+      @recipe = recipe
+      @cleaned_sections = []
+      @cwd = @recipe.global["kameleon_cwd"]
+      @build_recipe_path = File.join(@cwd, "kameleon_build_recipe.yaml")
+
+      build_recipe = load_build_recipe
+      # restore previous build uuid
+      unless build_recipe.nil?
+        # binding.pry
+        %w(kameleon_uuid kameleon_short_uuid).each do |key|
+          @recipe.global[key] = build_recipe["global"][key]
+        end
+      end
+
+      @enable_checkpoint = !@options[:no_checkpoint]
+      # Check if the recipe have checkpoint entry
+      @enable_checkpoint = !@recipe.checkpoint.nil? if @enable_checkpoint
+
+      @recipe.resolve!
+      @in_context = nil
+      begin
+        @logger.notice("Creating kameleon working directory : #{@cwd}")
+        FileUtils.mkdir_p @cwd
+      rescue
+        raise BuildError, "Failed to create working directory #{@cwd}"
+      end
+      @logger.notice("Building local context [local]")
+      @local_context = Context.new("local", "bash", @cwd, "", @cwd)
+      @logger.notice("Building external context [out]")
+      @out_context = Context.new("out",
+                                 @recipe.global["out_context"]["cmd"],
+                                 @recipe.global["out_context"]["workdir"],
+                                 @recipe.global["out_context"]["exec_prefix"],
+                                 @cwd)
+    end
+
+    def create_checkpoint(microstep_id)
+      cmd = @recipe.checkpoint["create"].gsub("@microstep_id", microstep_id)
+      create_cmd = Kameleon::Command.new({"exec_out" => cmd}, "checkpoint")
+      safe_exec_cmd(create_cmd, :log_level => "debug")
+    end
+
+    def apply_checkpoint(microstep_id)
+      cmd = @recipe.checkpoint["apply"].gsub("@microstep_id", microstep_id)
+      apply_cmd = Kameleon::Command.new({"exec_out" => cmd}, "checkpoint")
+      safe_exec_cmd(apply_cmd, :log_level => "debug")
+    end
+
+    def list_all_checkpoints
+      list = ""
+      cmd = Kameleon::Command.new({"exec_out" => @recipe.checkpoint['list']},
+                                  "checkpoint")
+      safe_exec_cmd(cmd, :stdout => list)
+      return list.split(/\r?\n/)
+    end
+
+    def list_checkpoints
+      if @list_checkpoints.nil?
+        checkpoints = list_all_checkpoints
+        all_microsteps_ids = @recipe.microsteps.map { |m| m.identifier }
+        # get sorted checkpoints by microsteps order
+        @list_checkpoints = []
+        all_microsteps_ids.each do |id|
+          @list_checkpoints.push(id) if checkpoints.include?(id)
+        end
+      end
+      return @list_checkpoints
+    end
+
+    def do_steps(section_name)
+      section = @recipe.sections.fetch(section_name)
+      section.sequence do |macrostep|
+        macrostep.sequence do |microstep|
+          @logger.notice("Step #{ microstep.order } : #{ microstep.slug }")
+          @logger.notice(" ---> #{ microstep.identifier }")
+          if @enable_checkpoint
+            if microstep.on_checkpoint == "skip"
+              @logger.notice(" ---> Skipped")
+              next
+            end
+            if microstep.in_cache && microstep.on_checkpoint == "use_cache"
+              @logger.notice(" ---> Using cache")
+            else
+              @logger.notice(" ---> Running step")
+              microstep.commands.each do |cmd|
+                safe_exec_cmd(cmd)
+              end
+              unless microstep.on_checkpoint == "redo"
+                @logger.notice(" ---> Creating checkpoint : #{ microstep.identifier }")
+                create_checkpoint(microstep.identifier)
+              end
+            end
+          else
+            @logger.notice(" ---> Running step")
+            microstep.commands.each do |cmd|
+              safe_exec_cmd(cmd)
+            end
+          end
+        end
+      end
+      @cleaned_sections.push(section.name)
+    end
+
+    def safe_exec_cmd(cmd, kwargs = {})
+      finished = false
+      begin
+        exec_cmd(cmd, kwargs)
+        finished = true
+      rescue ExecError
+        finished = rescue_exec_error(cmd)
+      end until finished
+    end
+
+    def exec_cmd(cmd, kwargs = {})
+      def skip_alert(cmd)
+        @logger.warn("Skipping cmd '#{cmd.string_cmd}'. The in_context is" \
+                     " not ready yet")
+      end
+      case cmd.key
+      when "breakpoint"
+        breakpoint(cmd.value, )
+      when "exec_in"
+        skip_alert(cmd) if @in_context.nil?
+        @in_context.execute(cmd.value, kwargs) unless @in_context.nil?
+      when "exec_out"
+        @out_context.execute(cmd.value, kwargs)
+      when "exec_local"
+        @local_context.execute(cmd.value, kwargs)
+      when "pipe"
+        first_cmd, second_cmd = cmd.value
+        if ((first_cmd.key == "exec_in" || second_cmd.key == "exec_in")\
+             && @in_context.nil?)
+          skip_alert(cmd)
+        else
+          expected_cmds = ["exec_in", "exec_out", "exec_local"]
+          [first_cmd.key, second_cmd.key].each do |key|
+            unless expected_cmds.include?(key)
+              @logger.error("Invalid pipe arguments. Expected "\
+                            "#{expected_cmds} commands")
+              fail ExecError
+            end
+          end
+          map = {"exec_in" => @in_context,
+                 "exec_out" => @out_context,
+                 "exec_local" => @local_context,}
+          first_context = map[first_cmd.key]
+          second_context = map[second_cmd.key]
+          first_context.pipe(first_cmd.value, second_cmd.value, second_context)
+        end
+      when "rescue"
+        first_cmd, second_cmd = cmd.value
+        begin
+          exec_cmd(first_cmd)
+        rescue ExecError
+          exec_cmd(second_cmd)
+        end
+      else
+        @logger.warn("Unknown command : #{cmd.key}")
+      end
+    end
+
+
+    def breakpoint(message, kwargs = {})
+      message.split( /\r?\n/ ).each {|m| @logger.error "#{m}" }
+      enable_retry = kwargs[:enable_retry]
+      msg = ""
+      msg << "Press [r] to retry\n" if enable_retry
+      msg << "Press [c] to continue with execution"
+      msg << "\nPress [a] to abort execution"
+      msg << "\nPress [l] to switch to local_context shell" unless @local_context.nil?
+      msg << "\nPress [o] to switch to out_context shell" unless @out_context.nil?
+      msg << "\nPress [i] to switch to in_context shell" unless @in_context.nil?
+      responses = {"c" => "continue", "a" => "abort"}
+      responses["r"] = "retry" if enable_retry
+      responses.merge!({"l" => "launch local_context"}) unless @out_context.nil?
+      responses.merge!({"o" => "launch out_context"}) unless @out_context.nil?
+      responses.merge!({"i" => "launch in_context"}) unless @in_context.nil?
+      while true
+        msg.split( /\r?\n/ ).each {|m| @logger.notice "#{m}" }
+        @logger.progress "answer ? [" + responses.keys().join("/") + "]: "
+        answer = $stdin.gets
+        raise AbortError, "Execution aborted..." if answer.nil?
+        answer.chomp!
+        if responses.keys.include?(answer)
+          @logger.notice("User choice : [#{answer}] #{responses[answer]}")
+          if ["o", "i", "l"].include?(answer)
+            if answer.eql? "l"
+              @local_context.start_shell
+            elsif answer.eql? "o"
+              @out_context.start_shell
+            else
+              @in_context.start_shell
+            end
+            @logger.notice("Getting back to Kameleon ...")
+          elsif answer.eql? "a"
+            raise AbortError, "Execution aborted..."
+          elsif answer.eql? "c"
+            ## resetting the exit status
+            @in_context.execute("true") unless @in_context.nil?
+            @out_context.execute("true") unless @out_context.nil?
+            return true
+          elsif answer.eql? "r"
+            @logger.notice("Retrying the previous command...")
+            return false
+          end
+        end
+      end
+    end
+
+    def rescue_exec_error(cmd)
+      message = "Error occured when executing the following command :\n"
+      cmd.string_cmd.split( /\r?\n/ ).each {|m| message << "\n> #{m}" }
+      return breakpoint(message, :enable_retry => true)
+    end
+
+    def finish_clean()
+      @recipe.sections.values.each do |section|
+        next if @cleaned_sections.include?(section.name)
+        begin
+          @logger.notice("Cleaning #{section.name} section")
+          section.clean_macrostep.sequence do |microstep|
+            microstep.commands.each do |cmd|
+              begin
+                exec_cmd(cmd)
+              rescue
+                @logger.warn("An error occurred while executing : #{cmd.value}")
+              end
+            end
+          end
+        end
+      end
+    end
+
+    def clear
+      @recipe.sections.values.each do |section|
+        @logger.notice("Cleaning #{section.name} section")
+        section.clean_macrostep.sequence do |microstep|
+          microstep.commands.each do |cmd|
+            if (cmd.key == "exec_out" || cmd.key == "exec_local")
+              begin
+                exec_cmd(cmd)
+              rescue
+                @logger.warn("An error occurred while executing : #{cmd.value}")
+              end
+            end
+          end
+        end
+      end
+      unless @recipe.checkpoint.nil?
+        @logger.notice("Removing all old checkpoints")
+        cmd = @recipe.checkpoint["clear"]
+        clear_cmd = Kameleon::Command.new({"exec_out" => cmd}, "checkpoint")
+        safe_exec_cmd(clear_cmd, :log_level => "info")
+      end
+    end
+
+    def build
+      if @enable_checkpoint
+        @from_checkpoint = @options[:from_checkpoint]
+        if @from_checkpoint.nil?
+          @from_checkpoint = list_checkpoints.last
+        else
+          unless list_checkpoints.include?@from_checkpoint
+            fail BuildError, "Unknown checkpoint hash : #{@from_checkpoint}." \
+                             " Use checkpoints command to find a valid" \
+                             " checkpoint"
+          end
+        end
+        unless @from_checkpoint.nil?
+          @logger.notice("Restoring last build from step : #{@from_checkpoint}")
+          apply_checkpoint @from_checkpoint
+          @recipe.microsteps.each do |microstep|
+            microstep.in_cache = true
+            if microstep.identifier == @from_checkpoint
+              break
+            end
+          end
+        end
+      end
+      dump_build_recipe
+      begin
+        do_steps("bootstrap")
+        @logger.notice("Building internal context [in]")
+        @in_context = Context.new("in",
+                                  @recipe.global["in_context"]["cmd"],
+                                  @recipe.global["in_context"]["workdir"],
+                                  @recipe.global["in_context"]["exec_prefix"],
+                                  @cwd)
+        do_steps("setup")
+        do_steps("export")
+      rescue Exception => e
+        @logger.warn("Waiting for cleanup before exiting...")
+        @out_context.reopen if !@out_context.nil?
+        @in_context.reopen if !@in_context.nil?
+        @local_context.reopen if !@local_context.nil?
+        finish_clean
+        @out_context.close! unless @out_context.nil?
+        @in_context.close! unless @in_context.nil?
+        @local_context.close! unless @local_context.nil?
+        raise e
+      end
+    end
+
+    def dump_build_recipe
+      File.open(@build_recipe_path, 'w') do |f|
+        f.write @recipe.to_hash.to_yaml
+      end
+    end
+
+    def load_build_recipe
+      if File.file?(@build_recipe_path)
+        result = YAML.load_file(@build_recipe_path)
+        return result if result
+      end
+      return nil
+    end
+
+    def pretty_checkpoints_list
+      def find_microstep_slug_by_id(id)
+        @recipe.microsteps.each do |m|
+          return m.slug if m.identifier == id
+        end
+      end
+      dict_checkpoints = []
+      if @enable_checkpoint
+        list_checkpoints.each do |id|
+          slug = find_microstep_slug_by_id id
+          unless slug.nil?
+            dict_checkpoints.push({
+              "id" => id,
+              "step" => slug
+            })
+          end
+        end
+      end
+      if dict_checkpoints.empty?
+        puts "No checkpoint available for the recipe '#{recipe.name}'"
+      else
+        puts "The following checkpoints are available for  " \
+                 "the recipe '#{recipe.name}':"
+        tp dict_checkpoints, {"id" => {:width => 20}}, { "step" => {:width => 60}}
+      end
+    end
+  end
+end