kameleon-builder 2.0.0.dev

data/lib/kameleon/environment.rb

@@ -0,0 +1,38 @@
+module Kameleon
+
+  # This class allows access to the recipes, CLI, etc. all in the scope of
+  # this environment
+  class Environment
+
+    attr_accessor :workspace
+    attr_accessor :templates_path
+    attr_accessor :recipes_path
+    attr_accessor :build_path
+    attr_accessor :log_file
+    attr_accessor :debug
+
+
+    def initialize(options = {})
+      @logger = Log4r::Logger.new("kameleon::[env]")
+      # symbolify commandline options
+      options = options.inject({}) {|result,(key,value)| result.update({key.to_sym => value})}
+      workspace = File.expand_path(options[:workspace])
+      build_path = File.expand_path(options[:build_path] || File.join(workspace, "builds"))
+      defaults = {
+        :workspace => Pathname.new(workspace),
+        :templates_path => Kameleon.templates_path,
+        :templates_names => Kameleon.templates_names,
+        :recipes_path => Pathname.new(File.join(workspace, "recipes")),
+        :build_path => Pathname.new(build_path),
+        :log_file => Pathname.new(File.join(workspace, "kameleon.log"))
+      }
+      options = defaults.merge(options)
+      @logger.debug("Environment initialized (#{self})")
+      # Injecting all variables of the options and assign the variables
+      options.each do |key, value|
+        instance_variable_set("@#{key}".to_sym, options[key])
+        @logger.debug("  @#{key} : #{options[key]}")
+      end
+    end
+  end
+end

data/lib/kameleon/error.rb

@@ -0,0 +1,51 @@
+require 'thor/error'
+
+module Kameleon
+  class KameleonError < ::StandardError
+    attr_accessor :object
+
+    def initialize(message=nil, object=nil)
+      super(message)
+      self.object = object
+    end
+
+    def self.status_code(code)
+      define_method(:status_code) { code }
+    end
+  end
+
+  class ExecError < KameleonError; status_code(2) ; end
+  class InternalError < KameleonError; status_code(3) ; end
+  class ContextError < KameleonError; status_code(4) ; end
+  class ShellError < KameleonError; status_code(5) ; end
+  class RecipeError < KameleonError; status_code(6) ; end
+  class BuildError < KameleonError; status_code(7) ; end
+  class AbortError < KameleonError; status_code(8) ; end
+
+  def self.with_friendly_errors
+    yield
+  rescue Kameleon::KameleonError => e
+    e.message.split( /\r?\n/ ).each {|m| Kameleon.logger.fatal m }
+    exit e.status_code
+  rescue Thor::UndefinedTaskError => e
+    $stderr << "#{e.message}\n"
+    e.backtrace.each {|m| Kameleon.logger.debug m }
+    exit 15
+  rescue Thor::Error => e
+    $stderr << "#{e.message}\n"
+    e.backtrace.each {|m| Kameleon.logger.debug m }
+    exit 15
+  rescue SystemExit, Interrupt => e
+    Kameleon.logger.fatal("Quitting...")
+    exit 1
+  rescue Exception => e
+    if ENV["KAMELEON_LOG"] != "debug"
+      $stderr << "Unfortunately, a fatal error has occurred : "\
+                 "#{e.message}.\nUse --debug option for more details\n"
+    else
+      Kameleon.logger.debug "Error : #{e}"
+      e.backtrace.each {|m| puts "==> #{m}" }
+    end
+    exit 666
+  end
+end

data/lib/kameleon/logger.rb

@@ -0,0 +1,53 @@
+require 'log4r-color'
+
+module Kameleon
+  # Custom Log4r formatter for the console
+  class ConsoleFormatter < Log4r::BasicFormatter
+    @@basicformat = "%*s"
+
+    def initialize(hash={})
+      super(hash)
+      @max_level_length = 11
+      @on_progress = false
+    end
+
+    def format(event)
+      buff = sprintf(@@basicformat, @max_level_length, event.name)
+      buff << (event.tracer.nil? ? "" : "(#{event.tracer[0]})") + ": "
+      unless Log4r::LNAMES[event.level].eql? "PROGRESS"
+        @on_progress = false
+        buff << format_object(event.data) + "\n"
+      else
+        if @on_progress
+          event.data
+        else
+          @on_progress = true
+          buff << format_object(event.data)
+        end
+      end
+    end
+  end
+
+  # Custom Log4r formatter for files
+  class FileFormatter < Log4r::BasicFormatter
+
+    def initialize(hash={})
+      super(hash)
+    end
+
+    def format(logevent)
+      if Log4r::LNAMES[logevent.level].eql? "PROGRESS"
+        # Formats the data as is with no newline, to allow progress bars to be logged.
+        sprintf("%s", logevent.data.to_s)
+      else
+        if logevent.data.kind_of? String
+          # remove ^M characters
+          logevent.data.gsub!(/\r/, "")
+          # Prevent two newlines in the log file
+          logevent.data.chop! if logevent.data =~ /\n$/
+        end
+        sprintf("[%8s %s] %s\n", Log4r::LNAMES[logevent.level], Time.now.strftime("%m/%d/%Y %I:%M:%S %p"), format_object(logevent.data))
+      end
+    end
+  end
+end

data/lib/kameleon/recipe.rb

@@ -0,0 +1,474 @@
+require 'kameleon/utils'
+require 'kameleon/step'
+
+module Kameleon
+
+  class Recipe
+    attr_accessor :path, :name, :global, :sections, :aliases, :aliases_path, \
+                  :checkpoint, :checkpoint_path, :metainfo
+
+    def initialize(path)
+      @logger = Log4r::Logger.new("kameleon::[recipe]")
+      @path = Pathname.new(path)
+      @name = (@path.basename ".yaml").to_s
+      @recipe_content = File.open(@path, 'r') { |f| f.read }
+      @sections = {
+        "bootstrap" => Section.new("bootstrap"),
+        "setup" => Section.new("setup"),
+        "export" => Section.new("export"),
+      }
+      @required_global = %w(out_context in_context)
+      kameleon_id = SecureRandom.uuid
+      @global = {
+        "kameleon_recipe_name" => @name,
+        "kameleon_recipe_dir" => File.dirname(@path),
+        "kameleon_uuid" => kameleon_id,
+        "kameleon_short_uuid" => kameleon_id.split("-").last,
+        "kameleon_cwd" => File.join(Kameleon.env.build_path, @name),
+      }
+      @aliases = {}
+      @checkpoint = nil
+      @files = []
+      @logger.debug("Initialize new recipe (#{path})")
+      load!
+    end
+
+    def load!
+      # Find recipe path
+      @logger.notice("Loading #{@path}")
+      fail RecipeError, "Could not find this following recipe : #{@path}" \
+         unless File.file? @path
+      yaml_recipe = YAML.load File.open @path
+      unless yaml_recipe.kind_of? Hash
+        fail RecipeError, "Invalid yaml error"
+      end
+      unless yaml_recipe.key? "global"
+        fail RecipeError, "Recipe misses 'global' section"
+      end
+
+      #Load Global variables
+      @global.merge!(yaml_recipe.fetch("global"))
+      # Resolve dynamically-defined variables !!
+      resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
+      @global.merge! YAML.load(resolved_global)
+
+      # Loads aliases
+      load_aliases(yaml_recipe)
+      # Loads checkpoint configuration
+      load_checkpoint_config(yaml_recipe)
+
+      #Find and load steps
+      steps_dir = File.join(File.dirname(@path), 'steps')
+      @global['include_steps'] ||= []
+      @global['include_steps'] = [global['include_steps']].push ''
+      @global['include_steps'].flatten!
+      @global['include_steps'].compact!
+      @sections.values.each do |section|
+        dir_to_search = @global['include_steps'].map do |path|
+          [File.join(steps_dir, section.name, path),
+            File.join(steps_dir, path)]
+        end
+        dir_to_search.flatten!
+        if yaml_recipe.key? section.name
+          yaml_section = yaml_recipe.fetch(section.name)
+          next unless yaml_section.kind_of? Array
+          yaml_section.each do |raw_macrostep|
+
+            # Get macrostep name and arguments if available
+            if raw_macrostep.kind_of? String
+              name = raw_macrostep
+              args = nil
+            elsif raw_macrostep.kind_of? Hash
+              name = raw_macrostep.keys[0]
+              args = raw_macrostep.values[0]
+            else
+              fail RecipeError, "Malformed yaml recipe in section: "\
+                                "#{section.name}"
+            end
+
+            # Load macrostep yaml
+            loaded = false
+            dir_to_search.each do |dir|
+              macrostep_path = Pathname.new(File.join(dir, name + '.yaml'))
+              if File.file?(macrostep_path)
+                @logger.notice("Loading macrostep #{macrostep_path}")
+                macrostep = load_macrostep(macrostep_path, name, args)
+                section.macrosteps.push(macrostep)
+                @files.push(macrostep_path)
+                @logger.debug("Macrostep '#{name}' found in this path: " \
+                              "#{macrostep_path}")
+                loaded = true
+                break
+              else
+                @logger.debug("Macrostep '#{name}' not found in this path: " \
+                              "#{macrostep_path}")
+              end
+            end
+            fail RecipeError, "Step #{name} not found" unless loaded
+          end
+        end
+      end
+      @logger.notice("Loading recipe metadata")
+      @metainfo = {
+        "description" => Utils.extract_meta_var("description", @recipe_content),
+        "recipe" => Utils.extract_meta_var("recipe", @recipe_content),
+        "template" => Utils.extract_meta_var("template", @recipe_content),
+      }
+    end
+
+    def load_aliases(yaml_recipe)
+      if yaml_recipe.keys.include? "aliases"
+        aliases = yaml_recipe.fetch("aliases")
+        if aliases.kind_of? Hash
+          @aliases = aliases
+        elsif aliases.kind_of? String
+          path = Pathname.new(File.join(File.dirname(@path), "aliases", aliases))
+          if File.file?(path)
+            @logger.notice("Loading aliases #{path}")
+            @aliases = YAML.load_file(path)
+            @files.push(path)
+          else
+            fail RecipeError, "Aliases file '#{path}' does not exists"
+          end
+        end
+      end
+    end
+
+    def load_checkpoint_config(yaml_recipe)
+      if yaml_recipe.keys.include? "checkpoint"
+        checkpoint = yaml_recipe.fetch("checkpoint")
+        if checkpoint.kind_of? Hash
+          @checkpoint = checkpoint
+          @checkpoint["path"] = @path
+        elsif checkpoint.kind_of? String
+          path = Pathname.new(File.join(File.dirname(@path),
+                              "checkpoints",
+                              checkpoint))
+          if File.file?(path)
+            @logger.notice("Loading checkpoint configuration #{path}")
+            @checkpoint = YAML.load_file(path)
+            @checkpoint["path"] = path.to_s
+            @files.push(path)
+          else
+            fail RecipeError, "Checkpoint configuraiton file '#{path}' " \
+                              "does not exists"
+          end
+        end
+      end
+    end
+
+    def load_macrostep(step_path, name, args)
+      macrostep_yaml = YAML.load_file(step_path)
+      local_variables = {}
+      loaded_microsteps = []
+      # Basic macrostep syntax check
+      if not macrostep_yaml.kind_of? Array
+        fail RecipeError, "The macrostep #{step_path} is not valid "
+                           "(should be a list of microsteps)"
+      end
+      # Load default local variables
+      macrostep_yaml.each do |yaml_microstep|
+        key = yaml_microstep.keys[0]
+        value = yaml_microstep[key]
+        # Set new variable if not defined yet
+        if value.kind_of? Array
+          loaded_microsteps.push Microstep.new(yaml_microstep)
+        else
+          local_variables[key] = @global.fetch(key, value)
+        end
+      end
+      selected_microsteps = []
+      if args
+        args.each do |entry|
+          if entry.kind_of? Hash
+            # resolve variable before using it
+            entry.each do |key, value|
+              local_variables[key] = value
+            end
+          elsif entry.kind_of? String
+            selected_microsteps.push entry
+          end
+        end
+      end
+      unless selected_microsteps.empty?
+        # Some steps are selected so remove the others
+        # WARN: Allow the user to define this list not in the original order
+        strip_microsteps = []
+        selected_microsteps.each do |microstep_name|
+          macrostep = find_microstep(microstep_name, loaded_microsteps)
+          if macrostep.nil?
+            fail RecipeError, "Can't find microstep '#{microstep_name}' "\
+                              "in macrostep file '#{step_path}'"
+          else
+            strip_microsteps.push(macrostep)
+          end
+        end
+        loaded_microsteps = strip_microsteps
+      end
+      return Macrostep.new(name, loaded_microsteps, local_variables, step_path)
+    end
+
+    def find_microstep(microstep_name, loaded_microsteps)
+      @logger.debug("Looking for microstep #{microstep_name}")
+      loaded_microsteps.each do |microstep|
+        if microstep_name.eql? microstep.name
+          return microstep
+        end
+      end
+    end
+
+    def resolve!
+      consistency_check
+      resolve_checkpoint unless @checkpoint.nil?
+
+      @logger.notice("Resolving variables")
+      @sections.values.each do |section|
+        section.macrosteps.each do |macrostep|
+          macrostep.resolve_variables!(@global)
+        end
+      end
+
+      @sections.values.each do |section|
+        section.macrosteps.each do |macrostep|
+          # First pass : resolve aliases
+          @logger.debug("Resolving aliases for macrostep '#{macrostep.name}'")
+          macrostep.microsteps.each do |microstep|
+            microstep.commands.map! do |cmd|
+              # resolve alias
+              @aliases.keys.include?(cmd.key) ? resolve_alias(cmd) : cmd
+            end
+          end
+          # flatten for multiple-command alias + variables
+          @logger.debug("Resolving check statements for macrostep '#{macrostep.name}'")
+          macrostep.microsteps.each { |microstep| microstep.commands.flatten! }
+          # Second pass : resolve variables + clean/init hooks
+          macrostep.microsteps.each do |microstep|
+            microstep.commands.map! do |cmd|
+              resolve_hooks(cmd, macrostep, microstep)
+            end
+          end
+          @logger.debug("Compacting macrostep '#{macrostep.name}'")
+          # remove empty steps
+          macrostep.microsteps.map! do |microstep|
+            microstep.commands.compact!
+            microstep.commands.empty? ? nil : microstep
+          end
+          # remove nil values
+          macrostep.microsteps.compact!
+          @logger.debug("Resolving commands for macrostep '#{macrostep.name}'")
+          macrostep.microsteps.each do |microstep|
+            microstep.resolve!
+          end
+        end
+      end
+      calculate_step_identifiers
+    end
+
+    def consistency_check()
+      # flatten list of hash to an a hash
+      %w(out_context in_context).each do |context_name|
+        if @global[context_name].kind_of? Array
+          old_context_args = @global[context_name].clone
+          @global[context_name] = {}
+          old_context_args.each do |arg|
+            @global[context_name].merge!(arg)
+          end
+        end
+      end
+      @logger.notice("Starting recipe consistency check")
+      missings = []
+      @required_global.each do |key|
+        missings.push key unless @global.key? key
+      end
+      fail RecipeError, "Required parameters missing in global section :" \
+                        " #{missings.join ' '}" unless missings.empty?
+      # check context args
+      required_args = %w(cmd)
+      missings = []
+      %w(out_context in_context).each do |context_name|
+        context = @global[context_name]
+        missings = required_args - (context.keys() & required_args)
+        fail RecipeError, "Required paramater missing for #{context_name}:" \
+                          " #{ missings.join ' ' }" unless missings.empty?
+      end
+      unless @checkpoint.nil?
+        required_args = %w(create apply list clear)
+        missings = []
+        missings = required_args - (@checkpoint.keys() & required_args)
+        fail RecipeError, "Required paramater missing for checkpoint:" \
+                          " #{ missings.join ' ' }" unless missings.empty?
+      end
+    end
+
+    def resolve_checkpoint()
+      %w(create apply list clear).each do |key|
+        @checkpoint[key] = Utils.resolve_vars(@checkpoint[key],
+                                              @checkpoint["path"],
+                                              @global)
+      end
+    end
+
+    def resolve_alias(cmd)
+      name = cmd.key
+      aliases_cmd = @aliases.fetch(name).clone
+      aliases_cmd_str = aliases_cmd.to_yaml
+      args = YAML.load(cmd.string_cmd)[name]
+      args = [].push(args).flatten  # convert args to array
+      expected_args_number = aliases_cmd_str.scan(/@\d+/).uniq.count
+      if expected_args_number != args.count
+        if args.length == 0
+          msg = "#{name} takes no arguments (#{args.count} given)"
+        else
+          msg = "#{name} takes exactly #{expected_args_number} arguments"
+                " (#{args.count} given)"
+        end
+        raise RecipeError, msg
+      end
+      microstep = Microstep.new({cmd.microstep_name => aliases_cmd})
+      args.each_with_index do |arg, i|
+        microstep.gsub!("@#{i+1}", arg)
+      end
+      microstep.commands.map do |escaped_cmd|
+        Command.new(YAML.load(escaped_cmd.string_cmd), cmd.microstep_name)
+      end
+    end
+
+    #handle clean methods
+    def resolve_hooks(cmd, macrostep, microstep)
+      if (cmd.key =~ /on_(.*)clean/ || cmd.key =~ /on_(.*)init/)
+        cmds = []
+        if cmd.value.kind_of?(Array)
+          cmds = cmd.value.map do |c|
+            @aliases.keys.include?(c.key) ? resolve_alias(c) : c
+          end
+          cmds = cmds.flatten
+        else
+          fail RecipeError, "Invalid #{cmd.key} arguments"
+        end
+        if cmd.key.eql? "on_clean"
+          microstep_name = "_clean_#{macrostep.clean_microsteps.count}" \
+                           "_#{microstep.name}"
+          new_clean_microstep = Microstep.new({microstep_name => []})
+          new_clean_microstep.on_checkpoint = microstep.on_checkpoint
+          new_clean_microstep.commands = cmds.clone
+          macrostep.clean_microsteps.unshift new_clean_microstep
+          return
+        elsif cmd.key.eql? "on_init"
+          microstep_name = "_init_#{macrostep.init_microsteps.count}"\
+                           "_#{microstep.name}"
+          new_init_microstep = Microstep.new({microstep_name=> []},
+                                             microstep)
+          new_init_microstep.on_checkpoint = microstep.on_checkpoint
+          new_init_microstep.commands = cmds.clone
+          macrostep.init_microsteps.unshift new_init_microstep
+          return
+        else
+          @sections.values.each do |section|
+            section.clean_macrostep
+            if cmd.key.eql? "on_#{section.name}_clean"
+              microstep_name = "_clean_#{section.clean_macrostep.microsteps.count}" \
+                               "_#{microstep.name}"
+              new_clean_microstep = Microstep.new({microstep_name=> []})
+              new_clean_microstep.commands = cmds.clone
+              new_clean_microstep.on_checkpoint = microstep.on_checkpoint
+              section.clean_macrostep.microsteps.unshift new_clean_microstep
+              return
+            elsif cmd.key.eql? "on_#{section.name}_init"
+              microstep_name = "_init_#{section.init_macrostep.microsteps.count}" \
+                               "_#{microstep.name}"
+              new_init_microstep = Microstep.new({microstep_name=> []})
+              new_init_microstep.commands = cmds.clone
+              new_init_microstep.on_checkpoint = microstep.on_checkpoint
+              section.init_macrostep.microsteps.push new_init_microstep
+              return
+            end
+          end
+        end
+        fail RecipeError, "Invalid command : '#{cmd.key}'"
+      else
+        return cmd
+      end
+    end
+
+    def microsteps
+      if @microsteps.nil?
+        microsteps = []
+        @sections.values.each do |section|
+          section.sequence do |macrostep|
+            macrostep.sequence do |microstep|
+              microsteps.push microstep
+            end
+          end
+        end
+        @microsteps = microsteps
+      end
+      return @microsteps
+    end
+
+    def calculate_step_identifiers
+      @logger.notice("Calculating microstep identifiers")
+      base_salt = ""
+      order = 0
+      @sections.values.each do |section|
+        section.sequence do |macrostep|
+          macrostep.sequence do |microstep|
+            base_salt = microstep.calculate_identifier base_salt
+            slug = "#{section.name}/#{macrostep.name}/#{microstep.name}"
+            microstep.slug = slug
+            microstep.order = (order += 1)
+            @logger.debug("  #{microstep.slug}: #{microstep.identifier}")
+          end
+        end
+      end
+    end
+
+    def to_hash
+      recipe_hash = {
+        "name" => @name,
+        "path" => @path.to_s,
+        "files" => @files.map {|p| p.to_s },
+        "global" => @global,
+        "required_global" => @required_global,
+        "aliases" => @aliases,
+      }
+      recipe_hash["checkpoint"] = @checkpoint unless @checkpoint.nil?
+      recipe_hash["steps"] = to_array
+      return recipe_hash
+    end
+
+    def to_array
+      array = []
+      @sections.values.each do |section|
+        section.to_array.each { |m|  array.push m }
+      end
+      return array
+    end
+
+  end
+
+  class RecipeTemplate < Recipe
+
+    def copy_template(dest_path, recipe_name, force)
+      Dir::mktmpdir do |tmp_dir|
+        recipe_path = File.join(tmp_dir, recipe_name + '.yaml')
+        FileUtils.cp(@path, recipe_path)
+        File.open(recipe_path, 'w+') do |file|
+          tpl = ERB.new(@recipe_content)
+          result = tpl.result(binding)
+          file.write(result)
+        end
+
+        @files.each do |path|
+          relative_path = path.relative_path_from(Kameleon.env.templates_path)
+          dst = File.join(tmp_dir, File.dirname(relative_path))
+          FileUtils.mkdir_p dst
+          FileUtils.cp(path, dst)
+          @logger.debug("Copying '#{path}' to '#{dst}'")
+        end
+        # Create recipe dir if not exists
+        FileUtils.mkdir_p Kameleon.env.recipes_path
+        FileUtils.cp_r(Dir[tmp_dir + '/*'], Kameleon.env.recipes_path)
+      end
+    end
+  end
+end