kameleon-builder 2.0.0 → 2.1.0

data/lib/kameleon/environment.rb

@@ -6,23 +6,21 @@ module Kameleon
 
     attr_accessor :workspace
     attr_accessor :templates_path
-    attr_accessor :recipes_path
     attr_accessor :build_path
     attr_accessor :log_file
     attr_accessor :debug
 
 
     def initialize(options = {})
-      @logger = Log4r::Logger.new("kameleon::[env]")
+      @logger = Log4r::Logger.new("kameleon::[kameleon]")
       # symbolify commandline options
       options = options.inject({}) {|result,(key,value)| result.update({key.to_sym => value})}
-      workspace = File.expand_path(options[:workspace])
-      build_path = File.expand_path(options[:build_path] || File.join(workspace, "builds"))
+      workspace = File.expand_path(Dir.pwd)
+      build_path = File.expand_path(options[:build_path] || File.join(workspace, "build"))
       defaults = {
         :workspace => Pathname.new(workspace),
         :templates_path => Kameleon.templates_path,
         :templates_names => Kameleon.templates_names,
-        :recipes_path => Pathname.new(File.join(workspace, "recipes")),
         :build_path => Pathname.new(build_path),
         :log_file => Pathname.new(File.join(workspace, "kameleon.log"))
       }

data/lib/kameleon/error.rb

@@ -1,7 +1,7 @@
 require 'thor/error'
 
 module Kameleon
-  class KameleonError < ::StandardError
+  class Error < ::StandardError
     attr_accessor :object
 
     def initialize(message=nil, object=nil)
@@ -14,17 +14,19 @@ module Kameleon
     end
   end
 
-  class ExecError < KameleonError; status_code(2) ; end
-  class InternalError < KameleonError; status_code(3) ; end
-  class ContextError < KameleonError; status_code(4) ; end
-  class ShellError < KameleonError; status_code(5) ; end
-  class RecipeError < KameleonError; status_code(6) ; end
-  class BuildError < KameleonError; status_code(7) ; end
-  class AbortError < KameleonError; status_code(8) ; end
+  class KameleonError < Error; status_code(1) ; end
+  class ExecError < Error; status_code(2) ; end
+  class InternalError < Error; status_code(3) ; end
+  class ContextError < Error; status_code(4) ; end
+  class ShellError < Error; status_code(5) ; end
+  class RecipeError < Error; status_code(6) ; end
+  class BuildError < Error; status_code(7) ; end
+  class AbortError < Error; status_code(8) ; end
+  class TemplateNotFound < Error; status_code(9) ; end
 
   def self.with_friendly_errors
     yield
-  rescue Kameleon::KameleonError => e
+  rescue Kameleon::Error => e
     e.message.split( /\r?\n/ ).each {|m| Kameleon.logger.fatal m }
     exit e.status_code
   rescue Thor::UndefinedTaskError => e
@@ -38,6 +40,10 @@ module Kameleon
   rescue SystemExit, Interrupt => e
     Kameleon.logger.fatal("Quitting...")
     exit 1
+  rescue Errno::ENOENT => e
+    $stderr << "#{e.message}\n"
+    e.backtrace.each {|m| Kameleon.logger.debug m }
+    exit 16
   rescue Exception => e
     if ENV["KAMELEON_LOG"] != "debug"
       $stderr << "Unfortunately, a fatal error has occurred : "\

data/lib/kameleon/logger.rb

@@ -14,7 +14,7 @@ module Kameleon
     def format(event)
       buff = sprintf(@@basicformat, @max_level_length, event.name)
       buff << (event.tracer.nil? ? "" : "(#{event.tracer[0]})") + ": "
-      unless Log4r::LNAMES[event.level].eql? "PROGRESS"
+      unless Log4r::LNAMES[event.level].include? "PROGRESS"
         @on_progress = false
         buff << format_object(event.data) + "\n"
       else
@@ -30,13 +30,14 @@ module Kameleon
 
   # Custom Log4r formatter for files
   class FileFormatter < Log4r::BasicFormatter
-
+    @@basicformat = "%*s"
     def initialize(hash={})
       super(hash)
+      @max_level_length = 11
     end
 
     def format(logevent)
-      if Log4r::LNAMES[logevent.level].eql? "PROGRESS"
+      if Log4r::LNAMES[logevent.level].include? "PROGRESS"
         # Formats the data as is with no newline, to allow progress bars to be logged.
         sprintf("%s", logevent.data.to_s)
       else
@@ -46,7 +47,9 @@ module Kameleon
           # Prevent two newlines in the log file
           logevent.data.chop! if logevent.data =~ /\n$/
         end
-        sprintf("[%8s %s] %s\n", Log4r::LNAMES[logevent.level], Time.now.strftime("%m/%d/%Y %I:%M:%S %p"), format_object(logevent.data))
+        tracer = sprintf(@@basicformat, @max_level_length, logevent.name)
+        tracer << (logevent.tracer.nil? ? "" : "(#{logevent.tracer[0]})") + ": "
+        sprintf("%s %s\n", tracer, format_object(logevent.data))
       end
     end
   end

data/lib/kameleon/persistent_cache.rb

@@ -0,0 +1,139 @@
+require 'childprocess'
+require 'singleton'
+require 'socket'
+
+module Kameleon
+  #This ruby class will control the execution of Polipo web proxy
+  class Persistent_cache
+
+    include Singleton
+    attr_reader :polipo_env, :cache_dir,:polipo_port
+    attr_writer :activated, :cwd, :polipo_path, :name
+    def initialize()
+      @logger = Log4r::Logger.new("kameleon::[kameleon]")
+      ## we must configure Polipo to be execute for the in and out context
+      ## we have to start polipo in the out context for debootstrap step
+
+      @polipo_env = File.join(Kameleon.source_root,
+                              "contrib",
+                              "polipo_env.sh")
+
+      @polipo_process = nil
+      @polipo_port = find_unused_port
+
+      @polipo_cmd_options = {:diskCacheRoot => "",
+                            :idleTime => "5",
+                            :chunkHighMark => "425165824",
+                            :proxyPort => @polipo_port,
+                            #:proxyOffline => "true"
+                            :relaxTransparency =>"true"
+                            }
+
+      @activated = false
+
+      @cache_dir = ""
+      @polipo_path = nil
+      @cwd = ""
+
+    end
+
+    def find_unused_port
+      ports = (8000..9000)
+      port = 0
+      tmp = nil
+      ports.each do |p|
+        begin
+          port = p
+          tmp = TCPServer.new('localhost',port)
+        rescue
+          port =0
+        end
+        break if(port>0)
+      end
+      tmp.close
+      port
+    end
+
+    def check_polipo_binary
+
+
+      @polipo_path ||= which("polipo")
+
+      if @polipo_path.nil? then
+        @logger.error("Polipo binary not found, make sure it is in your current PATH")
+        @logger.error("or use the option --proxy_path")
+        raise BuildError, "Failed to use persistent cache"
+      end
+    end
+
+    def activated?
+      @activated
+    end
+
+
+    def cwd=(dir)
+      @cwd = dir
+      @cache_dir = @cwd + "/cache/"
+    end
+
+    def create_cache_directory(step_name)
+      @logger.notice("Creating  cache directory #{step_name} for Polipo")
+      directory_name = @cache_dir + "/#{step_name}"
+      FileUtils.mkdir_p directory_name
+      directory_name
+    end
+
+    def start_web_proxy_in(directory)
+      ## This function assumes that the cache directory has already been created by the engine
+      ## Stopping first the previous proxy
+      ## have to check if polipo is running
+      @logger.notice("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
+      @polipo_process.stop unless @polipo_process.nil?
+      command = ["#{@polipo_path}/polipo"]
+      @polipo_cmd_options[:diskCacheRoot] = directory
+      @polipo_cmd_options.each{ |v,k| command.push("#{v}=#{k}") }
+      ChildProcess.posix_spawn = true
+      @polipo_process = ChildProcess.build(*command)
+      @polipo_process.io.stdout = Tempfile.new("polipo_output")
+      @polipo_process.start
+    end
+
+
+    def stop_web_proxy
+      @polipo_process.stop
+      @logger.notice("Stopping web proxy polipo")
+    end
+
+    def pack()
+      @logger.notice("Packing up the generated cache in #{@cwd}")
+      execute("tar","-cf #{@name}-cache.tar cache/",@cwd)
+      # The cache directory cannot be deleted due to the checkpoints
+    end
+
+    def unpack(cache_path)
+      @logger.notice("Unpacking persistent cache: #{cache_path}")
+      execute("tar","-xf #{cache_path} -C #{@cwd}")
+    end
+
+    def execute(cmd,args,dir=nil)
+      command = [cmd ] + args.split(" ")
+#      @logger.notice(" command generated: #{command}")
+      process = ChildProcess.build(*command)
+      process.cwd = dir unless dir.nil?
+      process.start
+      process.wait
+    end
+
+    def which(cmd)
+      ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+        exe = File.join(path, "#{cmd}")
+        return path if File.executable? exe
+      end
+      return nil
+    end
+
+
+  end
+
+end
+

data/lib/kameleon/recipe.rb

@@ -8,7 +8,7 @@ module Kameleon
                   :checkpoint, :checkpoint_path, :metainfo
 
     def initialize(path)
-      @logger = Log4r::Logger.new("kameleon::[recipe]")
+      @logger = Log4r::Logger.new("kameleon::[kameleon]")
       @path = Pathname.new(path)
       @name = (@path.basename ".yaml").to_s
       @recipe_content = File.open(@path, 'r') { |f| f.read }
@@ -17,7 +17,6 @@ module Kameleon
         "setup" => Section.new("setup"),
         "export" => Section.new("export"),
       }
-      @required_global = %w(out_context in_context)
       kameleon_id = SecureRandom.uuid
       @global = {
         "kameleon_recipe_name" => @name,
@@ -25,29 +24,32 @@ module Kameleon
         "kameleon_uuid" => kameleon_id,
         "kameleon_short_uuid" => kameleon_id.split("-").last,
         "kameleon_cwd" => File.join(Kameleon.env.build_path, @name),
+        "in_context" => {"cmd"=> "/bin/bash"},
+        "out_context" => {"cmd"=> "/bin/bash"}
       }
       @aliases = {}
       @checkpoint = nil
       @files = []
       @logger.debug("Initialize new recipe (#{path})")
+      @base_recipes_files = [@path]
       load!
     end
 
     def load!
       # Find recipe path
-      @logger.notice("Loading #{@path}")
+      @logger.debug("Loading #{@path}")
       fail RecipeError, "Could not find this following recipe : #{@path}" \
          unless File.file? @path
       yaml_recipe = YAML.load File.open @path
       unless yaml_recipe.kind_of? Hash
         fail RecipeError, "Invalid yaml error"
       end
-      unless yaml_recipe.key? "global"
-        fail RecipeError, "Recipe misses 'global' section"
-      end
+      # Load entended recipe variables
+      yaml_recipe = load_base_recipe(yaml_recipe)
+      yaml_recipe.delete("extend")
 
-      #Load Global variables
-      @global.merge!(yaml_recipe.fetch("global"))
+      # Load Global variables
+      @global.merge!(yaml_recipe.fetch("global", {}))
       # Resolve dynamically-defined variables !!
       resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
       @global.merge! YAML.load(resolved_global)
@@ -73,7 +75,7 @@ module Kameleon
           yaml_section = yaml_recipe.fetch(section.name)
           next unless yaml_section.kind_of? Array
           yaml_section.each do |raw_macrostep|
-
+            embedded_step = false
             # Get macrostep name and arguments if available
             if raw_macrostep.kind_of? String
               name = raw_macrostep
@@ -85,13 +87,28 @@ module Kameleon
               fail RecipeError, "Malformed yaml recipe in section: "\
                                 "#{section.name}"
             end
-
+            # Detect if step is embedded
+            if not args.nil?
+              args.each do |arg|
+                if arg.kind_of? Hash
+                  if arg.flatten[1].kind_of? Array
+                    embedded_step = true
+                  end
+                end
+              end
+            end
+            if embedded_step
+              @logger.debug("Loading embedded macrostep #{name}")
+              macrostep = load_macrostep(nil, name, args)
+              section.macrosteps.push(macrostep)
+              next
+            end
             # Load macrostep yaml
             loaded = false
             dir_to_search.each do |dir|
               macrostep_path = Pathname.new(File.join(dir, name + '.yaml'))
               if File.file?(macrostep_path)
-                @logger.notice("Loading macrostep #{macrostep_path}")
+                @logger.debug("Loading macrostep #{macrostep_path}")
                 macrostep = load_macrostep(macrostep_path, name, args)
                 section.macrosteps.push(macrostep)
                 @files.push(macrostep_path)
@@ -108,28 +125,81 @@ module Kameleon
           end
         end
       end
-      @logger.notice("Loading recipe metadata")
+      @logger.debug("Loading recipe metadata")
       @metainfo = {
-        "description" => Utils.extract_meta_var("description", @recipe_content),
-        "recipe" => Utils.extract_meta_var("recipe", @recipe_content),
-        "template" => Utils.extract_meta_var("template", @recipe_content),
+        "description" => Utils.extract_meta_var("description", @recipe_content)
       }
     end
 
+    def load_base_recipe(yaml_recipe)
+      base_recipe_name = yaml_recipe.fetch("extend", "")
+      return yaml_recipe if base_recipe_name.empty?
+
+      ## check that the recipe has not already been loaded
+      base_recipe_name << ".yaml" unless base_recipe_name.end_with? ".yaml"
+      base_recipe_path = File.join(File.dirname(@path), base_recipe_name)
+
+      ## check that the recipe has not already been loaded
+      return yaml_recipe if @base_recipes_files.include? base_recipe_path
+
+      base_recipe_path << ".yaml" unless base_recipe_path.end_with? ".yaml"
+      fail RecipeError, "Could not find this following recipe : #{@recipe_path}" \
+         unless File.file? @path
+      base_yaml_recipe = YAML.load File.open base_recipe_path
+      unless yaml_recipe.kind_of? Hash
+        fail RecipeError, "Invalid yaml error"
+      end
+      base_yaml_recipe.keys.each do |key|
+        if ["export", "bootstrap", "setup"].include? key
+          base_yaml_recipe.delete(key) unless yaml_recipe.keys.include? key
+        end
+      end
+      yaml_recipe.keys.each do |key|
+        if ["aliases", "checkpoint"].include? key
+          base_yaml_recipe[key] = yaml_recipe[key]
+        elsif ["export", "bootstrap", "setup"].include? key
+          base_section = base_yaml_recipe.fetch(key, [])
+          base_section = [] if base_section.nil?
+          recipe_section = yaml_recipe[key]
+          recipe_section = [] if recipe_section.nil?
+          index_base_steps = recipe_section.index("@base")
+          unless index_base_steps.nil?
+            recipe_section[index_base_steps] = base_section
+            recipe_section.flatten!
+          end
+          base_yaml_recipe[key] = recipe_section
+        elsif ["global"].include? key
+          base_section = base_yaml_recipe.fetch(key, {})
+          base_section = {} if base_section.nil?
+          recipe_section = yaml_recipe[key]
+          recipe_section = {} if recipe_section.nil?
+          base_yaml_recipe[key] = base_section.merge(recipe_section)
+        end
+      end
+      @base_recipes_files.push(Pathname.new(base_recipe_path))
+      return load_base_recipe(base_yaml_recipe)
+    end
+
     def load_aliases(yaml_recipe)
       if yaml_recipe.keys.include? "aliases"
         aliases = yaml_recipe.fetch("aliases")
         if aliases.kind_of? Hash
           @aliases = aliases
         elsif aliases.kind_of? String
-          path = Pathname.new(File.join(File.dirname(@path), "aliases", aliases))
-          if File.file?(path)
-            @logger.notice("Loading aliases #{path}")
-            @aliases = YAML.load_file(path)
-            @files.push(path)
-          else
-            fail RecipeError, "Aliases file '#{path}' does not exists"
+          dir_search = [
+            File.join(File.dirname(@path), "steps", "aliases"),
+            File.join(File.dirname(@path), "aliases")
+          ]
+          dir_search.each do |dir_path|
+            path = Pathname.new(File.join(dir_path, aliases))
+            if File.file?(path)
+              @logger.debug("Loading aliases #{path}")
+              @aliases = YAML.load_file(path)
+              @files.push(path)
+              return path
+            end
           end
+          fail RecipeError, "Aliases file '#{path}' does not exists"
         end
       end
     end
@@ -141,31 +211,39 @@ module Kameleon
           @checkpoint = checkpoint
           @checkpoint["path"] = @path
         elsif checkpoint.kind_of? String
-          path = Pathname.new(File.join(File.dirname(@path),
-                              "checkpoints",
-                              checkpoint))
-          if File.file?(path)
-            @logger.notice("Loading checkpoint configuration #{path}")
-            @checkpoint = YAML.load_file(path)
-            @checkpoint["path"] = path.to_s
-            @files.push(path)
-          else
-            fail RecipeError, "Checkpoint configuraiton file '#{path}' " \
-                              "does not exists"
+          dir_search = [
+            File.join(File.dirname(@path), "steps", "checkpoints"),
+            File.join(File.dirname(@path), "checkpoints")
+          ]
+          dir_search.each do |dir_path|
+            path = Pathname.new(File.join(dir_path, checkpoint))
+            if File.file?(path)
+              @logger.debug("Loading checkpoint configuration #{path}")
+              @checkpoint = YAML.load_file(path)
+              @checkpoint["path"] = path.to_s
+              @files.push(path)
+              return path
+            end
           end
+          fail RecipeError, "Checkpoint configuraiton file '#{path}' " \
+                            "does not exists"
         end
       end
     end
 
     def load_macrostep(step_path, name, args)
-      macrostep_yaml = YAML.load_file(step_path)
+      if step_path.nil?
+        macrostep_yaml = args
+      else
+        macrostep_yaml = YAML.load_file(step_path)
+        # Basic macrostep syntax check
+        if not macrostep_yaml.kind_of? Array
+          fail RecipeError, "The macrostep #{step_path} is not valid "
+                            "(should be a list of microsteps)"
+        end
+      end
       local_variables = {}
       loaded_microsteps = []
-      # Basic macrostep syntax check
-      if not macrostep_yaml.kind_of? Array
-        fail RecipeError, "The macrostep #{step_path} is not valid "
-                           "(should be a list of microsteps)"
-      end
       # Load default local variables
       macrostep_yaml.each do |yaml_microstep|
         key = yaml_microstep.keys[0]
@@ -177,33 +255,35 @@ module Kameleon
           local_variables[key] = @global.fetch(key, value)
         end
       end
-      selected_microsteps = []
-      if args
-        args.each do |entry|
-          if entry.kind_of? Hash
-            # resolve variable before using it
-            entry.each do |key, value|
-              local_variables[key] = value
+      unless step_path.nil?
+        selected_microsteps = []
+        if args
+          args.each do |entry|
+            if entry.kind_of? Hash
+              # resolve variable before using it
+              entry.each do |key, value|
+                local_variables[key] = value
+              end
+            elsif entry.kind_of? String
+              selected_microsteps.push entry
             end
-          elsif entry.kind_of? String
-            selected_microsteps.push entry
           end
         end
-      end
-      unless selected_microsteps.empty?
-        # Some steps are selected so remove the others
-        # WARN: Allow the user to define this list not in the original order
-        strip_microsteps = []
-        selected_microsteps.each do |microstep_name|
-          macrostep = find_microstep(microstep_name, loaded_microsteps)
-          if macrostep.nil?
-            fail RecipeError, "Can't find microstep '#{microstep_name}' "\
-                              "in macrostep file '#{step_path}'"
-          else
-            strip_microsteps.push(macrostep)
+        unless selected_microsteps.empty?
+          # Some steps are selected so remove the others
+          # WARN: Allow the user to define this list not in the original order
+          strip_microsteps = []
+          selected_microsteps.each do |microstep_name|
+            macrostep = find_microstep(microstep_name, loaded_microsteps)
+            if macrostep.nil?
+              fail RecipeError, "Can't find microstep '#{microstep_name}' "\
+                                "in macrostep file '#{step_path}'"
+            else
+              strip_microsteps.push(macrostep)
+            end
           end
+          loaded_microsteps = strip_microsteps
         end
-        loaded_microsteps = strip_microsteps
       end
       return Macrostep.new(name, loaded_microsteps, local_variables, step_path)
     end
@@ -276,12 +356,6 @@ module Kameleon
         end
       end
       @logger.notice("Starting recipe consistency check")
-      missings = []
-      @required_global.each do |key|
-        missings.push key unless @global.key? key
-      end
-      fail RecipeError, "Required parameters missing in global section :" \
-                        " #{missings.join ' '}" unless missings.empty?
       # check context args
       required_args = %w(cmd)
       missings = []
@@ -427,8 +501,8 @@ module Kameleon
         "name" => @name,
         "path" => @path.to_s,
         "files" => @files.map {|p| p.to_s },
+        "base_recipes_files" => @base_recipes_files.map {|p| p.to_s },
         "global" => @global,
-        "required_global" => @required_global,
         "aliases" => @aliases,
       }
       recipe_hash["checkpoint"] = @checkpoint unless @checkpoint.nil?
@@ -448,26 +522,71 @@ module Kameleon
 
   class RecipeTemplate < Recipe
 
-    def copy_template(dest_path, recipe_name, force)
+    def get_answer(msg)
+      while true
+        @logger.progress_notice msg
+        answer = $stdin.gets.downcase
+        raise AbortError, "Execution aborted..." if answer.nil?
+        answer.chomp!
+        if ["y", "n" , "", "a"].include?(answer)
+          if ["y", ""].include? answer
+            return true
+          elsif answer.eql? "a"
+            raise AbortError, "Aborted..."
+          end
+          return false
+        end
+      end
+    end
+
+    def safe_copy_file(src, dst, force)
+      if File.exists? dst
+        diff = Diffy::Diff.new(dst.to_s, src.to_s, :source => "files").to_s
+        unless diff.chomp.empty?
+          @logger.notice("conflict #{dst}")
+          @logger.notice("Differences between the old and the new :")
+          puts Diffy::Diff.new(dst.to_s, src.to_s,
+                               :source => "files",
+                               :context => 1,
+                               :include_diff_info => true).to_s
+          msg = "Overwrite #{dst}? [Y]es/[n]o/[a]bort : "
+          if force || get_answer(msg)
+            FileUtils.copy_file(src, dst)
+          end
+        else
+          @logger.notice("identical #{dst}")
+        end
+      else
+        unless File.dirname(dst).eql? "/"
+          FileUtils.mkdir_p File.dirname(dst)
+        end
+        @logger.notice("create #{dst}")
+        FileUtils.copy_file(src, dst)
+      end
+    end
+
+    def copy_extended_recipe(recipe_name, force)
       Dir::mktmpdir do |tmp_dir|
         recipe_path = File.join(tmp_dir, recipe_name + '.yaml')
-        FileUtils.cp(@path, recipe_path)
+        ## copying recipe
         File.open(recipe_path, 'w+') do |file|
-          tpl = ERB.new(@recipe_content)
+          extend_erb_tpl = File.join(Kameleon.env.templates_path, "extend.erb")
+          tpl = ERB.new(File.open(extend_erb_tpl, 'rb') { |f| f.read })
           result = tpl.result(binding)
           file.write(result)
         end
+        recipe_dst = File.join(Kameleon.env.workspace, recipe_name + '.yaml')
+        safe_copy_file(recipe_path, Pathname.new(recipe_dst), force)
+      end
+    end
 
-        @files.each do |path|
-          relative_path = path.relative_path_from(Kameleon.env.templates_path)
-          dst = File.join(tmp_dir, File.dirname(relative_path))
-          FileUtils.mkdir_p dst
-          FileUtils.cp(path, dst)
-          @logger.debug("Copying '#{path}' to '#{dst}'")
-        end
-        # Create recipe dir if not exists
-        FileUtils.mkdir_p Kameleon.env.recipes_path
-        FileUtils.cp_r(Dir[tmp_dir + '/*'], Kameleon.env.recipes_path)
+    def copy_template(force)
+      ## copying steps
+      files2copy = @base_recipes_files + @files
+      files2copy.each do |path|
+        relative_path = path.relative_path_from(Kameleon.env.templates_path)
+        dst = File.join(Kameleon.env.workspace, relative_path)
+        safe_copy_file(path, dst, force)
       end
     end
   end