kameleon-builder 2.1.0 → 2.1.1

data/docs/source/installation.rst

@@ -1,6 +1,14 @@
+.. _`installation`:
+
 ------------
 Installation
 ------------
+
+The easiest way to install and test Kameleon if you
+are using a Debian based linux distribution is by using the deb package available at `kameleon_deb`_.
+
+.. _kameleon_deb: http://kameleon.imag.fr/pkg/kameleon_2.1.0+20140612204940-1_amd64.deb
+
 .. note::
     On debian based distribution be sure to install the ``ruby-dev`` package first
 

data/docs/source/tau_install_g5k.yaml

@@ -0,0 +1,24 @@
+- get_tau:
+  - exec_local: cd /tmp/
+  - exec_local: wget  -q http://www.cs.uoregon.edu/research/tau/tau_releases/tau-2.22.2.tar.gz
+  - exec_local: wget -q http://www.cs.uoregon.edu/research/tau/pdt_releases/pdt-3.19.tar.gz
+  - local2in:
+      - /tmp/tau-2.22.2.tar.gz
+      - /tmp/tau-2.22.2.tar.gz
+  - local2in:
+      - /tmp/pdt-3.19.tar.gz
+      - /tmp/pdt-3.19.tar.gz
+- pdt_install:
+  - exec_in: cd /tmp/
+  - exec_in: tar -xzf pdt-3.19.tar.gz
+  - exec_in: cd /tmp/pdtoolkit-3.19
+  - exec_in: ./configure -prefix=/usr/local/pdt-install
+  - exec_in: make clean install
+
+- tau_install:
+  - exec_in: cd /tmp/
+  - exec_in: tar -xzf tau-2.22.2.tar.gz
+  - exec_in: cd /tmp/tau-2.22.2
+#  - exec_in: ./configure -prefix=/usr/local/tau-install -pdt=/usr/local/pdt-install/ -mpiinc=/usr/local/openmpi-install/include -mpilib=/usr/local/openmpi-install/lib
+  - exec_in: ./configure -prefix=/usr/local/tau-install -pdt=/usr/local/pdt-install/ -mpiinc=/usr/include/openmpi/ -mpilib=/usr/lib/openmpi/
+  - exec_in: make install

data/ext/mkrf_conf.rb

@@ -0,0 +1,58 @@
+# Based on: http://www.programmersparadox.com/2012/05/21/gemspec-loading-dependent-gems-based-on-the-users-system/
+
+# This file needs to be named mkrf_conf.rb
+# so that rubygems will recognize it as a ruby extension
+# file and not think it is a C extension file
+
+jruby = defined?(JRUBY_VERSION) || (defined?(RUBY_ENGINE) && 'jruby' == RUBY_ENGINE)
+rbx = defined?(RUBY_ENGINE) && 'rbx' == RUBY_ENGINE
+
+def already_installed(dep)
+  !Gem::DependencyInstaller.new(:domain => :local).find_gems_with_sources(dep).empty? ||
+  !Gem::DependencyInstaller.new(:domain => :local,:prerelease => true).find_gems_with_sources(dep).empty?
+end
+
+# Load up the rubygem's dependency installer to
+# installer the gems we want based on the version
+# of Ruby the user has installed
+
+unless jruby || rbx
+  require 'rubygems'
+  require 'rubygems/command.rb'
+  require 'rubygems/dependency.rb'
+  require 'rubygems/dependency_installer.rb'
+
+
+
+  begin
+    Gem::Command.build_args = ARGV
+    rescue NoMethodError
+  end
+
+  if RUBY_VERSION > "2.0"
+      dep = Gem::Dependency.new("syck", '> 0')
+  end
+
+  begin
+    puts "Installing base gem"
+    inst = Gem::DependencyInstaller.new
+    inst.install dep
+  rescue
+    inst = Gem::DependencyInstaller.new(:prerelease => true)
+    begin
+      inst.install dep
+    rescue Exception => e
+      puts e
+      puts e.backtrace.join "\n  "
+      exit(1)
+    end
+  end unless dep.nil? || already_installed(dep)
+end
+
+# If this was C, rubygems would attempt to run make
+# Since this is Ruby, rubygems will attempt to run rake
+# If it doesn't find and successfully run a rakefile, it errors out
+f = File.open(File.join(File.dirname(__FILE__), "Rakefile"), "w")
+f.write("task :default\n")
+f.close
+

data/kameleon-builder.gemspec

@@ -1,6 +1,4 @@
 # coding: utf-8
-RUBYONEX = RUBY_VERSION < "2.0"
-
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'kameleon/version'
@@ -26,18 +24,21 @@ Gem::Specification.new do |s|
   s.test_files    = s.files.grep(%r{^(tests|s|features)/})
   s.require_paths = ["lib"]
 
-  s.required_ruby_version     = RUBYONEX ? '>= 1.9.3' : '>= 2.0.0'
-
   s.add_dependency 'childprocess', '~> 0.5'
   s.add_dependency 'thor', '~> 0.15'
   s.add_dependency 'table_print', '~> 1.5'
   s.add_dependency 'log4r-color', '~> 1.2'
-  s.add_dependency 'syck', '~> 1.0.0' unless RUBYONEX
-  s.add_dependency 'diffy', '~> 3.0.4'
+  s.add_dependency 'diffy', '~> 3.0'
+  # Syck is also a dependency for Ruby 2 environments.
+  # It is installed at install-time if necessary,
+  # via ext/mkrf_conf.rb
 
   s.add_development_dependency 'pry', '~> 0.9'
   s.add_development_dependency 'pry-debugger', '~> 0.2'
   s.add_development_dependency 'rake', '~> 10.1'
   s.add_development_dependency 'minitest', '~> 4.7'
   s.add_development_dependency 'coveralls', '~> 0.7'
+
+  # To allow installing the Syck gem when Ruby version is >2.0.0
+  s.extensions = ["ext/mkrf_conf.rb"]
 end

data/lib/kameleon.rb

@@ -19,8 +19,12 @@ module Kameleon
                                     :FATAL)
 
   class << self
-    attr_writer :logger, :env, :source_root, :templates_path, :templates_names,
-                :templates_files
+    attr_writer :logger
+    attr_writer :env
+    attr_writer :source_root
+    attr_writer :templates_path
+    attr_writer :templates_names
+    attr_writer :templates_files
 
     # The source root is the path to the root directory of the kameleon gem.
     def source_root

data/lib/kameleon/compat.rb

@@ -1,7 +1,8 @@
 require 'securerandom'
 require 'yaml'
+require 'syck'
 
-YAML::ENGINE.yamler = 'syck' if (defined?(Syck) || defined?(YAML::Syck)) && defined?(YAML::ENGINE)
+YAML::ENGINE.yamler = 'syck' if defined?(YAML::ENGINE)
 
 if RUBY_VERSION < "1.9.3"
   # Backport of missing SecureRandom methods from 1.9
@@ -37,3 +38,24 @@ if RUBY_VERSION < "1.9.3"
     end
   end
 end
+
+
+class Object
+  ##
+  #   @person ? @person.name :nil
+  # vs
+  #   @person.try(:name)
+  def try(method)
+    send method if respond_to? method
+  end
+end
+
+class Hash
+  def self.try_convert(obj)
+    obj.try(:to_hash)
+  end
+
+  def flatten
+    to_a.flatten!
+  end
+end

data/lib/kameleon/context.rb

@@ -3,7 +3,8 @@ require 'kameleon/shell'
 module Kameleon
   class Context
 
-    attr_accessor :shell, :name
+    attr_accessor :shell
+    attr_accessor :name
 
     def initialize(name, cmd, workdir, exec_prefix, local_workdir)
       @name = name.downcase
@@ -80,10 +81,13 @@ module Kameleon
 
     def lazyload_shell()
       unless @shell.started?
+        @shell.restart
         # Start the shell process
-        @shell.start
         execute("echo The '#{name}_context' has been initialized", :log_level => "info")
       end
+    rescue
+      @shell.stop
+      raise
     end
 
     def start_shell
@@ -91,6 +95,8 @@ module Kameleon
       lazyload_shell
       @logger.info("Starting interactive shell")
       @shell.fork_and_wait
+    rescue ShellError => e
+      e.message.split( /\r?\n/ ).each {|m| @logger.error m }
     end
 
     def closed?

data/lib/kameleon/engine.rb

@@ -5,7 +5,10 @@ require 'kameleon/persistent_cache'
 module Kameleon
 
   class Engine
-    attr_accessor :recipe, :cwd, :build_recipe_path, :pretty_list_checkpoints
+    attr_accessor :recipe
+    attr_accessor :cwd
+    attr_accessor :build_recipe_path
+    attr_accessor :pretty_list_checkpoints
 
     def initialize(recipe, options)
       @options = options
@@ -250,9 +253,9 @@ module Kameleon
             raise AbortError, "Execution aborted..."
           elsif answer.eql? "c"
             ## resetting the exit status
-            @in_context.execute("true")
-            @out_context.execute("true")
-            @local_context.execute("true")
+            @in_context.execute("true") unless @in_context.closed?
+            @out_context.execute("true") unless @out_context.closed?
+            @local_context.execute("true") unless @local_context.closed?
             return true
           elsif answer.eql? "r"
             @logger.notice("Retrying the previous command...")
@@ -331,24 +334,23 @@ module Kameleon
           do_steps(section)
         end
         clean
-      rescue AbortError => e
-        @logger.error("Aborted...")
-        @logger.warn("Waiting for cleanup before exiting...")
-        clean
-        raise e
-      rescue SystemExit, Interrupt => e
-        @logger.error("Interrupted...")
-        @out_context.reopen
-        @in_context.reopen
-        @local_context.reopen
+      rescue Exception => e
+        if e.is_a?(AbortError)
+          @logger.error("Aborted...")
+        elsif e.is_a?(SystemExit) || e.is_a?(Interrupt)
+          @logger.error("Interrupted...")
+          @out_context.reopen
+          @in_context.reopen
+          @local_context.reopen
+        else
+          @logger.fatal("fatal error...")
+        end
         @logger.warn("Waiting for cleanup before exiting...")
         clean
-        raise e
-      rescue Exception => e
         @out_context.close!
         @in_context.close!
         @local_context.close!
-        raise e
+        raise
       end
     end
 

data/lib/kameleon/persistent_cache.rb

@@ -7,8 +7,14 @@ module Kameleon
   class Persistent_cache
 
     include Singleton
-    attr_reader :polipo_env, :cache_dir,:polipo_port
-    attr_writer :activated, :cwd, :polipo_path, :name
+    attr_reader :polipo_env
+    attr_reader :cache_dir
+    attr_reader :polipo_port
+    attr_writer :activated
+    attr_reader :cwd
+    attr_reader :polipo_path
+    attr_reader :name
+
     def initialize()
       @logger = Log4r::Logger.new("kameleon::[kameleon]")
       ## we must configure Polipo to be execute for the in and out context
@@ -25,7 +31,6 @@ module Kameleon
                             :idleTime => "5",
                             :chunkHighMark => "425165824",
                             :proxyPort => @polipo_port,
-                            #:proxyOffline => "true"
                             :relaxTransparency =>"true"
                             }
 
@@ -61,7 +66,7 @@ module Kameleon
 
       if @polipo_path.nil? then
         @logger.error("Polipo binary not found, make sure it is in your current PATH")
-        @logger.error("or use the option --proxy_path")
+        @logger.error("or use the option --proxy-path")
         raise BuildError, "Failed to use persistent cache"
       end
     end
@@ -77,7 +82,7 @@ module Kameleon
     end
 
     def create_cache_directory(step_name)
-      @logger.notice("Creating  cache directory #{step_name} for Polipo")
+      @logger.notice("Creating cache directory #{step_name} for Polipo")
       directory_name = @cache_dir + "/#{step_name}"
       FileUtils.mkdir_p directory_name
       directory_name
@@ -87,7 +92,7 @@ module Kameleon
       ## This function assumes that the cache directory has already been created by the engine
       ## Stopping first the previous proxy
       ## have to check if polipo is running
-      @logger.notice("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
+      @logger.debug("Starting web proxy Polipo in directory #{directory} using port: #{@polipo_port}")
       @polipo_process.stop unless @polipo_process.nil?
       command = ["#{@polipo_path}/polipo"]
       @polipo_cmd_options[:diskCacheRoot] = directory

data/lib/kameleon/recipe.rb

@@ -4,8 +4,15 @@ require 'kameleon/step'
 module Kameleon
 
   class Recipe
-    attr_accessor :path, :name, :global, :sections, :aliases, :aliases_path, \
-                  :checkpoint, :checkpoint_path, :metainfo
+    attr_accessor :path
+    attr_accessor :name
+    attr_accessor :global
+    attr_accessor :sections
+    attr_accessor :aliases
+    attr_accessor :aliases_path
+    attr_accessor :checkpoint
+    attr_accessor :checkpoint_path
+    attr_accessor :metainfo
 
     def initialize(path)
       @logger = Log4r::Logger.new("kameleon::[kameleon]")
@@ -51,7 +58,7 @@ module Kameleon
       # Load Global variables
       @global.merge!(yaml_recipe.fetch("global", {}))
       # Resolve dynamically-defined variables !!
-      resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
+      resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global, :strict => false)
       @global.merge! YAML.load(resolved_global)
 
       # Loads aliases
@@ -91,7 +98,7 @@ module Kameleon
             if not args.nil?
               args.each do |arg|
                 if arg.kind_of? Hash
-                  if arg.flatten[1].kind_of? Array
+                  if arg[arg.keys[0]].kind_of? Array
                     embedded_step = true
                   end
                 end
@@ -298,6 +305,10 @@ module Kameleon
     end
 
     def resolve!
+      # Resolve dynamically-defined variables !!
+      resolved_global = Utils.resolve_vars(@global.to_yaml, @path, @global)
+      @global.merge! YAML.load(resolved_global)
+
       consistency_check
       resolve_checkpoint unless @checkpoint.nil?
 

data/lib/kameleon/shell.rb

@@ -8,7 +8,9 @@ module Kameleon
     READ_CHUNK_SIZE = 1048576
     EXIT_TIMEOUT = 60
 
-    attr :exit_status, :process, :shell_cmd
+    attr :exit_status
+    attr :process
+    attr :shell_cmd
 
     def initialize(context_name, cmd, shell_workdir, local_workdir, kwargs = {})
       @logger = Log4r::Logger.new("kameleon::[kameleon]")
@@ -30,16 +32,6 @@ module Kameleon
 
       ## Changing the default bashrc if the cache is activated
       @cache = Kameleon::Persistent_cache.instance
-      if @cache.activated? then
-        new_kameleon_bashrc = Tempfile.new('kameleon_bashrc').path
-        FileUtils.cp @default_bashrc_file, new_kameleon_bashrc
-        tpl = ERB.new(File.read(@cache.polipo_env))
-        polipo_env_content = tpl.result(binding)
-        File.open(new_kameleon_bashrc,'a') do |f|
-          f.puts(polipo_env_content)
-        end
-        @default_bashrc_file = new_kameleon_bashrc
-      end
 
       @shell_cmd = "source #{@default_bashrc_file} 2> /dev/null; "\
                    "#{@cmd} --rcfile #{@bashrc_file}"
@@ -56,7 +48,10 @@ module Kameleon
     end
 
     def stop
-      @process.stop(0) unless @process.nil?
+      unless @process.nil?
+        @process.stop(0)
+        @process = nil
+      end
     end
 
     def started?
@@ -64,7 +59,7 @@ module Kameleon
     end
 
     def exited?
-      @process.exited?
+      return !@process.nil? && @process.exited?
     end
 
     def restart
@@ -98,6 +93,10 @@ module Kameleon
       if File.file?(@default_bashrc_file)
         tpl = ERB.new(File.read(@default_bashrc_file))
         bashrc_content = tpl.result(binding)
+        if @cache.activated? then
+          tpl = ERB.new(File.read(@cache.polipo_env))
+          bashrc_content << "\n" + ERB.new(File.read(@cache.polipo_env)).result(binding)
+        end
       end
       bashrc = Shellwords.escape(bashrc_content)
       if @shell_workdir

data/lib/kameleon/step.rb

@@ -1,7 +1,8 @@
 module Kameleon
 
   class Command
-    attr_accessor :string_cmd, :microstep_name
+    attr_accessor :string_cmd
+    attr_accessor :microstep_name
 
     def initialize(yaml_cmd, microstep_name)
       @string_cmd = YAML.dump(yaml_cmd).gsub("---", "").strip
@@ -72,8 +73,13 @@ module Kameleon
   end
 
   class Microstep
-    attr_accessor :commands, :name, :identifier, :slug, :in_cache,
-                  :on_checkpoint, :order
+    attr_accessor :commands
+    attr_accessor :name
+    attr_accessor :identifier
+    attr_accessor :slug
+    attr_accessor :in_cache
+    attr_accessor :on_checkpoint
+    attr_accessor :order
 
     def initialize(string_or_hash)
       @identifier = nil
@@ -128,8 +134,12 @@ module Kameleon
   end
 
   class Macrostep
-    attr_accessor :name, :clean_microsteps, :init_microsteps, :microsteps,
-                  :path, :variables
+    attr_accessor :name
+    attr_accessor :clean_microsteps
+    attr_accessor :init_microsteps
+    attr_accessor :microsteps
+    attr_accessor :path
+    attr_accessor :variables
 
     def initialize(name, microsteps, variables, path)
       @name = name
@@ -180,7 +190,10 @@ module Kameleon
   end
 
   class Section
-    attr_accessor :name, :clean_macrostep, :init_macrostep, :macrosteps
+    attr_accessor :name
+    attr_accessor :clean_macrostep
+    attr_accessor :init_macrostep
+    attr_accessor :macrosteps
 
     def initialize(name)
       @name = name