autoproj 1.9.4 → 1.9.5.rc1

data/bin/autoproj_bootstrap.in

@@ -133,7 +133,7 @@ OSDEPS_DEFAULTS
 EODEFS
 
 Autoproj::OSDependencies.define_osdeps_mode_option
-osdeps_mode = Autoproj::OSDependencies.osdeps_mode
+osdeps_mode = Autoproj::OSDependencies.osdeps_mode.join(",")
 ENV['AUTOPROJ_OSDEPS_MODE'] = osdeps_mode
 
 # First thing we do is install a proper ruby environment. We make sure that we

data/lib/autoproj/cmdline.rb

@@ -123,42 +123,22 @@ module Autoproj
                 @randomize_layout = Autoproj.user_config('randomize_layout')
             end
 
-            # If we are under rubygems, check that the GEM_HOME is right ...
-            if $LOADED_FEATURES.any? { |l| l =~ /rubygems/ }
-                if ENV['GEM_HOME'] != Autoproj.gem_home
-                    raise ConfigError.new, "RubyGems is already loaded with a different GEM_HOME, make sure you are loading the right #{ENV_FILENAME} script !"
-                end
+            # Make sure that the currently loaded env.sh is actually us
+            if ENV['AUTOPROJ_CURRENT_ROOT'] && (ENV['AUTOPROJ_CURRENT_ROOT'] != Autoproj.root_dir)
+                raise ConfigError.new, "the current environment is for #{ENV['AUTOPROJ_ROOT_DIR']}, but you are in #{Autoproj.root_dir}, make sure you are loading the right #{ENV_FILENAME} script !"
             end
 
-            # Set up some important autobuild parameters
-            Autoproj.env_inherit 'PATH', 'PKG_CONFIG_PATH', 'RUBYLIB', 'LD_LIBRARY_PATH', 'GEM_PATH', 'CMAKE_PREFIX_PATH', 'PYTHONPATH'
-            Autoproj.env_set 'GEM_HOME', Autoproj.gem_home
-            Autoproj.env_add 'GEM_PATH', Autoproj.gem_home
-            Autoproj.env_add 'PATH', File.join(Autoproj.gem_home, 'bin')
-            Autoproj.env_set 'RUBYOPT', "-rubygems"
+            Autoproj.manifest = Manifest.new
+            Autoproj.prepare_environment
             Autobuild.prefix  = Autoproj.build_dir
             Autobuild.srcdir  = Autoproj.root_dir
             Autobuild.logdir = File.join(Autobuild.prefix, 'log')
 
-            Autoproj.manifest = Manifest.new
-
             local_source = LocalPackageSet.new(Autoproj.manifest)
+            load_autoprojrc
 
-            home_dir =
-                if Dir.respond_to?(:home) # 1.9 specific
-                    Dir.home
-                else ENV['HOME']
-                end
-            # Load the user-wide autoproj RC file
-            if home_dir
-                Autoproj.load_if_present(local_source, home_dir, ".autoprojrc")
-            end
-
-            if Autoproj.has_config_key?('reused_autoproj_installations')
-                reused = Autoproj.user_config('reused_autoproj_installations')
-                reused.each do |path|
-                    Autoproj.manifest.reuse(path)
-                end
+            Autoproj.manifest.each_reused_autoproj_installation do |p|
+                Autoproj.manifest.reuse(path)
             end
 
             # We load the local init.rb first so that the manifest loading
@@ -208,6 +188,26 @@ module Autoproj
             end
         end
 
+        def self.load_autoprojrc
+            home_dir =
+                if Dir.respond_to?(:home) # 1.9 specific
+                    Dir.home
+                else ENV['HOME']
+                end
+
+            # Load the user-wide autoproj RC file
+            if home_dir
+                rcfile = File.join(home_dir, '.autoprojrc')
+                if File.file?(rcfile)
+                    begin
+                        Kernel.load rcfile
+                    rescue Interrupt
+                        raise
+                    end
+                end
+            end
+        end
+
         def self.update_myself
             return if !Autoproj::CmdLine.update_os_dependencies?
 
@@ -712,7 +712,7 @@ module Autoproj
                     # Run a filter now, to have errors as early as possible
                     selection.filter_excluded_and_ignored_packages(Autoproj.manifest)
                     # Delete this package from the current_packages set
-                    true
+                    next
                 end
 
                 Autoproj.each_post_import_block(pkg) do |block|
@@ -1554,34 +1554,18 @@ where 'mode' is one of:
             Autoproj.root_dir = Dir.pwd
             Autobuild.logdir = File.join(Autoproj.prefix, 'log')
 
-            # Check if GEM_HOME is set. If it is the case, assume that we are
-            # bootstrapping from another autoproj directory. We start by
-            # forcefully installing autoproj/autobuild so that the installation
-            # is self-contained.
-            #
-            # We don't use Autoproj.gem_home there as we might not be in an
-            # autoproj directory at all
-            gem_home = ENV['AUTOPROJ_GEM_HOME'] || File.join(Dir.pwd, ".gems")
-            if ENV['GEM_HOME'] && Autoproj.in_autoproj_installation?(ENV['GEM_HOME']) && ENV['GEM_HOME'] != gem_home
-                Autoproj::OSDependencies.define_osdeps_mode_option
-                osdeps = Autoproj::OSDependencies.load_default
-                if osdeps_forced_mode
-                    osdeps.osdeps_mode = osdeps_forced_mode
-                end
-                osdeps.osdeps_mode
-
-                Autoproj.message "autoproj: bootstrapping using another installation's autoproj gem"
-                ENV['GEM_HOME'] = gem_home
-                ENV.delete('GEM_PATH')
-                Autoproj.message "installing autoproj in #{ENV['GEM_HOME']} and restarting"
-                osdeps.install ['autoproj']
-                Autoproj.message "restarting bootstrapping from #{Dir.pwd}"
+            Autoproj.manifest = Manifest.new
+            load_autoprojrc
+            Autoproj.prepare_environment
 
-                require 'rbconfig'
-                ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']
-                ENV['AUTOPROJ_OSDEPS_MODE'] = osdeps.osdeps_mode
-                exec ruby, $0, 'bootstrap', *ARGV
+            Autoproj::OSDependencies.define_osdeps_mode_option
+            osdeps = Autoproj::OSDependencies.load_default
+            if osdeps_forced_mode
+                osdeps.osdeps_mode = osdeps_forced_mode
             end
+            osdeps.osdeps_mode
+
+            osdeps.install ['autoproj']
 
             reuse = []
             parser = lambda do |opt|
@@ -1605,12 +1589,6 @@ where 'mode' is one of:
 
             handle_ruby_version
 
-            Autobuild.env_set 'RUBYOPT', '-rubygems'
-            Autobuild.env_set 'GEM_HOME', Autoproj.gem_home
-            Autobuild.env_add_path 'PATH', File.join(Autoproj.gem_home, 'bin')
-            Autobuild.env_inherit 'PATH'
-            Autobuild.env_add_path 'GEM_PATH', Autoproj.gem_home
-            Autobuild.env_inherit 'GEM_PATH'
             Autoproj.export_env_sh
 
             if args.size == 1 # the user asks us to download a manifest
@@ -1888,8 +1866,8 @@ where 'mode' is one of:
         def self.initialize_root_directory
             Autoproj.root_dir
         rescue Autoproj::UserError => error
-            if ENV['GEM_HOME']
-                Dir.chdir(File.join(ENV['GEM_HOME'], '..'))
+            if ENV['AUTOPROJ_CURRENT_ROOT']
+                Dir.chdir(ENV['AUTOPROJ_CURRENT_ROOT'])
                 begin Autoproj.root_dir
                 rescue Autoproj::UserError
                     raise error

data/lib/autoproj/default.osdeps

@@ -132,5 +132,8 @@ cvs:
   fedora: cvs
   darwin: cvs
 
+pip:
+  debian,ubuntu: python-pip
+
 # vim: expandtab
 

data/lib/autoproj/manifest.rb

@@ -1080,7 +1080,7 @@ module Autoproj
 
 	def initialize
             @file = nil
-	    @data = nil
+	    @data = Hash.new
             @packages = Hash.new
             @package_manifests = Hash.new
             @automatic_exclusions = Hash.new
@@ -2388,6 +2388,12 @@ module Autoproj
             result
         end
 
+        def each_reused_autoproj_installation
+            if Autoproj.has_config_key?('reused_autoproj_installations')
+                Autoproj.user_config('reused_autoproj_installations').each(&proc)
+            end
+        end
+
         def reuse(*dir)
             dir = File.expand_path(File.join(*dir), Autoproj.root_dir)
             if reused_installations.any? { |mnf| mnf.path == dir }

data/lib/autoproj/osdeps.rb

@@ -24,6 +24,13 @@ module Autoproj
             def name
                 names.first
             end
+
+            # Overload to perform initialization of environment variables in
+            # order to have a properly functioning package manager
+            #
+            # This is e.g. needed for python pip or rubygems
+            def self.initialize_environment
+            end
         end
 
         # Dummy package manager used for unknown OSes. It simply displays a
@@ -314,6 +321,44 @@ fi
             @with_prerelease = false
             @with_doc = false
 
+            # Filters all paths that come from other autoproj installations out
+            # of GEM_PATH
+            def self.initialize_environment
+                Autobuild::ORIGINAL_ENV['GEM_PATH'] =
+                    (ENV['GEM_PATH'] || "").split(":").find_all do |p|
+                        !Autoproj.in_autoproj_installation?(p)
+                    end.join(":")
+                Autobuild.env_inherit 'GEM_PATH'
+                Autobuild.env_init_from_env 'GEM_PATH'
+
+                orig_gem_path = Autobuild::ORIGINAL_ENV['GEM_PATH'].split(':')
+                Gem.default_path.each do |p|
+                    if !orig_gem_path.include?(p)
+                        orig_gem_path << ":#{p}"
+                    end
+                end
+
+                Autoproj.manifest.each_reused_autoproj_installation do |p|
+                    p_gems = File.join(Autoproj.root_dir, '.gems')
+                    if File.directory?(p_gems)
+                        Autobuild.env_add_path 'GEM_PATH', p_gems
+                    end
+                end
+                Autobuild.env_add_path 'GEM_PATH', gem_home
+                Autobuild.env_set 'GEM_HOME', gem_home
+                Autobuild.env_add_path 'PATH', "#{gem_home}/bin"
+
+                # Now, reset the directories in our own RubyGems instance
+                if Gem::Specification.respond_to?(:dirs=)
+                    Gem::Specification.dirs = ENV['GEM_PATH'].split(':')
+                end
+            end
+
+            # Return the directory in which RubyGems package should be installed
+            def self.gem_home
+                ENV['AUTOPROJ_GEM_HOME'] || File.join(Autoproj.root_dir, ".gems")
+            end
+
             def initialize
                 super(['gem'])
                 @installed_gems = Set.new
@@ -477,8 +522,94 @@ fi
                 false
             end
         end
+    
+        # Using pip to install python packages
+        class PipManager < Manager
+
+            attr_reader :installed_gems
+
+            def self.initialize_environment
+                Autoproj.env_set 'PYTHONUSERBASE', pip_home
+            end
+
+            # Return the directory where python packages are installed to.
+            # The actual path is pip_home/lib/pythonx.y/site-packages.
+            def self.pip_home
+                ENV['AUTOPROJ_PYTHONUSERBASE'] || File.join(Autoproj.root_dir,".pip")
+            end
+
+
+            def initialize
+                super(['pip'])
+                @installed_pips = Set.new
+            end
+
+            def guess_pip_program
+                if Autobuild.programs['pip']
+                    return Autobuild.programs['pip']
+                end
+
+                Autobuild.programs['pip'] = "pip"
+            end
+
+            def install(pips)
+                guess_pip_program
+                if pips.is_a?(String)
+                    pips = [pips]
+                end
+
+                base_cmdline = [Autobuild.tool('pip'), 'install','--user']
+
+                cmdlines = [base_cmdline + pips]
+
+                if pips_interaction(pips, cmdlines)
+                    Autoproj.message "  installing/updating Python dependencies: "+
+                        "#{pips.sort.join(", ")}"
+
+                    cmdlines.each do |c|
+                        Autobuild::Subprocess.run 'autoproj', 'osdeps', *c
+                    end
+
+                    pips.each do |p|
+                        @installed_pips << p
+                    end
+                end
+            end
+            
+            def pips_interaction(pips, cmdlines)
+                if OSDependencies.force_osdeps
+                    return true
+                elsif enabled?
+                    return true
+                elsif silent?
+                    return false
+                end
+
+                # We're not supposed to install rubygem packages but silent is not
+                # set, so display information about them anyway
+                puts <<-EOMSG
+      #{Autoproj.color("The build process and/or the packages require some Python packages to be installed", :bold)}
+      #{Autoproj.color("and you required autoproj to not do it itself", :bold)}
+        The following command line can be used to install them manually
+        
+          #{cmdlines.map { |c| c.join(" ") }.join("\n      ")}
+        
+        Autoproj expects these Python packages to be installed in #{Autoproj.pip_home} This can
+        be overridden by setting the AUTOPROJ_PYTHONUSERBASE environment variable manually
+
+                EOMSG
+                print "    #{Autoproj.color("Press ENTER to continue ", :bold)}"
+
+                STDOUT.flush
+                STDIN.readline
+                puts
+                false
+            end
+        end
+
     end
 
+
     # Manager for packages provided by external package managers
     class OSDependencies
 	class << self
@@ -560,7 +691,8 @@ fi
             PackageManagers::EmergeManager,
             PackageManagers::PacmanManager,
             PackageManagers::YumManager,
-            PackageManagers::PortManager]
+            PackageManagers::PortManager,
+            PackageManagers::PipManager]
         OS_PACKAGE_HANDLERS = {
             'debian' => 'apt-dpkg',
             'gentoo' => 'emerge',
@@ -1181,27 +1313,22 @@ packages)
 This option is meant to allow you to control autoproj's behaviour while handling
 OS dependencies.
 
-* if you say "ruby", the RubyGem packages will be installed.
+* if you say "gem", the RubyGem packages will be installed.
 * if you say "none", autoproj will not do anything related to the OS
   dependencies.
 
 As any configuration value, the mode can be changed anytime by calling
-an autoproj operation with the --reconfigure option (e.g. autoproj update
---reconfigure).
+  autoproj reconfigure
 
-Finally, OS dependencies can be installed by calling "autoproj osdeps"
-with the corresponding option (--all, --ruby, --os or --none). Calling
-"autoproj osdeps" without arguments will also give you information as
-to what you should install to compile the software successfully.
+Finally, the "autoproj osdeps" command will give you the necessary information about the OS packages that you will need to install manually.
 
-So, what do you want ? (ruby or none)
+So, what do you want ? (either 'all', 'none', or a comma-separated list of 'gem' and 'pip' -- without the quotes)
             EOT
-            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (ruby, none) ?", long_doc.strip ]
+            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically ('all', 'none', or a comma-separated list of 'gem' and 'pip' -- without the quotes) ?", long_doc.strip ]
 
 	    Autoproj.configuration_option 'osdeps_mode', 'string',
 		:default => 'ruby',
 		:doc => message,
-                :possible_values => %w{ruby none},
                 :lowercase => true
         end
 
@@ -1219,7 +1346,9 @@ while handling OS dependencies.
 
 * if you say "all", it will install all packages automatically.
   This requires root access thru 'sudo'
-* if you say "ruby", only the Ruby packages will be installed.
+* if you say "pip", only the Ruby packages will be installed.
+  Installing these packages does not require root access.
+* if you say "gem", only the Ruby packages will be installed.
   Installing these packages does not require root access.
 * if you say "os", only the OS-provided packages will be installed.
   Installing these packages requires root access.
@@ -1227,20 +1356,17 @@ while handling OS dependencies.
   OS dependencies.
 
 As any configuration value, the mode can be changed anytime by calling
-an autoproj operation with the --reconfigure option (e.g. autoproj update
---reconfigure).
+  autoproj reconfigure
 
-Finally, OS dependencies can be installed by calling "autoproj osdeps"
-with the corresponding option (--all, --ruby, --os or --none).
+Finally, the "autoproj osdeps" command 
 
-So, what do you want ? (all, ruby, os or none)
+So, what do you want ? (either 'all', 'none', or a comma-separated list of 'os', 'gem' and 'pip' -- without the quotes)
             EOT
-            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, ruby, os, none) ?", long_doc.strip ]
+            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically ('all', 'none', or a comma-separated list of 'os', 'gem' and 'pip' -- without the quotes) ?", long_doc.strip ]
 
 	    Autoproj.configuration_option 'osdeps_mode', 'string',
 		:default => 'all',
 		:doc => message,
-                :possible_values => %w{all ruby os none},
                 :lowercase => true
         end
 
@@ -1255,11 +1381,11 @@ So, what do you want ? (all, ruby, os or none)
         def self.osdeps_mode_string_to_value(string)
             string = string.to_s.downcase
             case string
-            when 'all'  then HANDLE_ALL
-            when 'ruby' then HANDLE_RUBY
-            when 'os'   then HANDLE_OS
-            when 'none' then HANDLE_NONE
-            else raise ArgumentError, "invalid osdeps mode string '#{string}'"
+            when 'all'  then ['os', 'gem', 'pip']
+            when 'ruby' then ['gem']
+            when 'os'   then ['os']
+            when 'none' then ['none']
+            else string.split(',')
             end
         end
 
@@ -1328,20 +1454,26 @@ So, what do you want ? (all, ruby, os or none)
         # The set of packages that have already been installed
         attr_reader :installed_packages
 
-        def installs_os_packages?
-            osdeps_mode == HANDLE_ALL || osdeps_mode == HANDLE_OS
-        end
-
-        def installs_ruby_packages?
-            osdeps_mode == HANDLE_ALL || osdeps_mode == HANDLE_RUBY
-        end
-
-
         # Requests the installation of the given set of packages
-        def install(packages, package_osdeps = Hash.new)
-            os_package_handler.enabled = installs_os_packages?
+        def install(packages, options = Hash.new)
+            options =
+                if Kernel.respond_to?(:validate_options)
+                    Kernel.validate_options options, :osdeps_mode => osdeps_mode
+                else options.dup
+                end
+
+            os_package_handler.enabled = false
+            package_handlers.each_value do |handler|
+                handler.enabled = false
+            end
+            osdeps_mode.each do |m|
+                if m == 'os'
+                    os_package_handler.enabled = true
+                else
+                    package_handlers[m].enabled = true
+                end
+            end
             os_package_handler.silent = self.silent?
-            package_handlers['gem'].enabled = installs_ruby_packages?
             package_handlers.each_value do |v|
                 v.silent = self.silent?
             end