autoproj 2.0.0.rc3 → 2.0.0.rc4

data/lib/autoproj/ops/build.rb

@@ -19,10 +19,10 @@ def initialize(manifest)
             # It rebuilds (i.e. does a clean + build) of all packages declared
             # in the manifest's layout. It also performs a reinstall of all
             # non-OS-specific managers that support it (e.g. RubyGems) if
-            # {update_os_dependencies?} is set to true (the default)
+            # {update_os_packages?} is set to true (the default)
             def rebuild_all
                 packages = manifest.all_layout_packages
-                manifest.pristine_os_dependencies(packages)
+                manifest.pristine_os_packages(packages)
                 rebuild_packages(packages, packages)
             end
 

data/lib/autoproj/ops/configuration.rb

@@ -140,7 +140,7 @@ def update_remote_package_set(vcs, options = Hash.new)
                 Autoproj.message("autoproj: updating remote definitions of package sets", :bold)
                 @remote_update_message_displayed = true
             end
-            ws.osdeps.install([vcs.type])
+            ws.install_os_packages([vcs.type])
             update_configuration_repository(
                 vcs, name, raw_local_dir, options)
         end

data/lib/autoproj/ops/import.rb

@@ -28,7 +28,7 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
             end
 
             VALID_OSDEP_AVAILABILITY =
-                [OSDependencies::AVAILABLE, OSDependencies::IGNORE]
+                [OSPackageResolver::AVAILABLE, OSPackageResolver::IGNORE]
 
             def import_next_step(pkg, reverse_dependencies)
                 new_packages = []

data/lib/autoproj/ops/install.rb

@@ -0,0 +1,211 @@
+require 'optparse'
+require 'fileutils'
+require 'yaml'
+
+module Autoproj
+    module Ops
+        # This class contains the functionality necessary to install autoproj in a
+        # clean root
+        #
+        # It can be required standalone (i.e. does not depend on anything else than
+        # ruby and the ruby standard library)
+        class Install
+            # The directory in which to install autoproj
+            attr_reader :root_dir
+            # Content of the Gemfile generated to install autoproj itself
+            attr_accessor :gemfile
+
+            def initialize(root_dir)
+                @root_dir = root_dir
+                @gemfile  = default_gemfile_contents
+                @private_bundler  = false
+                @private_autoproj = false
+                @private_gems     = false
+            end
+
+            def dot_autoproj; File.join(root_dir, '.autoproj') end
+            def bin_dir; File.join(dot_autoproj, 'bin') end
+            def bundler_install_dir; File.join(dot_autoproj, 'bundler') end
+            def autoproj_install_dir; File.join(dot_autoproj, 'autoproj') end
+            # The path to the gemfile used to install autoproj
+            def autoproj_gemfile_path; File.join(autoproj_install_dir, 'Gemfile') end
+            def autoproj_config_path; File.join(dot_autoproj, 'config.yml') end
+
+            # Whether bundler should be installed locally in {#dot_autoproj}
+            def private_bundler?; @private_bundler end
+            # Whether autoproj should be installed locally in {#dot_autoproj}
+            def private_autoproj?; @private_autoproj end
+            # Whether bundler should be installed locally in the workspace
+            # prefix directory
+            def private_gems?; @private_gems end
+
+            def guess_gem_program
+                ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME']
+                ruby_bindir = RbConfig::CONFIG['bindir']
+
+                candidates = ['gem']
+                if ruby_bin =~ /^ruby(.+)$/
+                    candidates << "gem#{$1}" 
+                end
+
+                candidates.each do |gem_name|
+                    if File.file?(gem_full_path = File.join(ruby_bindir, gem_name))
+                        return gem_full_path
+                    end
+                end
+                raise ArgumentError, "cannot find a gem program (tried #{candidates.sort.join(", ")} in #{ruby_bindir})"
+            end
+
+            # The content of the default {#gemfile}
+            #
+            # @param [String] autoproj_version a constraint on the autoproj version
+            #   that should be used
+            # @return [String]
+            def default_gemfile_contents(autoproj_version = ">= 0")
+                ["source \"https://rubygems.org\"",
+                 "gem \"autoproj\", \"#{autoproj_version}\""].join("\n")
+            end
+
+            # Parse the provided command line options and returns the non-options
+            def parse_options(args = ARGV)
+                options = OptionParser.new do |opt|
+                    opt.on '--private-bundler', 'install bundler locally in the workspace' do
+                        @private_bundler = true
+                    end
+                    opt.on '--private-autoproj', 'install autoproj locally in the workspace' do
+                        @private_autoproj = true
+                    end
+                    opt.on '--private-gems', 'install gems locally in the prefix directory' do
+                        @private_gems = true
+                    end
+                    opt.on '--private', 'whether bundler, autoproj and the workspace gems should be installed locally in the workspace' do
+                        @private_bundler = true
+                        @private_autoproj = true
+                        @private_gems = true
+                    end
+                    opt.on '--version=VERSION_CONSTRAINT', String, 'use the provided string as a version constraint for autoproj' do |version|
+                        @gemfile = default_gemfile_contents(version)
+                    end
+                    opt.on '--gemfile=PATH', String, 'use the given Gemfile to install autoproj instead of the default' do |path|
+                        @gemfile = File.read(path)
+                    end
+                end
+                options.parse(ARGV)
+            end
+
+            def install_bundler
+                gem_program  = guess_gem_program
+                puts "Detected 'gem' to be #{gem_program}"
+
+                result = system(
+                    Hash['GEM_PATH' => nil,
+                         'GEM_HOME' => bundler_install_dir],
+                    gem_program, 'install', '--no-document', '--no-user-install', '--no-format-executable',
+                        "--bindir=#{File.join(bundler_install_dir, 'bin')}", 'bundler')
+
+                if !result
+                    STDERR.puts "FATAL: failed to install bundler in #{dot_autoproj}"
+                    exit 1
+                end
+                File.join(bin_dir, 'bundler')
+            end
+
+            def save_env_sh
+                env = Autobuild::Environment.new
+                path = []
+                if private_bundler?
+                    env.push_path 'PATH', File.join(bundler_install_dir, 'bin')
+                    env.push_path 'GEM_PATH', bundler_install_dir
+                end
+                env.push_path 'PATH', File.join(autoproj_install_dir, 'bin')
+                env.inherit 'PATH'
+                if private_autoproj?
+                    env.push_path 'GEM_PATH', autoproj_install_dir
+                end
+
+                # Generate environment files right now, we can at least use bundler
+                File.open(File.join(dot_autoproj, 'env.sh'), 'w') do |io|
+                    env.export_env_sh(io)
+                end
+
+                File.open(File.join(root_dir, 'env.sh'), 'w') do |io|
+                    io.write <<-EOSHELL
+source "#{File.join(dot_autoproj, 'env.sh')}"
+export AUTOPROJ_CURRENT_ROOT=#{root_dir}
+                    EOSHELL
+                end
+            end
+
+            def save_gemfile
+                FileUtils.mkdir_p File.dirname(autoproj_gemfile_path)
+                File.open(autoproj_gemfile_path, 'w') do |io|
+                    io.write gemfile
+                end
+            end
+            
+            def install_autoproj(bundler)
+                # Force bundler to update. If the user does not want this, let him specify a
+                # Gemfile with tighter version constraints
+                lockfile = File.join(File.dirname(autoproj_gemfile_path), 'Gemfile.lock')
+                if File.exist?(lockfile)
+                    FileUtils.rm lockfile
+                end
+
+                env = Hash['BUNDLE_GEMFILE' => nil, 'RUBYLIB' => nil]
+                opts = Array.new
+
+                if private_autoproj?
+                    env = Hash['GEM_PATH' => bundler_install_dir,
+                               'GEM_HOME' => nil]
+                    opts << "--clean" << "--path=#{autoproj_install_dir}"
+                end
+
+                result = system(env,
+                    bundler, 'install',
+                        "--gemfile=#{autoproj_gemfile_path}",
+                        "--binstubs=#{File.join(autoproj_install_dir, 'bin')}",
+                        *opts)
+                if !result
+                    STDERR.puts "FATAL: failed to install autoproj in #{dot_autoproj}"
+                    exit 1
+                end
+            end
+
+            def update_configuration
+                if File.exist?(autoproj_config_path)
+                    config = YAML.load(File.read(autoproj_config_path)) || Hash.new
+                else
+                    config = Hash.new
+                end
+                config['private_bundler']  = private_bundler?
+                config['private_autoproj'] = private_autoproj?
+                config['private_gems']     = private_gems?
+                File.open(autoproj_config_path, 'w') do |io|
+                    YAML.dump(config, io)
+                end
+            end
+
+            def install
+                if private_bundler?
+                    puts "Installing bundler in #{bundler_install_dir}"
+                    bundler = install_bundler
+                end
+                save_gemfile
+                puts "Installing autoproj in #{dot_autoproj}"
+                install_autoproj(bundler || 'bundler')
+            end
+
+            # Actually perform the install
+            def run
+                install
+                ENV['BUNDLE_GEMFILE'] = autoproj_gemfile_path
+                require 'bundler'
+                Bundler.setup
+                require 'autobuild'
+                save_env_sh
+                update_configuration
+            end
+        end
+    end
+end
+

data/lib/autoproj/ops/main_config_switcher.rb

@@ -81,10 +81,6 @@ def bootstrap(buildconf_info, options = Hash.new)
                 validate_autoproj_current_root(options[:reuse])
 
                 ws.config.validate_ruby_executable
-
-                PackageManagers::GemManager.with_prerelease(ws.config.use_prerelease?) do
-                    ws.osdeps.install(%w{autobuild autoproj})
-                end
                 ws.config.set 'reused_autoproj_installations', options[:reuse], true
                 ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
 
@@ -174,7 +170,7 @@ def do_switch_config(delete_current, type, url, *options)
                 vcs = VCSDefinition.from_raw(vcs_def)
 
                 # Install the OS dependencies required for this VCS
-                ws.osdeps.install([vcs.type])
+                ws.install_os_packages([vcs.type])
 
                 # Now check out the actual configuration
                 config_dir = File.join(ws.root_dir, "autoproj")

data/lib/autoproj/os_package_installer.rb

@@ -0,0 +1,348 @@
+require 'autoproj/package_managers/manager'
+require 'autoproj/package_managers/unknown_os_manager'
+require 'autoproj/package_managers/shell_script_manager'
+
+require 'autoproj/package_managers/apt_dpkg_manager'
+require 'autoproj/package_managers/emerge_manager'
+require 'autoproj/package_managers/homebrew_manager'
+require 'autoproj/package_managers/pacman_manager'
+require 'autoproj/package_managers/pkg_manager'
+require 'autoproj/package_managers/port_manager'
+require 'autoproj/package_managers/yum_manager'
+require 'autoproj/package_managers/zypper_manager'
+
+require 'autoproj/package_managers/bundler_manager'
+require 'autoproj/package_managers/pip_manager'
+
+module Autoproj
+    class OSPackageInstaller
+        attr_reader :ws
+
+        PACKAGE_MANAGERS = Hash[
+           'apt-dpkg' => PackageManagers::AptDpkgManager,
+           'gem'      => PackageManagers::BundlerManager,
+           'emerge'   => PackageManagers::EmergeManager,
+           'brew'     => PackageManagers::PacmanManager,
+           'pacman'   => PackageManagers::HomebrewManager,
+           'pip'      => PackageManagers::YumManager,
+           'pkg'      => PackageManagers::PortManager,
+           'macports' => PackageManagers::ZypperManager,
+           'yum'      => PackageManagers::PipManager ,
+           'zypper'   => PackageManagers::PkgManager
+        ]
+
+        attr_reader :os_package_resolver
+
+        # The set of packages that have already been installed
+        attr_reader :installed_packages
+
+        attr_writer :silent
+        def silent?; @silent end
+
+        class << self
+            attr_accessor :force_osdeps
+        end
+
+        def initialize(ws, os_package_resolver)
+            @ws = ws
+            @os_package_resolver = os_package_resolver
+            @installed_packages = Set.new
+            @silent = true
+            @filter_uptodate_packages = true
+        end
+
+        # Returns the package manager object for the current OS
+        def os_package_manager
+            if !@os_package_manager
+                name = os_package_resolver.os_package_manager
+                @os_package_manager = package_managers[name] ||
+                    PackageManagers::UnknownOSManager.new(ws)
+            end
+            return @os_package_manager
+        end
+
+        # Returns the set of package managers
+        def package_managers
+            if !@package_managers
+                @package_managers = Hash.new
+                PACKAGE_MANAGERS.each do |name, klass|
+                    @package_managers[name] = klass.new(ws)
+                end
+            end
+            @package_managers
+        end
+
+        def each_manager(&block)
+            package_managers.each_value(&block)
+        end
+
+        HANDLE_ALL  = 'all'
+        HANDLE_RUBY = 'ruby'
+        HANDLE_OS   = 'os'
+        HANDLE_NONE = 'none'
+
+        def osdeps_mode_option_unsupported_os(config)
+            long_doc =<<-EOT
+The software packages that autoproj will have to build may require other
+prepackaged softwares (a.k.a. OS dependencies) to be installed (RubyGems
+packages, packages from your operating system/distribution, ...). Autoproj is
+usually able to install those automatically, but unfortunately your operating
+system is not (yet) supported by autoproj's osdeps mechanism, it can only offer
+you some limited support.
+
+Some package handlers are cross-platform, and are therefore supported.  However,
+you will have to install the kind of OS dependencies (so-called OS packages)
+
+This option is meant to allow you to control autoproj's behaviour while handling
+OS dependencies.
+
+* if you say "all", all OS-independent packages are going to be installed.
+* if you say "gem", the RubyGem packages will be installed.
+* if you say "pip", the Pythin PIP packages will be installed.
+* if you say "none", autoproj will not do anything related to the OS
+  dependencies.
+
+As any configuration value, the mode can be changed anytime by calling
+  autoproj reconfigure
+
+Finally, the "autoproj osdeps" command will give you the necessary information
+about the OS packages that you will need to install manually.
+
+So, what do you want ? (all, none or a comma-separated list of: gem pip)
+            EOT
+            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, none or a comma-separated list of: gem pip) ?", long_doc.strip ]
+
+            config.declare 'osdeps_mode', 'string',
+                default: 'ruby',
+                doc: message,
+                lowercase: true
+        end
+
+        def osdeps_mode_option_supported_os(config)
+            long_doc =<<-EOT
+The software packages that autoproj will have to build may require other
+prepackaged softwares (a.k.a. OS dependencies) to be installed (RubyGems
+packages, packages from your operating system/distribution, ...). Autoproj
+is able to install those automatically for you.
+
+Advanced users may want to control this behaviour. Additionally, the
+installation of some packages require administration rights, which you may
+not have. This option is meant to allow you to control autoproj's behaviour
+while handling OS dependencies.
+
+* if you say "all", it will install all packages automatically.
+  This requires root access thru 'sudo'
+* if you say "pip", only the Ruby packages will be installed.
+  Installing these packages does not require root access.
+* if you say "gem", only the Ruby packages will be installed.
+  Installing these packages does not require root access.
+* if you say "os", only the OS-provided packages will be installed.
+  Installing these packages requires root access.
+* if you say "none", autoproj will not do anything related to the
+  OS dependencies.
+
+Finally, you can provide a comma-separated list of pip gem and os.
+
+As any configuration value, the mode can be changed anytime by calling
+  autoproj reconfigure
+
+Finally, the "autoproj osdeps" command will give you the necessary information
+about the OS packages that you will need to install manually.
+
+So, what do you want ? (all, none or a comma-separated list of: os gem pip)
+            EOT
+            message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, none or a comma-separated list of: os gem pip) ?", long_doc.strip ]
+
+            config.declare 'osdeps_mode', 'string',
+                default: 'all',
+                doc: message,
+                lowercase: true
+        end
+
+        def define_osdeps_mode_option
+            if OSPackageResolver.supported_operating_system?
+                osdeps_mode_option_supported_os(ws.config)
+            else
+                osdeps_mode_option_unsupported_os(ws.config)
+            end
+        end
+
+        def osdeps_mode_string_to_value(string)
+            string = string.to_s.downcase.split(',')
+            modes = []
+            string.map do |str|
+                case str
+                when 'all'  then modes.concat(['os', 'gem', 'pip'])
+                when 'ruby' then modes << 'gem'
+                when 'gem'  then modes << 'gem'
+                when 'pip'  then modes << 'pip'
+                when 'os'   then modes << 'os'
+                when 'none' then
+                else raise ArgumentError, "#{str} is not a known package handler"
+                end
+            end
+            modes
+        end
+
+        # If set to true (the default), #install will try to remove the list of
+        # already uptodate packages from the installed packages. Set to false to
+        # install all packages regardless of their status
+        attr_writer :filter_uptodate_packages
+
+        # If set to true (the default), #install will try to remove the list of
+        # already uptodate packages from the installed packages. Use
+        # #filter_uptodate_packages= to set it to false to install all packages
+        # regardless of their status
+        def filter_uptodate_packages?
+            !!@filter_uptodate_packages
+        end
+
+        # Override the osdeps mode
+        def osdeps_mode=(value)
+            @osdeps_mode = osdeps_mode_string_to_value(value)
+        end
+
+        # Returns the osdeps mode chosen by the user
+        def osdeps_mode
+            # This has two uses. It caches the value extracted from the
+            # AUTOPROJ_OSDEPS_MODE and/or configuration file. Moreover, it
+            # allows to override the osdeps mode by using
+            # OSPackageInstaller#osdeps_mode=
+            if @osdeps_mode
+                return @osdeps_mode
+            end
+
+            config = ws.config
+            while true
+                mode =
+                    if !config.has_value_for?('osdeps_mode') && mode_name = ENV['AUTOPROJ_OSDEPS_MODE']
+                        begin osdeps_mode_string_to_value(mode_name)
+                        rescue ArgumentError
+                            Autoproj.warn "invalid osdeps mode given through AUTOPROJ_OSDEPS_MODE (#{mode})"
+                            nil
+                        end
+                    else
+                        mode_name = config.get('osdeps_mode')
+                        begin osdeps_mode_string_to_value(mode_name)
+                        rescue ArgumentError
+                            Autoproj.warn "invalid osdeps mode stored in configuration file"
+                            nil
+                        end
+                    end
+
+                if mode
+                    @osdeps_mode = mode
+                    config.set('osdeps_mode', mode_name, true)
+                    return mode
+                end
+
+                # Invalid configuration values. Retry
+                config.reset('osdeps_mode')
+                ENV['AUTOPROJ_OSDEPS_MODE'] = nil
+            end
+        end
+
+        # Set up the registered package handlers according to the specified osdeps mode
+        #
+        # It enables/disables package handlers based on either the value
+        # returned by {#osdeps_mode} or the value passed as option (the latter
+        # takes precedence). Moreover, sets the handler's silent flag using
+        # {#silent?}
+        #
+        # @option options [Array<String>] the package handlers that should be
+        #   enabled. The default value is returned by {#osdeps_mode}
+        # @return [Array<PackageManagers::Manager>] the set of enabled package
+        #   managers
+        def setup_package_managers(options = Hash.new)
+            options = Kernel.validate_options options,
+                osdeps_mode: osdeps_mode
+
+            os_package_manager.enabled = false
+            package_managers.each_value do |handler|
+                handler.enabled = false
+            end
+            options[:osdeps_mode].each do |m|
+                if m == 'os'
+                    os_package_manager.enabled = true
+                elsif pkg = package_managers[m]
+                    pkg.enabled = true
+                else
+                    Autoproj.warn "osdep handler #{m.inspect} has no handler, available handlers are #{package_managers.keys.map(&:inspect).sort.join(", ")}"
+                end
+            end
+            os_package_manager.silent = self.silent?
+            package_managers.each_value do |v|
+                v.silent = self.silent?
+            end
+
+            enabled_handlers = []
+            if os_package_manager.enabled?
+                enabled_handlers << os_package_manager
+            end
+            package_managers.each_value do |v|
+                if v.enabled?
+                    enabled_handlers << v
+                end
+            end
+            enabled_handlers
+        end
+
+        # Requests that packages that are handled within the autoproj project
+        # (i.e. gems) are restored to pristine condition
+        #
+        # This is usually called as a rebuild step to make sure that all these
+        # packages are updated to whatever required the rebuild
+        def pristine(packages, options = Hash.new)
+            install(packages, options.merge(install_only: true))
+            packages = os_package_resolver.resolve_os_packages(packages)
+
+            _, other_packages =
+                packages.partition { |handler, list| handler == os_package_manager }
+            other_packages.each do |handler, list|
+                if handler.respond_to?(:pristine)
+                    handler.pristine(list)
+                end
+            end
+        end
+
+        # Requests the installation of the given set of packages
+        def install(packages, options = Hash.new)
+            # Remove the set of packages that have already been installed 
+            packages = packages.to_set - installed_packages
+            return false if packages.empty?
+
+            filter_options, options =
+                filter_options options, install_only: !Autobuild.do_update
+            setup_package_managers(options)
+
+            packages = os_package_resolver.resolve_os_packages(packages)
+            packages = packages.map do |handler_name, list|
+                [package_managers[handler_name], list]
+            end
+
+            needs_filter = (filter_uptodate_packages? || filter_options[:install_only])
+            packages = packages.map do |handler, list|
+                if needs_filter && handler.respond_to?(:filter_uptodate_packages)
+                    list = handler.filter_uptodate_packages(list, filter_options)
+                end
+
+                if !list.empty?
+                    [handler, list]
+                end
+            end.compact
+            return false if packages.empty?
+
+            # Install OS packages first, as the other package handlers might
+            # depend on OS packages
+            os_packages, other_packages = packages.partition { |handler, list| handler == os_package_manager }
+            [os_packages, other_packages].each do |packages|
+                packages.each do |handler, list|
+                    handler.install(list)
+                    @installed_packages |= list.to_set
+                end
+            end
+            true
+        end
+    end
+end 
+