autoproj 2.0.0.rc37 → 2.0.0.rc38

data/lib/autoproj/ops/install.rb

@@ -33,6 +33,7 @@ def initialize(root_dir)
                 @root_dir = root_dir
                 @gem_source = "https://rubygems.org"
                 @gemfile = nil
+                @skip_stage2 = false
 
                 @autoproj_options = Array.new
 
@@ -103,6 +104,11 @@ def autoproj_gemfile_path; File.join(dot_autoproj, 'Gemfile') end
             # @return [String]
             def autoproj_config_path; File.join(dot_autoproj, 'config.yml') end
 
+            # Whether the stage2 install should be called or not
+            def skip_stage2?; !!@skip_stage2 end
+            # (see #skip_stage2?)
+            def skip_stage2=(flag); @skip_stage2 = flag end
+
             # Whether we can access the network while installing
             def local?; !!@local end
             # (see #local?)
@@ -186,6 +192,9 @@ def parse_options(args = ARGV)
                     opt.on '--local', 'do not access the network (may fail)' do
                         @local = true
                     end
+                    opt.on '--skip-stage2', 'do not run the stage2 install' do
+                        @skip_stage2 = true
+                    end
                     opt.on '--gem-source=URL', String, "use this source for RubyGems instead of rubygems.org" do |url|
                         @gem_source = url
                     end
@@ -337,6 +346,7 @@ def self.shim_bundler(ruby_executable, autoproj_gemfile_path, gems_gem_home)
     end
 end
 
+ENV['BUNDLE_GEMFILE'] ||= '#{autoproj_gemfile_path}'
 ENV['GEM_HOME'] = '#{gems_gem_home}'
 ENV.delete('GEM_PATH')
 Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => '']
@@ -349,8 +359,10 @@ def self.shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load
 
 if defined?(Bundler)
     Bundler.with_clean_env do
-        exec($0, *ARGV)
+        exec(Hash['RUBYLIB' => nil], $0, *ARGV)
     end
+elsif ENV['RUBYLIB']
+    exec(Hash['RUBYLIB' => nil], $0, *ARGV)
 end
 
 ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}'
@@ -397,6 +409,18 @@ def save_gemfile
                         default_gemfile_contents
                     end
 
+                gemfile += [
+                    "",
+                    "config_path = File.join(__dir__, 'config.yml')",
+                    "if File.file?(config_path)",
+                    "    require 'yaml'",
+                    "    config = YAML.load(File.read(config_path))",
+                    "    (config['plugins'] || Hash.new).each do |plugin_name, (version, options)|",
+                    "        gem plugin_name, version, **options",
+                    "    end",
+                    "end"
+                ].join("\n")
+
                 FileUtils.mkdir_p File.dirname(autoproj_gemfile_path)
                 File.open(autoproj_gemfile_path, 'w') do |io|
                     io.write gemfile

data/lib/autoproj/ops/loader.rb

@@ -15,7 +15,10 @@ def initialize(root_dir)
         end
 
         def in_package_set(pkg_set, path)
-            @file_stack.push([pkg_set, File.expand_path(path).gsub(/^#{Regexp.quote(root_dir)}\//, '')])
+            if path
+                path = File.expand_path(path, root_dir)
+            end
+            @file_stack.push([pkg_set, path])
             yield
         ensure
             @file_stack.pop

data/lib/autoproj/ops/main_config_switcher.rb

@@ -107,7 +107,7 @@ def bootstrap(buildconf_info, options = Hash.new)
 
                 elsif buildconf_info.size >= 2 # is a VCS definition for the manifest itself ...
                     type, url, *options = *buildconf_info
-                    url = VCSDefinition.to_absolute_url(url, Dir.pwd)
+                    url = VCSDefinition.to_absolute_url(url, ws.root_dir)
                     do_switch_config(false, type, url, *options)
                 end
                 ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
@@ -125,7 +125,7 @@ def switch_config(*args)
                 end
                 options = args
 
-                url = VCSDefinition.to_absolute_url(url)
+                url = VCSDefinition.to_absolute_url(url, ws.root_dir)
 
                 if vcs && (vcs.type == type && vcs.url == url)
                     vcs = vcs.to_hash
@@ -157,7 +157,7 @@ def switch_config(*args)
             def do_switch_config(delete_current, type, url, *options)
                 vcs_def = Hash.new
                 vcs_def[:type] = type
-                vcs_def[:url]  = VCSDefinition.to_absolute_url(url)
+                vcs_def[:url]  = VCSDefinition.to_absolute_url(url, ws.root_dir)
                 options.each do |opt|
                     name, value = opt.split("=")
                     if value =~ /^\d+$/
@@ -170,7 +170,7 @@ def do_switch_config(delete_current, type, url, *options)
                 vcs = VCSDefinition.from_raw(vcs_def)
 
                 # Install the OS dependencies required for this VCS
-                ws.install_os_packages([vcs.type])
+                ws.install_os_packages([vcs.type], all: nil)
 
                 # Now check out the actual configuration
                 config_dir = File.join(ws.root_dir, "autoproj")

data/lib/autoproj/ops/snapshot.rb

@@ -134,7 +134,7 @@ def snapshot_packages(packages, target_dir = nil, options = Hash.new)
 
             result = Array.new
             packages.each do |package_name|
-                package  = manifest.packages[package_name]
+                package  = manifest.find_package_definition(package_name)
                 if !package
                     raise ArgumentError, "#{package_name} is not a known package"
                 end
@@ -217,8 +217,9 @@ def update_package_import_state(name, packages)
             if current_versions.empty?
                 # Do a full snapshot this time only
                 Autoproj.message "  building initial autoproj import log, this may take a while"
-                packages = manifest.all_selected_packages.
-                    find_all { |pkg| File.directory?(manifest.find_package(pkg).autobuild.srcdir) }
+                packages = manifest.all_selected_source_packages.
+                    find_all { |pkg| File.directory?(pkg.autobuild.srcdir) }.
+                    map(&:name)
             end
             versions  = snapshot_package_sets
             versions += snapshot_packages(packages)

data/lib/autoproj/os_package_installer.rb

@@ -33,9 +33,6 @@ class OSPackageInstaller
 
         attr_reader :os_package_resolver
 
-        # The set of packages that have already been installed
-        attr_reader :installed_packages
-
         # The set of resolved packages that have already been installed
         attr_reader :installed_resolved_packages
 
@@ -46,13 +43,18 @@ class << self
             attr_accessor :force_osdeps
         end
 
-        def initialize(ws, os_package_resolver)
+        def initialize(ws, os_package_resolver, package_managers: PACKAGE_MANAGERS)
             @ws = ws
             @os_package_resolver = os_package_resolver
-            @installed_packages = Set.new
+            @os_package_manager  = nil
             @installed_resolved_packages = Hash.new { |h, k| h[k] = Set.new }
             @silent = true
             @filter_uptodate_packages = true
+
+            @package_managers = Hash.new
+            package_managers.each do |name, klass|
+                @package_managers[name] = klass.new(ws)
+            end
         end
 
         # Returns the package manager object for the current OS
@@ -66,15 +68,7 @@ def os_package_manager
         end
 
         # Returns the set of package managers
-        def package_managers
-            if !@package_managers
-                @package_managers = Hash.new
-                PACKAGE_MANAGERS.each do |name, klass|
-                    @package_managers[name] = klass.new(ws)
-                end
-            end
-            @package_managers
-        end
+        attr_reader :package_managers
 
         def each_manager(&block)
             package_managers.each_value(&block)
@@ -172,9 +166,9 @@ def define_osdeps_mode_option
         end
 
         def osdeps_mode_string_to_value(string)
-            string = string.to_s.downcase.split(',')
+            user_modes = string.to_s.downcase.split(',')
             modes = []
-            string.map do |str|
+            user_modes.each do |str|
                 case str
                 when 'all'  then modes.concat(['os', 'gem', 'pip'])
                 when 'ruby' then modes << 'gem'
@@ -182,7 +176,12 @@ def osdeps_mode_string_to_value(string)
                 when 'pip'  then modes << 'pip'
                 when 'os'   then modes << 'os'
                 when 'none' then
-                else raise ArgumentError, "#{str} is not a known package handler"
+                else
+                    if package_managers.has_key?(str)
+                        modes << str
+                    else
+                        raise ArgumentError, "#{str} is not a known package handler, known handlers are #{package_managers.keys.sort.join(", ")}"
+                    end
                 end
             end
             modes
@@ -268,7 +267,7 @@ def setup_package_managers(osdeps_mode: self.osdeps_mode)
                 elsif pkg = package_managers[m]
                     pkg.enabled = true
                 else
-                    Autoproj.warn "osdep handler #{m.inspect} has no handler, available handlers are #{package_managers.keys.map(&:inspect).sort.join(", ")}"
+                    Autoproj.warn "osdep handler #{m.inspect} found in osdep_mode has no handler, available handlers are #{package_managers.keys.map(&:inspect).sort.join(", ")}"
                 end
             end
             os_package_manager.silent = self.silent?
@@ -314,45 +313,105 @@ def pristine(packages, options = Hash.new)
             end
         end
 
-        # Requests the installation of the given set of packages
-        def install(osdep_packages, install_only: false, run_package_managers_without_packages: false, **options)
-            osdep_packages = osdep_packages.to_set - installed_packages
-
-            setup_package_managers(**options)
-
-            managers = package_managers.dup
-            packages = os_package_resolver.resolve_os_packages(osdep_packages)
-            packages = packages.map do |handler_name, list|
-                if manager = managers.delete(handler_name)
-                    [manager, list]
+        # @api private
+        #
+        # Resolves the package managers from their name in a manager-to-package
+        # list mapping.
+        #
+        # This is a helper method to resolve the value returned by
+        # {OSPackageResolver#resolve_os_packages}
+        #
+        # @raise [ArgumentError] if a manager name cannot be resolved
+        def resolve_package_managers_in_mapping(mapping)
+            resolved = Hash.new
+            mapping.each do |manager_name, package_list|
+                if manager = package_managers[manager_name]
+                    resolved[manager] = package_list
                 else
                     raise ArgumentError, "no package manager called #{handler_name} found"
                 end
             end
+            resolved
+        end
 
-            managers.each_value do |pkg_manager|
-                packages << [pkg_manager, []]
+        # @api private
+        #
+        # Resolves and partitions osdep packages into the various package
+        # managers. The returned hash will have one entry per package manager
+        # that has a package, with additional entries for package managers that
+        # have an empty list but for which
+        # {PackageManagers::Manager#call_while_empty?} returns true
+        #
+        # @param [Array<String>] osdep_packages the list of osdeps to install
+        # @return [Hash<PackageManagers::Manager,Array<(String, String)>] the
+        #   resolved and partitioned osdeps
+        # @raise (see #resolve_package_managers_in_mapping)
+        # @raise [InternalError] if all_osdep_packages is not provided (is nil)
+        #   and some strict package managers need to be called to handle
+        #   osdep_packages
+        def resolve_and_partition_osdep_packages(osdep_packages, all_osdep_packages = nil)
+            packages = os_package_resolver.resolve_os_packages(osdep_packages)
+            packages = resolve_package_managers_in_mapping(packages)
+            all_packages = os_package_resolver.resolve_os_packages(all_osdep_packages || Array.new)
+            all_packages = resolve_package_managers_in_mapping(all_packages)
+
+            partitioned_packages = Hash.new
+            package_managers.each do |manager_name, manager|
+                manager_selected = packages.fetch(manager, Set.new).to_set
+                manager_all      = all_packages.fetch(manager, Set.new).to_set
+
+                next if manager_selected.empty? && !manager.call_while_empty?
+
+                # If the manager is strict, we need to bypass it if we did not
+                # get the complete list of osdep packages
+                if manager.strict? && !all_osdep_packages
+                    if !manager_selected.empty?
+                        raise InternalError, "requesting to install the osdeps #{partitioned_packages[manager].to_a.sort.join(", ")} through #{manager_name} but the complete list of osdep packages managed by this manager was not provided. This would break the workspace"
+                    end
+                    next
+                end
+
+                if manager.strict?
+                    manager_packages = manager_all | manager_selected
+                else
+                    manager_packages = manager_selected
+                end
+
+                partitioned_packages[manager] = manager_packages
             end
+            partitioned_packages
+        end
+
+        # Requests the installation of the given set of packages
+        def install(osdep_packages, all: nil, install_only: false, run_package_managers_without_packages: false, **options)
+            setup_package_managers(**options)
+            partitioned_packages =
+                resolve_and_partition_osdep_packages(osdep_packages, all)
 
             # Install OS packages first, as the other package handlers might
             # depend on OS packages
-            os_packages, other_packages = packages.partition do |handler, list|
-                handler == os_package_manager
+            if os_packages = partitioned_packages.delete(os_package_manager)
+                install_manager_packages(os_package_manager, os_packages,
+                                         install_only: install_only,
+                                         run_package_managers_without_packages: run_package_managers_without_packages)
             end
-            [os_packages, other_packages].each do |packages|
-                packages.each do |handler, list|
-                    list = list.to_set - installed_resolved_packages[handler]
-
-                    if !list.empty? || (run_package_managers_without_packages && handler.call_while_empty?)
-                        handler.install(
-                            list.to_a,
-                            filter_uptodate_packages: filter_uptodate_packages?,
-                            install_only: install_only)
-                        installed_resolved_packages[handler].merge(list)
-                    end
-                end
+            partitioned_packages.each do |manager, package_list|
+                install_manager_packages(manager, package_list,
+                                         install_only: install_only,
+                                         run_package_managers_without_packages: run_package_managers_without_packages)
+            end
+        end
+
+        def install_manager_packages(manager, package_list, install_only: false, run_package_managers_without_packages: false)
+            list = package_list.to_set - installed_resolved_packages[manager]
+
+            if !list.empty? || run_package_managers_without_packages
+                manager.install(
+                    list.to_a,
+                    filter_uptodate_packages: filter_uptodate_packages?,
+                    install_only: install_only)
+                installed_resolved_packages[manager].merge(list)
             end
-            installed_packages.merge(packages)
         end
     end
 end 

data/lib/autoproj/os_package_resolver.rb

@@ -17,7 +17,7 @@ class << self
 	end
 	@suffixes = []
 
-        def self.load(file)
+        def self.load(file, **options)
 	    if !File.file?(file)
 		raise ArgumentError, "no such file or directory #{file}"
 	    end
@@ -29,18 +29,18 @@ def self.load(file)
                       else ArgumentError
                       end
 
-	    result = new
-	    candidates.each do |file|
-                next if !File.file?(file)
-                file = File.expand_path(file)
+	    result = new(**options)
+	    candidates.each do |file_candidate|
+                next if !File.file?(file_candidate)
+                file_candidate = File.expand_path(file_candidate)
                 begin
-                    data = YAML.load(File.read(file)) || Hash.new
+                    data = YAML.load(File.read(file_candidate)) || Hash.new
                     verify_definitions(data)
                 rescue *error_t => e
-                    raise ConfigError.new, "error in #{file}: #{e.message}", e.backtrace
+                    raise ConfigError.new, "error in #{file_candidate}: #{e.message}", e.backtrace
                 end
 
-                result.merge(new(data, file))
+                result.merge(new(data, file_candidate, **options))
 	    end
 	    result
         end
@@ -128,15 +128,22 @@ def load_default
         def prefer_indep_over_os_packages?; !!@prefer_indep_over_os_packages end
         # (see prefer_indep_over_os_packages?)
         def prefer_indep_over_os_packages=(flag); @prefer_indep_over_os_packages = flag end
+        # Cached results of {#resolve_package}
+        attr_reader :resolve_package_cache
 
         # Use to override the autodetected OS-specific package handler
-        attr_writer :os_package_manager
+        def os_package_manager=(manager_name)
+            if manager_name && !package_managers.include?(manager_name)
+                raise ArgumentError, "#{manager_name} is not a known package manager"
+            end
+            @os_package_manager = manager_name
+        end
 
         # Returns the name of the package manager object for the current OS
         #
         # @return [String]
         def os_package_manager
-            if !instance_variable_defined?(:@os_package_manager)
+            if !@os_package_manager
                 os_names, _ = operating_system
                 os_name = os_names.find { |name| OS_PACKAGE_MANAGERS[name] }
                 @os_package_manager = OS_PACKAGE_MANAGERS[os_name] ||
@@ -152,20 +159,32 @@ def os_package_manager
 
         # The Gem::SpecFetcher object that should be used to query RubyGems, and
         # install RubyGems packages
-        def initialize(defs = Hash.new, file = nil)
+        def initialize(defs = Hash.new, file = nil,
+                       operating_system: nil,
+                       package_managers: PACKAGE_MANAGERS.dup,
+                       os_package_manager: nil)
             @definitions = defs.to_hash
+            @resolve_package_cache = Hash.new
             @all_definitions = Hash.new { |h, k| h[k] = Array.new }
-            @package_managers = PACKAGE_MANAGERS.dup
+            @package_managers = package_managers
+            self.os_package_manager = os_package_manager
+
             @prefer_indep_over_os_packages = false
 
             @sources     = Hash.new
             @installed_packages = Set.new
-            @operating_system = self.class.operating_system
+            @operating_system = operating_system
+            @supported_operating_system = nil
+            @odeps_mode = nil
             if file
                 defs.each_key do |package_name|
                     sources[package_name] = file
                     all_definitions[package_name] << [[file], defs[package_name]]
                 end
+            else
+                defs.each_key do |package_name|
+                    all_definitions[package_name] << [[], defs[package_name]]
+                end
             end
         end
 
@@ -174,7 +193,7 @@ def initialize(defs = Hash.new, file = nil)
         # It includes even the packages for which there are no definitions on
         # this OS
         def all_package_names
-            all_definitions.keys
+            definitions.keys
         end
 
         # Returns the full path to the osdeps file from which the package
@@ -183,16 +202,24 @@ def source_of(package_name)
             sources[package_name]
         end
 
+        def add_entries(entries, file: nil)
+            merge(self.class.new(entries, file))
+        end
+
+        def invalidate_resolve_package_cache
+            @resolve_package_cache.clear
+        end
+
         # Merges the osdeps information of +info+ into +self+. If packages are
         # defined in both OSPackageResolver objects, the information in +info+
         # takes precedence
         def merge(info)
-            root_dir = nil
+            invalidate_resolve_package_cache
+
             @definitions = definitions.merge(info.definitions) do |h, v1, v2|
                 if v1 != v2
-                    root_dir ||= "#{Autoproj.root_dir}/"
-                    old = source_of(h).gsub(root_dir, '')
-                    new = info.source_of(h).gsub(root_dir, '')
+                    old = source_of(h)
+                    new = info.source_of(h)
 
                     # Warn if the new osdep definition resolves to a different
                     # set of packages than the old one
@@ -247,12 +274,7 @@ def self.verify_definitions(hash, path = [])
         # system on which we are installed
         def supported_operating_system?
             if @supported_operating_system.nil?
-                os_names, _ = operating_system
-                @supported_operating_system =
-                    if !os_names then false
-                    else
-                        os_names.any? { |os_name| OS_PACKAGE_MANAGERS.has_key?(os_name) }
-                    end
+                @supported_operating_system = (os_package_manager != 'unknown')
             end
             return @supported_operating_system
         end
@@ -337,21 +359,18 @@ def self.normalize_os_representation(names, versions)
             return names, versions
         end
 
+        # The operating system self is targetting
+        #
+        # If unset in {#initialize} or by calling {#operating_system=}, it will
+        # attempt to autodetect it on the first call
         def operating_system
-            @operating_system || self.class.operating_system
+            @operating_system ||= self.class.autodetect_operating_system
         end
 
+        # Change the operating system this resolver is targetting
         def operating_system=(values)
             @supported_operating_system = nil
-            @operating_system = values
-        end
-
-        def self.operating_system
-            @operating_system
-        end
-        @operating_system = nil
-
-        def self.operating_system=(values)
+            @os_package_manager = nil
             @operating_system = values
         end
 
@@ -471,6 +490,10 @@ def self.resolve_name(name)
         # name and version. The package list might be empty even if status ==
         # FOUND_PACKAGES, for instance if the ignore keyword is used.
         def resolve_package(name)
+            if resolve_package_cache.has_key?(name)
+                return resolve_package_cache[name]
+            end
+
             path = self.class.resolve_name(name)
             name = path.last
 
@@ -484,7 +507,7 @@ def resolve_package(name)
 
             dep_def = definitions[name]
             if !dep_def
-                return nil
+                return (resolve_package_cache[name] = nil)
             end
 
             # Partition the found definition in all entries that are interesting
@@ -522,7 +545,7 @@ def resolve_package(name)
                 end
             end
 
-            result
+            resolve_package_cache[name] = result
         end
 
         # Value returned by #resolve_package and #partition_osdep_entry in
@@ -712,6 +735,10 @@ def resolve_os_packages(dependencies)
             return all_packages
         end
 
+        # Returns true if the given name has an entry in the osdeps
+        def include?(name)
+            definitions.has_key?(name)
+        end
 
         # Returns true if +name+ is an acceptable OS package for this OS and
         # version