autoproj 2.0.0.rc38 → 2.0.0.rc39

data/lib/autoproj/manifest.rb

@@ -174,6 +174,7 @@ def accept_unavailable_osdeps=(flag); @accept_unavailable_osdeps = flag end
 
         def initialize(ws, os_package_resolver: OSPackageResolver.new)
             @ws = ws
+            @vcs = VCSDefinition.none
             @file = nil
 	    @data = Hash.new
             @has_layout = false
@@ -181,6 +182,8 @@ def initialize(ws, os_package_resolver: OSPackageResolver.new)
             @packages = Hash.new
             @package_sets = []
             @os_package_resolver = os_package_resolver
+            # Cache for #ignored?
+            @ignored_package_names = nil
 
             @automatic_exclusions = Hash.new
             @constants_definitions = Hash.new

data/lib/autoproj/ops/configuration.rb

@@ -33,7 +33,7 @@ def remotes_user_dir
         #
         # @return [String]
         def manifest_path
-            File.join(ws.config_dir, 'manifest')
+            ws.manifest_file_path
         end
 
         # @param [Manifest] manifest
@@ -50,20 +50,17 @@ def initialize(workspace, update_from: nil)
         # Imports or updates a source (remote or otherwise).
         #
         # See create_autobuild_package for informations about the arguments.
-        def update_configuration_repository(vcs, name, into, options = Hash.new)
-            options = Kernel.validate_options options,
+        def update_configuration_repository(vcs, name, into,
                 only_local: false,
-                checkout_only: !Autobuild.do_update,
-                ignore_errors: false,
                 reset: false,
-                retry_count: nil
+                retry_count: nil)
 
             fake_package = Tools.create_autobuild_package(vcs, name, into)
             if update_from
                 # Define a package in the installation manifest that points to
                 # the desired folder in other_root
                 relative_path = Pathname.new(into).
-                    relative_path_from(Pathname.new(root_dir)).to_s
+                    relative_path_from(Pathname.new(ws.root_dir)).to_s
                 other_dir = File.join(update_from.path, relative_path)
                 if File.directory?(other_dir)
                     update_from.packages.unshift(
@@ -79,10 +76,10 @@ def update_configuration_repository(vcs, name, into, options = Hash.new)
                     fake_package.update = false
                 end
             end
-            if retry_count = options.delete(:retry_count)
+            if retry_count
                 fake_package.importer.retry_count = retry_count
             end
-            fake_package.import(options)
+            fake_package.import(only_local: only_local, reset: reset)
 
         rescue Autobuild::ConfigException => e
             raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
@@ -92,22 +89,24 @@ def update_configuration_repository(vcs, name, into, options = Hash.new)
         #
         # @return [Boolean] true if something got updated or checked out,
         #   and false otherwise
-        def update_main_configuration(options = Hash.new)
-            if !options.kind_of?(Hash)
-                options = Hash[only_local: options]
+        def update_main_configuration(keep_going: false, checkout_only: !Autobuild.do_update, only_local: false, reset: false, retry_count: nil)
+            if checkout_only && File.exist?(ws.config_dir)
+                return []
             end
-            options = validate_options options,
-                only_local: false,
-                checkout_only: !Autobuild.do_update,
-                ignore_errors: false,
-                reset: false,
-                retry_count: nil
 
             update_configuration_repository(
-                ws.manifest.vcs,
-                "autoproj main configuration",
-                ws.config_dir,
-                options)
+                ws.manifest.vcs, "autoproj main configuration", ws.config_dir,
+                only_local: only_local, reset: reset, retry_count: retry_count
+            )
+            []
+        rescue Interrupt
+            raise
+        rescue Exception => e
+            if keep_going
+                [e]
+            else
+                raise e
+            end
         end
 
         # Update or checkout a remote package set, based on its VCS definition
@@ -115,22 +114,18 @@ def update_main_configuration(options = Hash.new)
         # @param [VCSDefinition] vcs the package set VCS
         # @return [Boolean] true if something got updated or checked out,
         #   and false otherwise
-        def update_remote_package_set(vcs, options = Hash.new)
-            # BACKWARD
-            if !options.kind_of?(Hash)
-                options = Hash[only_local: options]
-            end
-            options = validate_options options,
-                only_local: false,
+        def update_remote_package_set(vcs,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false,
+                only_local: false,
                 reset: false,
-                retry_count: nil
+                retry_count: nil)
 
             name = PackageSet.name_of(ws, vcs)
             raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
 
-            return if options[:checkout_only] && File.exist?(raw_local_dir)
+            if checkout_only && File.exist?(raw_local_dir)
+                return
+            end
 
             # YUK. I am stopping there in the refactoring
             # TODO: figure out a better way
@@ -140,7 +135,10 @@ def update_remote_package_set(vcs, options = Hash.new)
             end
             ws.install_os_packages([vcs.type], all: nil)
             update_configuration_repository(
-                vcs, name, raw_local_dir, options)
+                vcs, name, raw_local_dir,
+                only_local: only_local,
+                reset: reset,
+                retry_count: retry_count)
         end
 
         # Create the user-visible directory for a remote package set
@@ -200,20 +198,16 @@ def queue_auto_imports_if_needed(queue, pkg_set, root_set)
         #
         # @yieldparam [String] osdep the name of an osdep required to import the
         #   package sets
-        def load_and_update_package_sets(root_pkg_set, options = Hash.new)
-            if !options.kind_of?(Hash)
-                options = Hash[only_local: options]
-            end
-            options = validate_options options,
-                only_local: false,
-                checkout_only: !Autobuild.do_update,
-                ignore_errors: false,
-                reset: false,
-                retry_count: nil
-
+        def load_and_update_package_sets(root_pkg_set,
+                                         only_local: false,
+                                         checkout_only: !Autobuild.do_update,
+                                         keep_going: false,
+                                         reset: false,
+                                         retry_count: nil)
             package_sets = [root_pkg_set]
             by_repository_id = Hash.new
             by_name = Hash.new
+            failures = Array.new
 
             required_remotes_dirs = Array.new
 
@@ -238,8 +232,21 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
                 # Make sure the package set has been already checked out to
                 # retrieve the actual name of the package set
                 if !vcs.local?
-                    update_remote_package_set(vcs, options)
-                    required_remotes_dirs << PackageSet.raw_local_dir_of(ws, vcs)
+                    failed = handle_keep_going(keep_going, vcs, failures) do
+                        update_remote_package_set(
+                            vcs, checkout_only: checkout_only,
+                            only_local: only_local, reset: reset,
+                            retry_count: retry_count)
+                    end
+                    raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
+
+                    # We really can't continue if the VCS was being checked out
+                    # and that failed
+                    if failed && !File.directory?(raw_local_dir)
+                        raise failures.last
+                    end
+
+                    required_remotes_dirs << raw_local_dir
                 end
 
                 name = PackageSet.name_of(ws, vcs)
@@ -277,7 +284,7 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
                 by_repository_id[repository_id][2] = pkg_set
                 package_sets << pkg_set
 
-                by_name[pkg_set.name] = [pkg_set, vcs, options, imported_from]
+                by_name[pkg_set.name] = [pkg_set, vcs, import_options, imported_from]
 
                 # Finally, queue the imports
                 queue_auto_imports_if_needed(queue, pkg_set, root_pkg_set)
@@ -286,7 +293,8 @@ def load_and_update_package_sets(root_pkg_set, options = Hash.new)
             required_user_dirs = by_name.collect { |k,v| k }
             cleanup_remotes_dir(package_sets, required_remotes_dirs)
             cleanup_remotes_user_dir(package_sets, required_user_dirs)
-            package_sets
+
+            return package_sets, failures
         end
 
         # Removes from {remotes_dir} the directories that do not match a package
@@ -369,44 +377,228 @@ def sort_package_sets_by_import_order(package_sets, root_pkg_set)
             result
         end
 
-        def load_package_sets(options = Hash.new)
-            options = validate_options options,
+        def load_package_sets(
                 only_local: false,
                 checkout_only: true,
-                ignore_errors: false,
+                keep_going: false,
                 reset: false,
-                retry_count: nil
-            update_configuration(options)
+                retry_count: nil,
+                mainline: nil)
+            update_configuration(
+                only_local: only_local,
+                checkout_only: checkout_only,
+                keep_going: keep_going,
+                reset: reset,
+                retry_count: retry_count,
+                mainline: nil)
+        end
+
+        def report_import_failure(what, reason)
+            if !reason.kind_of?(Interrupt)
+                Autoproj.message "import of #{what} failed", :red
+                Autoproj.message reason.to_s, :red
+            end
+        end
+
+        def handle_keep_going(keep_going, vcs, failures)
+            yield
+            false
+        rescue Interrupt
+            raise
+        rescue Exception => failure_reason
+            if keep_going
+                report_import_failure(vcs, failure_reason)
+                failures << failure_reason
+                true
+            else
+                raise
+            end
         end
 
         def update_configuration(
                 only_local: false,
                 checkout_only: !Autobuild.do_update,
-                ignore_errors: false,
+                keep_going: false,
                 reset: false,
-                retry_count: nil)
-
-            update_options = Hash[
-                only_local: only_local,
-                checkout_only: checkout_only,
-                ignore_errors: ignore_errors,
-                reset: reset,
-                retry_count: retry_count]
-
-            if ws.manifest.vcs && !ws.manifest.vcs.local?
-                update_main_configuration(**update_options)
+                retry_count: nil,
+                mainline: nil)
+
+            if ws.manifest.vcs.needs_import?
+                main_configuration_failure = update_main_configuration(
+                    keep_going: keep_going,
+                    checkout_only: checkout_only,
+                    only_local: only_local,
+                    reset: reset,
+                    retry_count: retry_count)
+
+                main_configuration_failure.each do |e|
+                    report_import_failure("main configuration", e)
+                end
+            else
+                main_configuration_failure = []
             end
             ws.load_main_initrb
             ws.manifest.load(manifest_path)
             root_pkg_set = ws.manifest.main_package_set
             root_pkg_set.load_description_file
             root_pkg_set.explicit = true
-            update_package_sets(**update_options)
+
+            package_sets_failure = update_package_sets(
+                only_local: only_local,
+                checkout_only: checkout_only,
+                keep_going: keep_going,
+                reset: reset,
+                retry_count: retry_count)
+
+            load_package_set_information(mainline: mainline)
+
+            if !main_configuration_failure.empty? && !package_sets_failure.empty?
+                raise ImportFailed.new(main_configuration_failure + package_sets_failure)
+            elsif !main_configuration_failure.empty?
+                raise ImportFailed.new(main_configuration_failure)
+            elsif !package_sets_failure.empty?
+                raise ImportFailed.new(package_sets_failure)
+            end
+        end
+
+        def load_package_set_information(mainline: nil)
+            manifest = ws.manifest
+            manifest.each_package_set do |pkg_set|
+                if Gem::Version.new(pkg_set.required_autoproj_version) > Gem::Version.new(Autoproj::VERSION)
+                    raise ConfigError.new(pkg_set.source_file), "the #{pkg_set.name} package set requires autoproj v#{pkg_set.required_autoproj_version} but this is v#{Autoproj::VERSION}"
+                end
+            end
+
+            # Loads OS package definitions once and for all
+            load_osdeps_from_package_sets
+
+            # Load the required autobuild definitions
+            Autoproj.message("autoproj: loading ...", :bold)
+            manifest.each_package_set do |pkg_set|
+                pkg_set.each_autobuild_file do |path|
+                    ws.import_autobuild_file pkg_set, path
+                end
+            end
+
+            # Now, load the package's importer configurations (from the various
+            # source.yml files)
+            if mainline.respond_to?(:to_str)
+                mainline = manifest.package_set(mainline)
+            end
+            manifest.load_importers(mainline: mainline)
+
+            auto_add_packages_from_layout
+
+            manifest.each_autobuild_package do |pkg|
+                Autobuild.each_utility do |uname, _|
+                    pkg.utility(uname).enabled =
+                        ws.config.utility_enabled_for?(uname, pkg.name)
+                end
+            end
+
+            mark_unavailable_osdeps_as_excluded
+        end
+
+        # @api private
+        #
+        # Attempts to find packages mentioned in the layout but that are not
+        # defined, and auto-define them if they can be found on disk
+        #
+        # It only warns about packages that can't be defined that way are on
+        def auto_add_packages_from_layout
+            manifest = ws.manifest
+
+            # Auto-add packages that are
+            #  * present on disk
+            #  * listed in the layout part of the manifest
+            #  * but have no definition
+            explicit = manifest.normalized_layout
+            explicit.each do |pkg_or_set, layout_level|
+                next if manifest.find_autobuild_package(pkg_or_set)
+                next if manifest.has_package_set?(pkg_or_set)
+                full_path = File.expand_path(File.join(ws.root_dir, layout_level, pkg_or_set))
+                next if !File.directory?(full_path)
+
+                if handler = auto_add_package(pkg_or_set, full_path)
+                    Autoproj.message "  auto-added #{pkg_or_set} #{"in #{layout_level} " if layout_level != "/"}using the #{handler.gsub(/_package/, '')} package handler"
+                else
+                    Autoproj.warn "cannot auto-add #{pkg_or_set}: unknown package type"
+                end
+
+            end
         end
 
-        def update_package_sets(**update_options)
+        # @api private
+        #
+        # Attempts to auto-add the package checked out at the given path
+        #
+        # @param [String] full_path
+        # @return [String,nil] either the name of the package handler used to
+        #   define the package, or nil if no handler could be found
+        def auto_add_package(name, full_path)
+            manifest = ws.manifest
+            handler, _srcdir = Autoproj.package_handler_for(full_path)
+            if handler
+                ws.set_as_main_workspace do
+                    ws.in_package_set(manifest.main_package_set, manifest.file) do
+                        send(handler, name)
+                    end
+                end
+                handler
+            end
+        end
+
+        def mark_unavailable_osdeps_as_excluded
+            os_package_resolver = ws.os_package_resolver
+            manifest = ws.manifest
+            os_package_resolver.all_package_names.each do |osdep_name|
+                # If the osdep can be replaced by source packages, there's
+                # nothing to do really. The exclusions of the source packages
+                # will work as expected
+                if manifest.osdeps_overrides[osdep_name] || manifest.find_autobuild_package(osdep_name)
+                    next
+                end
+
+                case os_package_resolver.availability_of(osdep_name)
+                when OSPackageResolver::UNKNOWN_OS
+                    manifest.exclude_package(osdep_name, "this operating system is unknown to autoproj")
+                when OSPackageResolver::WRONG_OS
+                    manifest.exclude_package(osdep_name, "there are definitions for it, but not for this operating system")
+                when OSPackageResolver::NONEXISTENT
+                    manifest.exclude_package(osdep_name, "it is marked as unavailable for this operating system")
+                end
+            end
+        end
+
+        # Load OS dependency information contained in our registered package
+        # sets into the provided osdep object
+        #
+        # This is included in {load_package_sets}
+        #
+        # @return [void]
+        def load_osdeps_from_package_sets
+            ws.manifest.each_package_set do |pkg_set|
+                pkg_set.each_osdeps_file do |file|
+                    file_osdeps = pkg_set.load_osdeps(
+                        file, operating_system: ws.operating_system)
+                    ws.os_package_resolver.merge(file_osdeps)
+                end
+            end
+        end
+
+        def update_package_sets(only_local: false,
+                                checkout_only: !Autobuild.do_update,
+                                keep_going: false,
+                                reset: false,
+                                retry_count: nil)
             root_pkg_set = ws.manifest.main_package_set
-            package_sets = load_and_update_package_sets(root_pkg_set, **update_options)
+            package_sets, failures = load_and_update_package_sets(
+                root_pkg_set,
+                only_local: only_local,
+                checkout_only: checkout_only,
+                keep_going: keep_going,
+                reset: reset,
+                retry_count: retry_count)
             root_pkg_set.imports.each do |pkg_set|
                 pkg_set.explicit = true
             end
@@ -420,6 +612,8 @@ def update_package_sets(**update_options)
             if @remote_update_message_displayed
                 Autoproj.message
             end
+
+            failures
         end
     end
     end