autoproj 2.10.1 → 2.13.0

data/lib/autoproj/ops/cached_env.rb

@@ -9,7 +9,7 @@ def self.cached_env_path(root_dir)
         def self.load_cached_env(root_dir)
             path = cached_env_path(root_dir)
             if File.file?(path)
-                env = YAML.load(File.read(path))
+                env = YAML.safe_load(File.read(path))
                 Autobuild::Environment::ExportedEnvironment.new(
                     env['set'], env['unset'], env['update'])
             end
@@ -20,7 +20,7 @@ def self.save_cached_env(root_dir, env)
             path = cached_env_path(root_dir)
             existing =
                 begin
-                    YAML.load(File.read(path))
+                    YAML.safe_load(File.read(path))
                 rescue Exception
                 end
 

data/lib/autoproj/ops/import.rb

@@ -12,9 +12,10 @@ def auto_exclude?
             end
             attr_writer :auto_exclude
 
-            def initialize(ws)
+            def initialize(ws, report_path: nil)
                 @ws = ws
                 @auto_exclude = false
+                @report_path = report_path
             end
 
             def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
@@ -22,24 +23,24 @@ def mark_exclusion_along_revdeps(pkg_name, revdeps, chain = [], reason = nil)
                 chain.unshift pkg_name
                 if root
                     reason = ws.manifest.exclusion_reason(pkg_name)
+                elsif chain.size == 1
+                    ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
                 else
-                    if chain.size == 1
-                        ws.manifest.exclude_package(pkg_name, "its dependency #{reason}")
-                    else
-                        ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: #{chain.join(">")})")
-                    end
+                    ws.manifest.exclude_package(pkg_name, "#{reason} (dependency chain: "\
+                                                          "#{chain.join('>')})")
                 end
 
-                return if !revdeps.has_key?(pkg_name)
+                return unless revdeps.key?(pkg_name)
+
                 revdeps[pkg_name].each do |dep_name|
-                    if !ws.manifest.excluded?(dep_name)
+                    unless ws.manifest.excluded?(dep_name)
                         mark_exclusion_along_revdeps(dep_name, revdeps, chain.dup, reason)
                     end
                 end
             end
 
             VALID_OSDEP_AVAILABILITY =
-                [OSPackageResolver::AVAILABLE, OSPackageResolver::IGNORE]
+                [OSPackageResolver::AVAILABLE, OSPackageResolver::IGNORE].freeze
 
             def import_next_step(pkg, reverse_dependencies)
                 new_packages = []
@@ -85,23 +86,32 @@ def pre_package_import(selection, manifest, pkg, reverse_dependencies)
                 elsif manifest.ignored?(pkg.name)
                     false
                 elsif !pkg.importer && !File.directory?(pkg.srcdir)
-                    raise ConfigError.new, "#{pkg.name} has no VCS, but is not checked out in #{pkg.srcdir}"
+                    raise ConfigError.new, "#{pkg.name} has no VCS, but is not "\
+                                           "checked out in #{pkg.srcdir}"
                 else
                     true
                 end
             end
 
-            def post_package_import(selection, manifest, pkg, reverse_dependencies, auto_exclude: auto_exclude?)
-                Rake::Task["#{pkg.name}-import"].instance_variable_set(:@already_invoked, true)
+            def post_package_import(selection, manifest, pkg, reverse_dependencies,
+                                    auto_exclude: auto_exclude?)
+                Rake::Task["#{pkg.name}-import"]
+                    .instance_variable_set(:@already_invoked, true)
                 if pkg.checked_out?
                     begin
                         manifest.load_package_manifest(pkg.name)
-                    rescue Exception => e
-                        raise if !auto_exclude
-                        manifest.add_exclusion(pkg.name, "#{pkg.name} failed to import with #{e} and auto_exclude was true")
+                    rescue StandardError => e
+                        raise unless auto_exclude
+
+                        manifest.add_exclusion(pkg.name, "#{pkg.name} failed to import "\
+                            "with #{e} and auto_exclude was true")
                     end
                 end
 
+                if !manifest.excluded?(pkg.name) && !manifest.ignored?(pkg.name)
+                    process_post_import_blocks(pkg) if pkg.checked_out?
+                end
+
                 # The package setup mechanisms might have added an exclusion
                 # on this package. Handle this.
                 if manifest.excluded?(pkg.name)
@@ -113,10 +123,8 @@ def post_package_import(selection, manifest, pkg, reverse_dependencies, auto_exc
                 elsif manifest.ignored?(pkg.name)
                     false
                 else
-                    if pkg.checked_out?
-                        process_post_import_blocks(pkg)
-                    end
-                    import_next_step(pkg, reverse_dependencies)
+                    pkg.apply_dependencies_from_manifest
+                    import_next_step(pkg.autobuild, reverse_dependencies)
                 end
             end
 
@@ -163,16 +171,16 @@ def install_internal_dependencies_for(*packages, **osdeps_options)
             # @param [Integer] retry_count the number of retries that are
             #   allowed. Set to zero for no retry
             # @param [Hash] import_options options passed to {Autobuild::Importer#import}
-            def queue_import_work(executor, completion_queue, pkg, retry_count: nil, **import_options)
-                import_future = Concurrent::Future.new(executor: executor, args: [pkg]) do |import_pkg|
+            def queue_import_work(executor, completion_queue, pkg,
+                                  retry_count: nil, **import_options)
+                import_future = Concurrent::Future
+                                .new(executor: executor, args: [pkg]) do |import_pkg|
                     ## COMPLETELY BYPASS RAKE HERE
                     # The reason is that the ordering of import/prepare between
                     # packages is not important BUT the ordering of import vs.
                     # prepare in one package IS important: prepare is the method
                     # that takes into account dependencies.
-                    if retry_count
-                        import_pkg.autobuild.importer.retry_count = retry_count
-                    end
+                    import_pkg.autobuild.importer.retry_count = retry_count if retry_count
                     import_pkg.autobuild.import(**import_options)
                 end
                 import_future.add_observer do |time, result, reason|
@@ -191,10 +199,13 @@ def import_selected_packages(selection,
                                          install_vcs_packages: Hash.new,
                                          non_imported_packages: :checkout,
                                          auto_exclude: auto_exclude?,
+                                         filter: ->(package) { true },
                                          **import_options)
 
-                if ![:checkout, :ignore, :return].include?(non_imported_packages)
-                    raise ArgumentError, "invalid value for 'non_imported_packages'. Expected one of :checkout, :ignore or :return but got #{non_imported_packages}"
+                unless %i[checkout ignore return].include?(non_imported_packages)
+                    raise ArgumentError, "invalid value for 'non_imported_packages'. "\
+                                         "Expected one of :checkout, :ignore or :return "\
+                                         "but got #{non_imported_packages}"
                 end
 
                 # This is used in the ensure block, initialize as early as
@@ -220,32 +231,44 @@ def import_selected_packages(selection,
                 all_processed_packages = Set.new
                 main_thread_imports = Array.new
                 package_queue = selected_packages.to_a.sort_by(&:name)
+
                 failures = Array.new
                 missing_vcs = Array.new
                 installed_vcs_packages = Set['none', 'local']
                 while failures.empty? || keep_going
+                    # Allow 'filter' to parallelize as well
+                    if filter.respond_to?(:lookahead)
+                        package_queue.each { |pkg| filter.lookahead(pkg) }
+                    end
+
                     # Queue work for all packages in the queue
                     package_queue.each do |pkg|
                         # Remove packages that have already been processed
                         next if all_processed_packages.include?(pkg)
-                        if (non_imported_packages != :checkout) && !File.directory?(pkg.autobuild.srcdir)
+
+                        vcs_installed = installed_vcs_packages.include?(pkg.vcs.type)
+                        if (non_imported_packages != :checkout) && !pkg.checked_out?
+                            all_processed_packages << pkg
                             if non_imported_packages == :return
-                                all_processed_packages << pkg
                                 completion_queue << [pkg, Time.now, false, nil]
-                                next
                             else
-                                all_processed_packages << pkg
                                 ws.manifest.ignore_package(pkg.name)
-                                next
                             end
-                        elsif install_vcs_packages && !installed_vcs_packages.include?(pkg.vcs.type)
+                            next
+                        elsif install_vcs_packages && !vcs_installed
                             missing_vcs << pkg
                             next
                         end
                         all_processed_packages << pkg
 
+                        unless filter.call(pkg)
+                            completion_queue << [pkg, Time.now]
+                            next
+                        end
+
                         importer = pkg.autobuild.importer
-                        if !pre_package_import(selection, manifest, pkg.autobuild, reverse_dependencies)
+                        if !pre_package_import(selection, manifest, pkg.autobuild,
+                                               reverse_dependencies)
                             next
                         elsif !importer
                             # The validity of this is checked in
@@ -255,28 +278,33 @@ def import_selected_packages(selection,
                         elsif importer.interactive?
                             main_thread_imports << pkg
                             next
-                        elsif pkg.autobuild.checked_out? && import_options[:checkout_only]
+                        elsif pkg.checked_out? && import_options[:checkout_only]
                             main_thread_imports << pkg
                             next
                         end
 
-                        pending_packages << pkg
                         begin
+                            pending_packages << pkg
                             queue_import_work(
-                                executor, completion_queue, pkg, retry_count: retry_count,
+                                executor, completion_queue, pkg,
+                                retry_count: retry_count,
                                 **import_options.merge(allow_interactive: false))
                         rescue Exception
                             pending_packages.delete(pkg)
                             raise
                         end
-                        true
                     end
                     package_queue.clear
 
                     if completion_queue.empty? && pending_packages.empty?
-                        if !missing_vcs.empty?
+                        unless missing_vcs.empty?
                             installed_vcs_packages.merge(
-                                install_vcs_packages_for(*missing_vcs, **install_vcs_packages))
+                                install_vcs_packages_for(
+                                    *missing_vcs,
+                                     install_only: import_options[:checkout_only],
+                                     **install_vcs_packages
+                                )
+                            )
                             package_queue.concat(missing_vcs)
                             missing_vcs.clear
                             next
@@ -289,15 +317,17 @@ def import_selected_packages(selection,
                             break
                         else
                             main_thread_imports.delete_if do |pkg|
+                                # rubocop:disable Lint/HandleExceptions
                                 begin
                                     if retry_count
                                         pkg.autobuild.importer.retry_count = retry_count
                                     end
                                     result = pkg.autobuild.import(
                                         **import_options.merge(allow_interactive: true))
-                                rescue Exception => reason
+                                rescue StandardError => e
                                 end
-                                completion_queue << [pkg, Time.now, result, reason]
+                                completion_queue << [pkg, Time.now, result, e]
+                                # rubocop:enable Lint/HandleExceptions
                             end
                         end
                     end
@@ -309,20 +339,21 @@ def import_selected_packages(selection,
                         if reason.kind_of?(Autobuild::InteractionRequired)
                             main_thread_imports << pkg
                         elsif auto_exclude
-                            manifest.add_exclusion(pkg.name, "#{pkg.name} failed to import with #{reason} and auto_exclude was true")
+                            manifest.add_exclusion(
+                                pkg.name, "#{pkg.name} failed to import with "\
+                                          "#{reason} and auto_exclude was true")
                             selection.filter_excluded_and_ignored_packages(manifest)
                         else
                             # One importer failed... terminate
                             Autoproj.error "import of #{pkg.name} failed"
-                            if !reason.kind_of?(Interrupt)
-                                Autoproj.error "#{reason}"
-                            end
+                            Autoproj.error reason.to_s unless reason.kind_of?(Interrupt)
                             failures << reason
                         end
                     else
-                        if new_packages = post_package_import(
-                                selection, manifest, pkg.autobuild, reverse_dependencies,
-                                auto_exclude: auto_exclude)
+                        new_packages = post_package_import(
+                            selection, manifest, pkg, reverse_dependencies,
+                            auto_exclude: auto_exclude)
+                        if new_packages
                             # Excluded dependencies might have caused the package to be
                             # excluded as well ... do not add any dependency to the
                             # processing queue if it is the case
@@ -336,10 +367,10 @@ def import_selected_packages(selection,
                 end
 
                 all_processed_packages.delete_if do |processed_pkg|
-                    ws.manifest.excluded?(processed_pkg.name) || ws.manifest.ignored?(processed_pkg.name)
+                    ws.manifest.excluded?(processed_pkg.name) ||
+                        ws.manifest.ignored?(processed_pkg.name)
                 end
-                return all_processed_packages, failures
-
+                [all_processed_packages, failures]
             ensure
                 if failures && !failures.empty? && !keep_going
                     Autoproj.error "waiting for pending import jobs to finish"
@@ -350,17 +381,22 @@ def import_selected_packages(selection,
                 end
             end
 
-            def finalize_package_load(processed_packages, auto_exclude: auto_exclude?)
+            def finalize_package_load(processed_packages,
+                                      ignore_optional_dependencies: false,
+                                      auto_exclude: auto_exclude?)
                 manifest = ws.manifest
 
                 all = Set.new
-                package_queue = manifest.all_layout_packages(false).each_source_package_name.to_a +
+                package_queue =
+                    manifest.all_layout_packages(false)
+                            .each_source_package_name.to_a +
                     processed_packages.map(&:name).to_a
-                while !package_queue.empty?
+
+                until package_queue.empty?
                     pkg_name = package_queue.shift
                     next if all.include?(pkg_name)
-                    all << pkg_name
 
+                    all << pkg_name
                     next if manifest.ignored?(pkg_name) || manifest.excluded?(pkg_name)
 
                     pkg_definition = manifest.find_package_definition(pkg_name)
@@ -370,27 +406,28 @@ def finalize_package_load(processed_packages, auto_exclude: auto_exclude?)
                             manifest.load_package_manifest(pkg.name)
                             process_post_import_blocks(pkg)
                         rescue Exception => e
-                            raise if !auto_exclude
-                            manifest.exclude_package(pkg.name, "#{pkg.name} had an error when being loaded (#{e.message}) and auto_exclude is true")
+                            raise unless auto_exclude
+
+                            manifest.exclude_package(
+                                pkg.name, "#{pkg.name} had an error when "\
+                                          "being loaded (#{e.message}) and "\
+                                          "auto_exclude is true")
                             next
                         end
                     end
 
                     packages, osdeps = pkg.partition_optional_dependencies
-                    packages.each do |dep_pkg_name|
-                        if !manifest.ignored?(dep_pkg_name) && !manifest.excluded?(dep_pkg_name)
-                            pkg.depends_on dep_pkg_name
-                        end
-                    end
-                    osdeps.each do |osdep_pkg_name|
-                        if !manifest.ignored?(osdep_pkg_name) && !manifest.excluded?(osdep_pkg_name)
-                            pkg.os_packages << osdep_pkg_name
-                        end
+                    (packages + osdeps).each do |dep_pkg_name|
+                        next if manifest.ignored?(dep_pkg_name)
+                        next if manifest.excluded?(dep_pkg_name)
+
+                        pkg.depends_on dep_pkg_name
                     end
 
                     if File.directory?(pkg.srcdir)
                         pkg.prepare
-                        Rake::Task["#{pkg.name}-prepare"].instance_variable_set(:@already_invoked, true)
+                        Rake::Task["#{pkg.name}-prepare"]
+                            .instance_variable_set(:@already_invoked, true)
                     end
                     pkg.update_environment
                     package_queue.concat(pkg.dependencies)
@@ -406,6 +443,7 @@ def import_packages(selection,
                                 keep_going: false,
                                 install_vcs_packages: Hash.new,
                                 auto_exclude: auto_exclude?,
+                                filter: ->(pkg) { true },
                                 **import_options)
 
                 manifest = ws.manifest
@@ -417,11 +455,10 @@ def import_packages(selection,
                     recursive: recursive,
                     install_vcs_packages: install_vcs_packages,
                     auto_exclude: auto_exclude,
+                    filter: filter,
                     **import_options)
 
-                if !keep_going && !failures.empty?
-                    raise failures.first
-                end
+                raise failures.first if !keep_going && !failures.empty?
 
                 install_internal_dependencies_for(*all_processed_packages)
                 finalize_package_load(all_processed_packages, auto_exclude: auto_exclude)
@@ -437,14 +474,17 @@ def import_packages(selection,
                 if warn_about_excluded_packages
                     selection.exclusions.each do |sel, pkg_names|
                         pkg_names.sort.each do |pkg_name|
-                            Autoproj.warn "#{pkg_name}, which was selected for #{sel}, cannot be built: #{manifest.exclusion_reason(pkg_name)}", :bold
+                            Autoproj.warn "#{pkg_name}, which was selected "\
+                                          "for #{sel}, cannot be built: "\
+                                          "#{manifest.exclusion_reason(pkg_name)}", :bold
                         end
                     end
                 end
                 if warn_about_ignored_packages
                     selection.ignores.each do |sel, pkg_names|
                         pkg_names.sort.each do |pkg_name|
-                            Autoproj.warn "#{pkg_name}, which was selected for #{sel}, is ignored", :bold
+                            Autoproj.warn "#{pkg_name}, which was selected for #{sel}, "\
+                                          "is ignored", :bold
                         end
                     end
                 end
@@ -452,41 +492,67 @@ def import_packages(selection,
                 if !failures.empty?
                     raise PackageImportFailed.new(
                         failures, source_packages: all_enabled_sources,
-                        osdep_packages: all_enabled_osdeps)
+                                  osdep_packages: all_enabled_osdeps)
                 else
                     return all_enabled_sources, all_enabled_osdeps
                 end
-
             ensure
-                if ws.config.import_log_enabled? && Autoproj::Ops::Snapshot.update_log_available?(manifest)
-                    update_log_for_processed_packages(all_processed_packages || Array.new, $!)
+                create_report(all_processed_packages || []) if @report_path
+
+                update_log = ws.config.import_log_enabled? &&
+                             Autoproj::Ops::Snapshot.update_log_available?(manifest)
+                if update_log
+                    update_log_for_processed_packages(
+                        all_processed_packages || Array.new, $!)
                 end
             end
 
+            # Call post-import blcoks registered for the given package
+            #
+            # @param [PackageDefinition] pkg
             def process_post_import_blocks(pkg)
                 Autoproj.each_post_import_block(pkg) do |block|
                     block.call(pkg)
                 end
             end
 
+            def create_report(package_list)
+                FileUtils.mkdir_p File.dirname(@report_path)
+
+                packages = package_list.each_with_object({}) do |pkg_name, h|
+                    pkg = @ws.manifest.find_autobuild_package(pkg_name)
+
+                    h[pkg.name] = {
+                        invoked: !!pkg.import_invoked?,
+                        success: !!pkg.imported?
+                    }
+                end
+
+                report = JSON.pretty_generate({
+                    import_report: {
+                        timestamp: Time.now,
+                        packages: packages
+                    }
+                })
+                IO.write(@report_path, report)
+            end
+
             def update_log_for_processed_packages(all_processed_packages, exception)
                 all_updated_packages = all_processed_packages.find_all do |processed_pkg|
                     processed_pkg.autobuild.updated?
                 end
 
-                if !all_updated_packages.empty?
+                unless all_updated_packages.empty?
                     failure_message =
                         if exception
                             " (#{exception.message.split("\n").first})"
                         end
                     ops = Ops::Snapshot.new(ws.manifest, keep_going: true)
                     ops.update_package_import_state(
-                        "#{$0} #{ARGV.join(" ")}#{failure_message}",
+                        "#{$0} #{ARGV.join(' ')}#{failure_message}",
                         all_updated_packages.map(&:name))
                 end
             end
         end
     end
 end
-
-