autoproj 2.8.8 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 52887a2af332210cca2a14090350718f88ca070be1a5ab7493cc82af38f7ba59
-  data.tar.gz: b3d84d28e9a283f0cd9efd2cf0cd254382caffabbd7144aff55633e917466c9e
+  metadata.gz: a79f0dc8faf8c992dedb96e65c997ab35f41d62107845dd9dd20bc260630f0a2
+  data.tar.gz: 2e3db38917f2cefa9defa70fa248ced0c05ad909817003b135d9b7f6179ca8bc
 SHA512:
-  metadata.gz: 03c32ce93aed3de8fad8bf3dfcbfe5529703315b83e0a8afe70bd1b54c3f0c91257a42f838e67866aca8beff348f4766b37eec82e70dfa04b243bc6d41df2747
-  data.tar.gz: 8fe58af39dd943a8841eadce3222cf32124d3befe1e909e683b34481d5211da660c1ba8d1ab25ee8fad6fbe5aca209cfe1d9927a1248fa8cf74bab99cd2e0198
+  metadata.gz: 346bdce3bf52c42187143908ef04767a03e1e71b24738b0c333591bd73d113355a871ccdf67bec9b69018d30e3302e6a53946e007922a4a3e01fb82fa371eae8
+  data.tar.gz: 470faaa51b2d58609f70e794173d07d41003feee472c3c9e3dd9bb8024e66746403a480becad1dc0222066084e76c0205104465e138afc714d8555fc32d60d55

data/bin/alog

@@ -24,10 +24,12 @@ Autoproj.report(silent: true) do
     if log_files.size == 1
         logfile = log_files.first
     elsif log_files.size > 1
-        logfile = cli.select_log_file(log_files)
+        begin
+            logfile = cli.select_log_file(log_files)
+            puts File.read(logfile)
+        rescue Interrupt
+        end
     elsif log_files.empty?
         raise "no logs found for #{string}"
     end
-
-    puts File.read(logfile)
 end

data/lib/autoproj.rb

@@ -30,6 +30,8 @@
 require 'autoproj/installation_manifest'
 require 'autoproj/os_package_installer'
 require 'autoproj/os_package_resolver'
+require 'autoproj/os_repository_resolver'
+require 'autoproj/os_repository_installer'
 require 'autoproj/system'
 require 'autoproj/build_option'
 require 'autoproj/configuration'

data/lib/autoproj/cli/base.rb

@@ -206,7 +206,14 @@ def self.validate_options(args, options)
             end
 
             def export_env_sh(shell_helpers: ws.config.shell_helpers?)
-                @env_sh_updated = ws.export_env_sh(shell_helpers: shell_helpers)
+                # @env_sh_updated == nil means "did not even export".
+                # make sure that it is set to 'true' or 'false'
+                @env_sh_updated =
+                    if ws.export_env_sh(shell_helpers: shell_helpers)
+                        true
+                    else
+                        false
+                    end
             end
 
             def notify_env_sh_updated

data/lib/autoproj/cli/build.rb

@@ -27,7 +27,10 @@ def run(selected_packages, options)
                     confirm: true
 
                 command_line_selection, source_packages, _osdep_packages =
-                    super(selected_packages, options.merge(checkout_only: true))
+                    super(selected_packages,
+                          ignore_errors: options[:keep_going],
+                          checkout_only: true,
+                          **options)
 
                 parallel = build_options[:parallel] || ws.config.parallel_build_level
 
@@ -41,7 +44,7 @@ def run(selected_packages, options)
 
                 Autobuild.ignore_errors = options[:keep_going]
 
-                ops = Ops::Build.new(ws.manifest)
+                ops = Ops::Build.new(ws.manifest, report_dir: ws.log_dir)
                 if build_options[:rebuild] || build_options[:force]
                     packages_to_rebuild =
                         if options[:deps] || command_line_selection.empty?
@@ -77,7 +80,6 @@ def run(selected_packages, options)
 
                 Autobuild.do_build = true
                 ops.build_packages(source_packages, parallel: parallel)
-                Autobuild.apply(source_packages, "autoproj-build", ['install'])
                 Main.run_post_command_hook(:build, ws, source_packages: source_packages)
             ensure
                 export_env_sh

data/lib/autoproj/cli/locate.rb

@@ -285,7 +285,11 @@ def select_log_file(log_files)
                 end
 
                 prompt = TTY::Prompt.new
-                prompt.select("Select the log file", choices)
+                begin
+                    prompt.select("Select the log file", choices)
+                rescue TTY::Reader::InputInterrupt
+                    raise Interrupt
+                end
             end
         end
     end

data/lib/autoproj/cli/osdeps.rb

@@ -24,6 +24,7 @@ def run(user_selection, update: true, **options)
 
                 shell_helpers = options.fetch(:shell_helpers, ws.config.shell_helpers?)
 
+                ws.install_os_repositories
                 ws.install_os_packages(
                     osdep_packages,
                     run_package_managers_without_packages: true,

data/lib/autoproj/cli/status.rb

@@ -73,6 +73,15 @@ def snapshot_overrides_vcs?(importer, vcs, snapshot)
                 end
             end
 
+            def report_exception(package_status, msg, e)
+                package_status.msg << Autoproj.color("  #{msg} (#{e})", :red)
+                if Autobuild.debug
+                    package_status.msg.concat(e.backtrace.map do |line|
+                        Autoproj.color("    #{line}", :red)
+                    end)
+                end
+            end
+
             PackageStatus = Struct.new :msg, :sync, :uncommitted, :local, :remote
             def status_of_package(package_description, only_local: false, snapshot: false)
                 pkg = package_description.autobuild
@@ -85,9 +94,9 @@ def status_of_package(package_description, only_local: false, snapshot: false)
                 elsif !File.directory?(pkg.srcdir)
                     package_status.msg << Autoproj.color("  is not imported yet", :magenta)
                 else
-                    begin status = importer.status(pkg, only_local)
+                    begin status = importer.status(pkg, only_local: only_local)
                     rescue StandardError => e
-                        package_status.msg << Autoproj.color("  failed to fetch status information (#{e})", :red)
+                        report_exception(package_status, "failed to fetch status information", e)
                         return package_status
                     end
 
@@ -99,7 +108,7 @@ def status_of_package(package_description, only_local: false, snapshot: false)
                             rescue Autobuild::PackageException
                                 Hash.new
                             rescue StandardError => e
-                                package_status.msg << Autoproj.color("  failed to fetch snapshotting information (#{e})", :red)
+                                report_exception(package_status, "failed to fetch snapshotting information", e)
                                 return package_status
                             end
                         if snapshot_overrides_vcs?(importer, package_description.vcs, snapshot_version)
@@ -227,7 +236,7 @@ def display_status(packages, parallel: ws.config.parallel_import_level, snapshot
                     end
                 end
 
-                result = each_package_status(packages, parallel: parallel, progress: progress) do |pkg, status|
+                result = each_package_status(packages, only_local: only_local, parallel: parallel, progress: progress) do |pkg, status|
                     if spinner
                         spinner.stop
                         spinner = nil

data/lib/autoproj/cli/update.rb

@@ -44,7 +44,7 @@ def validate_options(selection, options)
                     (options[:autoproj] || (
                         options[:autoproj] != false &&
                         !has_explicit_selection &&
-                        !options[:config] && 
+                        !options[:config] &&
                         !options[:checkout_only])
                     )
 
@@ -97,6 +97,8 @@ def run(selected_packages, run_hook: false, **options)
                     command_line_selection, selected_packages =
                         finish_loading_configuration(selected_packages)
                 else
+                    ws.setup_all_package_directories
+                    ws.finalize_package_setup
                     command_line_selection, selected_packages = [], PackageSelection.new
                 end
 
@@ -106,10 +108,11 @@ def run(selected_packages, run_hook: false, **options)
                     osdeps: options[:osdeps],
                     osdeps_filter_uptodate: options[:osdeps_filter_uptodate])
 
-                source_packages, osdep_packages, import_failure = 
+                source_packages, osdep_packages, import_failure =
                     update_packages(
                         selected_packages,
-                        osdeps: options[:osdeps], osdeps_options: osdeps_options,
+                        osdeps: options[:osdeps],
+                        osdeps_options: osdeps_options,
                         from: options[:from],
                         checkout_only: options[:checkout_only],
                         only_local: options[:only_local],
@@ -124,6 +127,7 @@ def run(selected_packages, run_hook: false, **options)
                 ws.export_installation_manifest
 
                 if options[:osdeps] && !osdep_packages.empty?
+                    ws.install_os_repositories
                     ws.install_os_packages(osdep_packages, **osdeps_options)
                 end
 
@@ -141,7 +145,7 @@ def run(selected_packages, run_hook: false, **options)
 
                 export_env_sh
 
-                if !options[:auto_exclude]
+                if !options[:auto_exclude] && !options[:ignore_errors]
                     if import_failure && configuration_import_failure
                         raise ImportFailed.new(configuration_import_failure.original_errors + import_failure.original_errors)
                     elsif import_failure
@@ -198,7 +202,7 @@ def update_packages(selected_packages,
                 end
 
                 ops = Autoproj::Ops::Import.new(ws)
-                source_packages, osdep_packages = 
+                source_packages, osdep_packages =
                         ops.import_packages(selected_packages,
                                         checkout_only: checkout_only,
                                         only_local: only_local,

data/lib/autoproj/default.osdeps

@@ -160,9 +160,23 @@ pip:
   opensuse: python-pip
   fedora: python-pip
   freebsd: pip
+  default: ignore # assume pip will be installed by the user
 
 sudo:
   macos-brew: ignore
   default: sudo
 
+archive-keyring:
+  ubuntu: ubuntu-keyring
+  debian: debian-archive-keyring
+  default: ignore
+
+gnupg:
+  ubuntu,debian: gnupg
+  default: ignore
+
+apt-transport-https:
+  ubuntu,debian: apt-transport-https
+  default: ignore
+
 # vim: expandtab

data/lib/autoproj/manifest.rb

@@ -892,16 +892,14 @@ def package_selected?(name, validate = true)
         #
         # @return [Array<PackageDefinition>]
         def all_selected_source_packages(validate = true)
-            result = Set.new
-            selection = default_packages(validate)
+            default_packages(validate).all_selected_source_packages(self)
+        end
 
-            root_sources = selection.each_source_package_name.to_set
-            root_sources.each do |pkg_name|
-                find_autobuild_package(pkg_name).all_dependencies(result)
-            end
-            result.merge(root_sources).map do |pkg_name|
-                find_package_definition(pkg_name)
-            end
+        # Returns the set of osdep packages that are selected by the layout
+        #
+        # @return [Array<String>]
+        def all_selected_osdep_packages(validate = true)
+            default_packages(validate).all_selected_osdep_packages(self)
         end
 
         # Returns the set of packages that are selected by the layout

data/lib/autoproj/ops/build.rb

@@ -1,3 +1,4 @@
+require 'erb'
 module Autoproj
     module Ops
         # Operations related to building packages
@@ -10,8 +11,11 @@ class Build
             # @return [Manifest]
             attr_reader :manifest
 
-            def initialize(manifest)
+            # @param [String] report_dir the log directory in which to build
+            #   the build report. If left to nil, no report will be generated
+            def initialize(manifest, report_dir: nil)
                 @manifest = manifest
+                @report_dir = report_dir
             end
 
             # Triggers a rebuild of all packages
@@ -73,6 +77,7 @@ def force_build_packages(selected_packages, all_enabled_packages)
                 build_packages(all_enabled_packages)
             end
 
+
             # Builds the listed packages
             #
             # Only build steps that are actually needed will be performed. See
@@ -84,7 +89,48 @@ def force_build_packages(selected_packages, all_enabled_packages)
             def build_packages(all_enabled_packages, options = Hash.new)
                 Autobuild.do_rebuild = false
                 Autobuild.do_forced_build = false
-                Autobuild.apply(all_enabled_packages, "autoproj-build", ['build'], options)
+                begin
+                    Autobuild.apply(all_enabled_packages, "autoproj-build", ['build'], options)
+                ensure
+                    build_report(all_enabled_packages) if @report_dir
+                end
+            end
+
+            REPORT_BASENAME = "build_report.json"
+
+            # The path to the report file
+            #
+            # @return [String,nil] the path, or nil if the report should not
+            #    be generated
+            def report_path
+                File.join(@report_dir, REPORT_BASENAME) if @report_dir
+            end
+
+            def build_report(package_list)
+                FileUtils.mkdir_p @report_dir
+
+                packages = package_list.map do |pkg_name|
+                    pkg = manifest.find_autobuild_package(pkg_name)
+
+                    {
+                        name: pkg.name,
+                        import_invoked: pkg.import_invoked?,
+                        prepare_invoked: pkg.prepare_invoked?,
+                        build_invoked: pkg.build_invoked?,
+                        failed: pkg.failed?,
+                        imported: pkg.imported?,
+                        prepared: pkg.prepared?,
+                        built: pkg.built?
+                    }
+                end
+
+                build_report = JSON.pretty_generate({
+                    build_report: {
+                        timestamp: Time.now,
+                        packages: packages
+                    }
+                })
+                IO.write(report_path, build_report)
             end
         end
     end

data/lib/autoproj/ops/configuration.rb

@@ -464,6 +464,9 @@ def load_package_set_information(mainline: nil)
                 end
             end
 
+            # Loads OS repository definitions once and for all
+            load_osrepos_from_package_sets
+
             # Loads OS package definitions once and for all
             load_osdeps_from_package_sets
 
@@ -580,6 +583,21 @@ def load_osdeps_from_package_sets
             end
         end
 
+        # Load OS repository information contained in our registered package
+        # sets into the provided osrepo object
+        #
+        # This is included in {load_package_sets}
+        #
+        # @return [void]
+        def load_osrepos_from_package_sets
+            ws.manifest.each_package_set do |pkg_set|
+                pkg_set.each_osrepos_file do |file|
+                    file_osrepos = pkg_set.load_osrepos(file)
+                    ws.os_repository_resolver.merge(file_osrepos)
+                end
+            end
+        end
+
         def update_package_sets(only_local: false,
                                 checkout_only: !Autobuild.do_update,
                                 keep_going: false,

data/lib/autoproj/os_package_installer.rb

@@ -188,6 +188,10 @@ def osdeps_mode_string_to_value(string)
             modes
         end
 
+        def configure_manager
+            os_package_manager.configure_manager if osdeps_mode.include?('os')
+        end
+
         # If set to true (the default), #install will try to remove the list of
         # already uptodate packages from the installed packages. Set to false to
         # install all packages regardless of their status
@@ -380,6 +384,29 @@ def resolve_and_partition_osdep_packages(osdep_packages, all_osdep_packages = ni
 
                 partitioned_packages[manager] = manager_packages
             end
+            resolve_managers_dependencies(partitioned_packages)
+        end
+
+        def resolve_managers_dependencies(partitioned_packages)
+            partitioned_packages.clone.each do |manager, packages|
+                # Skip if the manager is not being used
+                next if packages&.empty?
+
+                manager_dependencies = os_package_resolver.resolve_os_packages(manager.os_dependencies)
+                manager_dependencies = resolve_package_managers_in_mapping(manager_dependencies)
+                manager_dependencies.each_key do |nested_manager|
+                    deps = manager_dependencies.fetch(nested_manager, Set.new).to_set
+                    next if deps.empty?
+
+                    unless partitioned_packages[nested_manager]
+                        partitioned_packages[nested_manager] = Set.new
+                        enable_recursion = true
+                    end
+
+                    partitioned_packages[nested_manager] += deps
+                    partitioned_packages = resolve_managers_dependencies(partitioned_packages) if enable_recursion
+               end
+            end
             partitioned_packages
         end
 

data/lib/autoproj/os_repository_installer.rb

@@ -0,0 +1,53 @@
+require 'autoproj/repository_managers/manager'
+require 'autoproj/repository_managers/unknown_os_manager'
+require 'autoproj/repository_managers/apt'
+
+module Autoproj
+    class OSRepositoryInstaller
+        # The workspace object
+        attr_reader :ws
+
+        # Returns the set of repository managers
+        attr_reader :repository_managers
+
+        OS_REPOSITORY_MANAGERS = {
+            'debian' => RepositoryManagers::APT
+        }.freeze
+
+        def initialize(ws)
+            @ws = ws
+            @repository_managers = {}
+            OS_REPOSITORY_MANAGERS.each do |name, klass|
+                @repository_managers[name] = klass.new(ws)
+            end
+        end
+
+        def os_repository_resolver
+            ws.os_repository_resolver
+        end
+
+        # Returns the repository manager object for the current OS
+        def os_repository_manager
+            return @os_repository_manager if @os_repository_manager
+
+            os_names, = os_repository_resolver.operating_system
+            os_name = os_names.find { |name| OS_REPOSITORY_MANAGERS[name] }
+
+            @os_repository_manager =
+                repository_managers[os_name] ||
+                RepositoryManagers::UnknownOSManager.new(ws)
+        end
+
+        def each_manager(&block)
+            repository_managers.each_value(&block)
+        end
+
+        def install_os_repositories
+            return if os_repository_resolver.resolved_entries.empty?
+
+            deps = os_repository_manager.os_dependencies
+            ws.install_os_packages(deps, all: nil) unless deps.empty?
+            os_repository_manager.install(os_repository_resolver.resolved_entries)
+        end
+    end
+end