autoproj 2.11.0 → 2.14.0

data/lib/autoproj/cli/test.rb

@@ -10,7 +10,7 @@ module Autoproj
             end
 
             def package_metadata(package)
-                u = package.autobuild.test_utility
+                u = package.test_utility
                 super.merge(
                     'coverage_available' => !!u.coverage_available?,
                     'coverage_enabled' => !!u.coverage_enabled?,

data/lib/autoproj/cli/update.rb

@@ -1,5 +1,6 @@
 require 'autoproj/cli'
 require 'autoproj/cli/base'
+require 'autoproj/cli/status'
 require 'autoproj/ops/import'
 
 module Autoproj
@@ -66,17 +67,20 @@ module Autoproj
                 return selection, options
             end
 
-            def run(selected_packages, run_hook: false, report: true, **options)
+            def run(selected_packages, run_hook: false, report: true, ask: false, **options)
                 ws.manifest.accept_unavailable_osdeps = !options[:osdeps]
                 ws.setup
                 ws.autodetect_operating_system(force: true)
-                if options[:bundler]
-                    ws.update_bundler
-                end
-                if options[:autoproj]
-                    ws.update_autoproj
+
+                if ask
+                    prompt = TTY::Prompt.new
+                    options[:bundler] &&= prompt.yes?("Update bundler ?")
+                    options[:autoproj] &&= prompt.yes?("Update autoproj ?")
                 end
 
+                ws.update_bundler if options[:bundler]
+                ws.update_autoproj if options[:autoproj]
+
                 begin
                     ws.load_package_sets(
                         mainline: options[:mainline],
@@ -84,7 +88,8 @@ module Autoproj
                         checkout_only: !options[:config] || options[:checkout_only],
                         reset: options[:reset],
                         keep_going: options[:keep_going],
-                        retry_count: options[:retry_count])
+                        retry_count: options[:retry_count]
+                    )
                 rescue ImportFailed => configuration_import_failure
                     if !options[:keep_going]
                         raise
@@ -122,6 +127,7 @@ module Autoproj
                         parallel: options[:parallel] || ws.config.parallel_import_level,
                         retry_count: options[:retry_count],
                         auto_exclude: options[:auto_exclude],
+                        ask: ask,
                         report: report)
 
                 ws.finalize_setup
@@ -193,15 +199,62 @@ module Autoproj
                 osdeps_options
             end
 
+            class AskUpdateFilter
+                def initialize(prompt, parallel: 1, only_local: false)
+                    @prompt = prompt
+                    @only_local = only_local
+                    @executor = Concurrent::FixedThreadPool.new(parallel, max_length: 0)
+
+                    @parallel = parallel
+                    @futures = {}
+                    @lookahead_queue = []
+                end
+
+                def call(pkg)
+                    unless (status = @futures.delete(pkg).value)
+                        raise v.reason
+                    end
+
+                    clean = !status.unexpected &&
+                            (status.sync || (status.local && !status.remote))
+                    if clean
+                        msg = Autobuild.color('already up-to-date', :green)
+                        pkg.autobuild.message "#{msg} %s"
+                        return false
+                    end
+
+                    Autobuild.progress_display_synchronize do
+                        status.msg.each { |m| puts m }
+                        @prompt.yes?("Update #{pkg.name} ?")
+                    end
+                end
+
+                def lookahead(pkg)
+                    @futures[pkg] = Concurrent::Future.execute(executor: @executor) do
+                        Status.status_of_package(
+                            pkg, snapshot: false, only_local: @only_local
+                        )
+                    end
+                end
+            end
+
             def update_packages(selected_packages,
                 from: nil, checkout_only: false, only_local: false, reset: false,
                 deps: true, keep_going: false, parallel: 1,
                 retry_count: 0, osdeps: true, auto_exclude: false, osdeps_options: Hash.new,
-                report: true)
+                report: true, ask: false)
 
-                if from
-                    setup_update_from(from)
-                end
+                setup_update_from(from) if from
+
+                filter =
+                    if ask
+                        prompt = TTY::Prompt.new
+                        filter = AskUpdateFilter.new(
+                            prompt, parallel: parallel, only_local: only_local
+                        )
+                    else
+                        ->(pkg) { true }
+                    end
 
                 ops = Autoproj::Ops::Import.new(
                     ws, report_path: (ws.import_report_path if report))
@@ -215,15 +268,17 @@ module Autoproj
                                         parallel: parallel,
                                         retry_count: retry_count,
                                         install_vcs_packages: (osdeps_options if osdeps),
-                                        auto_exclude: auto_exclude)
-                return source_packages, osdep_packages, nil
+                                        auto_exclude: auto_exclude,
+                                        filter: filter)
+                [source_packages, osdep_packages, nil]
             rescue ExcludedSelection => e
                 raise CLIInvalidSelection, e.message, e.backtrace
             rescue PackageImportFailed => import_failure
-                if !keep_going
-                    raise
-                end
-                return import_failure.source_packages, import_failure.osdep_packages, import_failure
+                raise unless keep_going
+
+                [import_failure.source_packages,
+                 import_failure.osdep_packages,
+                 import_failure]
             end
 
             def setup_update_from(other_root)

data/lib/autoproj/cli/utility.rb

@@ -70,7 +70,7 @@ module Autoproj
                 out_format = "%-#{w}s %-7s %-9s"
                 puts format(out_format, 'Package Name', 'Enabled', 'Available')
                 lines.each do |name, enabled, available|
-                    puts(format(out_format, name, (!!enabled).to_s, (!!available).to_s))
+                    puts(format(out_format, name, (!!enabled).to_s, (!!available).to_s)) # rubocop:disable Style/DoubleNegation
                 end
             end
 
@@ -89,24 +89,37 @@ module Autoproj
                     raise CLIInvalidArguments, "autoproj: the provided package "\
                         "is not selected for build"
                 end
+                return if package_names.empty?
 
                 packages = package_names.map do |pkg_name|
                     ws.manifest.find_package_definition(pkg_name)
                 end
 
-                Autobuild.apply(
-                    package_names,
-                    "autoproj-#{utility_name}",
-                    [utility_name],
-                    parallel: options[:parallel]
-                )
+                apply_to_packages(packages, parallel: options[:parallel])
+            end
 
+            def apply_to_packages(packages, parallel: ws.config.parallel_build_level)
+                if @report_path
+                    reporting = Ops::PhaseReporting.new(
+                        @utility_name, @report_path,
+                        method(:package_metadata)
+                    )
+                end
+
+                reporting&.initialize_incremental_report
+                Autobuild.apply(
+                    packages.map(&:name), "autoproj-#{@utility_name}",
+                    [@utility_name], parallel: parallel
+                ) do |pkg, phase|
+                    reporting&.report_incremental(pkg) if phase == utility_name
+                end
             ensure
-                create_report(packages) if packages && @report_path
+                reporting&.create_report(packages.map(&:autobuild))
             end
 
-            def package_metadata(package)
-                u = package.autobuild.utility(@utility_name)
+            def package_metadata(autobuild_package)
+                # rubocop:disable Style/DoubleNegation
+                u = autobuild_package.utility(@utility_name)
                 {
                     'source_dir' => u.source_dir,
                     'target_dir' => u.target_dir,
@@ -114,25 +127,9 @@ module Autoproj
                     'enabled' => !!u.enabled?,
                     'invoked' => !!u.invoked?,
                     'success' => !!u.success?,
-                    'installed' => !!u.installed?,
+                    'installed' => !!u.installed?
                 }
-            end
-
-            def create_report(packages)
-                info = packages.each_with_object({}) do |p, map|
-                    map[p.name] = package_metadata(p)
-                end
-
-                FileUtils.mkdir_p File.dirname(@report_path)
-                File.open(@report_path, 'w') do |io|
-                    dump = JSON.dump(
-                        "#{@utility_name}_report" => {
-                            'timestamp' => Time.now,
-                            'packages' => info
-                        }
-                    )
-                    io.write dump
-                end
+                # rubocop:enable Style/DoubleNegation
             end
         end
     end

data/lib/autoproj/configuration.rb

@@ -196,7 +196,9 @@ module Autoproj
                     Autoproj.info "       using: #{value} (noninteractive mode)"
                 end
                 @modified = true
-                if !is_default
+                if is_default
+                    value = opt.validate(value)
+                else
                     config[option_name] = [value, true]
                     displayed_options[option_name] = value
                 end
@@ -242,10 +244,10 @@ module Autoproj
             @modified = false
         end
 
-        def each_reused_autoproj_installation
+        def each_reused_autoproj_installation(&block)
             if has_value_for?('reused_autoproj_installations')
-                get('reused_autoproj_installations').each(&proc)
-            else [].each(&proc)
+                get('reused_autoproj_installations').each(&block)
+            else [].each(&block)
             end
         end
 
@@ -353,6 +355,10 @@ module Autoproj
             set 'shell_helpers', flag, true
         end
 
+        def bundler_version
+            get 'bundler_version', nil
+        end
+
         def apply_autobuild_configuration
             if has_value_for?('autobuild')
                 params = get('autobuild')
@@ -369,6 +375,11 @@ module Autoproj
             get('importer_cache_dir', nil)
         end
 
+        # Set import and gem cache directory
+        def importer_cache_dir=(path)
+            set('importer_cache_dir', path, true)
+        end
+
         # Sets the directory in which packages will be installed
         def prefix_dir=(path)
             set('prefix', path, true)

data/lib/autoproj/default.osdeps

@@ -92,6 +92,12 @@ ruby24:
 ruby25:
   default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
 
+ruby26:
+  default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
+
+ruby27:
+  default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
+
 build-essential:
   debian,ubuntu: build-essential
   gentoo: ignore
@@ -132,15 +138,19 @@ archive:
   debian,ubuntu:
   - tar
   - unzip
+  - ca-certificates
   gentoo:
   - app-arch/tar
   - app-arch/unzip
+  - app-misc/ca-certificates
   arch:
   - tar
   - unzip
+  - ca-certificates
   fedora:
   - tar
   - unzip
+  - ca-certificates
   macos-port:
   - gnutar
   - unzip
@@ -149,13 +159,19 @@ archive:
   opensuse:
   - tar
   - unzip
+  - ca-certificates
   default: ignore
 
 cvs:
   default: cvs
 
 pip:
-  debian,ubuntu: python-pip
+  debian:
+    '8,jessie,9,stretch': python-pip
+    default: python3-pip
+  ubuntu:
+    '16.04,18.04': python-pip
+    default: python3-pip
   arch: python2-pip
   opensuse: python-pip
   fedora: python-pip
@@ -164,7 +180,12 @@ pip:
 
 python:
   arch: python2
-  debian,ubuntu: python-dev
+  debian:
+    '8,jessie,9,stretch': python-dev
+    default: python3-dev
+  ubuntu:
+    '16.04,18.04': python-dev
+    default: python3-dev
   fedora: python-devel
   freebsd: python-devel
   gentoo: dev-lang/python
@@ -173,7 +194,12 @@ python:
 
 python-setuptools:
   arch: python2-distribute
-  debian,ubuntu: python-setuptools
+  debian:
+     '8,jessie,9,stretch': python-setuptools
+     default: python3-setuptools
+  ubuntu:
+    '16.04,18.04': python-setuptools
+    default: python3-setuptools
   fedora: python-setuptools
   gentoo: dev-python/setuptools
   default: ignore # will be installed manually by the user

data/lib/autoproj/environment.rb

@@ -17,6 +17,22 @@ module Autoproj
             Autoproj.filter_out_paths_in_workspace(env)
         end
 
+        def env_filename(shell, *subdir)
+            env_filename = if shell == 'sh'
+                               ENV_FILENAME
+                           else
+                               (Pathname(ENV_FILENAME).sub_ext '').to_s.concat(".#{shell}")
+                           end
+
+            File.join(root_dir, *subdir, env_filename)
+        end
+
+        def each_env_filename(*subdir)
+            (['sh'] + Autoproj.workspace.config.user_shells).to_set.each do |shell|
+                yield shell, env_filename(shell, *subdir)
+            end
+        end
+
         def export_env_sh(subdir = nil, options = Hash.new)
             if subdir.kind_of?(Hash)
                 subdir, options = nil, subdir
@@ -28,19 +44,7 @@ module Autoproj
             completion_dir = File.join(shell_dir, 'completion')
             env_updated = false
 
-            (['sh'] + Autoproj.workspace.config.user_shells).to_set.each do |shell|
-                env_filename = if shell == 'sh'
-                                   ENV_FILENAME
-                               else
-                                   (Pathname(ENV_FILENAME).sub_ext '').to_s.concat(".#{shell}")
-                               end
-
-                filename = if subdir
-                    File.join(root_dir, subdir, env_filename)
-                else
-                    File.join(root_dir, env_filename)
-                end
-
+            each_env_filename(*[subdir].compact) do |shell, filename|
                 helper = File.join(shell_dir, "autoproj_#{shell}")
                 if options[:shell_helpers]
                     source_after(helper, shell: shell) if File.file?(helper)

data/lib/autoproj/installation_manifest.rb

@@ -1,7 +1,8 @@
 module Autoproj
     # Manifest of installed packages imported from another autoproj installation
     class InstallationManifest
-        Package = Struct.new :name, :type, :vcs, :srcdir, :prefix, :builddir, :logdir, :dependencies
+        Package = Struct.new :name, :type, :vcs, :srcdir, :importdir,
+                             :prefix, :builddir, :logdir, :dependencies
         PackageSet = Struct.new :name, :vcs, :raw_local_dir, :user_local_dir
 
         attr_reader :path
@@ -28,11 +29,11 @@ module Autoproj
         def each_package_set(&block)
             package_sets.each_value(&block)
         end
-            
+
         def each_package(&block)
             packages.each_value(&block)
         end
-            
+
         def load
             @packages = Hash.new
             raw = YAML.load(File.open(path))
@@ -51,8 +52,8 @@ module Autoproj
                         package_sets[pkg_set.name] = pkg_set
                     else
                         pkg = Package.new(
-                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['prefix'],
-                            entry['builddir'], entry['logdir'], entry['dependencies'])
+                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['importdir'],
+                            entry['prefix'], entry['builddir'], entry['logdir'], entry['dependencies'])
                         packages[pkg.name] = pkg
                     end
                 end
@@ -74,6 +75,7 @@ module Autoproj
                          'type' => v.class.name,
                          'vcs' => package_def.vcs.to_hash,
                          'srcdir' => v.srcdir,
+                         'importdir' => (v.importdir if v.respond_to?(:importdir)),
                          'builddir' => (v.builddir if v.respond_to?(:builddir)),
                          'logdir' => v.logdir,
                          'prefix' => v.prefix,