autoproj 2.11.0 → 2.12.0

data/lib/autoproj/package_managers/apt_dpkg_manager.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'autoproj/package_managers/debian_version'
 
 module Autoproj
@@ -32,7 +34,9 @@ def keep_uptodate=(flag)
                 ws.config.set('apt_dpkg_update', flag, true)
             end
 
-            def self.parse_package_status(installed_packages, installed_versions, paragraph)
+            def self.parse_package_status(
+                installed_packages, installed_versions, paragraph, virtual: true
+            )
                 if paragraph =~ /^Status: install ok installed$/
                     if paragraph =~ /^Package: (.*)$/
                         package_name = $1
@@ -41,13 +45,13 @@ def self.parse_package_status(installed_packages, installed_versions, paragraph)
                             installed_versions[package_name] = DebianVersion.new($1)
                         end
                     end
-                    if paragraph =~ /^Provides: (.*)$/
+                    if virtual && paragraph =~ /^Provides: (.*)$/
                         installed_packages.merge($1.split(',').map(&:strip))
                     end
                 end
             end
 
-            def self.parse_dpkg_status(status_file)
+            def self.parse_dpkg_status(status_file, virtual: true)
                 installed_packages = Set.new
                 installed_versions = {}
                 dpkg_status = File.read(status_file)
@@ -60,9 +64,11 @@ def self.parse_dpkg_status(status_file)
 
                 while paragraph_end = dpkg_status.scan_until(/Package: /)
                     paragraph = "Package: #{paragraph_end[0..-10]}"
-                    parse_package_status(installed_packages, installed_versions, paragraph)
+                    parse_package_status(installed_packages, installed_versions,
+                                         paragraph, virtual: virtual)
                 end
-                parse_package_status(installed_packages, installed_versions, "Package: #{dpkg_status.rest}")
+                parse_package_status(installed_packages, installed_versions,
+                                     "Package: #{dpkg_status.rest}", virtual: virtual)
                 [installed_packages, installed_versions]
             end
 

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -24,6 +24,14 @@ def self.with_prerelease(*value)
                 end
             end
 
+            # Directory with cached .gem packages
+            #
+            # The directory must exist, but may be empty.
+            # It is initialized with {BundlerManager.cache_dir}
+            #
+            # @return [String]
+            attr_accessor :cache_dir
+
             # (see Manager#call_while_empty?)
             def call_while_empty?
                 !workspace_configuration_gemfiles.empty?
@@ -45,14 +53,31 @@ def initialize_environment
                 env.add_path 'PATH', File.join(ws.dot_autoproj_dir, 'bin')
                 env.set 'GEM_HOME', config.gems_gem_home
                 env.clear 'GEM_PATH'
+                if (bundler_version = config.bundler_version)
+                    env.set 'BUNDLER_VERSION', bundler_version
+                else
+                    env.clear 'BUNDLER_VERSION'
+                end
 
                 gemfile_path = File.join(ws.prefix_dir, 'gems', 'Gemfile')
                 env.set('BUNDLE_GEMFILE', gemfile_path) if File.file?(gemfile_path)
 
-                Autobuild.programs['bundler'] = File.join(ws.dot_autoproj_dir,
-                                                          'bin', 'bundle')
-                Autobuild.programs['bundle'] = File.join(ws.dot_autoproj_dir,
-                                                         'bin', 'bundle')
+                if cache_dir && File.exist?(cache_dir)
+                    vendor_dir = File.join(File.dirname(gemfile_path), 'vendor')
+                    FileUtils.mkdir_p vendor_dir
+                    bundler_cache_dir = File.join(vendor_dir, 'cache')
+                    if File.writable?(cache_dir)
+                        create_cache_symlink(cache_dir, bundler_cache_dir)
+                    else
+                        Autoproj.warn "BundlerManager: #{cache_dir} is read-only "\
+                                      "copying the cache instead of symlinking it"
+                        create_cache_copy(cache_dir, bundler_cache_dir)
+                    end
+                end
+
+                Autobuild.programs['bundler'] =
+                    Autobuild.programs['bundle'] =
+                    File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
 
                 env.init_from_env 'RUBYLIB'
                 env.inherit 'RUBYLIB'
@@ -96,6 +121,43 @@ def initialize_environment
                 end
             end
 
+            def create_cache_symlink(cache_dir, bundler_cache_dir)
+                valid = !File.exist?(bundler_cache_dir) ||
+                        File.symlink?(bundler_cache_dir)
+
+                unless valid
+                    Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                  "#{bundler_cache_dir} already exists"
+                    return
+                end
+
+                FileUtils.rm_f bundler_cache_dir
+                FileUtils.ln_s cache_dir, bundler_cache_dir
+            end
+
+            def create_cache_copy(cache_dir, bundler_cache_dir)
+                valid = !File.exist?(bundler_cache_dir) ||
+                        File.directory?(bundler_cache_dir) ||
+                        File.symlink?(bundler_cache_dir)
+
+                unless valid
+                    Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                  "#{bundler_cache_dir} already exists"
+                    return
+                end
+
+                # Gracefully upgrade from the symlinks
+                FileUtils.rm_f bundler_cache_dir if File.symlink?(bundler_cache_dir)
+                FileUtils.mkdir_p bundler_cache_dir
+
+                Dir.glob(File.join(cache_dir, '*.gem')) do |path_src|
+                    path_dest = File.join(bundler_cache_dir, File.basename(path_src))
+                    next if File.exist?(path_dest)
+
+                    FileUtils.cp path_src, path_dest
+                end
+            end
+
             # Enumerate the per-gem build configurations
             def self.per_gem_build_config(ws)
                 ws.config.get('bundler.build', {})
@@ -248,10 +310,13 @@ def backup_clean(mapping)
                 end
             end
 
-            def self.run_bundler_install(ws, gemfile, *options,
-                                         update: true, binstubs: nil,
-                                         gem_home: ws.config.gems_gem_home,
-                                         gem_path: ws.config.gems_install_path)
+            def self.run_bundler_install(
+                ws, gemfile, *options,
+                update: true, binstubs: nil,
+                bundler_version: ws.config.bundler_version,
+                gem_home: ws.config.gems_gem_home,
+                gem_path: ws.config.gems_install_path
+            )
                 FileUtils.rm "#{gemfile}.lock" if update && File.file?("#{gemfile}.lock")
 
                 options << '--path' << gem_path
@@ -262,6 +327,7 @@ def self.run_bundler_install(ws, gemfile, *options,
 
                 connections = Set.new
                 run_bundler(ws, 'install', *options,
+                            bundler_version: bundler_version,
                             gem_home: gem_home, gemfile: gemfile) do |line|
                     case line
                     when /Installing (.*)/
@@ -276,11 +342,13 @@ def self.run_bundler_install(ws, gemfile, *options,
                 end
             end
 
-            def self.bundle_gem_path(ws, gem_name, gem_home: nil, gemfile: nil)
+            def self.bundle_gem_path(ws, gem_name,
+                                     bundler_version: ws.config.bundler_version,
+                                     gem_home: nil, gemfile: nil)
                 path = String.new
-                PackageManagers::BundlerManager.run_bundler(
+                run_bundler(
                     ws, 'show', gem_name,
-                    gem_home: gem_home,
+                    bundler_version: bundler_version, gem_home: gem_home,
                     gemfile: gemfile) { |line| path << line }
                 path.chomp
             end
@@ -289,17 +357,26 @@ def self.default_bundler(ws)
                 File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
             end
 
-            def self.run_bundler(ws, *commandline, gem_home: nil, gemfile: nil)
+            def self.run_bundler(ws, *commandline,
+                                 bundler_version: ws.config.bundler_version,
+                                 gem_home: ws.config.gems_gem_home,
+                                 gemfile: default_gemfile_path(ws))
                 bundle = Autobuild.programs['bundle'] || default_bundler(ws)
 
-                Bundler.with_clean_env do
+                Autoproj.bundler_with_unbundled_env do
+                    bundler_version_env =
+                        if bundler_version
+                            { 'BUNDLER_VERSION' => bundler_version }
+                        else
+                            {}
+                        end
                     target_env = Hash[
                         'GEM_HOME' => gem_home,
                         'GEM_PATH' => nil,
                         'BUNDLE_GEMFILE' => gemfile,
                         'RUBYOPT' => nil,
-                        'RUBYLIB' => rubylib_for_bundler
-                    ]
+                        'RUBYLIB' => rubylib_for_bundler,
+                    ].merge(bundler_version_env)
                     ws.run('autoproj', 'osdeps',
                            bundle, *commandline,
                            working_directory: File.dirname(gemfile),
@@ -390,9 +467,13 @@ def workspace_configuration_gemfiles
                 gemfiles
             end
 
+            def self.default_gemfile_path(ws)
+                File.join(ws.prefix_dir, 'gems', 'Gemfile')
+            end
+
             def install(gems, filter_uptodate_packages: false, install_only: false)
-                root_dir     = File.join(ws.prefix_dir, 'gems')
-                gemfile_path = File.join(root_dir, 'Gemfile')
+                gemfile_path = self.class.default_gemfile_path(ws)
+                root_dir = File.dirname(gemfile_path)
                 gemfile_lock_path = "#{gemfile_path}.lock"
                 backups = Hash[
                     gemfile_path => "#{gemfile_path}.orig",
@@ -466,7 +547,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
             def discover_rubylib
                 require 'bundler'
                 Tempfile.open 'autoproj-rubylib' do |io|
-                    result = Bundler.clean_system(
+                    result = Autoproj.bundler_unbundled_system(
                         Hash['RUBYLIB' => nil],
                         Autobuild.tool('ruby'), '-e', 'puts $LOAD_PATH',
                         out: io,
@@ -490,7 +571,7 @@ def discover_bundle_rubylib(silent_errors: false)
                 silent_redirect[:err] = :close if silent_errors
                 env = ws.env.resolved_env
                 Tempfile.open 'autoproj-rubylib' do |io|
-                    result = Bundler.clean_system(
+                    result = Autoproj.bundler_unbundled_system(
                         Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'],
                              'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil,
                              'RUBYLIB' => self.class.rubylib_for_bundler],

data/lib/autoproj/package_managers/homebrew_manager.rb

@@ -15,7 +15,7 @@ def filter_uptodate_packages(packages)
                 # somewhere else
                 packages = packages.uniq
                 command_line = "brew info --json=v1 #{packages.join(' ')}"
-                result = Bundler.with_clean_env do
+                result = Autoproj.bundler_with_unbundled_env do
                     (Autobuild::Subprocess.run 'autoproj', 'osdeps', command_line).first
                 end
 
@@ -30,7 +30,7 @@ def filter_uptodate_packages(packages)
                     end
                     return packages
                 end
-                
+
                 # fall back if something else went wrong
                 if packages.size != result.size
                     Autoproj.warn "brew info returns less or more packages when requested. Falling back to install all packages"

data/lib/autoproj/test.rb

@@ -71,6 +71,7 @@ def setup
             Autobuild.logdir = make_tmpdir
             ws_define_package_manager 'os'
             ws_define_package_manager 'os_indep'
+            Autobuild.progress_display_period = 0
 
             super
         end
@@ -80,6 +81,7 @@ def teardown
             @tmpdir.each do |dir|
                 FileUtils.remove_entry_secure dir
             end
+            Rake::Task.clear
             Autobuild::Package.clear
             Autoproj.silent = false
 
@@ -94,6 +96,11 @@ def teardown
             end
         end
 
+        def data_path(*args)
+            File.expand_path(File.join(*args),
+                             File.join(__dir__, '..', '..', 'test', 'data'))
+        end
+
         def create_bootstrap
             ws_create
         end
@@ -170,7 +177,7 @@ def invoke_test_script(name, *arguments,
                     'PACKAGE_BASE_DIR' => package_base_dir,
                     'RUBY' => Gem.ruby
                 ]
-                result = Bundler.clean_system(
+                result = Autoproj.bundler_unbundled_system(
                     default_env.merge(env), script, *arguments,
                     in: :close, **Hash[chdir: dir].merge(system_options))
             end
@@ -201,7 +208,7 @@ def prepare_fixture_gem_home
             cached_bundler_gem = File.join(vendor, bundler_filename)
             unless File.file?(cached_bundler_gem)
                 FileUtils.mkdir_p vendor
-                Bundler.clean_system(
+                Autoproj.bundler_unbundled_system(
                     Ops::Install.guess_gem_program, 'fetch', '-v',
                     Bundler::VERSION, 'bundler', chdir: vendor)
 
@@ -211,7 +218,7 @@ def prepare_fixture_gem_home
             end
 
             capture_subprocess_io do
-                Bundler.clean_system(
+                Autoproj.bundler_unbundled_system(
                     Hash['GEM_HOME' => fixture_gem_home, 'GEM_PATH' => nil],
                     Ops::Install.guess_gem_program, 'install', '--no-document',
                     cached_bundler_gem)
@@ -256,7 +263,7 @@ def capture_deprecation_message
 
         def find_bundled_gem_path(bundler, gem_name, gemfile)
             out_r, out_w = IO.pipe
-            result = Bundler.clean_system(
+            result = Autoproj.bundler_unbundled_system(
                 bundler, 'show', gem_name,
                 out: out_w,
                 chdir: File.dirname(gemfile))
@@ -370,9 +377,16 @@ def ws_add_osdep_entries_to_layout(entries)
 
         def ws_define_package(package_type, package_name,
                               package_set: ws.manifest.main_package_set,
-                              create: true)
+                              create: true, &block)
             package = Autobuild.send(package_type, package_name)
-            package.srcdir = File.join(ws.root_dir, package_name.to_s)
+            ws_setup_package(
+                package, package_set: package_set, create: create, &block
+            )
+        end
+
+        def ws_setup_package(package, package_set: ws.manifest.main_package_set,
+                                      create: true)
+            package.srcdir = File.join(ws.root_dir, package.name.to_s)
             FileUtils.mkdir_p package.srcdir if create
             autoproj_package = ws.register_package(package, nil, package_set)
             yield(package) if block_given?

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "2.11.0"
+    VERSION = "2.12.0"
 end

data/lib/autoproj/workspace.rb

@@ -370,6 +370,14 @@ def setup
             Autobuild.logdir = log_dir
             if (cache_dir = config.importer_cache_dir)
                 Autobuild::Importer.default_cache_dirs = cache_dir
+                os_package_installer.each_manager_with_name do |name, manager|
+                    next unless manager.respond_to?(:cache_dir=)
+
+                    manager_cache_path = File.join(cache_dir, 'package_managers', name)
+                    if File.directory?(manager_cache_path)
+                        manager.cache_dir = manager_cache_path
+                    end
+                end
             end
             setup_os_package_installer
             install_ruby_shims
@@ -416,7 +424,11 @@ def update_bundler
             gem_program = Ops::Install.guess_gem_program
             install = Ops::Install.new(root_dir)
             Autoproj.message "  updating bundler"
-            install.install_bundler(gem_program, silent: true)
+            install.install_bundler(
+                gem_program,
+                version: config.bundler_version,
+                silent: true
+            )
         end
 
         def update_autoproj(restart_on_update: true)
@@ -430,18 +442,21 @@ def update_autoproj(restart_on_update: true)
             binstubs = File.join(dot_autoproj_dir, 'bin')
             if restart_on_update
                 old_autoproj_path = PackageManagers::BundlerManager.bundle_gem_path(
-                    self, 'autoproj', gem_home: config.gems_gem_home, gemfile: gemfile)
+                    self, 'autoproj', gemfile: gemfile
+                )
             end
             begin
                 Autoproj.message "  updating autoproj"
                 PackageManagers::BundlerManager.run_bundler_install(
-                    self, gemfile, binstubs: binstubs)
+                    self, gemfile, binstubs: binstubs
+                )
             ensure
                 rewrite_shims
             end
             if restart_on_update
                 new_autoproj_path = PackageManagers::BundlerManager.bundle_gem_path(
-                    self, 'autoproj', gem_home: config.gems_gem_home, gemfile: gemfile)
+                    self, 'autoproj', gemfile: gemfile
+                )
             end
 
             # First things first, see if we need to update ourselves

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoproj
 version: !ruby/object:Gem::Version
-  version: 2.11.0
+  version: 2.12.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-10-03 00:00:00.000000000 Z
+date: 2020-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -376,6 +376,7 @@ files:
 - lib/autoproj/ops/install.rb
 - lib/autoproj/ops/loader.rb
 - lib/autoproj/ops/main_config_switcher.rb
+- lib/autoproj/ops/phase_reporting.rb
 - lib/autoproj/ops/snapshot.rb
 - lib/autoproj/ops/tools.rb
 - lib/autoproj/ops/watch.rb