autoproj 2.11.0 → 2.14.0

data/lib/autoproj/ops/phase_reporting.rb

@@ -0,0 +1,49 @@
+module Autoproj
+    module Ops
+        # Common logic to generate build/import/utility reports
+        class PhaseReporting
+            def initialize(name, path, metadata_get)
+                @name = name
+                @path = path
+                @metadata_get = metadata_get
+            end
+
+            def create_report(autobuild_packages)
+                info = autobuild_packages.each_with_object({}) do |p, map|
+                    map[p.name] = @metadata_get.call(p)
+                end
+
+                dump = JSON.dump(
+                    "#{@name}_report" => {
+                        'timestamp' => Time.now,
+                        'packages' => info
+                    }
+                )
+
+                FileUtils.mkdir_p File.dirname(@path)
+                File.open(@path, 'w') do |io|
+                    io.write dump
+                end
+            end
+
+            def initialize_incremental_report
+                FileUtils.mkdir_p File.dirname(@path)
+                @incremental_report = ""
+            end
+
+            def report_incremental(autobuild_package)
+                new_metadata = @metadata_get.call(autobuild_package)
+                prefix = @incremental_report.empty? ? "\n" : ",\n"
+                @incremental_report.concat(
+                    "#{prefix}\"#{autobuild_package.name}\": #{JSON.dump(new_metadata)}"
+                )
+                File.open(@path, 'w') do |io|
+                    io.write "{ \"#{@name}_report\": "\
+                             "{\"timestamp\": #{JSON.dump(Time.now)}, \"packages\": {"
+                    io.write(@incremental_report)
+                    io.write "}}}"
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/ops/snapshot.rb

@@ -119,6 +119,7 @@ module Autoproj
 
         def snapshot_packages(packages, target_dir = nil, only_local: true, fingerprint: false)
             result = Array.new
+            fingerprint_memo = Hash.new
             packages.each do |package_name|
                 package  = manifest.find_package_definition(package_name)
                 if !package
@@ -141,7 +142,7 @@ module Autoproj
                     end
 
                 if fingerprint
-                    vcs_info['fingerprint'] = package.autobuild.fingerprint
+                    vcs_info['fingerprint'] = package.autobuild.fingerprint(memo: fingerprint_memo)
                 end
 
                 if vcs_info

data/lib/autoproj/ops/tools.rb

@@ -24,8 +24,8 @@ module Autoproj
                 @@packages.delete(text_name)
             end
 
-            def import(options = Hash.new)
-                importer.import(self, options)
+            def import(**options)
+                importer.import(self, **options)
             end
 
             def add_stat(*args)

data/lib/autoproj/os_package_installer.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'autoproj/package_managers/manager'
 require 'autoproj/package_managers/unknown_os_manager'
 require 'autoproj/package_managers/shell_script_manager'
@@ -37,7 +39,9 @@ module Autoproj
         attr_reader :installed_resolved_packages
 
         attr_writer :silent
-        def silent?; @silent end
+        def silent?
+            @silent
+        end
 
         class << self
             attr_accessor :force_osdeps
@@ -60,12 +64,12 @@ module Autoproj
 
         # Returns the package manager object for the current OS
         def os_package_manager
-            if !@os_package_manager
+            unless @os_package_manager
                 name = os_package_resolver.os_package_manager
                 @os_package_manager = package_managers[name] ||
-                    PackageManagers::UnknownOSManager.new(ws)
+                                      PackageManagers::UnknownOSManager.new(ws)
             end
-            return @os_package_manager
+            @os_package_manager
         end
 
         # Returns the set of package managers
@@ -75,13 +79,17 @@ module Autoproj
             package_managers.each_value(&block)
         end
 
+        def each_manager_with_name(&block)
+            package_managers.each(&block)
+        end
+
         HANDLE_ALL  = 'all'
         HANDLE_RUBY = 'ruby'
         HANDLE_OS   = 'os'
         HANDLE_NONE = 'none'
 
         def osdeps_mode_option_unsupported_os(config)
-            long_doc =<<-EOT
+            long_doc = <<-EOT
 The software packages that autoproj will have to build may require other
 prepackaged softwares (a.k.a. OS dependencies) to be installed (RubyGems
 packages, packages from your operating system/distribution, ...). Autoproj is
@@ -153,9 +161,9 @@ So, what do you want ? (all, none or a comma-separated list of: os gem pip)
             message = [ "Which prepackaged software (a.k.a. 'osdeps') should autoproj install automatically (all, none or a comma-separated list of: os gem pip) ?", long_doc.strip ]
 
             config.declare 'osdeps_mode', 'string',
-                default: 'all',
-                doc: message,
-                lowercase: true
+                           default: 'all',
+                           doc: message,
+                           lowercase: true
         end
 
         def define_osdeps_mode_option
@@ -171,14 +179,14 @@ So, what do you want ? (all, none or a comma-separated list of: os gem pip)
             modes = []
             user_modes.each do |str|
                 case str
-                when 'all'  then modes.concat(['os', 'gem', 'pip'])
+                when 'all'  then modes.concat(%w[os gem pip])
                 when 'ruby' then modes << 'gem'
                 when 'gem'  then modes << 'gem'
                 when 'pip'  then modes << 'pip'
                 when 'os'   then modes << 'os'
                 when 'none' then
                 else
-                    if package_managers.has_key?(str)
+                    if package_managers.key?(str)
                         modes << str
                     else
                         raise ArgumentError, "#{str} is not a known package handler, known handlers are #{package_managers.keys.sort.join(", ")}"
@@ -442,5 +450,5 @@ So, what do you want ? (all, none or a comma-separated list of: os gem pip)
             end
         end
     end
-end 
+end
 

data/lib/autoproj/package_definition.rb

@@ -90,7 +90,7 @@ module Autoproj
                         autobuild.depends_on name
                     end
                 rescue ConfigError => e
-                    raise ConfigError.new(manifest.path),
+                    raise PackageNotFound.new(manifest.path),
                           "manifest #{manifest.path} of #{self.name} from "\
                           "#{package_set.name} lists '#{name}' as dependency, "\
                           'but it is neither a normal package nor an osdeps '\

data/lib/autoproj/package_managers/apt_dpkg_manager.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'autoproj/package_managers/debian_version'
 
 module Autoproj
@@ -12,15 +14,17 @@ module Autoproj
                 @installed_packages = nil
                 @installed_versions = nil
                 super(ws, true,
-                      %w{apt-get install},
-                      %w{DEBIAN_FRONTEND=noninteractive apt-get install -y})
+                      %w[apt-get install],
+                      %w[DEBIAN_FRONTEND=noninteractive apt-get install -y])
             end
 
             def configure_manager
                 super
-                ws.config.declare 'apt_dpkg_update', 'boolean',
+                ws.config.declare(
+                    'apt_dpkg_update', 'boolean',
                     default: 'yes',
                     doc: ['Would you like autoproj to keep apt packages up-to-date?']
+                )
                 keep_uptodate?
             end
 
@@ -32,7 +36,9 @@ module Autoproj
                 ws.config.set('apt_dpkg_update', flag, true)
             end
 
-            def self.parse_package_status(installed_packages, installed_versions, paragraph)
+            def self.parse_package_status(
+                installed_packages, installed_versions, paragraph, virtual: true
+            )
                 if paragraph =~ /^Status: install ok installed$/
                     if paragraph =~ /^Package: (.*)$/
                         package_name = $1
@@ -41,38 +47,40 @@ module Autoproj
                             installed_versions[package_name] = DebianVersion.new($1)
                         end
                     end
-                    if paragraph =~ /^Provides: (.*)$/
+                    if virtual && paragraph =~ /^Provides: (.*)$/
                         installed_packages.merge($1.split(',').map(&:strip))
                     end
                 end
             end
 
-            def self.parse_dpkg_status(status_file)
+            def self.parse_dpkg_status(status_file, virtual: true)
                 installed_packages = Set.new
                 installed_versions = {}
                 dpkg_status = File.read(status_file)
                 dpkg_status << "\n"
 
                 dpkg_status = StringScanner.new(dpkg_status)
-                if !dpkg_status.scan(/Package: /)
-                    raise ArgumentError, "expected #{status_file} to have Package: lines but found none"
+                unless dpkg_status.scan(/Package: /)
+                    raise ArgumentError, "expected #{status_file} to have Package: "\
+                                         "lines but found none"
                 end
 
-                while paragraph_end = dpkg_status.scan_until(/Package: /)
+                while (paragraph_end = dpkg_status.scan_until(/Package: /))
                     paragraph = "Package: #{paragraph_end[0..-10]}"
-                    parse_package_status(installed_packages, installed_versions, paragraph)
+                    parse_package_status(installed_packages, installed_versions,
+                                         paragraph, virtual: virtual)
                 end
-                parse_package_status(installed_packages, installed_versions, "Package: #{dpkg_status.rest}")
+                parse_package_status(installed_packages, installed_versions,
+                                     "Package: #{dpkg_status.rest}", virtual: virtual)
                 [installed_packages, installed_versions]
             end
 
             def self.parse_apt_cache_paragraph(paragraph)
                 version = '0'
-                if paragraph =~ /^Package: (.*)$/
-                    package_name = $1
-                    if paragraph =~ /^Version: (.*)$/
-                        version = $1
-                    end
+                if (paragraph_m = /^Package: (.*)$/.match(paragraph))
+                    package_name = paragraph_m[1]
+                    version_m = /^Version: (.*)$/.match(paragraph)
+                    version = version_m[1] if version_m
                 end
                 [package_name, version]
             end
@@ -81,23 +89,24 @@ module Autoproj
                 packages_versions = {}
                 apt_cache_show = `apt-cache show --no-all-versions #{packages.join(' ')}`
                 apt_cache_show = StringScanner.new(apt_cache_show)
-                if !apt_cache_show.scan(/Package: /)
-                    return packages_versions
-                end
+                return packages_versions unless apt_cache_show.scan(/Package: /)
 
-                while paragraph_end = apt_cache_show.scan_until(/Package: /)
+                while (paragraph_end = apt_cache_show.scan_until(/Package: /))
                     paragraph = "Package: #{paragraph_end[0..-10]}"
                     package_name, version = parse_apt_cache_paragraph(paragraph)
                     packages_versions[package_name] = DebianVersion.new(version)
                 end
-                package_name, version = parse_apt_cache_paragraph("Package: #{apt_cache_show.rest}")
+                package_name, version = parse_apt_cache_paragraph(
+                    "Package: #{apt_cache_show.rest}"
+                )
                 packages_versions[package_name] = DebianVersion.new(version)
                 packages_versions
             end
 
             def updated?(package, available_version)
-                # Consider up-to-date if the package is provided by another package (purely virtual)
-                # Ideally, we should check the version of the package that provides it
+                # Consider up-to-date if the package is provided by another
+                # package (purely virtual) Ideally, we should check the version
+                # of the package that provides it
                 return true unless available_version && @installed_versions[package]
 
                 (available_version <= @installed_versions[package])
@@ -105,8 +114,13 @@ module Autoproj
 
             # On a dpkg-enabled system, checks if the provided package is installed
             # and returns true if it is the case
-            def installed?(package_name, filter_uptodate_packages: false, install_only: false)
-                @installed_packages, @installed_versions = self.class.parse_dpkg_status(status_file) unless @installed_packages && @installed_versions
+            def installed?(package_name, filter_uptodate_packages: false,
+                                         install_only: false)
+                unless @installed_packages && @installed_versions
+                    @installed_packages, @installed_versions =
+                        self.class.parse_dpkg_status(status_file)
+                end
+
                 if package_name =~ /^(\w[a-z0-9+-.]+)/
                     @installed_packages.include?($1)
                 else
@@ -116,11 +130,18 @@ module Autoproj
             end
 
             def install(packages, filter_uptodate_packages: false, install_only: false)
-                packages_versions = self.class.parse_packages_versions(packages)
                 if filter_uptodate_packages || install_only
-                    packages = packages.find_all do |package_name|
-                        !installed?(package_name) || (keep_uptodate? && !updated?(package_name, packages_versions[package_name]))
+                    already_installed, missing = packages.partition do |package_name|
+                        installed?(package_name)
+                    end
+
+                    if keep_uptodate? && !install_only
+                        packages_versions = self.class.parse_packages_versions(already_installed)
+                        need_update = already_installed.find_all do |package_name|
+                            !updated?(package_name, packages_versions[package_name])
+                        end
                     end
+                    packages = missing + (need_update || [])
                 end
 
                 if super(packages)

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -24,6 +24,14 @@ module Autoproj
                 end
             end
 
+            # Directory with cached .gem packages
+            #
+            # The directory must exist, but may be empty.
+            # It is initialized with {BundlerManager.cache_dir}
+            #
+            # @return [String]
+            attr_accessor :cache_dir
+
             # (see Manager#call_while_empty?)
             def call_while_empty?
                 !workspace_configuration_gemfiles.empty?
@@ -45,14 +53,31 @@ module Autoproj
                 env.add_path 'PATH', File.join(ws.dot_autoproj_dir, 'bin')
                 env.set 'GEM_HOME', config.gems_gem_home
                 env.clear 'GEM_PATH'
+                if (bundler_version = config.bundler_version)
+                    env.set 'BUNDLER_VERSION', bundler_version
+                else
+                    env.clear 'BUNDLER_VERSION'
+                end
 
                 gemfile_path = File.join(ws.prefix_dir, 'gems', 'Gemfile')
                 env.set('BUNDLE_GEMFILE', gemfile_path) if File.file?(gemfile_path)
 
-                Autobuild.programs['bundler'] = File.join(ws.dot_autoproj_dir,
-                                                          'bin', 'bundle')
-                Autobuild.programs['bundle'] = File.join(ws.dot_autoproj_dir,
-                                                         'bin', 'bundle')
+                if cache_dir && File.exist?(cache_dir)
+                    vendor_dir = File.join(File.dirname(gemfile_path), 'vendor')
+                    FileUtils.mkdir_p vendor_dir
+                    bundler_cache_dir = File.join(vendor_dir, 'cache')
+                    if File.writable?(cache_dir)
+                        create_cache_symlink(cache_dir, bundler_cache_dir)
+                    else
+                        Autoproj.warn "BundlerManager: #{cache_dir} is read-only "\
+                                      "copying the cache instead of symlinking it"
+                        create_cache_copy(cache_dir, bundler_cache_dir)
+                    end
+                end
+
+                Autobuild.programs['bundler'] =
+                    Autobuild.programs['bundle'] =
+                    File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
 
                 env.init_from_env 'RUBYLIB'
                 env.inherit 'RUBYLIB'
@@ -64,6 +89,7 @@ module Autoproj
                             Bundler.rubygems.gem_path
                                    .none? { |gem_p| p.start_with?(gem_p) }
                     end
+
                 # And discover the system's rubylib
                 if (system_rubylib = discover_rubylib)
                     # Do not explicitely add the system rubylib to the
@@ -96,6 +122,44 @@ module Autoproj
                 end
             end
 
+            def create_cache_symlink(cache_dir, bundler_cache_dir)
+                if File.exist?(bundler_cache_dir)
+                    if !File.symlink?(bundler_cache_dir)
+                        Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                      "#{bundler_cache_dir} already exists"
+                        return
+                    elsif File.readlink(bundler_cache_dir) == cache_dir
+                        return
+                    end
+                end
+
+                FileUtils.rm_f bundler_cache_dir
+                FileUtils.ln_s cache_dir, bundler_cache_dir
+            end
+
+            def create_cache_copy(cache_dir, bundler_cache_dir)
+                valid = !File.exist?(bundler_cache_dir) ||
+                        File.directory?(bundler_cache_dir) ||
+                        File.symlink?(bundler_cache_dir)
+
+                unless valid
+                    Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                  "#{bundler_cache_dir} already exists"
+                    return
+                end
+
+                # Gracefully upgrade from the symlinks
+                FileUtils.rm_f bundler_cache_dir if File.symlink?(bundler_cache_dir)
+                FileUtils.mkdir_p bundler_cache_dir
+
+                Dir.glob(File.join(cache_dir, '*.gem')) do |path_src|
+                    path_dest = File.join(bundler_cache_dir, File.basename(path_src))
+                    next if File.exist?(path_dest)
+
+                    FileUtils.cp path_src, path_dest
+                end
+            end
+
             # Enumerate the per-gem build configurations
             def self.per_gem_build_config(ws)
                 ws.config.get('bundler.build', {})
@@ -248,10 +312,13 @@ module Autoproj
                 end
             end
 
-            def self.run_bundler_install(ws, gemfile, *options,
-                                         update: true, binstubs: nil,
-                                         gem_home: ws.config.gems_gem_home,
-                                         gem_path: ws.config.gems_install_path)
+            def self.run_bundler_install(
+                ws, gemfile, *options,
+                update: true, binstubs: nil,
+                bundler_version: ws.config.bundler_version,
+                gem_home: ws.config.gems_gem_home,
+                gem_path: ws.config.gems_install_path
+            )
                 FileUtils.rm "#{gemfile}.lock" if update && File.file?("#{gemfile}.lock")
 
                 options << '--path' << gem_path
@@ -262,6 +329,7 @@ module Autoproj
 
                 connections = Set.new
                 run_bundler(ws, 'install', *options,
+                            bundler_version: bundler_version,
                             gem_home: gem_home, gemfile: gemfile) do |line|
                     case line
                     when /Installing (.*)/
@@ -276,11 +344,13 @@ module Autoproj
                 end
             end
 
-            def self.bundle_gem_path(ws, gem_name, gem_home: nil, gemfile: nil)
+            def self.bundle_gem_path(ws, gem_name,
+                                     bundler_version: ws.config.bundler_version,
+                                     gem_home: nil, gemfile: nil)
                 path = String.new
-                PackageManagers::BundlerManager.run_bundler(
+                run_bundler(
                     ws, 'show', gem_name,
-                    gem_home: gem_home,
+                    bundler_version: bundler_version, gem_home: gem_home,
                     gemfile: gemfile) { |line| path << line }
                 path.chomp
             end
@@ -289,17 +359,26 @@ module Autoproj
                 File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
             end
 
-            def self.run_bundler(ws, *commandline, gem_home: nil, gemfile: nil)
+            def self.run_bundler(ws, *commandline,
+                                 bundler_version: ws.config.bundler_version,
+                                 gem_home: ws.config.gems_gem_home,
+                                 gemfile: default_gemfile_path(ws))
                 bundle = Autobuild.programs['bundle'] || default_bundler(ws)
 
-                Bundler.with_clean_env do
+                Autoproj.bundler_with_unbundled_env do
+                    bundler_version_env =
+                        if bundler_version
+                            { 'BUNDLER_VERSION' => bundler_version }
+                        else
+                            {}
+                        end
                     target_env = Hash[
                         'GEM_HOME' => gem_home,
                         'GEM_PATH' => nil,
                         'BUNDLE_GEMFILE' => gemfile,
                         'RUBYOPT' => nil,
-                        'RUBYLIB' => rubylib_for_bundler
-                    ]
+                        'RUBYLIB' => rubylib_for_bundler,
+                    ].merge(bundler_version_env)
                     ws.run('autoproj', 'osdeps',
                            bundle, *commandline,
                            working_directory: File.dirname(gemfile),
@@ -390,9 +469,13 @@ module Autoproj
                 gemfiles
             end
 
+            def self.default_gemfile_path(ws)
+                File.join(ws.prefix_dir, 'gems', 'Gemfile')
+            end
+
             def install(gems, filter_uptodate_packages: false, install_only: false)
-                root_dir     = File.join(ws.prefix_dir, 'gems')
-                gemfile_path = File.join(root_dir, 'Gemfile')
+                gemfile_path = self.class.default_gemfile_path(ws)
+                root_dir = File.dirname(gemfile_path)
                 gemfile_lock_path = "#{gemfile_path}.lock"
                 backups = Hash[
                     gemfile_path => "#{gemfile_path}.orig",
@@ -466,7 +549,7 @@ module Autoproj
             def discover_rubylib
                 require 'bundler'
                 Tempfile.open 'autoproj-rubylib' do |io|
-                    result = Bundler.clean_system(
+                    result = Autoproj.bundler_unbundled_system(
                         Hash['RUBYLIB' => nil],
                         Autobuild.tool('ruby'), '-e', 'puts $LOAD_PATH',
                         out: io,
@@ -490,7 +573,7 @@ module Autoproj
                 silent_redirect[:err] = :close if silent_errors
                 env = ws.env.resolved_env
                 Tempfile.open 'autoproj-rubylib' do |io|
-                    result = Bundler.clean_system(
+                    result = Autoproj.bundler_unbundled_system(
                         Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'],
                              'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil,
                              'RUBYLIB' => self.class.rubylib_for_bundler],