autoproj 2.12.1 → 2.15.1

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -1,4 +1,4 @@
-require 'bundler'
+require "bundler"
 module Autoproj
     module PackageManagers
         # Package manager interface for the RubyGems system
@@ -49,23 +49,23 @@ module Autoproj
 
                 config = ws.config
 
-                env.add_path 'PATH', File.join(ws.prefix_dir, 'gems', 'bin')
-                env.add_path 'PATH', File.join(ws.dot_autoproj_dir, 'bin')
-                env.set 'GEM_HOME', config.gems_gem_home
-                env.clear 'GEM_PATH'
+                env.add_path "PATH", File.join(ws.prefix_dir, "gems", "bin")
+                env.add_path "PATH", File.join(ws.dot_autoproj_dir, "bin")
+                env.set "GEM_HOME", config.gems_gem_home
+                env.clear "GEM_PATH"
                 if (bundler_version = config.bundler_version)
-                    env.set 'BUNDLER_VERSION', bundler_version
+                    env.set "BUNDLER_VERSION", bundler_version
                 else
-                    env.clear 'BUNDLER_VERSION'
+                    env.clear "BUNDLER_VERSION"
                 end
 
-                gemfile_path = File.join(ws.prefix_dir, 'gems', 'Gemfile')
-                env.set('BUNDLE_GEMFILE', gemfile_path) if File.file?(gemfile_path)
+                gemfile_path = File.join(ws.prefix_dir, "gems", "Gemfile")
+                env.set("BUNDLE_GEMFILE", gemfile_path) if File.file?(gemfile_path)
 
                 if cache_dir && File.exist?(cache_dir)
-                    vendor_dir = File.join(File.dirname(gemfile_path), 'vendor')
+                    vendor_dir = File.join(File.dirname(gemfile_path), "vendor")
                     FileUtils.mkdir_p vendor_dir
-                    bundler_cache_dir = File.join(vendor_dir, 'cache')
+                    bundler_cache_dir = File.join(vendor_dir, "cache")
                     if File.writable?(cache_dir)
                         create_cache_symlink(cache_dir, bundler_cache_dir)
                     else
@@ -75,16 +75,16 @@ module Autoproj
                     end
                 end
 
-                Autobuild.programs['bundler'] =
-                    Autobuild.programs['bundle'] =
-                    File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
+                Autobuild.programs["bundler"] =
+                    Autobuild.programs["bundle"] =
+                        File.join(ws.dot_autoproj_dir, "bin", "bundle")
 
-                env.init_from_env 'RUBYLIB'
-                env.inherit 'RUBYLIB'
+                env.init_from_env "RUBYLIB"
+                env.inherit "RUBYLIB"
                 # Sanitize the rubylib we get from the environment by removing
                 # anything that comes from Gem or Bundler
                 original_rubylib =
-                    (env['RUBYLIB'] || "").split(File::PATH_SEPARATOR).find_all do |p|
+                    (env["RUBYLIB"] || "").split(File::PATH_SEPARATOR).find_all do |p|
                         !p.start_with?(Bundler.rubygems.gem_dir) &&
                             Bundler.rubygems.gem_path
                                    .none? { |gem_p| p.start_with?(gem_p) }
@@ -97,22 +97,22 @@ module Autoproj
                     #
                     # This allows to use a binstub generated for one of ruby
                     # interpreter version on our workspace
-                    env.system_env['RUBYLIB'] = []
-                    env.original_env['RUBYLIB'] = (original_rubylib - system_rubylib)
+                    env.system_env["RUBYLIB"] = []
+                    env.original_env["RUBYLIB"] = (original_rubylib - system_rubylib)
                                                   .join(File::PATH_SEPARATOR)
                 end
 
                 ws.config.each_reused_autoproj_installation do |p|
                     reused_w = ws.new(p)
-                    env.add_path 'PATH', File.join(reused_w.prefix_dir, 'gems', 'bin')
+                    env.add_path "PATH", File.join(reused_w.prefix_dir, "gems", "bin")
                 end
 
                 prefix_gems = File.join(ws.prefix_dir, "gems")
                 FileUtils.mkdir_p prefix_gems
-                gemfile = File.join(prefix_gems, 'Gemfile')
+                gemfile = File.join(prefix_gems, "Gemfile")
                 unless File.exist?(gemfile)
                     Ops.atomic_write(gemfile) do |io|
-                        dot_autoproj_gemfile = File.join(ws.dot_autoproj_dir, 'Gemfile')
+                        dot_autoproj_gemfile = File.join(ws.dot_autoproj_dir, "Gemfile")
                         io.puts "eval_gemfile \"#{dot_autoproj_gemfile}\""
                     end
                 end
@@ -123,13 +123,14 @@ module Autoproj
             end
 
             def create_cache_symlink(cache_dir, bundler_cache_dir)
-                valid = !File.exist?(bundler_cache_dir) ||
-                        File.symlink?(bundler_cache_dir)
-
-                unless valid
-                    Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
-                                  "#{bundler_cache_dir} already exists"
-                    return
+                if File.exist?(bundler_cache_dir)
+                    if !File.symlink?(bundler_cache_dir)
+                        Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                      "#{bundler_cache_dir} already exists"
+                        return
+                    elsif File.readlink(bundler_cache_dir) == cache_dir
+                        return
+                    end
                 end
 
                 FileUtils.rm_f bundler_cache_dir
@@ -151,7 +152,7 @@ module Autoproj
                 FileUtils.rm_f bundler_cache_dir if File.symlink?(bundler_cache_dir)
                 FileUtils.mkdir_p bundler_cache_dir
 
-                Dir.glob(File.join(cache_dir, '*.gem')) do |path_src|
+                Dir.glob(File.join(cache_dir, "*.gem")) do |path_src|
                     path_dest = File.join(bundler_cache_dir, File.basename(path_src))
                     next if File.exist?(path_dest)
 
@@ -161,7 +162,7 @@ module Autoproj
 
             # Enumerate the per-gem build configurations
             def self.per_gem_build_config(ws)
-                ws.config.get('bundler.build', {})
+                ws.config.get("bundler.build", {})
             end
 
             # Add new build configuration arguments for a given gem
@@ -169,9 +170,9 @@ module Autoproj
             # This is meant to be used from the Autoproj configuration files,
             # e.g. overrides.rb or package configuration
             def self.add_build_configuration_for(gem_name, build_config, ws: Autoproj.workspace)
-                c = ws.config.get('bundler.build', {})
+                c = ws.config.get("bundler.build", {})
                 c[gem_name] = [c[gem_name], build_config].compact.join(" ")
-                ws.config.set('bundler.build', c)
+                ws.config.set("bundler.build", c)
             end
 
             # Set the build configuration for the given gem
@@ -179,9 +180,9 @@ module Autoproj
             # This is meant to be used from the Autoproj configuration files,
             # e.g. overrides.rb or package configuration
             def self.configure_build_for(gem_name, build_config, ws: Autoproj.workspace)
-                c = ws.config.get('bundler.build', {})
+                c = ws.config.get("bundler.build", {})
                 c[gem_name] = build_config
-                ws.config.set('bundler.build', c)
+                ws.config.set("bundler.build", c)
             end
 
             # Removes build configuration flags for the given gem
@@ -189,9 +190,9 @@ module Autoproj
             # This is meant to be used from the Autoproj configuration files,
             # e.g. overrides.rb or package configuration
             def self.remove_build_configuration_for(gem_name, ws: Autoproj.workspace)
-                c = ws.config.get('bundler.build', {})
+                c = ws.config.get("bundler.build", {})
                 c.delete(gem_name)
-                ws.config.set('bundler.build', c)
+                ws.config.set("bundler.build", c)
             end
 
             # @api private
@@ -202,7 +203,7 @@ module Autoproj
             #   should be updated
             # @return [void]
             def self.apply_build_config(ws)
-                root_dir = File.join(ws.prefix_dir, 'gems')
+                root_dir = File.join(ws.prefix_dir, "gems")
                 current_config_path = File.join(root_dir, ".bundle", "config")
                 current_config =
                     if File.file?(current_config_path)
@@ -220,10 +221,10 @@ module Autoproj
                     next(line) unless (m = line.match(/BUNDLE_BUILD__(.*): "(.*)"$/))
                     next unless (desired_config = build_config.delete(m[1]))
 
-                    if m[2] != desired_config
-                        "BUNDLE_BUILD__#{m[1]}: \"#{desired_config}\""
-                    else
+                    if m[2] == desired_config
                         line
+                    else
+                        "BUNDLE_BUILD__#{m[1]}: \"#{desired_config}\""
                     end
                 end.compact
 
@@ -233,7 +234,7 @@ module Autoproj
 
                 if new_config != current_config
                     FileUtils.mkdir_p File.dirname(current_config_path)
-                    File.open(current_config_path, 'w') do |io|
+                    File.open(current_config_path, "w") do |io|
                         io.write new_config.join
                     end
                 end
@@ -249,28 +250,10 @@ module Autoproj
             # @param [Array<String>] system_rubylib the rubylib entries that are
             #   set by the underlying ruby interpreter itself
             def update_env_rubylib(bundle_rubylib, system_rubylib = discover_rubylib)
-                current = (ws.env.resolved_env['RUBYLIB'] || '')
+                current = (ws.env.resolved_env["RUBYLIB"] || "")
                           .split(File::PATH_SEPARATOR) + system_rubylib
                 (bundle_rubylib - current).each do |p|
-                    ws.env.add_path('RUBYLIB', p)
-                end
-            end
-
-            # @api private
-            #
-            # Parse an osdep entry into a gem name and gem version
-            #
-            # The 'gem' entries in the osdep files can contain a version
-            # specification. This method parses the two parts and return them
-            #
-            # @param [String] entry the osdep entry
-            # @return [(String,String),(String,nil)] the gem name, and an
-            #   optional version specification
-            def parse_package_entry(entry)
-                if entry =~ /^([^><=~]*)([><=~]+.*)$/
-                    [$1.strip, $2.strip]
-                else
-                    [entry]
+                    ws.env.add_path("RUBYLIB", p)
                 end
             end
 
@@ -320,21 +303,21 @@ module Autoproj
             )
                 FileUtils.rm "#{gemfile}.lock" if update && File.file?("#{gemfile}.lock")
 
-                options << '--path' << gem_path
+                options << "--path" << gem_path
                 options << "--shebang" << Gem.ruby
                 options << "--binstubs" << binstubs if binstubs
 
                 apply_build_config(ws)
 
                 connections = Set.new
-                run_bundler(ws, 'install', *options,
+                run_bundler(ws, "install", *options,
                             bundler_version: bundler_version,
                             gem_home: gem_home, gemfile: gemfile) do |line|
                     case line
                     when /Installing (.*)/
                         Autobuild.message "  bundler: installing #{$1}"
                     when /Fetching.*from (.*)/
-                        host = $1.gsub(/\.+$/, '')
+                        host = $1.gsub(/\.+$/, "")
                         unless connections.include?(host)
                             Autobuild.message "  bundler: connected to #{host}"
                             connections << host
@@ -344,41 +327,42 @@ module Autoproj
             end
 
             def self.bundle_gem_path(ws, gem_name,
-                                     bundler_version: ws.config.bundler_version,
-                                     gem_home: nil, gemfile: nil)
+                bundler_version: ws.config.bundler_version,
+                gem_home: nil, gemfile: nil)
                 path = String.new
                 run_bundler(
-                    ws, 'show', gem_name,
+                    ws, "show", gem_name,
                     bundler_version: bundler_version, gem_home: gem_home,
-                    gemfile: gemfile) { |line| path << line }
+                    gemfile: gemfile
+                ) { |line| path << line }
                 path.chomp
             end
 
             def self.default_bundler(ws)
-                File.join(ws.dot_autoproj_dir, 'bin', 'bundle')
+                File.join(ws.dot_autoproj_dir, "bin", "bundle")
             end
 
             def self.run_bundler(ws, *commandline,
-                                 bundler_version: ws.config.bundler_version,
-                                 gem_home: ws.config.gems_gem_home,
-                                 gemfile: default_gemfile_path(ws))
-                bundle = Autobuild.programs['bundle'] || default_bundler(ws)
+                bundler_version: ws.config.bundler_version,
+                gem_home: ws.config.gems_gem_home,
+                gemfile: default_gemfile_path(ws))
+                bundle = Autobuild.programs["bundle"] || default_bundler(ws)
 
                 Autoproj.bundler_with_unbundled_env do
                     bundler_version_env =
                         if bundler_version
-                            { 'BUNDLER_VERSION' => bundler_version }
+                            { "BUNDLER_VERSION" => bundler_version }
                         else
                             {}
                         end
                     target_env = Hash[
-                        'GEM_HOME' => gem_home,
-                        'GEM_PATH' => nil,
-                        'BUNDLE_GEMFILE' => gemfile,
-                        'RUBYOPT' => nil,
-                        'RUBYLIB' => rubylib_for_bundler,
+                        "GEM_HOME" => gem_home,
+                        "GEM_PATH" => nil,
+                        "BUNDLE_GEMFILE" => gemfile,
+                        "RUBYOPT" => nil,
+                        "RUBYLIB" => rubylib_for_bundler,
                     ].merge(bundler_version_env)
-                    ws.run('autoproj', 'osdeps',
+                    ws.run("autoproj", "osdeps",
                            bundle, *commandline,
                            working_directory: File.dirname(gemfile),
                            env: target_env) { |line| yield(line) if block_given? }
@@ -386,7 +370,7 @@ module Autoproj
             end
 
             # Parse the contents of a gemfile into a set of
-            def merge_gemfiles(*path, unlock: [])
+            def merge_gemfiles(*path, ruby_version: nil, unlock: [])
                 gems_remotes = Set.new
                 dependencies = Hash.new do |h, k|
                     h[k] = Hash.new do |i, j|
@@ -400,21 +384,21 @@ module Autoproj
                         begin Bundler::Dsl.evaluate(gemfile, nil, [])
                         rescue Exception => e
                             cleaned_message = e
-                                .message
-                                .gsub(/There was an error parsing([^:]+)/,
-                                      "Error in gem definitions")
-                                .gsub(/#  from.*/, '')
+                                              .message
+                                              .gsub(/There was an error parsing([^:]+)/,
+                                                    "Error in gem definitions")
+                                              .gsub(/#  from.*/, "")
                             raise ConfigError, cleaned_message
                         end
                     gems_remotes |= bundler_def.send(:sources).rubygems_remotes.to_set
                     bundler_def.dependencies.each do |d|
                         d.groups.each do |group_name|
-                            if !d.platforms.empty?
+                            if d.platforms.empty?
+                                dependencies[group_name][""][d.name] = d
+                            else
                                 d.platforms.each do |platform_name|
                                     dependencies[group_name][platform_name][d.name] = d
                                 end
-                            else
-                                dependencies[group_name][''][d.name] = d
                             end
                         end
                     end
@@ -423,9 +407,12 @@ module Autoproj
                 contents = []
                 gems_remotes.each do |g|
                     g = g.to_s
-                    g = g[0..-2] if g.end_with?('/')
+                    g = g[0..-2] if g.end_with?("/")
                     contents << "source '#{g}'"
                 end
+                if ruby_version
+                    contents << "ruby \"#{ruby_version}\" if respond_to?(:ruby)"
+                end
                 valid_keys = %w[group groups git path glob name branch ref tag
                                 require submodules platform platforms type
                                 source install_if]
@@ -444,11 +431,11 @@ module Autoproj
                                 options = options.map { |k, v| "#{k}: \"#{v}\"" }
                             end
                             contents << ["  #{platform_indent}gem \"#{d.name}\",
-                                         \"#{d.requirement}\"", *options].join(', ')
+                                         \"#{d.requirement}\"", *options].join(", ")
                         end
-                        contents << '  end' unless platform_name.empty?
+                        contents << "  end" unless platform_name.empty?
                     end
-                    contents << 'end'
+                    contents << "end"
                 end
                 contents.join("\n")
             end
@@ -456,7 +443,7 @@ module Autoproj
             def workspace_configuration_gemfiles
                 gemfiles = []
                 ws.manifest.each_package_set do |source|
-                    pkg_set_gemfile = File.join(source.local_dir, 'Gemfile')
+                    pkg_set_gemfile = File.join(source.local_dir, "Gemfile")
                     if source.local_dir && File.file?(pkg_set_gemfile)
                         gemfiles << pkg_set_gemfile
                     end
@@ -469,7 +456,58 @@ module Autoproj
             end
 
             def self.default_gemfile_path(ws)
-                File.join(ws.prefix_dir, 'gems', 'Gemfile')
+                File.join(ws.prefix_dir, "gems", "Gemfile")
+            end
+
+            GemEntry = Struct.new :name, :version, :options do
+                def self.parse(object)
+                    if object.respond_to?(:to_str)
+                        parse_from_string(object)
+                    elsif object.respond_to?(:to_hash)
+                        parse_from_hash(object)
+                    else
+                        raise ArgumentError,
+                              "expected #{object} to either be a string or a map"
+                    end
+                end
+
+                # Parse an osdep entry string into a gem name and gem version
+                #
+                # The 'gem' entries in the osdep files can contain a version
+                # specification. This method parses the two parts and return them
+                #
+                # @param [String] entry the osdep entry
+                # @return [(String,String),(String,nil)] the gem name, and an
+                #   optional version specification
+                def self.parse_from_string(entry)
+                    if entry =~ /^([^><=~]*)([><=~]+.*)$/
+                        GemEntry.new($1.strip, $2.strip, {})
+                    else
+                        GemEntry.new(entry, nil, {})
+                    end
+                end
+
+                # Parse an option hash into a GemEntry
+                def self.parse_from_hash(hash)
+                    hash = hash.dup
+                    unless (name = hash.delete("name"))
+                        raise ArgumentError,
+                              "expected gem entry #{hash} to have at least a 'name' key"
+                    end
+
+                    version = hash.delete("version")
+                    GemEntry.new(name, version, hash)
+                end
+
+                def to_gemfile_line
+                    options_s = options.map { |k, v| "#{k}: \"#{v}\"" }.join(", ")
+                    entries = [
+                        "\"#{name}\"",
+                        ("\"#{version}\"" if version),
+                        (options_s unless options_s.empty?)
+                    ].compact
+                    "gem #{entries.join(', ')}"
+                end
             end
 
             def install(gems, filter_uptodate_packages: false, install_only: false)
@@ -486,40 +524,38 @@ module Autoproj
                 backup_files(backups)
                 unless File.file?("#{gemfile_path}.orig")
                     Ops.atomic_write("#{gemfile_path}.orig") do |io|
-                        dot_autoproj_gemfile = File.join(ws.dot_autoproj_dir, 'Gemfile')
+                        dot_autoproj_gemfile = File.join(ws.dot_autoproj_dir, "Gemfile")
                         io.puts "eval_gemfile \"#{dot_autoproj_gemfile}\""
                     end
                 end
 
-                gemfiles = workspace_configuration_gemfiles
-                gemfiles << File.join(ws.dot_autoproj_dir, 'Gemfile')
+                gemfiles = []
 
-                # Save the osdeps entries in a temporary gemfile and finally
-                # merge the whole lot of it
-                gemfile_contents = Tempfile.open 'autoproj-gemfile' do |io|
-                    gems.sort.each do |name|
-                        name, version = parse_package_entry(name)
-                        io.puts "gem \"#{name}\", \"#{version || '>= 0'}\""
-                    end
-                    io.flush
-                    gemfiles.unshift io.path
-                    # The autoproj gemfile needs to be last, we really don't
-                    # want to mess it up
-                    merge_gemfiles(*gemfiles)
+                unless gems.empty?
+                    osdeps_gemfile_io = prepare_osdeps_gemfile(gems)
+                    gemfiles << osdeps_gemfile_io.path
                 end
 
+                gemfiles += workspace_configuration_gemfiles
+                # The autoproj gemfile needs to be last, we really don't
+                # want to mess it up
+                gemfiles << File.join(ws.dot_autoproj_dir, "Gemfile")
+
+                ruby_version = RUBY_VERSION.gsub(/\.\d+$/, ".0")
+                gemfile_contents =
+                    merge_gemfiles(*gemfiles, ruby_version: "~> #{ruby_version}")
+
                 FileUtils.mkdir_p root_dir
                 updated = (!File.exist?(gemfile_path) ||
                            File.read(gemfile_path) != gemfile_contents)
                 if updated
                     Ops.atomic_write(gemfile_path) do |io|
-                        io.puts "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)"
                         io.puts gemfile_contents
                     end
                 end
 
-                options = Array.new
-                binstubs_path = File.join(root_dir, 'bin')
+                options = []
+                binstubs_path = File.join(root_dir, "bin")
                 if updated || !install_only || !File.file?("#{gemfile_path}.lock")
                     self.class.run_bundler_install(ws, gemfile_path, *options,
                                                    binstubs: binstubs_path)
@@ -539,20 +575,40 @@ module Autoproj
                 raise
             ensure
                 if binstubs_path
-                    FileUtils.rm_f File.join(binstubs_path, 'bundle')
-                    FileUtils.rm_f File.join(binstubs_path, 'bundler')
+                    FileUtils.rm_f File.join(binstubs_path, "bundle")
+                    FileUtils.rm_f File.join(binstubs_path, "bundler")
                 end
                 backup_clean(backups)
             end
 
+            # Prepare a Gemfile that contains osdeps gem declarations
+            #
+            # @param [Array<String,Hash>] gems osdeps declarations as understood
+            #   by {GemEntry.parse}
+            # @return [File]
+            def prepare_osdeps_gemfile(gems)
+                io = Tempfile.open "autoproj-gemfile"
+                io.puts "source \"https://rubygems.org\""
+                gems.map { |entry| GemEntry.parse(entry) }
+                    .sort_by(&:name)
+                    .each { |entry| io.puts entry.to_gemfile_line }
+
+                io.flush
+                io
+            rescue Exception
+                io&.close
+                raise
+            end
+
             def discover_rubylib
-                require 'bundler'
-                Tempfile.open 'autoproj-rubylib' do |io|
+                require "bundler"
+                Tempfile.open "autoproj-rubylib" do |io|
                     result = Autoproj.bundler_unbundled_system(
-                        Hash['RUBYLIB' => nil],
-                        Autobuild.tool('ruby'), '-e', 'puts $LOAD_PATH',
+                        Hash["RUBYLIB" => nil],
+                        Autobuild.tool("ruby"), "-e", "puts $LOAD_PATH",
                         out: io,
-                        err: '/dev/null')
+                        err: "/dev/null"
+                    )
                     if result
                         io.rewind
                         io.readlines.map(&:chomp).find_all { |l| !l.empty? }
@@ -566,19 +622,20 @@ module Autoproj
             end
 
             def discover_bundle_rubylib(silent_errors: false)
-                require 'bundler'
-                gemfile = File.join(ws.prefix_dir, 'gems', 'Gemfile')
+                require "bundler"
+                gemfile = File.join(ws.prefix_dir, "gems", "Gemfile")
                 silent_redirect = Hash.new
                 silent_redirect[:err] = :close if silent_errors
                 env = ws.env.resolved_env
-                Tempfile.open 'autoproj-rubylib' do |io|
+                Tempfile.open "autoproj-rubylib" do |io|
                     result = Autoproj.bundler_unbundled_system(
-                        Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'],
-                             'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil,
-                             'RUBYLIB' => self.class.rubylib_for_bundler],
-                        Autobuild.tool('ruby'), '-rbundler/setup',
-                        '-e', 'puts $LOAD_PATH',
-                        out: io, **silent_redirect)
+                        Hash["GEM_HOME" => env["GEM_HOME"], "GEM_PATH" => env["GEM_PATH"],
+                             "BUNDLE_GEMFILE" => gemfile, "RUBYOPT" => nil,
+                             "RUBYLIB" => self.class.rubylib_for_bundler],
+                        Autobuild.tool("ruby"), "-rbundler/setup",
+                        "-e", "puts $LOAD_PATH",
+                        out: io, **silent_redirect
+                    )
 
                     if result
                         io.rewind