autoproj 2.0.0.rc32 → 2.0.0.rc33

data/lib/autoproj/autobuild.rb

@@ -103,7 +103,7 @@ def post_import(&block)
         end
 
         def autoproj_name # :nodoc:
-            srcdir.gsub /^#{Regexp.quote(Autoproj.root_dir)}\//, ''
+            srcdir.gsub(/^#{Regexp.quote(Autoproj.root_dir)}\//, '')
         end
 
         alias __depends_on__ depends_on

data/lib/autoproj/configuration.rb

@@ -217,28 +217,6 @@ def self.gems_path_suffix
                 relative_path_from(Pathname.new(dot_gem_dir)).to_s
         end
 
-        # The gem install root into which the bundler and autoproj gems, as well
-        # as autoproj's dependencies, are installed
-        #
-        # Note that while this setting is separated from the other gems path,
-        # the only way to reliably isolate the gems of an autoproj workspace is
-        # to separate both the autoproj gems and the workspace gems.
-        #
-        # The gems are actually installed under a platform and version-specific
-        # subdirectory (returned by {#gems_path_suffix})
-        #
-        # @return [String]
-        def autoproj_install_path
-            get('autoproj_install_path')
-        end
-
-        # The GEM_HOME into which the autoproj gem and its dependencies are installed
-        #
-        # @return [String]
-        def autoproj_gem_home
-            File.join(autoproj_install_path, self.class.gems_path_suffix)
-        end
-
         # The gem install root into which the workspace gems are installed
         #
         # Note that while this setting is separated from the other gems path,
@@ -251,16 +229,16 @@ def autoproj_gem_home
         #
         # @param [Workspace] ws the workspace whose gems are being considered
         # @return [String]
-        def gems_install_path(ws)
-            File.expand_path(get('gems_install_path'), ws.prefix_dir)
+        def gems_install_path
+            get('gems_install_path')
         end
 
         # The GEM_HOME into which the workspace gems are installed
         #
         # @param [Workspace] ws the workspace whose gems are being considered
         # @return [String]
-        def gems_gem_home(ws)
-            File.join(gems_install_path(ws), self.class.gems_path_suffix)
+        def gems_gem_home
+            File.join(gems_install_path, self.class.gems_path_suffix)
         end
 
         # The full path to the expected ruby executable

data/lib/autoproj/ops/install.rb

@@ -31,12 +31,8 @@ class UnexpectedBinstub < RuntimeError; end
 
             def initialize(root_dir)
                 @root_dir = root_dir
-                if File.file?(autoproj_gemfile_path)
-                    @gemfile = File.read(autoproj_gemfile_path)
-                else
-                    @gemfile = default_gemfile_contents
-                end
                 @gem_source = "https://rubygems.org"
+                @gemfile = nil
 
                 @autoproj_options = Array.new
 
@@ -56,7 +52,6 @@ def initialize(root_dir)
 
                 default_gem_path = File.join(Dir.home, '.autoproj', 'gems')
                 @gems_install_path     = default_gem_path
-                @autoproj_install_path = default_gem_path
             end
 
             def env_for_child
@@ -126,33 +121,10 @@ def gem_path_suffix
                     relative_path_from(Pathname.new(dot_gem_dir)).to_s
             end
 
-            # The path into which autoproj and its dependencies should be installed
-            #
-            # They are installed in a versioned subdirectory of this path, e.g.
-            # {#gem_path_suffix}. It is always absolute.
-            #
-            # @return [String]
-            attr_reader :autoproj_install_path
-            # The GEM_HOME into which the autoproj gems should be installed
-            def autoproj_gem_home; File.join(autoproj_install_path, gem_path_suffix) end
-            # Sets the place where autoproj should be installed
-            #
-            # @param [String] path Sets the path given to bundler, i.e. the
-            #   gems will be installed under the {#gem_path_suffix}
-            def autoproj_install_path=(path)
-                @autoproj_install_path = File.expand_path(path)
-            end
-            # Install autoproj in Gem's default user dir
-            def install_autoproj_in_gem_user_dir
-                @autoproj_install_path = File.join(Gem.user_home, '.gem')
-            end
-
             # The path into which the workspace's gems should be installed
             #
             # They are installed in a versioned subdirectory of this path, e.g.
-            # {#gem_path_suffix}. Unlike {#autoproj_install_path}, it can be
-            # relative, in which case it is relative to the workspace's prefix
-            # directory.
+            # {#gem_path_suffix}.
             # 
             # @return [String]
             attr_reader :gems_install_path
@@ -180,7 +152,7 @@ def prefer_indep_over_os_packages?; @prefer_indep_over_os_packages end
             # (see #prefer_index_over_os_packages?)
             def prefer_indep_over_os_packages=(flag); @prefer_indep_over_os_packages = !!flag end
 
-            def guess_gem_program
+            def self.guess_gem_program
                 ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME']
                 ruby_bindir = RbConfig::CONFIG['bindir']
 
@@ -215,25 +187,24 @@ def parse_options(args = ARGV)
                         @local = true
                     end
                     opt.on '--gem-source=URL', String, "use this source for RubyGems instead of rubygems.org" do |url|
-                        self.gem_source = url
+                        @gem_source = url
                     end
-                    opt.on '--shared-gems[=PATH]', "install gems in a shared location. By default,  uses the default RubyGems locations" do |path|
-                        if path
-                            self.autoproj_install_path = path
-                            self.gems_install_path     = path
-                        else
-                            self.install_autoproj_in_gem_user_dir
-                            self.install_gems_in_gem_user_dir
-                        end
+                    opt.on '--gems-path=PATH', "install gems under this path instead of ~/.autoproj/gems" do |path|
+                        self.gems_install_path     = path
                     end
-                    opt.on '--private-gems', "install bundler, autoproj and the workspace gems in dedicated locations within the workspace" do |path|
-                        self.autoproj_install_path = File.join(dot_autoproj, 'gems')
-                        self.gems_install_path     = 'gems'
+                    opt.on '--public-gems', "install gems in the default gem location" do
+                        self.install_gems_in_gem_user_dir
                     end
                     opt.on '--version=VERSION_CONSTRAINT', String, 'use the provided string as a version constraint for autoproj' do |version|
+                        if @gemfile
+                            raise "cannot give both --version and --gemfile"
+                        end
                         @gemfile = default_gemfile_contents(version)
                     end
                     opt.on '--gemfile=PATH', String, 'use the given Gemfile to install autoproj instead of the default' do |path|
+                        if @gemfile
+                            raise "cannot give both --version and --gemfile"
+                        end
                         @gemfile = File.read(path)
                     end
                     opt.on '--seed-config=PATH', String, 'path to a seed file that should be used to initialize the configuration' do |path|
@@ -264,7 +235,7 @@ def find_bundler(gem_program)
                     out: '/dev/null')
                 return if !result
 
-                bundler_path = File.join(autoproj_gem_home, 'bin', 'bundler')
+                bundler_path = File.join(gems_gem_home, 'bin', 'bundler')
                 if File.exist?(bundler_path)
                     bundler_path
                 end
@@ -277,14 +248,14 @@ def install_bundler(gem_program)
                     env_for_child,
                     Gem.ruby, gem_program, 'install', '--env-shebang', '--no-document', '--no-format-executable', '--clear-sources', '--source', gem_source,
                         *local,
-                        "--bindir=#{File.join(autoproj_gem_home, 'bin')}", 'bundler')
+                        "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler')
 
                 if !result
-                    STDERR.puts "FATAL: failed to install bundler in #{autoproj_gem_home}"
+                    STDERR.puts "FATAL: failed to install bundler in #{gems_gem_home}"
                     nil
                 end
 
-                bundler_path = File.join(autoproj_gem_home, 'bin', 'bundler')
+                bundler_path = File.join(gems_gem_home, 'bin', 'bundler')
                 if File.exist?(bundler_path)
                     bundler_path
                 else
@@ -305,7 +276,7 @@ def install_autoproj(bundler)
 
                 opts = Array.new
                 opts << '--local' if local?
-                opts << "--path=#{autoproj_install_path}"
+                opts << "--path=#{gems_install_path}"
                 shims_path = File.join(dot_autoproj, 'bin')
                 result = system(clean_env,
                     Gem.ruby, bundler, 'install',
@@ -319,39 +290,69 @@ def install_autoproj(bundler)
                     exit 1
                 end
             ensure
-                self.class.clean_binstubs(shims_path, ruby_executable, bundler)
+                self.class.rewrite_shims(shims_path, ruby_executable, autoproj_gemfile_path, gems_gem_home)
             end
 
-            def self.clean_binstubs(shim_path, ruby_executable, bundler_path)
-                %w{bundler bundle rake thor ruby}.each do |bundler_bin|
-                    path = File.join(shim_path, bundler_bin)
-                    if File.file?(path)
-                        FileUtils.rm path
-                    end
+            def self.rewrite_shims(shim_path, ruby_executable, autoproj_gemfile_path, gems_gem_home)
+                FileUtils.mkdir_p shim_path
+                File.open(File.join(shim_path, 'ruby'), 'w') do |io|
+                    io.puts "#! /bin/sh"
+                    io.puts "exec #{ruby_executable} \"$@\""
                 end
+                FileUtils.chmod 0755, File.join(shim_path, 'ruby')
 
-                # Now tune the binstubs to force the usage of the autoproj
-                # gemfile. Otherwise, they get BUNDLE_GEMFILE from the
-                # environment by default
-                Dir.glob(File.join(shim_path, '*')) do |path|
-                    next if !File.file?(path)
-
-                    lines = File.readlines(path)
-                    matched = false
-                    filtered = lines.map do |l|
-                        matched ||= (ENV_BUNDLE_GEMFILE_RX === l)
-                        l.gsub(ENV_BUNDLE_GEMFILE_RX, '\\1=')
-                    end
-                    if !matched
-                        raise UnexpectedBinstub, "expected #{path} to contain a line looking like ENV['BUNDLE_GEMFILE'] ||= but could not find one"
-                    end
-                    File.open(path, 'w') do |io|
-                        io.write filtered.join("")
+                FileUtils.touch File.join(shim_path, 'bundler')
+                FileUtils.touch File.join(shim_path, 'bundle')
+                Dir.glob(File.join(shim_path, '*')) do |bin_script|
+                    next if !File.file?(bin_script)
+                    bin_name = File.basename(bin_script)
+                    next if bin_name == 'ruby'
+
+                    bin_shim = File.join(shim_path, bin_name)
+                    bin_script_lines = File.readlines(bin_script)
+                    File.open(bin_shim, 'w') do |io|
+                        if bin_name == 'bundler' || bin_name == 'bundle'
+                            io.puts shim_bundler(ruby_executable, autoproj_gemfile_path, gems_gem_home)
+                        else
+                            load_line = bin_script_lines.grep(/load Gem.bin_path/).first
+                            io.puts shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load_line)
+                        end
                     end
+                    FileUtils.chmod 0755, bin_shim
                 end
+            end
 
-            ensure
-                save_ruby_and_bundler_shims(shim_path, ruby_executable, bundler_path)
+            def self.shim_bundler(ruby_executable, autoproj_gemfile_path, gems_gem_home)
+"#! #{ruby_executable}
+
+if defined?(Bundler)
+    Bundler.with_clean_env do
+        exec($0, *ARGV)
+    end
+end
+
+ENV.delete('BUNDLE_GEMFILE')
+ENV['GEM_HOME'] = '#{gems_gem_home}'
+ENV.delete('GEM_PATH')
+Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => '']
+
+load Gem.bin_path('bundler', 'bundler')"
+            end
+            
+            def self.shim_script(ruby_executable, autoproj_gemfile_path, gems_gem_home, load_line)
+"#! #{ruby_executable}
+
+if defined?(Bundler)
+    Bundler.with_clean_env do
+        exec($0, *ARGV)
+    end
+end
+
+ENV['BUNDLE_GEMFILE'] = '#{autoproj_gemfile_path}'
+require 'rubygems'
+Gem.paths = Hash['GEM_HOME' => '#{gems_gem_home}', 'GEM_PATH' => '']
+require 'bundler/setup'
+#{load_line}"
             end
 
             def save_env_sh(*vars)
@@ -382,6 +383,15 @@ def save_env_sh(*vars)
             end
 
             def save_gemfile
+                gemfile =
+                    if @gemfile
+                        @gemfile
+                    elsif File.file?(autoproj_gemfile_path)
+                        File.read(autoproj_gemfile_path)
+                    else
+                        default_gemfile_contents
+                    end
+
                 FileUtils.mkdir_p File.dirname(autoproj_gemfile_path)
                 File.open(autoproj_gemfile_path, 'w') do |io|
                     io.write gemfile
@@ -424,46 +434,32 @@ def gem_bindir
                 end
             end
 
-            def self.save_ruby_and_bundler_shims(shim_path, ruby_executable, bundler_path)
-                FileUtils.mkdir_p shim_path
-                bundler_rubylib = File.expand_path(File.join('..', '..', 'lib'), bundler_path)
-                File.open(File.join(shim_path, 'bundler'), 'w') do |io|
-                    io.puts "#! /bin/sh"
-                    io.puts "exec #{ruby_executable} #{bundler_path} \"$@\""
-                end
-                FileUtils.chmod 0755, File.join(shim_path, 'bundler')
-                File.open(File.join(shim_path, 'ruby'), 'w') do |io|
-                    io.puts "#! /bin/sh"
-                    io.puts "exec #{ruby_executable} \"$@\""
-                end
-                FileUtils.chmod 0755, File.join(shim_path, 'ruby')
-            end
-
             def install
                 if ENV['BUNDLER_GEMFILE']
                     raise "cannot run autoproj_install or autoproj_bootstrap while under a 'bundler exec' subcommand or having loaded an env.sh. Open a new console and try again"
                 end
 
-                gem_program  = guess_gem_program
+                gem_program  = self.class.guess_gem_program
                 puts "Detected 'gem' to be #{gem_program}"
-                env['GEM_HOME'] = [autoproj_gem_home]
+                env['GEM_HOME'] = [gems_gem_home]
 
                 if bundler = find_bundler(gem_program)
                     puts "Detected bundler at #{bundler}"
                 else
-                    puts "Installing bundler in #{autoproj_gem_home}"
+                    puts "Installing bundler in #{gems_gem_home}"
                     if !(bundler = install_bundler(gem_program))
                         exit 1
                     end
                 end
-                self.class.save_ruby_and_bundler_shims(
+                self.class.rewrite_shims(
                     File.join(dot_autoproj, 'bin'),
                     ruby_executable,
-                    bundler)
+                    autoproj_gemfile_path,
+                    gems_gem_home)
                 env['PATH'].unshift File.join(dot_autoproj, 'bin')
                 save_gemfile
 
-                puts "Installing autoproj in #{autoproj_gem_home}"
+                puts "Installing autoproj in #{gems_gem_home}"
                 install_autoproj(bundler)
             end
 
@@ -490,13 +486,12 @@ def load_config
                 end
 
                 @config = config
-                %w{autoproj_install_path gems_install_path prefer_indep_over_os_packages}.each do |flag|
+                %w{gems_install_path prefer_indep_over_os_packages}.each do |flag|
                     instance_variable_set "@#{flag}", config.fetch(flag, false)
                 end
             end
 
             def save_config
-                config['autoproj_install_path'] = autoproj_install_path
                 config['gems_install_path']     = gems_install_path
                 config['prefer_indep_over_os_packages'] = prefer_indep_over_os_packages?
                 File.open(autoproj_config_path, 'w') { |io| YAML.dump(config, io) }
@@ -541,13 +536,13 @@ def stage2(*vars)
                 require 'autobuild'
                 puts "saving env.sh and .autoproj/env.sh"
                 save_env_sh(*vars)
-                if !system(Gem.ruby, autoproj_path, 'envsh', *autoproj_options)
+                if !system(Gem.ruby, autoproj_path, 'envsh', *autoproj_options, '--debug')
                     STDERR.puts "failed to run autoproj envsh on the newly installed autoproj (#{autoproj_path})"
                     exit 1
                 end
                 # This is really needed on an existing install to install the
                 # gems that were present in the v1 layout
-                if !system(Gem.ruby, autoproj_path, 'osdeps')
+                if !system(Gem.ruby, autoproj_path, 'osdeps', '--debug')
                     STDERR.puts "failed to run autoproj osdeps on the newly installed autoproj (#{autoproj_path})"
                     exit 1
                 end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -38,8 +38,8 @@ def initialize_environment
 
                 env.add_path 'PATH', File.join(ws.prefix_dir, 'gems', 'bin')
                 env.add_path 'PATH', File.join(ws.dot_autoproj_dir, 'bin')
-                env.set 'GEM_HOME', config.gems_gem_home(ws)
-                env.set 'GEM_PATH', config.autoproj_gem_home
+                env.set 'GEM_HOME', config.gems_gem_home
+                env.clear 'GEM_PATH'
 
                 gemfile_path = File.join(ws.prefix_dir, 'gems', 'Gemfile')
                 if File.file?(gemfile_path)
@@ -128,34 +128,54 @@ def backup_clean(mapping)
                 end
             end
 
-            def self.run_bundler_install(ws, gemfile, *options, update: true, binstubs: nil)
+            def self.run_bundler_install(ws, gemfile, *options, update: true, binstubs: nil, gem_path: ws.config.gems_install_path)
                 if update && File.file?("#{gemfile}.lock")
                     FileUtils.rm "#{gemfile}.lock"
                 end
 
+                options << '--path' << gem_path
                 options << "--shebang" << Gem.ruby
                 if binstubs
                     options << "--binstubs" << binstubs
                 end
 
+                connections = Set.new
+                run_bundler(ws, 'install', *options, gemfile: gemfile) do |line|
+                    case line
+                    when /Installing (.*)/
+                        Autobuild.message "  bundler: installing #{$1}"
+                    when /Fetching.*from (.*)/
+                        host = $1.gsub(/\.+$/, '')
+                        if !connections.include?(host)
+                            Autobuild.message "  bundler: connected to #{host}"
+                            connections << host
+                        end
+                    end
+                end
+            end
+
+            def self.bundle_gem_path(ws, gem_name, gemfile: nil)
+                path = String.new
+                PackageManagers::BundlerManager.run_bundler(ws, 'show', gem_name, gemfile: gemfile) do |line|
+                    path << line
+                end
+                path.chomp
+            end
+
+            def self.run_bundler(ws, *commandline, gemfile: nil)
                 Bundler.with_clean_env do
-                    connections = Set.new
+                    target_env = Hash[
+                        'GEM_HOME' => nil,
+                        'GEM_PATH' => nil,
+                        'BUNDLE_GEMFILE' => gemfile,
+                        'RUBYOPT' => nil,
+                        'RUBYLIB' => nil
+                    ]
                     ws.run 'autoproj', 'osdeps',
-                        Autobuild.tool('bundler'), 'install',
-                            *options,
-                            working_directory: File.dirname(gemfile), env: Hash['BUNDLE_GEMFILE' => nil, 'RUBYOPT' => nil] do |line|
-
-                        case line
-                        when /Installing (.*)/
-                            Autobuild.message "  bundler: installing #{$1}"
-                        when /Fetching.*from (.*)/
-                            host = $1.gsub(/\.+$/, '')
-                            if !connections.include?(host)
-                                Autobuild.message "  bundler: connected to #{host}"
-                                connections << host
-                            end
+                        Autobuild.tool('bundler'), *commandline,
+                        working_directory: File.dirname(gemfile), env: target_env do |line|
+                            yield(line) if block_given?
                         end
-                    end
                 end
             end
 
@@ -313,12 +333,13 @@ def discover_bundle_rubylib(silent_errors: false)
                 gemfile = File.join(ws.prefix_dir, 'gems', 'Gemfile')
                 silent_redirect = Hash.new
                 if silent_errors
-                    silent_redirect[:err] = '/dev/null'
+                    silent_redirect[:err] = :close
                 end
+                env = ws.env.resolved_env
                 Tempfile.open 'autoproj-rubylib' do |io|
                     result = Bundler.clean_system(
-                        Hash['BUNDLE_GEMFILE' => gemfile, 'RUBYLIB' => nil],
-                        Autobuild.tool('bundler'), 'exec', Autobuild.tool('ruby'), '-rbundler/setup', '-e', 'puts $LOAD_PATH',
+                        Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'], 'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil, 'RUBYLIB' => nil],
+                        Autobuild.tool('ruby'), '-rbundler/setup', '-e', 'puts $LOAD_PATH',
                         out: io, **silent_redirect)
                         
                     if result