autoproj 2.0.0.rc5 → 2.0.0.rc6

data/bin/autoproj_install.in

@@ -11,4 +11,4 @@ ENV.delete('BUNDLE_GEMFILE')
 ENV.delete('RUBYLIB')
 ops = Autoproj::Ops::Install.new(Dir.pwd)
 ops.parse_options(ARGV)
-ops.run
+ops.stage1

data/lib/autoproj/autobuild.rb

@@ -469,7 +469,7 @@ def only_on(*architectures)
         end
     end
 
-    os_names, os_versions = Autoproj::OSPackageResolver.operating_system
+    os_names, os_versions = Autoproj.workspace.operating_system
     matching_archs = architectures.find_all { |arch| os_names.include?(arch[0].downcase) }
     if matching_archs.empty?
         return
@@ -494,7 +494,7 @@ def not_on(*architectures)
         end
     end
 
-    os_names, os_versions = Autoproj::OSPackageResolver.operating_system
+    os_names, os_versions = Autoproj.workspace.operating_system
     matching_archs = architectures.find_all { |arch| os_names.include?(arch[0].downcase) }
     if matching_archs.empty?
         return yield

data/lib/autoproj/cli/main.rb

@@ -315,24 +315,24 @@ def query(query_string = nil)
                 desc: 'do not access the network (will fail if some gems are missing)'
             option :gemfile, type: :string,
                 desc: 'path to a gemfile that should be used to install autoproj'
-            option :private_bundler, type: :boolean,
-                desc: 'install bundler inside the workspace instead of using the default Gem location'
-            option :private_gems, type: :boolean,
-                desc: 'install gems inside the workspace instead of using the default Gem location'
-            option :private_autoproj, type: :boolean,
-                desc: 'install bundler inside the workspace instead of using the default Gem location'
-            option :private, type: :boolean,
-                desc: 'equivalent to --private-bundler --private-autoproj --private-gems'
             def upgrade
-                require 'autoproj/cli/upgrade'
-                Autoproj::CLI::Upgrade.new.run(options)
+                require 'autoproj/ops/install'
+                installer = Autoproj::Ops::Install.new(Dir.pwd)
+
+                if options.has_key?('local')
+                    installer.local = options['local']
+                end
+                if options[:gemfile]
+                    installer.gemfile = File.read(options[:gemfile])
+                end
+                installer.stage1
             end
 
-            desc 'install_stage2 ROOT_DIR', 'used by autoproj_install to finalize the installation',
+            desc 'install_stage2 ROOT_DIR [ENVVAR=VALUE ...]', 'used by autoproj_install to finalize the installation',
                 hide: true
-            def install_stage2(root_dir)
+            def install_stage2(root_dir, *vars)
                 require 'autoproj/ops/install'
-                Autoproj::Ops::Install.new(root_dir).run(stage2: true)
+                Autoproj::Ops::Install.new(root_dir).stage2(*vars)
             end
         end
     end

data/lib/autoproj/cli/switch_config.rb

@@ -1,5 +1,6 @@
 require 'autoproj'
 require 'autoproj/cli/base'
+require 'autoproj/cli/update'
 require 'autoproj/ops/main_config_switcher'
 require 'autoproj/ops/configuration'
 
@@ -7,18 +8,22 @@ module Autoproj
     module CLI
         class SwitchConfig < Base
             def run(args, options = Hash.new)
-                if Dir.pwd.start_with?(ws.remotes_dir) || Dir.pwd.start_with?(ws.config_dir)
+                if !File.directory?(ws.config_dir)
+                    raise ConfigError, "there's no autoproj/ directory in this workspace, use autoproj bootstrap to check out one"
+                elsif Dir.pwd.start_with?(ws.remotes_dir) || Dir.pwd.start_with?(ws.config_dir)
                     raise ConfigError, "you cannot run autoproj switch-config from autoproj's configuration directory or one of its subdirectories"
                 end
 
                 ws.load_config
+                ws.setup_os_package_installer
 
                 # We must switch to the root dir first, as it is required by the
                 # configuration switch code. This is acceptable as long as we
                 # quit just after the switch
                 switcher = Ops::MainConfigSwitcher.new(ws)
                 if switcher.switch_config(*args)
-                    CLI::Main.start(['update', '--config'])
+                    updater = Update.new(ws)
+                    updater.run([], config: true)
                 end
             end
         end

data/lib/autoproj/cli/update.rb

@@ -49,14 +49,7 @@ def run(selected_packages, options)
                 ws.setup
                 parallel = options[:parallel] || ws.config.parallel_import_level
 
-                # Do that AFTER we have properly setup ws.osdeps as to avoid
-                # unnecessarily redetecting the operating system
-                if options[:osdeps]
-                    ws.config.set(
-                        'operating_system',
-                        Autoproj::OSPackageResolver.operating_system(:force => true),
-                        true)
-                end
+                ws.autodetect_operating_system(force: true)
 
                 if options[:autoproj]
                     ws.update_autoproj

data/lib/autoproj/configuration.rb

@@ -429,6 +429,16 @@ def utility_disable(utility, *packages)
         def merge(conf)
             config.merge!(conf.config)
         end
+
+        # Whether the OS package handler should prefer installing OS-independent
+        # packages (as e.g. RubyGems) as opposed to the binary packages
+        # equivalent (e.g. thor as a gem vs. thor as the ruby-thor Ubuntu
+        # package)
+        #
+        # This is false by default
+        def prefer_indep_over_os_packages?
+            get('prefer_indep_over_os_packages', false)
+        end
     end
 end
 

data/lib/autoproj/environment.rb

@@ -12,12 +12,10 @@ def prepare(root_dir)
 
             @root_dir = root_dir
             set 'AUTOPROJ_CURRENT_ROOT', root_dir
+        end
 
-            @original_env = original_env.map_value do |name, value|
-                filtered = value.split(File::PATH_SEPARATOR).
-                    find_all { |p| !Workspace.in_autoproj_project?(p) }
-                filtered.join(File::PATH_SEPARATOR)
-            end
+        def filter_original_env(name, env)
+            env.find_all { |p| !Workspace.in_autoproj_project?(p) }
         end
 
         def expand(value)

data/lib/autoproj/ops/import.rb

@@ -266,7 +266,11 @@ def finalize_package_load(processed_packages)
                             pkg.depends_on pkg_name
                         end
                     end
-                    pkg.os_packages.merge(osdeps)
+                    osdeps.each do |pkg_name|
+                        if !manifest.ignored?(pkg_name) && !manifest.excluded?(pkg_name)
+                            pkg.os_packages << pkg_name
+                        end
+                    end
                     pkg.prepare
                     Rake::Task["#{pkg.name}-prepare"].instance_variable_set(:@already_invoked, true)
                     pkg.update_environment

data/lib/autoproj/ops/install.rb

@@ -19,6 +19,8 @@ class UnexpectedBinstub < RuntimeError; end
             attr_accessor :gemfile
             # The environment that is passed to the bundler installs
             attr_reader :env
+            # The configuration hash
+            attr_reader :config
 
             def initialize(root_dir)
                 @root_dir = root_dir
@@ -28,9 +30,7 @@ def initialize(root_dir)
                     @gemfile = default_gemfile_contents
                 end
 
-                @private_bundler  = false
-                @private_autoproj = false
-                @private_gems     = false
+                load_config
                 @local = false
                 @env = self.class.clean_env
             end
@@ -42,7 +42,7 @@ def self.clean_env
                 %w{PATH GEM_HOME}.each do |name|
                     env[name] = sanitize_env(ENV[name] || "")
                 end
-                env['BUNDLE_GEMFILE'] = nil
+                env['BUNDLE_GEMFILE'] = []
                 env
             end
 
@@ -53,6 +53,16 @@ def env_for_child
                 end
             end
 
+            def apply_env(env)
+                env.each do |k, v|
+                    if v
+                        ENV[k] = v
+                    else
+                        ENV.delete(k)
+                    end
+                end
+            end
+
             def self.sanitize_env(value)
                 value.split(File::PATH_SEPARATOR).
                     find_all { |p| !in_workspace?(p) }
@@ -71,7 +81,6 @@ def self.in_workspace?(base_dir)
 
 
             def dot_autoproj; File.join(root_dir, '.autoproj') end
-            def bin_dir; File.join(dot_autoproj, 'bin') end
             def bundler_install_dir; File.join(dot_autoproj, 'bundler') end
             def autoproj_install_dir; File.join(dot_autoproj, 'autoproj') end
             # The path to the gemfile used to install autoproj
@@ -86,16 +95,22 @@ def local=(flag); @local = flag end
             # Whether bundler should be installed locally in {#dot_autoproj}
             def private_bundler?; @private_bundler end
             # (see #private_bundler?)
-            def private_bundler=(flag); @private_bundler = flag end
+            def private_bundler=(flag); @private_bundler = !!flag end
             # Whether autoproj should be installed locally in {#dot_autoproj}
             def private_autoproj?; @private_autoproj end
             # (see #private_autoproj?)
-            def private_autoproj=(flag); @private_autoproj = flag end
+            def private_autoproj=(flag); @private_autoproj = !!flag end
             # Whether bundler should be installed locally in the workspace
             # prefix directory
             def private_gems?; @private_gems end
             # (see #private_gems?)
-            def private_gems=(flag); @private_gems = flag end
+            def private_gems=(flag); @private_gems = !!flag end
+            # Whether autoproj should prefer OS-independent packages over their
+            # OS-packaged equivalents (e.g. the thor gem vs. the ruby-thor
+            # Debian package)
+            def prefer_indep_over_os_packages?; @prefer_indep_over_os_packages end
+            # (see #private_gems?)
+            def prefer_indep_over_os_packages=(flag); @prefer_indep_over_os_packages = !!flag end
 
             def guess_gem_program
                 ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME']
@@ -151,6 +166,9 @@ def parse_options(args = ARGV)
                     opt.on '--gemfile=PATH', String, 'use the given Gemfile to install autoproj instead of the default' do |path|
                         @gemfile = File.read(path)
                     end
+                    opt.on '--prefer-os-independent-packages', 'prefer OS-independent packages (such as a RubyGem) over their OS-packaged equivalent (e.g. the thor gem vs. the ruby-thor debian package)' do
+                        @prefer_indep_over_os_packages = true
+                    end
                 end
                 options.parse(ARGV)
             end
@@ -171,77 +189,72 @@ def install_bundler
                     STDERR.puts "FATAL: failed to install bundler in #{dot_autoproj}"
                     exit 1
                 end
-                env['GEM_PATH'] << bundler_install_dir
-                env['PATH'] << File.join(bundler_install_dir, 'bin')
-                File.join(bin_dir, 'bundler')
+                env['PATH'].unshift File.join(bundler_install_dir, 'bin')
+                File.join(bundler_install_dir, 'bin', 'bundler')
             end
 
-            def save_env_sh
-                env = Autobuild::Environment.new
-                env.prepare
+            def find_bundler
+                self.env['PATH'].unshift gem_bindir
+                clean_env = env_for_child
+                Gem.paths = Hash[
+                    'GEM_HOME' => clean_env['GEM_HOME'] || Gem.default_dir,
+                    'GEM_PATH' => clean_env['GEM_PATH'] || nil
+                ]
 
-                %w{GEM_HOME GEM_PATH}.each do |name|
-                    value = self.env[name]
-                    if value.empty?
-                        env.unset name
-                    else
-                        env.set name, *value
+                bundler = find_in_clean_path('bundler')
+                if !bundler
+                    clean_path = env_for_child['PATH']
+                    STDERR.puts "cannot find 'bundler' in PATH=#{clean_path}"
+                    STDERR.puts "installing it now ..."
+                    result = system(clean_env, Gem.ruby, '-S', 'gem', 'install', 'bundler')
+                    if !result
+                        if ENV['PATH'] != clean_path
+                            STDERR.puts "  it appears that you already have some autoproj-generated env.sh loaded"
+                            STDERR.puts "  - if you are running 'autoproj upgrade', please contact the autoproj author at https://github.com/rock-core/autoproj/issues/new"
+                            STDERR.puts "  - if you are running an install, try again in a console where the env.sh is not loaded"
+                            exit 1
+                        else
+                            STDERR.puts "  the recommended action is to install it manually first by running 'gem install bundler'"
+                            STDERR.puts "  or call this command again with --private-bundler to have it installed in the workspace"
+                            exit 1
+                        end
                     end
-                end
-                env.push_path 'PATH', File.join(autoproj_install_dir, 'bin')
-
-                if private_autoproj?
-                    env.push_path 'GEM_PATH', autoproj_install_dir
-                end
-
-                # Generate environment files right now, we can at least use bundler
-                File.open(File.join(dot_autoproj, 'env.sh'), 'w') do |io|
-                    env.export_env_sh(io)
-                end
 
-                File.open(File.join(root_dir, 'env.sh'), 'w') do |io|
-                    io.write <<-EOSHELL
-source "#{File.join(dot_autoproj, 'env.sh')}"
-export AUTOPROJ_CURRENT_ROOT=#{root_dir}
-                    EOSHELL
+                    bundler = find_in_clean_path('bundler')
+                    if !bundler
+                        STDERR.puts "FATAL: gem install bundler returned successfully, but still cannot find bundler"
+                        STDERR.puts "FATAL: in #{clean_path}"
+                    end
                 end
-            end
 
-            def save_gemfile
-                FileUtils.mkdir_p File.dirname(autoproj_gemfile_path)
-                File.open(autoproj_gemfile_path, 'w') do |io|
-                    io.write gemfile
-                end
+                bundler
             end
 
-            ENV_BUNDLE_GEMFILE_RX = /^(\s*ENV\[['"]BUNDLE_GEMFILE['"]\]\s*)(?:\|\|)?=/
-
             def install_autoproj(bundler)
                 # Force bundler to update. If the user does not want this, let him specify a
                 # Gemfile with tighter version constraints
-                lockfile = File.join(File.dirname(autoproj_gemfile_path), 'Gemfile.lock')
+                lockfile = File.join(File.dirname(autoproj_install_dir), 'Gemfile.lock')
                 if File.exist?(lockfile)
                     FileUtils.rm lockfile
                 end
 
                 opts = Array.new
-                opts << '--local' if local?
+                clean_env = env_for_child.merge('BUNDLE_GEMFILE' => nil)
 
-                env = env_for_child
+                opts << '--local' if local?
                 if private_autoproj?
-                    env = env.merge(
-                        'GEM_PATH' => bundler_install_dir,
-                        'GEM_HOME' => nil)
+                    clean_env['GEM_PATH'] = (bundler_install_dir if private_bundler?)
+                    clean_env['GEM_HOME'] = nil
                     opts << "--clean" << "--path=#{autoproj_install_dir}"
                 end
-
                 binstubs_path = File.join(autoproj_install_dir, 'bin')
-
-                result = system(env,
+                result = system(clean_env,
                     Gem.ruby, bundler, 'install',
                         "--gemfile=#{autoproj_gemfile_path}",
+                        "--shebang=#{Gem.ruby}",
                         "--binstubs=#{binstubs_path}",
-                        *opts)
+                        *opts, chdir: autoproj_install_dir)
+
                 if !result
                     STDERR.puts "FATAL: failed to install autoproj in #{dot_autoproj}"
                     exit 1
@@ -252,10 +265,6 @@ def install_autoproj(bundler)
                 # environment by default
                 Dir.glob(File.join(binstubs_path, '*')) do |path|
                     next if !File.file?(path)
-                    # Do NOT do that for bundler, otherwise it will fail with an
-                    # "already loaded gemfile" message once we e.g. try to do
-                    # 'bundler install --gemfile=NEW_GEMFILE'
-                    next if File.basename(path) == 'bundler'
 
                     lines = File.readlines(path)
                     matched = false
@@ -271,26 +280,53 @@ def install_autoproj(bundler)
                     end
                 end
 
-                env['PATH'] << File.join(autoproj_install_dir, 'bin')
+                env['PATH'].unshift File.join(autoproj_install_dir, 'bin')
                 if private_autoproj?
-                    env['GEM_PATH'] << autoproj_install_dir
+                    env['GEM_PATH'].unshift autoproj_install_dir
+                end
+            ensure
+                if binstubs_path
+                    FileUtils.rm_f File.join(binstubs_path, 'bundler')
                 end
             end
 
-            def update_configuration
-                if File.exist?(autoproj_config_path)
-                    config = YAML.load(File.read(autoproj_config_path)) || Hash.new
-                else
-                    config = Hash.new
+            def save_env_sh(*vars)
+                env = Autobuild::Environment.new
+                env.prepare
+                vars.each do |kv|
+                    k, *v = kv.split("=")
+                    v = v.join("=")
+
+                    if v.empty?
+                        env.unset k
+                    else
+                        env.set k, *v.split(File::PATH_SEPARATOR)
+                    end
                 end
-                config['private_bundler']  = private_bundler?
-                config['private_autoproj'] = private_autoproj?
-                config['private_gems']     = private_gems?
-                File.open(autoproj_config_path, 'w') do |io|
-                    YAML.dump(config, io)
+                # Generate environment files right now, we can at least use bundler
+                File.open(File.join(dot_autoproj, 'env.sh'), 'w') do |io|
+                    env.export_env_sh(io)
+                end
+
+                # And now the root envsh
+                env = Autobuild::Environment.new
+                env.source_before File.join(dot_autoproj, 'env.sh')
+                env.set('AUTOPROJ_CURRENT_ROOT', root_dir)
+                File.open(File.join(root_dir, 'env.sh'), 'w') do |io|
+                    env.export_env_sh(io)
                 end
             end
 
+            def save_gemfile
+                FileUtils.mkdir_p File.dirname(autoproj_gemfile_path)
+                File.open(autoproj_gemfile_path, 'w') do |io|
+                    io.write gemfile
+                end
+            end
+
+            ENV_BUNDLE_GEMFILE_RX = /^(\s*ENV\[['"]BUNDLE_GEMFILE['"]\]\s*)(?:\|\|)?=/
+
+
             def find_in_clean_path(command)
                 clean_path = env_for_child['PATH'].split(File::PATH_SEPARATOR)
                 clean_path.each do |p|
@@ -302,12 +338,10 @@ def find_in_clean_path(command)
                 nil
             end
 
-            def find_bundler
-                clean_env = env_for_child
-                Gem.paths = Hash[
-                    'GEM_HOME' => clean_env['GEM_HOME'] || Gem.default_dir,
-                    'GEM_PATH' => clean_env['GEM_PATH'] || nil
-                ]
+            # The path of the bin/ folder for installed gems
+            def gem_bindir
+                return @gem_bindir if @gem_bindir
+
                 # Here, we're getting into the esotheric
                 #
                 # The problem is that e.g. Ubuntu and Debian install an
@@ -318,30 +352,12 @@ def find_bundler
                 #
                 # So, we're calling 'gem' as a subcommand to discovery the
                 # actual bindir
-                bindir = IO.popen(clean_env, [Gem.ruby, '-e', 'puts Gem.bindir']).read
+                bindir = IO.popen(env_for_child, [Gem.ruby, '-e', 'puts Gem.bindir']).read
                 if bindir
-                    env['PATH'].unshift bindir.chomp
+                    @gem_bindir = bindir.chomp
                 else
-                    STDERR.puts "FATAL: cannot run #{Gem.ruby} -e 'puts Gem.bindir'"
-                    exit 1
+                    raise "FATAL: cannot run #{Gem.ruby} -e 'puts Gem.bindir'"
                 end
-
-                bundler = find_in_clean_path('bundler')
-                if !bundler
-                    clean_path = env_for_child['PATH']
-                    STDERR.puts "FATAL: cannot find 'bundler' in PATH=#{clean_path}"
-                    if ENV['PATH'] != clean_path
-                        STDERR.puts "  it appears that you already have some autoproj-generated env.sh loaded"
-                        STDERR.puts "  - if you are running 'autoproj upgrade', please contact the autoproj author at https://github.com/rock-core/autoproj/issues/new"
-                        STDERR.puts "  - if you are running an install, try again in a console where the env.sh is not loaded"
-                        exit 1
-                    else
-                        STDERR.puts "  the recommended action is to install it manually first by running 'gem install bundler'"
-                        STDERR.puts "  or call this command again with --private-bundler to have it installed in the workspace"
-                        exit 1
-                    end
-                end
-                bundler
             end
 
             def install
@@ -357,27 +373,54 @@ def install
                 install_autoproj(bundler)
             end
 
-            # Actually perform the install
-            def run(stage2: false)
-                if stage2
-                    require 'autobuild'
-                    save_env_sh
-                else
-                    install
+            def load_config
+                v1_config_path = File.join(root_dir, 'autoproj', 'config.yml')
+                
+                config = Hash.new
+                if File.file?(v1_config_path)
+                    config.merge!(YAML.load(File.read(v1_config_path)))
+                end
+                if File.file?(autoproj_config_path)
+                    config.merge!(YAML.load(File.read(autoproj_config_path)))
+                end
 
-                    env_for_child.each do |k, v|
-                        if v
-                            ENV[k] = v
-                        else
-                            ENV.delete(k)
-                        end
-                    end
-                    ENV['BUNDLE_GEMFILE'] = autoproj_gemfile_path
-                    update_configuration
-                    exec Gem.ruby, File.join(autoproj_install_dir, 'bin', 'autoproj'),
-                        'install-stage2', root_dir
+                @config = config
+                %w{private_bundler private_gems private_autoproj prefer_indep_over_os_packages}.each do |flag|
+                    instance_variable_set "@#{flag}", config.fetch(flag, false)
                 end
             end
+
+            def save_config
+                config['private_bundler']  = private_bundler?
+                config['private_autoproj'] = private_autoproj?
+                config['private_gems']     = private_gems?
+                config['prefer_indep_over_os_packages'] = prefer_indep_over_os_packages?
+                File.open(autoproj_config_path, 'w') { |io| YAML.dump(config, io) }
+            end
+
+            def stage1
+                FileUtils.mkdir_p dot_autoproj
+                save_config
+                install
+
+                clean_env = env_for_child
+                stage2_vars = clean_env.map { |k, v| "#{k}=#{v}" }
+                puts "starting the newly installed autoproj for stage2 install"
+                puts [clean_env.merge('BUNDLE_GEMFILE' => autoproj_gemfile_path),
+                    Gem.ruby, File.join(autoproj_install_dir, 'bin', 'autoproj'),
+                    'install-stage2', root_dir, *stage2_vars].inspect
+                exec clean_env.merge('BUNDLE_GEMFILE' => autoproj_gemfile_path),
+                    Gem.ruby, File.join(autoproj_install_dir, 'bin', 'autoproj'),
+                    'install-stage2', root_dir, *stage2_vars
+            end
+
+            def stage2(*vars)
+                require 'autobuild'
+                puts "saving env.sh and .autoproj/env.sh"
+                save_env_sh(*vars)
+                puts "calling autoproj envsh"
+                system(Gem.ruby, $0, 'envsh')
+            end
         end
     end
 end