autoproj 2.0.3 → 2.1.0.rc1

data/lib/autoproj/cli/update.rb

@@ -12,7 +12,11 @@ def validate_options(selection, options)
                     options[:from] = Autoproj::InstallationManifest.from_workspace_root(from)
                 end
 
-                if options[:aup] && !options[:all] && selection.empty?
+                if options[:no_deps_shortcut]
+                    options[:deps] = false
+                end
+
+                if options[:aup] && !options[:config] && !options[:all] && selection.empty?
                     if Dir.pwd == ws.root_dir
                         options[:all] = true
                     else
@@ -55,6 +59,7 @@ def validate_options(selection, options)
                     (has_explicit_selection && !selection.empty?) ||
                     (!has_explicit_selection && !options[:config] && !options[:autoproj])
 
+                options[:bundler] = update_autoproj
                 options[:autoproj] = update_autoproj
                 options[:config]   = update_config
                 options[:packages] = update_packages
@@ -63,10 +68,11 @@ def validate_options(selection, options)
 
             def run(selected_packages, options)
                 ws.manifest.accept_unavailable_osdeps = !options[:osdeps]
-
                 ws.setup
                 ws.autodetect_operating_system(force: true)
-
+                if options[:bundler]
+                    ws.update_bundler
+                end
                 if options[:autoproj]
                     ws.update_autoproj
                 end
@@ -80,15 +86,18 @@ def run(selected_packages, options)
                         keep_going: options[:keep_going],
                         retry_count: options[:retry_count])
                 rescue ImportFailed => configuration_import_failure
-                    if !options[:keep_going] || !options[:packages]
+                    if !options[:keep_going]
                         raise
                     end
                 ensure
                     ws.config.save
                 end
 
-                if !options[:packages]
-                    return [], [], []
+                if options[:packages]
+                    command_line_selection, selected_packages =
+                        finish_loading_configuration(selected_packages)
+                else
+                    command_line_selection, selected_packages = [], PackageSelection.new
                 end
 
                 osdeps_options = normalize_osdeps_options(
@@ -97,8 +106,6 @@ def run(selected_packages, options)
                     osdeps: options[:osdeps],
                     osdeps_filter_uptodate: options[:osdeps_filter_uptodate])
 
-                command_line_selection, selected_packages =
-                    finish_loading_configuration(selected_packages)
                 source_packages, osdep_packages, import_failure = 
                     update_packages(
                         selected_packages,
@@ -119,8 +126,7 @@ def run(selected_packages, options)
                     ws.install_os_packages(osdep_packages, **osdeps_options)
                 end
 
-                ws.export_env_sh(source_packages)
-                Autoproj.message "  updated #{ws.root_dir}/#{Autoproj::ENV_FILENAME}", :green
+                export_env_sh
 
                 if import_failure && configuration_import_failure
                     raise ImportFailed.new(configuration_import_failure.original_errors + import_failure.original_errors)
@@ -195,22 +201,6 @@ def update_packages(selected_packages,
                 return import_failure.source_packages, import_failure.osdep_packages, import_failure
             end
 
-            def load_all_available_package_manifests
-                # Load the manifest for packages that are already present on the
-                # file system
-                ws.manifest.each_autobuild_package do |pkg|
-                    if pkg.checked_out?
-                        begin
-                            ws.manifest.load_package_manifest(pkg.name)
-                        rescue Interrupt
-                            raise
-                        rescue Exception => e
-                            Autoproj.warn "cannot load package manifest for #{pkg.name}: #{e.message}"
-                        end
-                    end
-                end
-            end
-
             def setup_update_from(other_root)
                 manifest.each_autobuild_package do |pkg|
                     if pkg.importer.respond_to?(:pick_from_autoproj_root)

data/lib/autoproj/configuration.rb

@@ -467,5 +467,10 @@ def merge(conf)
         def prefer_indep_over_os_packages?
             get('prefer_indep_over_os_packages', false)
         end
+
+        # The configuration as a key => value map
+        def to_hash
+            @config.dup
+        end
     end
 end

data/lib/autoproj/environment.rb

@@ -35,7 +35,12 @@ def export_env_sh(subdir = nil, options = Hash.new)
                 source_after(File.join(shell_dir, "autoproj_sh"))
             end
 
-            File.open(filename, "w") do |io|
+            existing_content =
+                begin File.read(filename)
+                rescue SystemCallError
+                end
+
+            StringIO.open(new_content = String.new, 'w') do |io|
                 if inherit?
                     io.write <<-EOF
                     if test -n "$AUTOPROJ_CURRENT_ROOT" && test "$AUTOPROJ_CURRENT_ROOT" != "#{root_dir}"; then
@@ -46,6 +51,11 @@ def export_env_sh(subdir = nil, options = Hash.new)
                 end
                 super(io)
             end
+
+            if new_content != existing_content
+                File.open(filename, 'w') { |io| io.write new_content }
+                true
+            end
         end
     end
 

data/lib/autoproj/local_package_set.rb

@@ -71,6 +71,8 @@ def raw_description_file
                 overrides_data = Autoproj.in_file(overrides_file_path, Autoproj::YAML_LOAD_ERROR) do
                     YAML.load(File.read(overrides_file_path)) || Hash.new
                 end
+                overrides_data = PackageSet.validate_and_normalize_source_file(
+                    overrides_file_path, overrides_data)
                 description = description.merge(overrides_data)
             end
 

data/lib/autoproj/manifest.rb

@@ -1024,6 +1024,22 @@ def load_package_manifest(pkg)
             manifest
         end
 
+        def load_all_available_package_manifests
+            # Load the manifest for packages that are already present on the
+            # file system
+            each_package_definition do |pkg|
+                if pkg.checked_out?
+                    begin
+                        load_package_manifest(pkg)
+                    rescue Interrupt
+                        raise
+                    rescue Exception => e
+                        Autoproj.warn "cannot load package manifest for #{pkg.name}: #{e.message}"
+                    end
+                end
+            end
+        end
+
         # The set of overrides added with #add_osdeps_overrides
         attr_reader :osdeps_overrides
 

data/lib/autoproj/ops/cache.rb

@@ -46,6 +46,9 @@ def cache_git(pkg, options = Hash.new)
                 with_retry(10) do
                     Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')
                 end
+                with_retry(10) do
+                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', '--tags')
+                end
                 Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'gc', '--prune=all')
             end
 

data/lib/autoproj/ops/configuration.rb

@@ -120,13 +120,14 @@ def update_remote_package_set(vcs,
                 reset: false,
                 retry_count: nil)
 
-            name = PackageSet.name_of(ws, vcs)
             raw_local_dir = PackageSet.raw_local_dir_of(ws, vcs)
-
             if checkout_only && File.exist?(raw_local_dir)
                 return
             end
 
+            # name_of does minimal validation of source.yml, so do it here
+            # even though we don't really need it
+            name = PackageSet.name_of(ws, vcs, ignore_load_errors: true)
             ws.install_os_packages([vcs.type], all: nil)
             update_configuration_repository(
                 vcs, name, raw_local_dir,

data/lib/autoproj/ops/install.rb

@@ -38,6 +38,7 @@ def initialize(root_dir)
                 @autoproj_options = Array.new
 
                 @env = Hash.new
+                env['RUBYOPT'] = []
                 env['RUBYLIB'] = []
                 env['GEM_PATH'] = []
                 env['GEM_HOME'] = []
@@ -182,6 +183,7 @@ def self.guess_gem_program
             # @return [String]
             def default_gemfile_contents(autoproj_version = ">= 2.0.0")
                 ["source \"#{gem_source}\"",
+                 "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)",
                  "gem \"autoproj\", \"#{autoproj_version}\"",
                  "gem \"utilrb\", \">= 3.0.1\""].join("\n")
             end
@@ -250,14 +252,19 @@ def find_bundler(gem_program)
                 end
             end
 
-            def install_bundler(gem_program)
+            def install_bundler(gem_program, silent: false)
                 local = ['--local'] if local?
 
+                redirection = Hash.new
+                if silent
+                    redirection = Hash[out: :close]
+                end
+
                 result = system(
-                    env_for_child,
+                    env_for_child.merge('GEM_HOME' => gems_gem_home),
                     Gem.ruby, gem_program, 'install', '--env-shebang', '--no-document', '--no-format-executable', '--clear-sources', '--source', gem_source,
                         *local,
-                        "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler')
+                        "--bindir=#{File.join(gems_gem_home, 'bin')}", 'bundler', **redirection)
 
                 if !result
                     STDERR.puts "FATAL: failed to install bundler in #{gems_gem_home}"
@@ -563,14 +570,16 @@ def call_stage2
 
             def stage2(*vars)
                 require 'autobuild'
-                puts "saving env.sh and .autoproj/env.sh"
+                puts "saving temporary env.sh and .autoproj/env.sh"
                 save_env_sh(*vars)
+                puts "running 'autoproj envsh' to generate a proper env.sh"
                 if !system(Gem.ruby, autoproj_path, 'envsh', *autoproj_options)
                     STDERR.puts "failed to run autoproj envsh on the newly installed autoproj (#{autoproj_path})"
                     exit 1
                 end
                 # This is really needed on an existing install to install the
                 # gems that were present in the v1 layout
+                puts "running 'autoproj osdeps' to re-install missing gems"
                 if !system(Gem.ruby, autoproj_path, 'osdeps')
                     STDERR.puts "failed to run autoproj osdeps on the newly installed autoproj (#{autoproj_path})"
                     exit 1

data/lib/autoproj/ops/main_config_switcher.rb

@@ -14,7 +14,7 @@ def initialize(ws)
             #
             # @see check_root_dir_empty
             EXPECTED_ROOT_ENTRIES = [".", "..", "autoproj_bootstrap",
-                                     ".gems", "bootstrap.sh", ENV_FILENAME].to_set
+                                     ".autoproj", "bootstrap.sh", ENV_FILENAME].to_set
 
             # Verifies that {#root_dir} contains only expected entries, to make
             # sure that the user bootstraps into a new directory
@@ -82,7 +82,7 @@ def bootstrap(buildconf_info, options = Hash.new)
 
                 ws.config.validate_ruby_executable
                 ws.config.set 'reused_autoproj_installations', options[:reuse], true
-                ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
+                ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
 
                 # If we are not getting the installation setup from a VCS, copy the template
                 # files
@@ -110,7 +110,7 @@ def bootstrap(buildconf_info, options = Hash.new)
                     url = VCSDefinition.to_absolute_url(url, ws.root_dir)
                     do_switch_config(false, type, url, *options)
                 end
-                ws.env.export_env_sh(nil, shell_helpers: ws.config.shell_helpers?)
+                ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
                 ws.config.save
             end
 

data/lib/autoproj/ops/snapshot.rb

@@ -19,15 +19,12 @@ class Snapshot
         # @param [Array<String=>Hash>] state the current state
         # @param [Hash] the updated information
         def self.merge_packets( overrides, state )
-            result = overrides.dup
             overriden = overrides.map { |entry| entry.keys.first }.to_set
-            state.each do |pkg|
+            filtered_state = state.find_all do |pkg|
                 name, _ = pkg.first
-                if !overriden.include?(name)
-                    result << pkg
-                end
+                !overriden.include?(name)
             end
-            result
+            filtered_state + overrides
         end
 
         def self.update_log_available?(manifest)
@@ -54,7 +51,7 @@ def save_versions( versions, versions_file, replace: false)
             FileUtils.mkdir_p(File.dirname( versions_file ))
 
             # augment the versions file with the updated versions
-            Snapshot.merge_packets( versions, existing_versions )
+            versions = Snapshot.merge_packets( versions, existing_versions )
 
             versions = sort_versions(versions)
 

data/lib/autoproj/package_definition.rb

@@ -69,5 +69,15 @@ def add_setup_block(block)
                 block.call(autobuild)
             end
         end
+
+        # Whether this package is already checked out
+        def checked_out?
+            autobuild.checked_out?
+        end
+
+        # Add another package as a dependency of this one
+        def depends_on(pkg)
+            autobuild.depends_on(pkg.autobuild)
+        end
     end
 end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -332,6 +332,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
                 FileUtils.mkdir_p root_dir
                 if updated = (!File.exist?(gemfile_path) || File.read(gemfile_path) != gemfile_contents)
                     File.open(gemfile_path, 'w') do |io|
+                        io.puts "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)"
                         io.puts gemfile_contents
                     end
                 end

data/lib/autoproj/package_managers/pip_manager.rb

@@ -6,6 +6,7 @@ class PipManager < Manager
 
             def initialize_environment
                 ws.env.set 'PYTHONUSERBASE', pip_home
+                ws.env.add_path 'PATH', File.join(pip_home, 'bin')
             end
 
             # Return the directory where python packages are installed to.

data/lib/autoproj/package_set.rb

@@ -225,12 +225,15 @@ def snapshot(target_dir, options = Hash.new)
         # checked out, and the vcs (as a string) otherwise
         #
         # @return [String]
-        def self.name_of(ws, vcs, raw_local_dir: raw_local_dir_of(ws, vcs))
+        def self.name_of(ws, vcs, raw_local_dir: raw_local_dir_of(ws, vcs), ignore_load_errors: false)
             if File.directory?(raw_local_dir)
-                raw_description_file(raw_local_dir, package_set_name: "#{vcs.type}:#{vcs.url}")['name']
-            else
-                vcs.to_s
+                begin
+                    return raw_description_file(raw_local_dir, package_set_name: "#{vcs.type}:#{vcs.url}")['name']
+                rescue ConfigError
+                    raise if !ignore_load_errors
+                end
             end
+            vcs.to_s
         end
 
         # Returns the local directory in which the given package set should be
@@ -258,7 +261,7 @@ def self.resolve_definition(ws, raw_spec, from: nil, raw: Array.new)
             spec = VCSDefinition.normalize_vcs_hash(raw_spec, base_dir: ws.config_dir)
             options, vcs_spec = Kernel.filter_options spec, auto_imports: true
 
-            vcs_spec = Autoproj.expand(vcs_spec, ws.manifest.constant_definitions)
+            vcs_spec = Autoproj.expand(vcs_spec, ws.config.to_hash.merge(ws.manifest.constant_definitions))
             return VCSDefinition.from_raw(vcs_spec, from: from, raw: raw), options
         end
 
@@ -309,6 +312,34 @@ def local_dir
             end
         end
 
+        # @api private
+        #
+        # Validate and normalizes a raw source file
+        def self.validate_and_normalize_source_file(yaml_path, yaml)
+            yaml = yaml.dup
+            %w{imports version_control}.each do |entry_name|
+                yaml[entry_name] ||= Array.new
+                if !yaml[entry_name].respond_to?(:to_ary)
+                    raise ConfigError.new(yaml_path), "expected the '#{entry_name}' entry to be an array"
+                end
+            end
+
+            %w{constants}.each do |entry_name|
+                yaml[entry_name] ||= Hash.new
+                if !yaml[entry_name].respond_to?(:to_h)
+                    raise ConfigError.new(yaml_path), "expected the '#{entry_name}' entry to be a map"
+                end
+            end
+
+            if yaml.has_key?('overrides')
+                yaml['overrides'] ||= Array.new
+                if !yaml['overrides'].respond_to?(:to_ary)
+                    raise ConfigError.new(yaml_path), "expected the 'overrides' entry to be an array"
+                end
+            end
+            yaml
+        end
+
         # @api private
         #
         # Read the description information for a package set in a given
@@ -330,7 +361,8 @@ def self.raw_description_file(raw_local_dir, package_set_name: nil)
                 newdefs = Autoproj.in_file(source_file, Autoproj::YAML_LOAD_ERROR) do
                     YAML.load(File.read(source_file))
                 end
-                source_definition.merge!(newdefs || Hash.new) do |k, old, new|
+                newdefs = validate_and_normalize_source_file(source_file, newdefs || Hash.new)
+                source_definition.merge!(newdefs) do |k, old, new|
                     if old.respond_to?(:to_ary)
                         old + new
                     else new
@@ -340,6 +372,7 @@ def self.raw_description_file(raw_local_dir, package_set_name: nil)
             if !source_definition['name']
                 raise ConfigError.new(master_source_file), "#{master_source_file} does not have a 'name' field"
             end
+
             source_definition
         end
 

data/lib/autoproj/test.rb

@@ -246,6 +246,12 @@ def workspace_env(varname)
             stdout.chomp
         end
 
+        def in_ws
+            Dir.chdir(@ws.root_dir) do
+                yield
+            end
+        end
+
         attr_reader :ws_os_package_resolver
 
         def ws_define_package_manager(name, strict: false, call_while_empty: false)
@@ -276,6 +282,8 @@ def ws_create
                 dir, os_package_resolver: ws_os_package_resolver,
                      package_managers: ws_package_managers)
             ws.config.set 'osdeps_mode', 'all'
+            ws.config.set 'GITHUB', 'http,ssh', true
+            ws.config.set 'GITORIOUS', 'http,ssh', true
             ws.config.set 'gems_install_path', File.join(dir, 'gems')
             ws.config.save
             ws.prefix_dir = make_tmpdir