autoproj 2.11.0 → 2.12.0

data/bin/autoproj_install.in

@@ -13,8 +13,13 @@ require 'autoproj/ops/install'
 ENV.delete('BUNDLE_GEMFILE')
 ENV.delete('RUBYLIB')
 ops = Autoproj::Ops::Install.new(Dir.pwd)
+
+existing_config = File.join(Dir.pwd, '.autoproj', 'config.yml')
+if File.file?(existing_config)
+    puts "Found existing configuration, using it as seed"
+    puts "use --no-seed-config to avoid this behavior"
+    ops.add_seed_config(existing_config)
+end
 ops.parse_options(ARGV)
 ops.stage1
-if !ops.skip_stage2?
-    ops.call_stage2
-end
+ops.call_stage2 unless ops.skip_stage2?

data/lib/autoproj.rb

@@ -44,6 +44,7 @@
 require 'autoproj/source_package_query'
 require 'autoproj/os_package_query'
 
+require 'autoproj/ops/phase_reporting'
 require 'autoproj/ops/install'
 require 'autoproj/ops/tools'
 require 'autoproj/ops/loader'

data/lib/autoproj/autobuild_extensions/dsl.rb

@@ -149,11 +149,13 @@ def ignore(*paths)
 
 # Adds a new setup block to an existing package
 def setup_package(package_name, workspace: Autoproj.workspace, &block)
-    raise ConfigError.new, "you must give a block to #setup_package" unless block
+    unless block
+        raise Autoproj::ConfigError.new, 'you must give a block to #setup_package'
+    end
 
     package_definition = workspace.manifest.find_package_definition(package_name)
     if !package_definition
-        raise ConfigError.new, "#{package_name} is not a known package"
+        raise Autoproj::ConfigError.new, "#{package_name} is not a known package"
     elsif package_definition.autobuild.kind_of?(Autobuild::DummyPackage)
         # Nothing to do!
     else

data/lib/autoproj/base.rb

@@ -56,4 +56,22 @@ def self.post_import(*packages, &block)
             end
         end
     end
+
+    # Shim for a smooth upgrade path to bundler 2.1+
+    def self.bundler_unbundled_system(*args, **options)
+        if Bundler.respond_to?(:unbundled_system)
+            Bundler.unbundled_system(*args, **options)
+        else
+            Bundler.clean_system(*args, **options)
+        end
+    end
+
+    # Shim for a smooth upgrade path to bundler 2.1+
+    def self.bundler_with_unbundled_env(&block)
+        if Bundler.respond_to?(:with_unbundled_env)
+            Bundler.with_unbundled_env(&block)
+        else
+            Bundler.with_clean_env(&block)
+        end
+    end
 end

data/lib/autoproj/cli/cache.rb

@@ -10,21 +10,50 @@ def validate_options(argv, options = Hash.new)
                 if argv.empty?
                     default_cache_dirs = Autobuild::Importer.default_cache_dirs
                     if !default_cache_dirs || default_cache_dirs.empty?
-                        raise CLIInvalidArguments, "no cache directory defined with e.g. the AUTOBUILD_CACHE_DIR environment variable, expected one cache directory as argument"
+                        raise CLIInvalidArguments,
+                              "no cache directory defined with e.g. the "\
+                              "AUTOBUILD_CACHE_DIR environment variable, "\
+                              "expected one cache directory as argument"
                     end
-                    Autoproj.warn "using cache directory #{default_cache_dirs.first} from the autoproj configuration"
+                    Autoproj.warn "using cache directory #{default_cache_dirs.first} "\
+                                  "from the autoproj configuration"
                     argv << default_cache_dirs.first
                 end
 
-                return File.expand_path(argv.first, ws.root_dir), *argv[1..-1], options
+                if (compile = options[:gems_compile])
+                    options[:gems_compile] = compile.map do |name|
+                        name, *artifacts = name.split('+')
+                        [name, artifacts: artifacts]
+                    end
+                end
+
+                [File.expand_path(argv.first, ws.root_dir), *argv[1..-1], options]
             end
 
-            def run(cache_dir, *package_names, all: true, keep_going: false, checkout_only: false)
+            def run(cache_dir, *package_names,
+                    keep_going: false,
+                    packages: true, all: true, checkout_only: false,
+                    gems: false, gems_compile: [], gems_compile_force: false)
                 initialize_and_load
                 finalize_setup
 
-                cache_op = Autoproj::Ops::Cache.new(cache_dir, ws.manifest)
-                cache_op.create_or_update(*package_names, all: all, keep_going: keep_going, checkout_only: checkout_only)
+                cache_op = Autoproj::Ops::Cache.new(cache_dir, ws)
+                if packages
+                    cache_op.create_or_update(
+                        *package_names,
+                        all: all, keep_going: keep_going,
+                        checkout_only: checkout_only
+                    )
+                end
+
+                if gems
+                    Autoproj.message "caching gems in #{cache_op.gems_cache_dir}"
+                    cache_op.create_or_update_gems(
+                        keep_going: keep_going,
+                        compile: gems_compile,
+                        compile_force: gems_compile_force
+                    )
+                end
             end
         end
     end

data/lib/autoproj/cli/main.rb

@@ -300,13 +300,34 @@ def build(*packages)
                 end
             end
 
-            desc 'cache CACHE_DIR', 'create or update a cache directory that can be given to AUTOBUILD_CACHE_DIR'
-            option :keep_going, aliases: :k,
-                desc: 'do not stop on errors'
-            option :checkout_only, aliases: :c, type: :boolean, default: false,
-                desc: "only checkout packages, do not update already-cached ones"
-            option :all, type: :boolean, default: true,
-                desc: "cache all defined packages (the default) or only the selected ones"
+            desc 'cache CACHE_DIR', 'create or update a cache directory that '\
+                                    'can be given to AUTOBUILD_CACHE_DIR'
+            option :keep_going,
+                   aliases: :k,
+                   desc: 'do not stop on errors'
+            option :checkout_only,
+                   aliases: :c, type: :boolean, default: false,
+                   desc: 'only checkout packages, do not update already-cached ones'
+            option :all,
+                   type: :boolean, default: true,
+                   desc: 'cache all defined packages (the default), '\
+                         ' or only the selected ones'
+            option :packages,
+                   type: :boolean, default: true,
+                   desc: 'update the package cache'
+            option :gems,
+                   type: :boolean, default: false,
+                   desc: 'update the gems cache'
+            option :gems_compile_force,
+                   type: :boolean, default: false,
+                   desc: 'with --gems-compile, recompile existing gems as well'
+            option :gems_compile,
+                   type: :array,
+                   desc: 'pre-compile the following gems. This requires gem-compiler '\
+                         'to be available in the workspace. Use GEM_NAME+ARTIFACT'\
+                         '[+ARTIFACT] to add files or directories to the precompiled '\
+                         'gems beyond what gem-compiler auto-adds (which is mostly '\
+                         'dynamic libraries)'
             def cache(*args)
                 run_autoproj_cli(:cache, :Cache, Hash[], *args)
             end

data/lib/autoproj/cli/test.rb

@@ -10,7 +10,7 @@ def initialize(ws = Workspace.default,
             end
 
             def package_metadata(package)
-                u = package.autobuild.test_utility
+                u = package.test_utility
                 super.merge(
                     'coverage_available' => !!u.coverage_available?,
                     'coverage_enabled' => !!u.coverage_enabled?,

data/lib/autoproj/cli/utility.rb

@@ -89,24 +89,37 @@ def run(user_selection, options = {})
                     raise CLIInvalidArguments, "autoproj: the provided package "\
                         "is not selected for build"
                 end
+                return if package_names.empty?
 
                 packages = package_names.map do |pkg_name|
                     ws.manifest.find_package_definition(pkg_name)
                 end
 
+                apply_to_packages(packages, parallel: options[:parallel])
+            end
+
+            def apply_to_packages(packages, parallel: ws.config.parallel_build_level)
+                if @report_path
+                    reporting = Ops::PhaseReporting.new(
+                        @utility_name, @report_path,
+                        method(:package_metadata)
+                    )
+                end
+
+                reporting&.initialize_incremental_report
                 Autobuild.apply(
-                    package_names,
-                    "autoproj-#{utility_name}",
-                    [utility_name],
-                    parallel: options[:parallel]
-                )
+                    packages.map(&:name), "autoproj-#{@utility_name}",
+                    [@utility_name], parallel: parallel
+                ) do |pkg, phase|
+                    reporting&.report_incremental(pkg) if phase == utility_name
+                end
 
             ensure
-                create_report(packages) if packages && @report_path
+                reporting&.create_report(packages.map(&:autobuild))
             end
 
-            def package_metadata(package)
-                u = package.autobuild.utility(@utility_name)
+            def package_metadata(autobuild_package)
+                u = autobuild_package.utility(@utility_name)
                 {
                     'source_dir' => u.source_dir,
                     'target_dir' => u.target_dir,
@@ -117,23 +130,6 @@ def package_metadata(package)
                     'installed' => !!u.installed?,
                 }
             end
-
-            def create_report(packages)
-                info = packages.each_with_object({}) do |p, map|
-                    map[p.name] = package_metadata(p)
-                end
-
-                FileUtils.mkdir_p File.dirname(@report_path)
-                File.open(@report_path, 'w') do |io|
-                    dump = JSON.dump(
-                        "#{@utility_name}_report" => {
-                            'timestamp' => Time.now,
-                            'packages' => info
-                        }
-                    )
-                    io.write dump
-                end
-            end
         end
     end
 end

data/lib/autoproj/configuration.rb

@@ -196,7 +196,9 @@ def configure(option_name)
                     Autoproj.info "       using: #{value} (noninteractive mode)"
                 end
                 @modified = true
-                if !is_default
+                if is_default
+                    value = opt.validate(value)
+                else
                     config[option_name] = [value, true]
                     displayed_options[option_name] = value
                 end
@@ -353,6 +355,10 @@ def shell_helpers=(flag)
             set 'shell_helpers', flag, true
         end
 
+        def bundler_version
+            get 'bundler_version', nil
+        end
+
         def apply_autobuild_configuration
             if has_value_for?('autobuild')
                 params = get('autobuild')
@@ -369,6 +375,11 @@ def importer_cache_dir
             get('importer_cache_dir', nil)
         end
 
+        # Set import and gem cache directory
+        def importer_cache_dir=(path)
+            set('importer_cache_dir', path, true)
+        end
+
         # Sets the directory in which packages will be installed
         def prefix_dir=(path)
             set('prefix', path, true)

data/lib/autoproj/installation_manifest.rb

@@ -1,7 +1,8 @@
 module Autoproj
     # Manifest of installed packages imported from another autoproj installation
     class InstallationManifest
-        Package = Struct.new :name, :type, :vcs, :srcdir, :prefix, :builddir, :logdir, :dependencies
+        Package = Struct.new :name, :type, :vcs, :srcdir, :importdir,
+                             :prefix, :builddir, :logdir, :dependencies
         PackageSet = Struct.new :name, :vcs, :raw_local_dir, :user_local_dir
 
         attr_reader :path
@@ -28,11 +29,11 @@ def add_package_set(pkg_set)
         def each_package_set(&block)
             package_sets.each_value(&block)
         end
-            
+
         def each_package(&block)
             packages.each_value(&block)
         end
-            
+
         def load
             @packages = Hash.new
             raw = YAML.load(File.open(path))
@@ -51,8 +52,8 @@ def load
                         package_sets[pkg_set.name] = pkg_set
                     else
                         pkg = Package.new(
-                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['prefix'],
-                            entry['builddir'], entry['logdir'], entry['dependencies'])
+                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['importdir'],
+                            entry['prefix'], entry['builddir'], entry['logdir'], entry['dependencies'])
                         packages[pkg.name] = pkg
                     end
                 end
@@ -74,6 +75,7 @@ def save(path = self.path)
                          'type' => v.class.name,
                          'vcs' => package_def.vcs.to_hash,
                          'srcdir' => v.srcdir,
+                         'importdir' => (v.importdir if v.respond_to?(:importdir)),
                          'builddir' => (v.builddir if v.respond_to?(:builddir)),
                          'logdir' => v.logdir,
                          'prefix' => v.prefix,

data/lib/autoproj/ops/build.rb

@@ -87,34 +87,36 @@ def force_build_packages(selected_packages, all_enabled_packages)
             #   names of the packages that should be rebuilt
             # @return [void]
             def build_packages(all_enabled_packages, options = Hash.new)
+                if @report_path
+                    reporting = Ops::PhaseReporting.new(
+                        'build', @report_path, method(:package_metadata)
+                    )
+                end
+
                 Autobuild.do_rebuild = false
                 Autobuild.do_forced_build = false
+                reporting&.initialize_incremental_report
                 begin
-                    Autobuild.apply(all_enabled_packages, "autoproj-build", ['build'], options)
+                    Autobuild.apply(
+                        all_enabled_packages,
+                        "autoproj-build", ['build'], options
+                    ) do |pkg, phase|
+                        reporting&.report_incremental(pkg) if phase == 'build'
+                    end
+
                 ensure
-                    create_report(all_enabled_packages) if @report_path
+                    packages = all_enabled_packages.map do |name|
+                        @manifest.find_autobuild_package(name)
+                    end
+                    reporting&.create_report(packages)
                 end
             end
 
-            def create_report(package_list)
-                FileUtils.mkdir_p File.dirname(@report_path)
-
-                packages = package_list.each_with_object({}) do |pkg_name, h|
-                    pkg = manifest.find_autobuild_package(pkg_name)
-
-                    h[pkg.name] = {
-                        invoked: !!pkg.install_invoked?,
-                        success: !!pkg.installed?
-                    }
-                end
-
-                report = JSON.pretty_generate({
-                    build_report: {
-                        timestamp: Time.now,
-                        packages: packages
-                    }
-                })
-                IO.write(@report_path, report)
+            def package_metadata(autobuild_package)
+                {
+                    invoked: !!autobuild_package.install_invoked?,
+                    success: !!autobuild_package.installed?
+                }
             end
         end
     end

data/lib/autoproj/ops/cache.rb

@@ -4,9 +4,10 @@ class Cache
             attr_reader :cache_dir
             attr_reader :manifest
 
-            def initialize(cache_dir, manifest)
+            def initialize(cache_dir, ws)
                 @cache_dir = cache_dir
-                @manifest  = manifest
+                @ws = ws
+                @manifest = ws.manifest
             end
 
             def with_retry(count)
@@ -27,29 +28,39 @@ def git_cache_dir
                 File.join(cache_dir, 'git')
             end
 
-            def cache_git(pkg, options = Hash.new)
+            def cache_git(pkg, checkout_only: false)
                 pkg.importdir = File.join(git_cache_dir, pkg.name)
-                if options[:checkout_only] && File.directory?(pkg.importdir)
-                    return
-                end
+                return if checkout_only && File.directory?(pkg.importdir)
 
                 pkg.importer.local_branch = nil
                 pkg.importer.remote_branch = nil
                 pkg.importer.remote_name = 'autobuild'
 
-                if !File.directory?(pkg.importdir)
+                unless File.directory?(pkg.importdir)
                     FileUtils.mkdir_p File.dirname(pkg.importdir)
-                    Autobuild::Subprocess.run("autoproj-cache", "import", Autobuild.tool(:git), "--git-dir", pkg.importdir, 'init', "--bare")
+                    Autobuild::Subprocess.run(
+                        "autoproj-cache", "import", Autobuild.tool(:git),
+                        "--git-dir", pkg.importdir, 'init', "--bare"
+                    )
                 end
                 pkg.importer.update_remotes_configuration(pkg)
 
                 with_retry(10) do
-                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')
+                    Autobuild::Subprocess.run(
+                        'autoproj-cache', :import, Autobuild.tool('git'),
+                        '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild'
+                    )
                 end
                 with_retry(10) do
-                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', 'autobuild', '--tags')
+                    Autobuild::Subprocess.run(
+                        'autoproj-cache', :import, Autobuild.tool('git'),
+                        '--git-dir', pkg.importdir, 'fetch', 'autobuild', '--tags'
+                    )
                 end
-                Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'gc', '--prune=all')
+                Autobuild::Subprocess.run(
+                    'autoproj-cache', :import, Autobuild.tool('git'),
+                    '--git-dir', pkg.importdir, 'gc', '--prune=all'
+                )
             end
 
             def archive_cache_dir
@@ -63,7 +74,8 @@ def cache_archive(pkg)
                 end
             end
 
-            def create_or_update(*package_names, all: true, keep_going: false, checkout_only: false)
+            def create_or_update(*package_names, all: true, keep_going: false,
+                                 checkout_only: false)
                 FileUtils.mkdir_p cache_dir
 
                 if package_names.empty?
@@ -75,7 +87,7 @@ def create_or_update(*package_names, all: true, keep_going: false, checkout_only
                         end
                 else
                     packages = package_names.map do |name|
-                        if pkg = manifest.find_autobuild_package(name)
+                        if (pkg = manifest.find_autobuild_package(name))
                             pkg
                         else
                             raise PackageNotFound, "no package named #{name}"
@@ -88,7 +100,10 @@ def create_or_update(*package_names, all: true, keep_going: false, checkout_only
                 total = packages.size
                 Autoproj.message "Handling #{total} packages"
                 packages.each_with_index do |pkg, i|
-                    next if pkg.srcdir != pkg.importdir # No need to process this one, it is uses another package's import
+                    # No need to process this one, it is uses another package's
+                    # import
+                    next if pkg.srcdir != pkg.importdir
+
                     begin
                         case pkg.importer
                         when Autobuild::Git
@@ -103,27 +118,117 @@ def create_or_update(*package_names, all: true, keep_going: false, checkout_only
                     rescue Interrupt
                         raise
                     rescue ::Exception => e
-                        if keep_going
-                            pkg.error "       failed to cache #{pkg.name}, but going on as requested"
-                            lines = e.to_s.split("\n")
-                            if lines.empty?
-                                lines = e.message.split("\n")
-                            end
-                            if lines.empty?
-                                lines = ["unknown error"]
-                            end
-                            pkg.message(lines.shift, :red, :bold)
-                            lines.each do |line|
-                                pkg.message(line)
+                        raise unless keep_going
+
+                        pkg.error "       failed to cache #{pkg.name}, "\
+                                  'but going on as requested'
+                        lines = e.to_s.split('\n')
+                        lines = e.message.split('\n') if lines.empty?
+                        lines = ['unknown error'] if lines.empty?
+                        pkg.message(lines.shift, :red, :bold)
+                        lines.each do |line|
+                            pkg.message(line)
+                        end
+                        nil
+                    end
+                end
+            end
+
+            def gems_cache_dir
+                File.join(cache_dir, 'package_managers', 'gem')
+            end
+
+            def create_or_update_gems(keep_going: true, compile_force: false, compile: [])
+                # Note: this might directly copy into the cache directoy, and
+                # we support it later
+                cache_dir = File.join(@ws.prefix_dir, 'gems', 'vendor', 'cache')
+                PackageManagers::BundlerManager.run_bundler(
+                    @ws, 'cache'
+                )
+
+                FileUtils.mkdir_p(gems_cache_dir) unless File.exist?(gems_cache_dir)
+
+                needs_copy =
+                    if File.exist?(cache_dir)
+                        real_cache_dir = File.realpath(cache_dir)
+                        real_target_dir = File.realpath(gems_cache_dir)
+                        (real_cache_dir != real_target_dir)
+                    end
+
+                synchronize_gems_cache_dirs(real_cache_dir, real_target_dir) if needs_copy
+
+                platform_suffix = "-#{Gem::Platform.local}.gem"
+                failed = []
+                compile.each do |gem_name, artifacts: []|
+                    Dir.glob(File.join(cache_dir, "#{gem_name}*.gem")) do |gem|
+                        next if gem.end_with?(platform_suffix)
+
+                        gem_basename = File.basename(gem, ".gem")
+                        expected_platform_gem = File.join(
+                            real_target_dir, "#{gem_basename}#{platform_suffix}"
+                        )
+                        next if !compile_force && File.file?(expected_platform_gem)
+
+                        begin
+                            compile_gem(gem, artifacts: artifacts, output: real_target_dir)
+                        rescue CompilationFailed
+                            unless keep_going
+                                raise CompilationFailed, "#{gem} failed to compile"
                             end
-                            nil
-                        else
-                            raise
+
+                            failed << gem
                         end
                     end
                 end
+
+                unless failed.empty?
+                    raise CompilationFailed, "#{failed.sort.join(', ')} failed to compile"
+                end
+            end
+
+            class CompilationFailed < RuntimeError; end
+
+            def synchronize_gems_cache_dirs(source, target)
+                Dir.glob(File.join(source, "*.gem")) do |source_file|
+                    basename = File.basename(source_file)
+                    target_file = File.join(target, basename)
+                    next if File.file?(target_file)
+
+                    Autoproj.message "gems: caching #{basename}"
+                    FileUtils.cp source_file, target_file
+                end
+            end
+
+            def guess_gem_program
+                if Autobuild.programs['gem']
+                    return Autobuild.programs['gem']
+                end
+
+                ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME']
+                ruby_bindir = RbConfig::CONFIG['bindir']
+
+                candidates = ['gem']
+                if ruby_bin =~ /^ruby(.+)$/
+                    candidates << "gem#{$1}"
+                end
+
+                candidates.each do |gem_name|
+                    if File.file?(gem_full_path = File.join(ruby_bindir, gem_name))
+                        Autobuild.programs['gem'] = gem_full_path
+                        return Autobuild.programs['gem']
+                    end
+                end
+
+                raise ArgumentError, "cannot find a gem program (tried #{candidates.sort.join(", ")} in #{ruby_bindir})"
+            end
+
+            private def compile_gem(gem_path, output:, artifacts: [])
+                artifacts = artifacts.flat_map { |n| ["--artifact", n] }
+                unless system(Autobuild.tool('ruby'), '-S', guess_gem_program,
+                              'compile', '--output', output, *artifacts, gem_path)
+                    raise CompilationFailed, "#{gem_path} failed to compile"
+                end
             end
         end
     end
 end
-