autoproj-ci 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 807429c3a708c91b9d1f061bd3138d6ce7f6bfa63b60cb56505b8c817dc0ea9a
-  data.tar.gz: 0cc4aa3955d61d2b30d5aad85a49c888b4312dd12c9685ca17e8f5fc261d8fde
+  metadata.gz: 7607b476bb61d53de2e2992f1d8ca7d91c2164090d9e22fe136aedce9004c735
+  data.tar.gz: 8f77955a24f0bc516b9251c1dd7cbad4326a22d5d1b42a50010b4975696f3adc
 SHA512:
-  metadata.gz: dfd1ed123ab40e60b71f41925940dbb947b16d91abb5e6c8e68f66ef18283ebb97e9953c4a4f1151776d0cfe09f25d807ee884dbfcc314cee56852e5f9ccfdb2
-  data.tar.gz: 3824b52d7b226df53b6304d2a751357081052e50478fcd8cab4d844a5384b4d1b6b79b98fd61060f96557b8286b9eb16626836094c86768c37552b901af6f608
+  metadata.gz: 62be513aba6072efdbe008fcedea7e95044298c4b46d53d624db42e32a8be121272e10e322ed2b3ab31c41cf8a1e05291e1ca51ae613eeec98cbb44d067954a7
+  data.tar.gz: 2b24d8860e43bb9de8330fd5989ff8920892c320c894369efc46ed4941dad9e5fb52e3cb36311f698b013bb8337dd07ef0dac8bd27fbc11e4ebecc2602a28cb4

data/Gemfile

@@ -8,6 +8,7 @@ group :vscode do
     gem 'ruby-debug-ide', '>= 0.6.0'
     gem 'debase', '>= 0.2.2.beta10'
     gem 'solargraph'
+    gem 'rubocop-rock'
 end
 
 git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }

data/autoproj-ci.gemspec

@@ -25,13 +25,14 @@ Gem::Specification.new do |spec|
   spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.bindir        = "exe"
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
   spec.add_dependency 'autoproj'
   spec.add_development_dependency "flexmock"
   spec.add_development_dependency "bundler"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "timecop"
   spec.add_development_dependency "minitest", "~> 5.0"
 end

data/bin/autoproj-ci

@@ -0,0 +1,5 @@
+#! /usr/bin/env ruby
+
+require 'autoproj/cli/standalone_ci'
+Autoproj::CLI::StandaloneCI.start(ARGV)
+

data/lib/autoproj-ci.rb

@@ -1,6 +1,13 @@
+# frozen_string_literal: true
+
 require 'autoproj/cli/main_ci'
 
-class Autoproj::CLI::Main
-    desc 'ci', 'subcommands tuned for usage in CI environments'
-    subcommand 'ci', Autoproj::CLI::MainCI
-end
+module Autoproj
+    module CLI
+        # Toplevel CLI interface from Autoproj
+        class Main
+            desc 'ci', 'subcommands tuned for usage in CI environments'
+            subcommand 'ci', Autoproj::CLI::MainCI
+        end
+    end
+end

data/lib/autoproj/ci/rebuild.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'autoproj'
+require 'tmpdir'
+
+module Autoproj
+    module CI
+        # Utilities to re-create a system image from the results of a CI build
+        module Rebuild
+            # Create a single tarball containing all the artifacts of a given build
+            #
+            # The generated tarball is 'rooted' at the filesystem root, i.e. it is meant
+            # to be unpacked from /
+            def self.prepare_synthetic_buildroot(
+                installation_manifest_path, versions_path, cache_root_path, output_dir
+            )
+                manifest = Autoproj::InstallationManifest.new(installation_manifest_path)
+                manifest.load
+                versions = YAML.safe_load(File.read(versions_path))
+
+                versions.each do |entry|
+                    name, entry = entry.first
+                    next if /^pkg_set:/.match?(name)
+
+                    unpack_package(
+                        output_dir,
+                        cache_root_path, name,
+                        manifest.packages.fetch(name),
+                        entry.fetch('fingerprint')
+                    )
+                end
+            end
+
+            def self.dpkg_create_package_install(status_path, rules, orig: nil)
+                installed, = Autoproj::PackageManagers::AptDpkgManager
+                             .parse_dpkg_status(status_path, virtual: false)
+
+                if orig
+                    orig_installed, = Autoproj::PackageManagers::AptDpkgManager
+                                      .parse_dpkg_status(orig, virtual: false)
+                    installed -= orig_installed
+                end
+
+                installed.find_all do |pkg_name|
+                    package_matches_rules?(pkg_name, rules)
+                end
+            end
+
+            def self.package_matches_rules?(pkg_name, rules)
+                rules.each do |mode, r|
+                    return mode if r.match?(pkg_name)
+                end
+                true
+            end
+
+            # Unpack a single package in its place within the
+            def self.unpack_package(output_path, cache_root_path, name, pkg, fingerprint)
+                cache_file_path = File.join(cache_root_path, name, fingerprint)
+                unless File.file?(cache_file_path)
+                    raise "no cache file found for fingerprint '#{fingerprint}', "\
+                          "package '#{name}' in #{cache_root_path}"
+                end
+
+                package_prefix = File.join(output_path, pkg.prefix)
+                FileUtils.mkdir_p(package_prefix)
+                unless system('tar', 'xzf', cache_file_path,
+                              chdir: package_prefix, out: '/dev/null')
+                    raise "failed to unpack #{cache_file_path} in #{package_prefix} "\
+                          "for package #{name} failed"
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/ci/version.rb

@@ -1,7 +1,9 @@
+# frozen_string_literal: true
+
 module Autoproj
     # autoproj-ci is an Autoproj plugin to provide functionality useful for CI
     # (automated builds & tests)
     module CI
-        VERSION = "0.2.0"
+        VERSION = '0.3.0'
     end
 end

data/lib/autoproj/cli/autoproj-stub.sh.erb

@@ -0,0 +1,12 @@
+#! /bin/sh -e
+
+if test "x$1" != "xexec"; then
+    echo "This is a minimal stub of autoproj meant only to"\
+         "allow for running `autoproj exec`" >& 2
+    exit 1
+fi
+
+shift
+
+. <%= workspace_dir %>/env.sh
+exec "$@"

data/lib/autoproj/cli/ci.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'autoproj/cli/inspection_tool'
 require 'tmpdir'
 
@@ -10,51 +12,102 @@ module Autoproj
         # same pattern, and registers its subcommand in {MainCI} while implementing
         # the functionality in this class
         class CI < InspectionTool
+            PHASES = %w[import build test].freeze
+
             def resolve_packages
                 initialize_and_load
                 source_packages, * = finalize_setup(
-                    [], non_imported_packages: :ignore)
+                    [], non_imported_packages: :ignore
+                )
                 source_packages.map do |pkg_name|
                     ws.manifest.find_autobuild_package(pkg_name)
                 end
             end
 
-            def cache_pull(dir, silent: true)
+            def cache_state(dir, ignore: [])
+                packages = resolve_packages
+
+                memo = {}
+                packages.each_with_object({}) do |pkg, h|
+                    state = package_cache_state(dir, pkg, memo: memo)
+                    if ignore.include?(pkg.name)
+                        state = state.merge('cached' => false, 'metadata' => false)
+                    end
+
+                    h[pkg.name] = state
+                end
+            end
+
+            def cache_pull(dir, ignore: [])
                 packages = resolve_packages
 
-                memo   = Hash.new
+                memo = {}
                 results = packages.each_with_object({}) do |pkg, h|
-                    state, fingerprint = pull_package_from_cache(dir, pkg, memo: memo)
-                    puts "pulled #{pkg.name} (#{fingerprint})" if state && !silent
+                    if ignore.include?(pkg.name)
+                        pkg.message '%s: ignored by command line'
+                        fingerprint = pkg.fingerprint(memo: memo)
+                        h[pkg.name] = {
+                            'cached' => false,
+                            'fingerprint' => fingerprint
+                        }
+                        next
+                    end
 
-                    h[pkg.name] = {
+                    state, fingerprint, metadata =
+                        pull_package_from_cache(dir, pkg, memo: memo)
+                    if state
+                        pkg.message "%s: pulled #{fingerprint}", :green
+                    else
+                        pkg.message "%s: #{fingerprint} not in cache, "\
+                                    'or not pulled from cache'
+                    end
+
+                    h[pkg.name] = metadata.merge(
                         'cached' => state,
                         'fingerprint' => fingerprint
-                    }
+                    )
                 end
 
-                unless silent
-                    hit = results.count { |_, info| info['cached'] }
-                    puts "#{hit} hits, #{results.size - hit} misses"
-                end
+                hit = results.count { |_, info| info['cached'] }
+                Autoproj.message "#{hit} hits, #{results.size - hit} misses"
 
                 results
             end
 
-            def cache_push(dir, silent: true)
+            def cache_push(dir)
                 packages = resolve_packages
+                metadata = consolidated_report['packages']
 
-                built = load_built_flags
-
-                memo   = Hash.new
+                memo = {}
                 results = packages.each_with_object({}) do |pkg, h|
-                    unless built[pkg.name]
+                    if !(pkg_metadata = metadata[pkg.name])
+                        pkg.message '%s: no metadata in build report', :magenta
+                        next
+                    elsif !(build_info = pkg_metadata['build'])
+                        pkg.message '%s: no build info in build report', :magenta
+                        next
+                    elsif build_info['cached']
+                        pkg.message '%s: was pulled from cache, not pushing'
+                        next
+                    elsif !build_info['success']
+                        pkg.message '%s: build failed, not pushing', :magenta
                         next
                     end
 
-                    state, fingerprint = push_package_to_cache(dir, pkg, memo: memo)
-                    puts "pushed #{pkg.name} (#{fingerprint})" if state && !silent
+                    # Remove cached flags before saving
+                    pkg_metadata = pkg_metadata.dup
+                    PHASES.each do |phase_name|
+                        pkg_metadata[phase_name]&.delete('cached')
+                    end
 
+                    state, fingerprint = push_package_to_cache(
+                        dir, pkg, pkg_metadata, force: true, memo: memo
+                    )
+                    if state
+                        pkg.message "%s: pushed #{fingerprint}", :green
+                    else
+                        pkg.message "%s: #{fingerprint} already in cache"
+                    end
 
                     h[pkg.name] = {
                         'updated' => state,
@@ -62,34 +115,81 @@ module Autoproj
                     }
                 end
 
-                unless silent
-                    hit = results.count { |_, info| info['updated'] }
-                    puts "#{hit} updated packages, #{results.size - hit} reused entries"
-                end
+                hit = results.count { |_, info| info['updated'] }
+                Autoproj.message "#{hit} updated packages, #{results.size - hit} "\
+                                 'reused entries'
 
                 results
             end
 
+            # Checks if a package's test results should be processed with xunit-viewer
+            #
+            # @param [String] results_dir the directory where the
+            # @param [String] xunit_output path to the xunit-viewer output. An
+            #   existing file is re-generated only if force is true
+            # @param [Boolean] force re-generation of the xunit-viewer output
+            def need_xunit_processing?(results_dir, xunit_output, force: false)
+                # We don't re-generate if the xunit-processed files were cached
+                return if !force && File.file?(xunit_output)
+
+                # We only check whether there are xml files in the
+                # package's test dir. That's the only check we do ... if
+                # the XML files are not JUnit, we'll finish with an empty
+                # xunit html file
+                Dir.enum_for(:glob, File.join(results_dir, '*.xml'))
+                   .first
+            end
+
+            # Process the package's test results with xunit-viewer
+            #
+            # @param [String] xunit_viewer path to xunit-viewer
+            # @param [Boolean] force re-generation of the xunit-viewer output. If
+            #   false, packages that already have a xunit-viewer output will be skipped
+            def process_test_results_xunit(force: false, xunit_viewer: 'xunit-viewer')
+                consolidated_report['packages'].each_value do |info|
+                    next unless info['test']
+                    next unless (results_dir = info['test']['target_dir'])
+
+                    xunit_output = "#{results_dir}.html"
+                    next unless need_xunit_processing?(results_dir, xunit_output,
+                                                       force: force)
+
+                    success = system(xunit_viewer,
+                                     "--results=#{results_dir}",
+                                     "--output=#{xunit_output}")
+                    unless success
+                        Autoproj.warn 'xunit-viewer conversion failed '\
+                                      "for '#{results_dir}'"
+                    end
+                end
+            end
+
+            # Post-processing of test results
+            def process_test_results(force: false, xunit_viewer: 'xunit-viewer')
+                process_test_results_xunit(force: force, xunit_viewer: xunit_viewer)
+            end
+
             # Build a report in a given directory
             #
             # The method itself will not archive the directory, only gather the
             # information in a consistent way
-            def build_report(dir)
+            def create_report(dir)
                 initialize_and_load
                 finalize_setup([], non_imported_packages: :ignore)
 
                 report = consolidated_report
-                FileUtils.mkdir_p dir
+                FileUtils.mkdir_p(dir)
                 File.open(File.join(dir, 'report.json'), 'w') do |io|
                     JSON.dump(report, io)
                 end
 
-                installation_manifest = InstallationManifest.
-                    from_workspace_root(@ws.root_dir)
+                installation_manifest = InstallationManifest
+                                        .from_workspace_root(@ws.root_dir)
                 logs = File.join(dir, 'logs')
+
                 # Pre-create the logs, or cp_r will have a different behavior
                 # if the directory exists or not
-                FileUtils.mkdir_p logs
+                FileUtils.mkdir_p(logs)
                 installation_manifest.each_package do |pkg|
                     glob = Dir.glob(File.join(pkg.logdir, '*'))
                     FileUtils.cp_r(glob, logs) if File.directory?(pkg.logdir)
@@ -101,79 +201,153 @@ module Autoproj
                 File.join(dir, pkg.name, fingerprint)
             end
 
+            def package_cache_state(dir, pkg, memo: {})
+                fingerprint = pkg.fingerprint(memo: memo)
+                path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)
+
+                {
+                    'path' => path,
+                    'cached' => File.file?(path),
+                    'metadata' => File.file?("#{path}.json"),
+                    'fingerprint' => fingerprint
+                }
+            end
+
+            class PullError < RuntimeError
+            end
+
             def pull_package_from_cache(dir, pkg, memo: {})
                 fingerprint = pkg.fingerprint(memo: memo)
                 path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)
-                unless File.file?(path)
-                    return [false, fingerprint]
+                return [false, fingerprint, {}] unless File.file?(path)
+
+                metadata_path = "#{path}.json"
+                metadata =
+                    if File.file?(metadata_path)
+                        JSON.parse(File.read(metadata_path))
+                    else
+                        {}
+                    end
+
+                # Do not pull packages for which we should run tests
+                tests_enabled = pkg.test_utility.enabled?
+                tests_invoked = metadata['test'] && metadata['test']['invoked']
+                if tests_enabled && !tests_invoked
+                    pkg.message '%s: has tests that have never '\
+                                'been invoked, not pulling from cache'
+                    return [false, fingerprint, {}]
                 end
 
-                FileUtils.mkdir_p pkg.prefix
-                result = system("tar", "xzf", path, chdir: pkg.prefix, out: '/dev/null')
-                unless result
-                    raise "tar failed when pulling cache file for #{pkg.name}"
+                FileUtils.mkdir_p(pkg.prefix)
+                unless system('tar', 'xzf', path, chdir: pkg.prefix, out: '/dev/null')
+                    raise PullError, "tar failed when pulling cache file for #{pkg.name}"
                 end
-                [true, pkg.fingerprint(memo: memo)]
+
+                [true, fingerprint, metadata]
             end
 
-            def push_package_to_cache(dir, pkg, memo: {})
+            def push_package_to_cache(dir, pkg, metadata, force: false, memo: {})
                 fingerprint = pkg.fingerprint(memo: memo)
                 path = package_cache_path(dir, pkg, fingerprint: fingerprint, memo: memo)
-                if File.file?(path)
-                    return [false, fingerprint]
+                temppath = "#{path}.#{Process.pid}.#{rand(256)}"
+
+                FileUtils.mkdir_p(File.dirname(path))
+                if force || !File.file?("#{path}.json")
+                    File.open(temppath, 'w') { |io| JSON.dump(metadata, io) }
+                    FileUtils.mv(temppath, "#{path}.json")
                 end
 
-                temppath = "#{path}.#{Process.pid}.#{rand(256)}"
-                FileUtils.mkdir_p File.dirname(path)
-                result = system("tar", "czf", temppath, ".",
-                    chdir: pkg.prefix, out: '/dev/null')
-                unless result
-                    raise "tar failed when pushing cache file for #{pkg.name}"
+                if !force && File.file?(path)
+                    # Update modification time for the cleanup process
+                    FileUtils.touch(path)
+                    return [false, fingerprint]
                 end
 
-                FileUtils.mv temppath, path
+                result = system('tar', 'czf', temppath, '.',
+                                chdir: pkg.prefix, out: '/dev/null')
+                raise "tar failed when pushing cache file for #{pkg.name}" unless result
+
+                FileUtils.mv(temppath, path)
                 [true, fingerprint]
             end
 
+            def cleanup_build_cache(dir, size_limit)
+                all_files = Find.enum_for(:find, dir).map do |path|
+                    next unless File.file?(path) && File.file?("#{path}.json")
+
+                    [path, File.stat(path)]
+                end.compact
+
+                total_size = all_files.map { |_, s| s.size }.sum
+                lru = all_files.sort_by { |_, s| s.mtime }
+
+                while total_size > size_limit
+                    path, stat = lru.shift
+                    Autoproj.message "removing #{path} (size=#{stat.size}, mtime=#{stat.mtime})"
+
+                    FileUtils.rm_f path
+                    FileUtils.rm_f "#{path}.json"
+                    total_size -= stat.size
+                end
+
+                Autoproj.message format("current build cache size: %.1f GB", Float(total_size) / 1_000_000_000)
+                total_size
+            end
+
             def load_built_flags
                 path = @ws.build_report_path
                 return {} unless File.file?(path)
 
-                report = JSON.load(File.read(path))
-                report['build_report']['packages'].
-                    each_with_object({}) do |pkg_report, h|
+                report = JSON.parse(File.read(path))
+                report['build_report']['packages']
+                    .each_with_object({}) do |pkg_report, h|
                         h[pkg_report['name']] = pkg_report['built']
                     end
             end
 
+            def load_report(path, root_name, default: { 'packages' => {} })
+                return default unless File.file?(path)
+
+                JSON.parse(File.read(path)).fetch(root_name)
+            end
+
             def consolidated_report
-                cache_pull = File.join(@ws.root_dir, 'cache-pull.json')
-                cache_report = if File.file?(cache_pull)
-                    JSON.load(File.read(cache_pull))
-                else
-                    {}
-                end
+                # NOTE: keys must match PHASES
+                new_reports = {
+                    'import' => @ws.import_report_path,
+                    'build' => @ws.build_report_path,
+                    'test' => @ws.utility_report_path('test')
+                }
 
-                packages =
-                    if File.file?(@ws.build_report_path)
-                        path = @ws.build_report_path
-                        report = JSON.load(File.read(path))
-                        report['build_report']['packages']
-                    elsif File.file?(@ws.import_report_path)
-                        path = @ws.import_report_path
-                        report = JSON.load(File.read(path))
-                        report['import_report']['packages']
+                # We start with the cached info (if any) and override with
+                # information from the other phase reports
+                cache_report_path = File.join(@ws.root_dir, 'cache-pull.json')
+                result = load_report(cache_report_path, 'cache_pull_report')['packages']
+                result.delete_if do |_name, info|
+                    next true unless info.delete('cached')
+
+                    PHASES.each do |phase_name|
+                        if (phase_info = info[phase_name])
+                            phase_info['cached'] = true
+                        end
                     end
-                return unless packages
+                    false
+                end
 
-                packages = packages.each_with_object({}) do |pkg_info, h|
-                    name = pkg_info.delete('name')
-                    h[name] = cache_report[name] || { 'cached' => false }
-                    h[name].merge!(pkg_info)
+                new_reports.each do |phase_name, path|
+                    report = load_report(path, "#{phase_name}_report")
+                    report['packages'].each do |pkg_name, pkg_info|
+                        result[pkg_name] ||= {}
+                        if pkg_info['invoked']
+                            result[pkg_name][phase_name] = pkg_info.merge(
+                                'cached' => false,
+                                'timestamp' => report['timestamp']
+                            )
+                        end
+                    end
                 end
-                { 'packages' => packages }
+                { 'packages' => result }
             end
         end
     end
 end
-

data/lib/autoproj/cli/main_ci.rb

@@ -1,95 +1,185 @@
+# frozen_string_literal: true
+
 require 'autoproj'
+require 'autoproj/cli/standalone_ci'
 
 module Autoproj
     module CLI
         # CLI interface for autoproj-ci
-        class MainCI < Thor
-            desc 'build [ARGS]', "Just like autoproj build, but can use a build cache"
+        class MainCI < StandaloneCI
+            desc 'build [ARGS]', 'Just like autoproj build, but can use a build cache'
             option :cache, type: 'string',
-                desc: 'path to the build cache'
+                           desc: 'path to the build cache'
+            option :cache_ignore, type: :array, default: [],
+                                  desc: 'list of packages to not pull from cache'
             option :report, type: 'string', default: 'cache-pull.json',
-                desc: 'a file which describes what has been pulled'
+                            desc: 'a file which describes what has been pulled'
             def build(*args)
                 if (cache = options.delete(:cache))
                     cache = File.expand_path(cache)
-                    results = cache_pull(cache)
-                    pulled_packages = results.
-                        map { |name, pkg| name if pkg['cached'] }.
-                        compact
+                    require 'autoproj/cli/base'
+                    Autoproj::CLI::Base.validate_options(args, options)
+                    results = cache_pull(cache, ignore: options.delete(:cache_ignore))
+                    pulled_packages = results
+                                      .map { |name, pkg| name if pkg['cached'] }
+                                      .compact
                     not_args = ['--not', *pulled_packages] unless pulled_packages.empty?
                 end
 
-                Process.exec(Gem.ruby, $PROGRAM_NAME, 'build', "--interactive=f", *args, *not_args)
+                args << "--progress=#{options[:progress] ? 't' : 'f'}"
+                args << "--color=#{options[:color] ? 't' : 'f'}"
+                Process.exec(Gem.ruby, $PROGRAM_NAME, 'build',
+                             '--interactive=f', *args, *not_args)
+            end
+
+            desc 'test [ARGS]', 'Like autoproj test, but selects only packages '\
+                                'that have been built'
+            def test(*args)
+                require 'autoproj/cli/ci'
+                cli = CI.new
+                cli.validate_options([], options.dup)
+                report = cli.consolidated_report
+
+                built_packages = report['packages'].find_all do |_name, info|
+                    info['build'] && !info['build']['cached'] && info['build']['success']
+                end
+                return if built_packages.empty?
+
+                built_package_names = built_packages.map(&:first)
+                Process.exec(Gem.ruby, $PROGRAM_NAME, 'test',
+                             'exec', '--interactive=f', *args, *built_package_names)
+            end
+
+            desc 'process-test-results [ARGS]',
+                 'Process test output (assumed to be in JUnit XML) through xunit-viewer'
+            option :force, desc: 're-generates existing output', default: false
+            option :xunit_viewer, desc: 'path to xunit-viewer', default: 'xunit-viewer'
+            def process_test_results
+                require 'autoproj/cli/ci'
+                cli = CI.new
+                cli.validate_options([], options.dup)
+                cli.process_test_results(
+                    force: options[:force],
+                    xunit_viewer: options[:xunit_viewer]
+                )
+            end
+
+            desc 'status DIR', 'Display the cache status'
+            option :cache, type: 'string',
+                           desc: 'path to the build cache'
+            def status(dir)
+                cache = File.expand_path(dir)
+                require 'autoproj/cli/ci'
+                cli = CI.new
+                cli.validate_options(dir, options)
+                results = cli.cache_state(cache)
+                results.keys.sort.each do |name|
+                    status = results[name]
+                    fields = []
+                    fields <<
+                        if status['cached']
+                            Autoproj.color('cache hit', :green)
+                        else
+                            Autoproj.color('cache miss', :red)
+                        end
+                    fields << "fingerprint=#{status['fingerprint']}"
+                    puts "#{name}: #{fields.join(', ')}"
+                end
             end
 
             desc 'cache-pull CACHE_DIR',
-                "This command gets relevant artifacts from a build cache and "\
-                "populates the current workspace's prefix with them. It is meant "\
-                "to be executed after a full checkout of the workspace"
+                 'This command gets relevant artifacts from a build cache and '\
+                 'populates the current workspace\'s prefix with them. It is meant '\
+                 'to be executed after a full checkout of the workspace'
             option :report, type: 'string', default: 'cache-pull.json',
-                desc: 'a file which describes what has been done'
-            def cache_pull(dir)
+                            desc: 'a file which describes what has been done'
+            option :ignore, type: :array, default: [],
+                            desc: 'list of packages to not pull from cache'
+            def cache_pull(dir, ignore: [])
                 dir = File.expand_path(dir)
 
                 require 'autoproj/cli/ci'
                 results = nil
-                Autoproj.report(silent: true) do
-                    cli = CI.new
-                    args, options = cli.validate_options(dir, self.options)
-                    report = options.delete(:report)
 
-                    results = cli.cache_pull(*dir, silent: false, **options)
+                cli = CI.new
+                _, options = cli.validate_options(dir, self.options)
+                report = options.delete(:report)
 
-                    if report && !report.empty?
-                        File.open(report, 'w') do |io|
-                            JSON.dump(results, io)
-                        end
+                # options[:ignore] is not set if we call from another
+                # command, e.g. build
+                ignore += (options.delete(:ignore) || [])
+                results = cli.cache_pull(*dir, ignore: ignore, **options)
+
+                if report && !report.empty?
+                    File.open(report, 'w') do |io|
+                        JSON.dump(
+                            {
+                                'cache_pull_report' => {
+                                    'packages' => results
+                                }
+                            }, io
+                        )
                     end
                 end
                 results
             end
 
             desc 'cache-push CACHE_DIR',
-                "This command writes the packages successfully built in the last "\
-                "build to the given build cache, so that they can be reused with "\
-                "cache-pull"
+                 'This command writes the packages successfully built in the last '\
+                 'build to the given build cache, so that they can be reused with '\
+                 'cache-pull'
             option :report, type: 'string', default: 'cache-push.json',
-                desc: 'a file which describes what has been done'
+                            desc: 'a file which describes what has been done'
             def cache_push(dir)
                 dir = File.expand_path(dir)
 
                 require 'autoproj/cli/ci'
-                Autoproj.report(silent: true) do
-                    cli = CI.new
+                cli = CI.new
 
-                    args, options = cli.validate_options(dir, self.options)
-                    report = options.delete(:report)
+                _, options = cli.validate_options(dir, self.options)
+                report = options.delete(:report)
 
-                    results = cli.cache_push(*dir, silent: false, **options)
+                results = cli.cache_push(dir, **options)
 
-                    if report && !report.empty?
-                        File.open(report, 'w') do |io|
-                            JSON.dump(results, io)
-                        end
+                if report && !report.empty?
+                    File.open(report, 'w') do |io|
+                        JSON.dump(
+                            {
+                                'cache_push_report' => {
+                                    'packages' => results
+                                }
+                            }, io
+                        )
                     end
                 end
             end
 
-            desc "build-report PATH",
-                "Create a tarball containing all the information about this "\
-                "build, such as cache information (from cache-pull), Autoproj\'s "\
-                "build report and installation manifest, and the package\'s logfiles"
-            def build_report(path)
+            desc 'build-cache-cleanup CACHE_DIR',
+                 'Remove the oldest entries in the cache until it is under a given size limit'
+            option :max_size, type: 'numeric', default: 10,
+                              desc: 'approximate target size limit (in GB, defaults to 10)'
+            def build_cache_cleanup(dir)
+                dir = File.expand_path(dir)
+
+                require 'autoproj/cli/ci'
+                cli = CI.new
+
+                _, options = cli.validate_options(dir, self.options)
+                cli.cleanup_build_cache(dir, options[:max_size] * 1_000_000_000)
+            end
+
+            desc 'build-report PATH',
+                 'create a directory containing all the information about this '\
+                 'build, such as cache information (from cache-pull), Autoproj\'s '\
+                 'build report and installation manifest, and the package\'s logfiles'
+            def create_report(path)
                 path = File.expand_path(path)
 
                 require 'autoproj/cli/ci'
-                Autoproj.report(silent: true) do
-                    cli = CI.new
-                    args, options = cli.validate_options(path, self.options)
-                    cli.build_report(*args, **options)
-                end
+                cli = CI.new
+                args, options = cli.validate_options(path, self.options)
+                cli.create_report(*args, **options)
             end
         end
     end
 end
-

data/lib/autoproj/cli/standalone_ci.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+require 'erb'
+require 'thor'
+
+require 'autoproj'
+require 'autoproj/ci/rebuild'
+
+module Autoproj
+    module CLI
+        # CI-related commands that can be executed without an Autoproj installation
+        class StandaloneCI < Thor
+            desc 'rebuild-root CONFIG_DIR CACHE_ROOT OUTPUT',
+                 'creates a compressed tarball containing the build products of a '\
+                 'finished build, pulled from build cache'
+            option 'workspace',
+                   desc: 'if given, setup a minimal workspace-like structure to '\
+                         'support execution in the given path',
+                   type: :string, default: nil
+            def rebuild_root(config_dir, cache_root, output)
+                dir = Dir.mktmpdir
+                Autoproj::CI::Rebuild.prepare_synthetic_buildroot(
+                    File.join(config_dir, 'installation-manifest'),
+                    File.join(config_dir, 'versions.yml'),
+                    cache_root,
+                    dir
+                )
+
+                if options[:workspace]
+                    prepare_workspace(config_dir, dir, options[:workspace])
+                end
+
+                output = File.expand_path(output)
+                unless system('tar', 'caf', output, '--owner=root', '--group=root',
+                              '.', chdir: dir)
+                    raise "failed to create #{output}"
+                end
+            ensure
+                FileUtils.rm_rf(dir) if dir && File.directory?(dir)
+            end
+
+            desc 'dpkg-filter-status STATUS_PATH [RULES]',
+                 'outputs the list of APT packages to install based on a dpkg status '\
+                 'file and a set of inclusion/exclusion rules of the form "+ pattern" '\
+                 'and "- pattern"'
+            option :orig, desc: 'a status file whose installed packages are removed',
+                          type: :string
+            option :file, desc: 'read the rules from a file',
+                          type: :string
+            def dpkg_filter_status(status_path, *rules)
+                rules += File.readlines(options[:file]).map(&:strip) if options[:file]
+                rules = rules.map do |line|
+                    next if line.empty? || line.start_with?('#')
+
+                    parse_rule(line)
+                end
+
+                packages = Autoproj::CI::Rebuild.dpkg_create_package_install(
+                    status_path, rules, orig: options[:orig]
+                )
+                puts packages.join("\n")
+            end
+
+            no_commands do # rubocop:disable Metrics/BlockLength
+                def parse_rule(line)
+                    unless (m = /^([+-])\s+(.*)/.match(line))
+                        raise ArgumentError, "invalid rule line '#{line}'"
+                    end
+
+                    mode = (m[1] == '+')
+                    begin
+                        [mode, Regexp.new(m[2])]
+                    rescue RegexpError => e
+                        raise ArgumentError, "invalid regexp in '#{line}': #{e}"
+                    end
+                end
+
+                def prepare_workspace(config_dir, output_dir, workspace_dir)
+                    FileUtils.mkdir_p File.join(output_dir, workspace_dir, '.autoproj')
+
+                    if File.file?(envsh = File.join(config_dir, 'env.sh'))
+                        filter_envsh(envsh, output_dir, workspace_dir)
+                        generate_autoproj_stub(output_dir, workspace_dir)
+                    end
+
+                    if File.file?(file = File.join(config_dir, 'source.yml'))
+                        FileUtils.cp(
+                            file, File.join(output_dir, workspace_dir, '.autoproj')
+                        )
+                    end
+
+                    if File.file?(file = File.join(config_dir, 'installation-manifest'))
+                        FileUtils.cp(
+                            file,
+                            File.join(output_dir, workspace_dir, 'installation-manifest')
+                        )
+                    end
+                end
+
+                AUTOPROJ_STUB_PATH = File.join(__dir__, 'autoproj-stub.sh.erb').freeze
+
+                def filter_envsh(source_path, output_dir, workspace_dir)
+                    filtered = File.readlines(source_path)
+                                   .find_all { |l| !/^(source|\.)/.match?(l) }
+                    File.open(File.join(output_dir, workspace_dir, 'env.sh'), 'w') do |io|
+                        io.write(filtered.join)
+                    end
+                end
+
+                def generate_autoproj_stub(output_dir, workspace_dir)
+                    dot_autoproj = File.join(output_dir, workspace_dir, '.autoproj')
+                    FileUtils.mkdir File.join(dot_autoproj, 'bin')
+                    autoproj_path = File.join(dot_autoproj, 'bin', 'autoproj')
+
+                    erb = ERB.new(File.read(AUTOPROJ_STUB_PATH))
+                    erb.location = [AUTOPROJ_STUB_PATH, 0]
+                    File.open(autoproj_path, 'w') do |io|
+                        io.write erb.result_with_hash(workspace_dir: "/#{workspace_dir}")
+                    end
+                    FileUtils.chmod 0o755, autoproj_path
+                end
+            end
+        end
+    end
+end

data/rubocop.yml

@@ -0,0 +1,4 @@
+---
+inherit_gem:
+  rubocop-rock: defaults.yml
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoproj-ci
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2019-08-30 00:00:00.000000000 Z
+date: 2020-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: autoproj
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +97,8 @@ dependencies:
 description: 
 email:
 - sylvain.joyeux@13robotics.com
-executables: []
+executables:
+- autoproj-ci
 extensions: []
 extra_rdoc_files: []
 files:
@@ -96,12 +111,15 @@ files:
 - README.md
 - Rakefile
 - autoproj-ci.gemspec
-- bin/console
-- bin/setup
+- bin/autoproj-ci
 - lib/autoproj-ci.rb
+- lib/autoproj/ci/rebuild.rb
 - lib/autoproj/ci/version.rb
+- lib/autoproj/cli/autoproj-stub.sh.erb
 - lib/autoproj/cli/ci.rb
 - lib/autoproj/cli/main_ci.rb
+- lib/autoproj/cli/standalone_ci.rb
+- rubocop.yml
 homepage: https://github.com/rock-core/autoproj-ci
 licenses:
 - MIT
@@ -124,8 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: plugin that provide subcommand useful in CI environments

data/bin/console

@@ -1,14 +0,0 @@
-#!/usr/bin/env ruby
-
-require "bundler/setup"
-require "autoproj/ci"
-
-# You can add fixtures and/or initialization code here to make experimenting
-# with your gem easier. You can also use a different console, if you like.
-
-# (If you use this, don't forget to add pry to your Gemfile!)
-# require "pry"
-# Pry.start
-
-require "irb"
-IRB.start(__FILE__)

data/bin/setup

@@ -1,8 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-IFS=$'\n\t'
-set -vx
-
-bundle install
-
-# Do any other automated setup that you need to do here