autoproj 2.10.1 → 2.13.0

data/lib/autoproj/default.osdeps

@@ -92,6 +92,12 @@ ruby24:
 ruby25:
   default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
 
+ruby26:
+  default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
+
+ruby27:
+  default: ignore # we assume that if the user has a ruby 2.5 runtime, it is usable
+
 build-essential:
   debian,ubuntu: build-essential
   gentoo: ignore
@@ -132,15 +138,19 @@ archive:
   debian,ubuntu:
   - tar
   - unzip
+  - ca-certificates
   gentoo:
   - app-arch/tar
   - app-arch/unzip
+  - app-misc/ca-certificates
   arch:
   - tar
   - unzip
+  - ca-certificates
   fedora:
   - tar
   - unzip
+  - ca-certificates
   macos-port:
   - gnutar
   - unzip
@@ -149,6 +159,7 @@ archive:
   opensuse:
   - tar
   - unzip
+  - ca-certificates
   default: ignore
 
 cvs:
@@ -171,6 +182,13 @@ python:
   opensuse: python-devel
   default: ignore # will be installed manually by the user
 
+python-setuptools:
+  arch: python2-distribute
+  debian,ubuntu: python-setuptools
+  fedora: python-setuptools
+  gentoo: dev-python/setuptools
+  default: ignore # will be installed manually by the user
+
 sudo:
   macos-brew: ignore
   default: sudo

data/lib/autoproj/installation_manifest.rb

@@ -1,7 +1,8 @@
 module Autoproj
     # Manifest of installed packages imported from another autoproj installation
     class InstallationManifest
-        Package = Struct.new :name, :type, :vcs, :srcdir, :prefix, :builddir, :logdir, :dependencies
+        Package = Struct.new :name, :type, :vcs, :srcdir, :importdir,
+                             :prefix, :builddir, :logdir, :dependencies
         PackageSet = Struct.new :name, :vcs, :raw_local_dir, :user_local_dir
 
         attr_reader :path
@@ -28,11 +29,11 @@ def add_package_set(pkg_set)
         def each_package_set(&block)
             package_sets.each_value(&block)
         end
-            
+
         def each_package(&block)
             packages.each_value(&block)
         end
-            
+
         def load
             @packages = Hash.new
             raw = YAML.load(File.open(path))
@@ -51,8 +52,8 @@ def load
                         package_sets[pkg_set.name] = pkg_set
                     else
                         pkg = Package.new(
-                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['prefix'],
-                            entry['builddir'], entry['logdir'], entry['dependencies'])
+                            entry['name'], entry['type'], entry['vcs'], entry['srcdir'], entry['importdir'],
+                            entry['prefix'], entry['builddir'], entry['logdir'], entry['dependencies'])
                         packages[pkg.name] = pkg
                     end
                 end
@@ -74,6 +75,7 @@ def save(path = self.path)
                          'type' => v.class.name,
                          'vcs' => package_def.vcs.to_hash,
                          'srcdir' => v.srcdir,
+                         'importdir' => (v.importdir if v.respond_to?(:importdir)),
                          'builddir' => (v.builddir if v.respond_to?(:builddir)),
                          'logdir' => v.logdir,
                          'prefix' => v.prefix,

data/lib/autoproj/manifest.rb

@@ -50,12 +50,14 @@ def self.load(file)
 
         # Load the manifest data contained in +file+
         def load(file)
-            if !File.exist?(file)
-                raise ConfigError.new(File.dirname(file)), "expected an autoproj configuration in #{File.dirname(file)}, but #{file} does not exist"
+            unless File.exist?(file)
+                raise ConfigError.new(File.dirname(file)),
+                      "expected an autoproj configuration in #{File.dirname(file)}, "\
+                      "but #{file} does not exist"
             end
 
             data = Autoproj.in_file(file, Autoproj::YAML_LOAD_ERROR) do
-                YAML.load(File.read(file)) || Hash.new
+                YAML.safe_load(File.read(file)) || {}
             end
 
             @file = file
@@ -81,7 +83,8 @@ def initialize_from_hash(data)
             @has_layout = !!data['layout']
 
             if data['constants']
-                @constant_definitions = Autoproj.resolve_constant_definitions(data['constants'])
+                @constant_definitions =
+                    Autoproj.resolve_constant_definitions(data['constants'])
             end
         end
 
@@ -1020,23 +1023,14 @@ def load_package_manifest(pkg)
             if manifest
                 pkg.autobuild.description = manifest
             else
-                Autoproj.warn "#{package.name} from #{package_set.name} does not have a manifest"
+                Autoproj.warn "#{package.name} from #{package_set.name} "\
+                              "does not have a manifest"
             end
 
-            manifest = pkg.autobuild.description
-            manifest.each_dependency(pkg.modes) do |name, is_optional|
-                begin
-                    if is_optional
-                        package.optional_dependency name
-                    else
-                        package.depends_on name
-                    end
-                rescue ConfigError => e
-                    raise ConfigError.new(manifest_path),
-                        "manifest #{manifest_path} of #{package.name} from #{package_set.name} lists '#{name}' as dependency, but it is neither a normal package nor an osdeps package. osdeps reports: #{e.message}", e.backtrace
-                end
-            end
-            manifest
+            pkg.apply_dependencies_from_manifest
+            # #description is initialized with a null package manifest
+            # return it even if we haven't overriden it
+            pkg.autobuild.description
         end
 
         def load_all_available_package_manifests
@@ -1243,7 +1237,7 @@ def self.config
         workspace.config
     end
 
-    def self.add_osdeps_overrides(*args, &block)
-        manifest.add_osdeps_overrides(*args, &block)
+    def self.add_osdeps_overrides(*args, **kw, &block)
+        manifest.add_osdeps_overrides(*args, **kw, &block)
     end
 end

data/lib/autoproj/ops/build.rb

@@ -87,40 +87,36 @@ def force_build_packages(selected_packages, all_enabled_packages)
             #   names of the packages that should be rebuilt
             # @return [void]
             def build_packages(all_enabled_packages, options = Hash.new)
+                if @report_path
+                    reporting = Ops::PhaseReporting.new(
+                        'build', @report_path, method(:package_metadata)
+                    )
+                end
+
                 Autobuild.do_rebuild = false
                 Autobuild.do_forced_build = false
+                reporting&.initialize_incremental_report
                 begin
-                    Autobuild.apply(all_enabled_packages, "autoproj-build", ['build'], options)
+                    Autobuild.apply(
+                        all_enabled_packages,
+                        "autoproj-build", ['build'], options
+                    ) do |pkg, phase|
+                        reporting&.report_incremental(pkg) if phase == 'build'
+                    end
+
                 ensure
-                    build_report(all_enabled_packages) if @report_path
+                    packages = all_enabled_packages.map do |name|
+                        @manifest.find_autobuild_package(name)
+                    end
+                    reporting&.create_report(packages)
                 end
             end
 
-            def build_report(package_list)
-                FileUtils.mkdir_p File.dirname(@report_path)
-
-                packages = package_list.map do |pkg_name|
-                    pkg = manifest.find_autobuild_package(pkg_name)
-
-                    {
-                        name: pkg.name,
-                        import_invoked: pkg.import_invoked?,
-                        prepare_invoked: pkg.prepare_invoked?,
-                        build_invoked: pkg.build_invoked?,
-                        failed: pkg.failed?,
-                        imported: pkg.imported?,
-                        prepared: pkg.prepared?,
-                        built: pkg.built?
-                    }
-                end
-
-                build_report = JSON.pretty_generate({
-                    build_report: {
-                        timestamp: Time.now,
-                        packages: packages
-                    }
-                })
-                IO.write(@report_path, build_report)
+            def package_metadata(autobuild_package)
+                {
+                    invoked: !!autobuild_package.install_invoked?,
+                    success: !!autobuild_package.installed?
+                }
             end
         end
     end

data/lib/autoproj/ops/cache.rb

@@ -4,9 +4,10 @@ class Cache
             attr_reader :cache_dir
             attr_reader :manifest
 
-            def initialize(cache_dir, manifest)
+            def initialize(cache_dir, ws)
                 @cache_dir = cache_dir
-                @manifest  = manifest
+                @ws = ws
+                @manifest = ws.manifest
             end
 
             def with_retry(count)
@@ -27,29 +28,39 @@ def git_cache_dir
                 File.join(cache_dir, 'git')
             end
 
-            def cache_git(pkg, options = Hash.new)
+            def cache_git(pkg, checkout_only: false)
                 pkg.importdir = File.join(git_cache_dir, pkg.name)
-                if options[:checkout_only] && File.directory?(pkg.importdir)
-                    return
-                end
+                return if checkout_only && File.directory?(pkg.importdir)
 
                 pkg.importer.local_branch = nil
                 pkg.importer.remote_branch = nil
                 pkg.importer.remote_name = 'autobuild'
 
-                if !File.directory?(pkg.importdir)
+                unless File.directory?(pkg.importdir)
                     FileUtils.mkdir_p File.dirname(pkg.importdir)
-                    Autobuild::Subprocess.run("autoproj-cache", "import", Autobuild.tool(:git), "--git-dir", pkg.importdir, 'init', "--bare")
+                    Autobuild::Subprocess.run(
+                        "autoproj-cache", "import", Autobuild.tool(:git),
+                        "--git-dir", pkg.importdir, 'init', "--bare"
+                    )
                 end
                 pkg.importer.update_remotes_configuration(pkg)
 
                 with_retry(10) do
-                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')
+                    Autobuild::Subprocess.run(
+                        'autoproj-cache', :import, Autobuild.tool('git'),
+                        '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild'
+                    )
                 end
                 with_retry(10) do
-                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', 'autobuild', '--tags')
+                    Autobuild::Subprocess.run(
+                        'autoproj-cache', :import, Autobuild.tool('git'),
+                        '--git-dir', pkg.importdir, 'fetch', 'autobuild', '--tags'
+                    )
                 end
-                Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'gc', '--prune=all')
+                Autobuild::Subprocess.run(
+                    'autoproj-cache', :import, Autobuild.tool('git'),
+                    '--git-dir', pkg.importdir, 'gc', '--prune=all'
+                )
             end
 
             def archive_cache_dir
@@ -63,7 +74,8 @@ def cache_archive(pkg)
                 end
             end
 
-            def create_or_update(*package_names, all: true, keep_going: false, checkout_only: false)
+            def create_or_update(*package_names, all: true, keep_going: false,
+                                 checkout_only: false)
                 FileUtils.mkdir_p cache_dir
 
                 if package_names.empty?
@@ -75,7 +87,7 @@ def create_or_update(*package_names, all: true, keep_going: false, checkout_only
                         end
                 else
                     packages = package_names.map do |name|
-                        if pkg = manifest.find_autobuild_package(name)
+                        if (pkg = manifest.find_autobuild_package(name))
                             pkg
                         else
                             raise PackageNotFound, "no package named #{name}"
@@ -88,42 +100,148 @@ def create_or_update(*package_names, all: true, keep_going: false, checkout_only
                 total = packages.size
                 Autoproj.message "Handling #{total} packages"
                 packages.each_with_index do |pkg, i|
-                    next if pkg.srcdir != pkg.importdir # No need to process this one, it is uses another package's import
+                    # No need to process this one, it is uses another package's
+                    # import
+                    next if pkg.srcdir != pkg.importdir
+
                     begin
                         case pkg.importer
                         when Autobuild::Git
-                            Autoproj.message "  [#{i}/#{total}] caching #{pkg.name} (git)"
+                            Autoproj.message(
+                                "  [#{i}/#{total}] caching #{pkg.name} (git)"
+                            )
                             cache_git(pkg, checkout_only: checkout_only)
                         when Autobuild::ArchiveImporter
-                            Autoproj.message "  [#{i}/#{total}] caching #{pkg.name} (archive)"
+                            Autoproj.message(
+                                "  [#{i}/#{total}] caching #{pkg.name} (archive)"
+                            )
                             cache_archive(pkg)
                         else
-                            Autoproj.message "  [#{i}/#{total}] not caching #{pkg.name} (cannot cache #{pkg.importer.class})"
+                            Autoproj.message(
+                                "  [#{i}/#{total}] not caching #{pkg.name} "\
+                                "(cannot cache #{pkg.importer.class})"
+                            )
                         end
                     rescue Interrupt
                         raise
                     rescue ::Exception => e
-                        if keep_going
-                            pkg.error "       failed to cache #{pkg.name}, but going on as requested"
-                            lines = e.to_s.split("\n")
-                            if lines.empty?
-                                lines = e.message.split("\n")
-                            end
-                            if lines.empty?
-                                lines = ["unknown error"]
-                            end
-                            pkg.message(lines.shift, :red, :bold)
-                            lines.each do |line|
-                                pkg.message(line)
+                        raise unless keep_going
+
+                        pkg.error "       failed to cache #{pkg.name}, "\
+                                  'but going on as requested'
+                        lines = e.to_s.split('\n')
+                        lines = e.message.split('\n') if lines.empty?
+                        lines = ['unknown error'] if lines.empty?
+                        pkg.message(lines.shift, :red, :bold)
+                        lines.each do |line|
+                            pkg.message(line)
+                        end
+                        nil
+                    end
+                end
+            end
+
+            def gems_cache_dir
+                File.join(cache_dir, 'package_managers', 'gem')
+            end
+
+            def create_or_update_gems(keep_going: true, compile_force: false, compile: [])
+                # Note: this might directly copy into the cache directoy, and
+                # we support it later
+                cache_dir = File.join(@ws.prefix_dir, 'gems', 'vendor', 'cache')
+                PackageManagers::BundlerManager.run_bundler(
+                    @ws, 'cache'
+                )
+
+                FileUtils.mkdir_p(gems_cache_dir) unless File.exist?(gems_cache_dir)
+
+                needs_copy =
+                    if File.exist?(cache_dir)
+                        real_cache_dir = File.realpath(cache_dir)
+                        real_target_dir = File.realpath(gems_cache_dir)
+                        (real_cache_dir != real_target_dir)
+                    end
+
+                synchronize_gems_cache_dirs(real_cache_dir, real_target_dir) if needs_copy
+
+                platform_suffix = "-#{Gem::Platform.local}.gem"
+                failed = []
+                compile.each do |gem_name, artifacts: []|
+                    Dir.glob(File.join(cache_dir, "#{gem_name}*.gem")) do |gem|
+                        next if gem.end_with?(platform_suffix)
+
+                        gem_basename = File.basename(gem, ".gem")
+                        expected_platform_gem = File.join(
+                            real_target_dir, "#{gem_basename}#{platform_suffix}"
+                        )
+                        next if !compile_force && File.file?(expected_platform_gem)
+
+                        begin
+                            compile_gem(
+                                gem, artifacts: artifacts, output: real_target_dir
+                            )
+                        rescue CompilationFailed
+                            unless keep_going
+                                raise CompilationFailed, "#{gem} failed to compile"
                             end
-                            nil
-                        else
-                            raise
+
+                            failed << gem
                         end
                     end
                 end
+
+                unless failed.empty?
+                    raise CompilationFailed, "#{failed.sort.join(', ')} failed to compile"
+                end
+            end
+
+            class CompilationFailed < RuntimeError; end
+
+            def synchronize_gems_cache_dirs(source, target)
+                Dir.glob(File.join(source, "*.gem")) do |source_file|
+                    basename = File.basename(source_file)
+                    target_file = File.join(target, basename)
+                    next if File.file?(target_file)
+
+                    Autoproj.message "gems: caching #{basename}"
+                    FileUtils.cp source_file, target_file
+                end
+            end
+
+            def guess_gem_program
+                return Autobuild.programs['gem'] if Autobuild.programs['gem']
+
+                ruby_bin = RbConfig::CONFIG['RUBY_INSTALL_NAME']
+                ruby_bindir = RbConfig::CONFIG['bindir']
+
+                candidates = ['gem']
+                candidates << "gem#{$1}" if ruby_bin =~ /^ruby(.+)$/
+
+                candidates.each do |gem_name|
+                    if File.file?(gem_full_path = File.join(ruby_bindir, gem_name))
+                        Autobuild.programs['gem'] = gem_full_path
+                        return Autobuild.programs['gem']
+                    end
+                end
+
+                raise ArgumentError,
+                      'cannot find a gem program (tried '\
+                      "#{candidates.sort.join(', ')} in #{ruby_bindir})"
+            end
+
+            private def compile_gem(gem_path, output:, artifacts: [])
+                artifacts = artifacts.flat_map do |include, n|
+                    if include
+                        ["--include", n]
+                    else
+                        ["--exclude", n]
+                    end
+                end
+                unless system(Autobuild.tool('ruby'), '-S', guess_gem_program,
+                              'compile', '--output', output, *artifacts, gem_path)
+                    raise CompilationFailed, "#{gem_path} failed to compile"
+                end
             end
         end
     end
 end
-