autoproj 2.8.8 → 2.9.0

data/lib/autoproj/package_selection.rb

@@ -50,6 +50,35 @@ def empty?
                 selection.empty?
             end
 
+            # Returns the source packages selected explicitely or through
+            # dependencies
+            #
+            # @param [Manifest] manifest
+            # @return [Array<PackageDefinition>]
+            def all_selected_source_packages(manifest)
+                names = Set.new
+                roots = each_source_package_name.to_set
+                roots.each do |pkg_name|
+                    manifest.find_autobuild_package(pkg_name).all_dependencies(names)
+                end
+                names.merge(roots).map do |pkg_name|
+                    manifest.find_package_definition(pkg_name)
+                end
+            end
+
+            # Returns the source packages selected explicitely or through
+            # dependencies
+            #
+            # @param [Manifest] manifest
+            # @return [Array<String>]
+            def all_selected_osdep_packages(manifest)
+                all_sources = all_selected_source_packages(manifest)
+                from_source = all_sources.each_with_object(Set.new) do |pkg, s|
+                    s.merge(pkg.autobuild.os_packages)
+                end
+                from_source | osdeps
+            end
+
             def each(&block)
                 Autoproj.warn_deprecated "PackageSelection#each", "use PackageSelection#each_source_package_name instead", 0
                 each_source_package_name(&block)
@@ -78,27 +107,24 @@ def packages
                 source_packages
             end
 
-            def select(sel, packages, options = Hash.new)
-                if !options.kind_of?(Hash)
+            def select(sel, packages, *_backward, weak: false, osdep: false)
+                unless _backward.empty?
                     Autoproj.warn_deprecated "calling PackageSelection#select with a boolean as third argument", "use e.g. weak: true instead", 0
-                    options = Hash[weak: options]
+                    weak = _backward.first
                 end
-                options = Kernel.validate_options options,
-                    weak: false,
-                    osdep: false
 
                 packages = Array(packages).to_set
                 matches[sel].merge(packages)
                 packages.each do |pkg_name|
                     selection[pkg_name] << sel
                 end
-                if options[:osdep]
+                if osdep
                     osdeps.merge(packages)
                 else
                     source_packages.merge(packages)
                 end
 
-                weak_dependencies[sel] = options[:weak]
+                weak_dependencies[sel] = weak
             end
 
             def initialize_copy(old)
@@ -160,10 +186,10 @@ def filter_excluded_and_ignored_packages(manifest)
                     end
                 end
 
-                source_packages.delete_if do |pkg_name| 
+                source_packages.delete_if do |pkg_name|
                     manifest.excluded?(pkg_name) || manifest.ignored?(pkg_name)
                 end
-                osdeps.delete_if do |pkg_name| 
+                osdeps.delete_if do |pkg_name|
                     manifest.excluded?(pkg_name) || manifest.ignored?(pkg_name)
                 end
                 selection.delete_if do |pkg_name, _|

data/lib/autoproj/package_set.rb

@@ -5,7 +5,7 @@ module Autoproj
     # dependencies that are provided by the operating system (.osdeps file).
     class PackageSet
         # Exception raised when an operation that needs the source.yml to be
-        # loaded is called before {PackageSet#load_description_file} is called 
+        # loaded is called before {PackageSet#load_description_file} is called
         class NotLoaded < RuntimeError
             attr_reader :package_set
             def initialize(package_set)
@@ -34,7 +34,7 @@ def add_source_file(name)
 
         # The manifest this package set is registered on
         #
-        # @return [Manifest] 
+        # @return [Manifest]
         def manifest
             ws.manifest
         end
@@ -64,10 +64,21 @@ def manifest
         #   and the corresponding OSPackageResolver object
         attr_reader :all_osdeps
 
+        # The set of OSRepositoryResolver object that represent the osrepos files
+        # available in this package set
+        #
+        # @return [Array<(String,OSRepositoryResolver)>] the list of osdep files
+        #   and the corresponding OSRepositoryResolver object
+        attr_reader :all_osrepos
+
         # The OSPackageResolver which is a merged version of all OSdeps in
         # {#all_osdeps}
         attr_reader :os_package_resolver
 
+        # The OSRepositoryResolver which is a merged version of all OSrepos in
+        # {#all_osrepos}
+        attr_reader :os_repository_resolver
+
         # If this package set has been imported from another package set, this
         # is the other package set object
         attr_accessor :imported_from
@@ -144,12 +155,14 @@ def initialize(
                 raise ArgumentError, "cannot create a package set with a nil vcs, create a null VCS using VCSDefinition.none"
             end
             @name = name
+            @os_repository_resolver = OSRepositoryResolver.new
             @os_package_resolver = OSPackageResolver.new(
                 operating_system: ws.os_package_resolver.operating_system,
                 package_managers: ws.os_package_resolver.package_managers,
                 os_package_manager: ws.os_package_resolver.os_package_manager)
             @importer_definitions_cache = Hash.new
             @all_osdeps = []
+            @all_osrepos = []
             @constants_definitions = Hash.new
             @required_autoproj_version = '0'
             @version_control = Array.new
@@ -175,11 +188,24 @@ def load_osdeps(file, **options)
             new_osdeps
         end
 
+        # Load a new osrepos file for this package set
+        def load_osrepos(file)
+            new_osrepos = OSRepositoryResolver.load(file)
+            all_osrepos << new_osrepos
+            os_repository_resolver.merge(all_osrepos.last)
+            new_osrepos
+        end
+
         # Enumerate all osdeps package names from this package set
         def each_osdep(&block)
             os_package_resolver.all_package_names.each(&block)
         end
 
+        # Enumerate all osrepos entries from this package set
+        def each_osrepo(&block)
+            os_repository_resolver.all_entries.each(&block)
+        end
+
         # True if this source has already been checked out on the local autoproj
         # installation
         def present?; File.directory?(raw_local_dir) end
@@ -302,7 +328,7 @@ def repository_id
         # For local sources, is simply returns the path to the source directory.
         def user_local_dir
             if local?
-                return vcs.url 
+                return vcs.url
             else
                 File.join(ws.config_dir, 'remotes', name)
             end
@@ -714,7 +740,7 @@ def overrides_for(package, vcs, require_existing: true)
         # This is a helper for {#overrides_for}
         def resolve_overrides(key, vcs)
             overrides.each do |file, file_overrides|
-                new_spec, new_raw_entry = 
+                new_spec, new_raw_entry =
                     Autoproj.in_file file do
                         version_control_field(key, file_overrides, validate: false, file: file)
                     end
@@ -762,6 +788,15 @@ def each_osdeps_file
                 yield(file)
             end
         end
+
+        # Yields each osdeps definition files that are present in this package
+        # set
+        def each_osrepos_file
+            return enum_for(__method__) if !block_given?
+            Dir.glob(File.join(local_dir, "*.osrepos")).each do |file|
+                yield(file)
+            end
+        end
     end
 end
 

data/lib/autoproj/repository_managers/apt.rb

@@ -0,0 +1,289 @@
+# frozen_string_literal: false
+
+require 'open3'
+require 'pathname'
+require 'open-uri'
+
+module Autoproj
+    module RepositoryManagers
+        # Apt repository manager class
+        class APT < Manager
+            attr_reader :source_files
+            attr_reader :source_entries
+            attr_reader :sources_dir
+            attr_reader :autoproj_sources
+
+            SOURCES_DIR = '/etc/apt'.freeze
+            SOURCE_TYPES = ['deb', 'deb-src'].freeze
+            AUTOPROJ_SOURCES = '/etc/apt/sources.list.d/autoproj.list'.freeze
+
+            def initialize(ws, sources_dir: SOURCES_DIR, autoproj_sources: AUTOPROJ_SOURCES)
+                @sources_dir = sources_dir
+                @autoproj_sources = autoproj_sources
+                @source_files = Dir[File.join(sources_dir, '**', '*.list')]
+                @source_entries = {}
+
+                source_files.each { |file| load_sources_from_file(file) }
+                super(ws)
+            end
+
+            def os_dependencies
+                super + ['archive-keyring', 'gnupg', 'apt-transport-https']
+            end
+
+            def load_sources_from_file(file)
+                contents = File.open(file).read
+                contents.gsub!(/\r\n?/, "\n")
+
+                contents.each_line do |line|
+                    @source_entries[file] ||= []
+                    @source_entries[file] << parse_source_line(line, raise_if_invalid: false)
+                end
+            end
+
+            def parse_source_line(line, raise_if_invalid: true)
+                entry = {}
+                entry[:valid] = false
+                entry[:enabled] = true
+                entry[:source] = ''
+                entry[:comment] = ''
+
+                line.strip!
+                if line.start_with?('#')
+                    entry[:enabled] = false
+                    line = line[1..-1]
+                end
+
+                i = line.index('#')
+                if i&.positive?
+                    entry[:comment] = line[(i + 1)..-1].strip
+                    line = line[0..(i - 1)]
+                end
+
+                entry[:source] = line.strip
+                chunks = entry[:source].split
+                entry[:valid] = true if SOURCE_TYPES.include?(chunks[0])
+                entry[:source] = chunks.join(' ')
+
+                if raise_if_invalid && (!entry[:valid] || !entry[:enabled])
+                    raise ConfigError, "Invalid source line: #{entry[:source]}"
+                end
+
+                entry
+            end
+
+            def add_source(source, file = nil)
+                file = if file
+                           File.join(sources_dir, 'sources.list.d', file)
+                       else
+                           autoproj_sources
+                       end
+
+                new_entry = parse_source_line(source)
+                found = entry_exist?(new_entry)
+
+                if found
+                    file = found.first
+                    entry = found.last
+                    return false if entry[:enabled]
+
+                    enable_entry_in_file(file, entry)
+                else
+                    add_entry_to_file(file, new_entry)
+                end
+            end
+
+            def append_entry(contents, entry)
+                unless entry[:enabled]
+                    contents << '#'
+                    contents << ' ' unless entry[:source].start_with?('#')
+                end
+
+                contents << entry[:source]
+                contents << "# #{entry[:comment]}" unless entry[:comment].empty?
+                contents << "\n"
+            end
+
+            def enable_entry_in_file(file, enable_entry)
+                contents = ''
+                source_entries[file].each do |entry|
+                    entry[:enabled] = true if enable_entry[:source] == entry[:source]
+                    append_entry(contents, entry)
+                end
+                run_tee_command(['sudo', 'tee', file], contents)
+                true
+            end
+
+            def add_entry_to_file(file, entry)
+                run_tee_command(['sudo', 'tee', '-a', file], entry[:source])
+                @source_entries[file] ||= []
+                @source_entries[file] << entry
+                true
+            end
+
+            def run_tee_command(command, contents)
+                contents = StringIO.new("#{contents}\n")
+                Autobuild::Subprocess.run('autoproj', 'osrepos', *command, input_streams: [contents])
+            end
+
+            def entry_exist?(new_entry)
+                source_entries.each_pair do |file, entries|
+                    entry = entries.find { |e| e[:source] == new_entry[:source] }
+                    return [file, entry] if entry
+                end
+                nil
+            end
+
+            def source_exist?(source)
+                entry_exist?(parse_source_line(source))
+            end
+
+            def key_exist?(key)
+                exist = false
+                Open3.popen3("apt-key export #{key}") do |_, _, stderr, wait_thr|
+                    exist = true if wait_thr.value.success? && stderr.read.empty?
+                end
+                exist
+            end
+
+            def apt_update
+                Autobuild::Subprocess.run(
+                    'autoproj',
+                    'osrepos',
+                    'sudo',
+                    'apt-get',
+                    'update'
+                )
+            end
+
+            def add_apt_key(id, origin, type: :keyserver)
+                if type == :keyserver
+                    Autobuild::Subprocess.run(
+                        'autoproj',
+                        'osrepos',
+                        'sudo',
+                        'apt-key',
+                        'adv',
+                        '--keyserver',
+                        origin,
+                        '--recv-key',
+                        id
+                    )
+                else
+                    open(origin) do |io|
+                        Autobuild::Subprocess.run(
+                            'autoproj',
+                            'osrepos',
+                            'sudo',
+                            'apt-key',
+                            'add',
+                            '-',
+                            input_streams: [io]
+                        )
+                    end
+                end
+            rescue Errno::ENOENT, SocketError => e
+                raise ConfigError, e.message
+            end
+
+            def filter_installed_definitions(definitions)
+                definitions = definitions.dup.reject do |definition|
+                    if definition['type'] == 'repo'
+                        _, entry = source_exist?(definition['repo'])
+                        entry && entry[:enabled]
+                    else
+                        key_exist?(definition['id'])
+                    end
+                end
+                definitions
+            end
+
+            def print_installing_definitions(definitions)
+                repos = definitions.select { |definition| definition['type'] == 'repo' }
+                keys = definitions.select { |definition| definition['type'] == 'key' }
+
+                unless repos.empty?
+                    Autoproj.message '  adding apt repositories:'
+                    repos.each do |repo|
+                        if repo['file']
+                            Autoproj.message "    #{repo['repo']}, file: #{repo['file']}"
+                        else
+                            Autoproj.message "    #{repo['repo']}"
+                        end
+                    end
+                end
+                return if keys.empty?
+
+                Autoproj.message '  adding apt keys:'
+                keys.each do |key|
+                    if key['keyserver']
+                        Autoproj.message "    id: #{key['id']}, keyserver: #{key['keyserver']}"
+                    else
+                        Autoproj.message "    id: #{key['id']}, url: #{key['url']}"
+                    end
+                end
+            end
+
+            # Validates repositories definitions from .osrepos files
+            #
+            # Examples:
+            #
+            # - ubuntu:
+            #   - xenial:
+            #     type: repo
+            #     repo: 'deb http://archive.ubuntu.com/ubuntu/ xenial main restricted'
+            #
+            # - ubuntu:
+            #   - xenial:
+            #     type: key
+            #     id: 630239CC130E1A7FD81A27B140976EAF437D05B5
+            #     keyserver: 'hkp://ha.pool.sks-keyservers.net:80'
+            #
+            # - ubuntu:
+            #   - xenial:
+            #     type: key
+            #     id: D2486D2DD83DB69272AFE98867170598AF249743
+            #     url: 'http://packages.osrfoundation.org/gazebo.key'
+            #
+            def validate_definitions(definitions)
+                invalid_string = 'Invalid apt repository definition'
+                definitions.each do |definition|
+                    raise ConfigError, "#{invalid_string} type: #{definition['type']}" unless %w[repo key].include?(definition['type'])
+
+                    if definition['type'] == 'repo'
+                        raise ConfigError, "#{invalid_string}: 'repo' key missing" if definition['repo'].nil?
+                        raise ConfigError, "#{invalid_string}: 'repo' should be a String" unless definition['repo'].is_a?(String)
+                        raise ConfigError, "#{invalid_string}: 'file' should be a String" if definition['file'] && !definition['file'].is_a?(String)
+                        if definition['file'] && Pathname.new(definition['file']).absolute?
+                            raise ConfigError, "#{invalid_string}: 'file' should be a relative to #{File.join(SOURCES_DIR, 'sources.list.d')}"
+                        end
+                    else
+                        raise ConfigError, "#{invalid_string}: 'id' key missing" if definition['id'].nil?
+                        raise ConfigError, "#{invalid_string}: 'id' should be a String" unless definition['id'].is_a?(String)
+                        raise ConfigError, "#{invalid_string}: 'url' conflicts with 'keyserver'" if definition['url'] && definition['keyserver']
+                        raise ConfigError, "#{invalid_string}: 'url' should be a String" if definition['url'] && !definition['url'].is_a?(String)
+                        raise ConfigError, "#{invalid_string}: 'keyserver' should be a String" if definition['keyserver'] && !definition['keyserver'].is_a?(String)
+                    end
+                end
+            end
+
+            def install(definitions)
+                super
+                validate_definitions(definitions)
+                definitions = filter_installed_definitions(definitions)
+                print_installing_definitions(definitions)
+
+                definitions.each do |definition|
+                    if definition['type'] == 'repo'
+                        add_source(definition['repo'], definition['file'])
+                    else
+                        type = definition['url'] ? 'url' : 'keyserver'
+                        origin = definition[type]
+                        add_apt_key(definition['id'], origin, type: type.to_sym)
+                    end
+                end
+                apt_update unless definitions.empty?
+            end
+        end
+    end
+end