autoproj 2.8.8 → 2.9.0

data/lib/autoproj/os_repository_resolver.rb

@@ -0,0 +1,137 @@
+require 'tempfile'
+require 'json'
+
+module Autoproj
+    # Manager for OS repository provided by package sets
+    class OSRepositoryResolver
+        # All the information contained in all the OSrepos files
+        attr_reader :all_definitions
+
+        # The operating system
+        attr_accessor :operating_system
+
+        def self.load(file)
+            raise ArgumentError, "no such file or directory: #{file}" unless File.file?(file)
+
+            error_t = if defined? Psych::SyntaxError
+                          [ArgumentError, Psych::SyntaxError]
+                      else
+                          ArgumentError
+                      end
+
+            result = new
+            file = File.expand_path(file)
+            begin
+                data = YAML.safe_load(File.read(file)) || {}
+                verify_definitions(data)
+            rescue *error_t => e
+                raise ConfigError.new, "error in #{file}: #{e.message}", e.backtrace
+            end
+            result.merge(new(data, file))
+            result
+        end
+
+        def initialize(defs = [], file = nil, operating_system: nil)
+            @operating_system = operating_system
+            @all_definitions = Set.new
+            if file
+                defs.each do |def_|
+                    all_definitions << [[file], def_]
+                end
+            else
+                defs.each do |def_|
+                    all_definitions << [[], def_]
+                end
+            end
+        end
+
+        def merge(info)
+            @all_definitions += info.all_definitions
+        end
+
+        def definitions
+            all_definitions.map(&:last).uniq
+        end
+
+        def all_entries
+            definitions.map do |distribution|
+                distribution.values.map do |release|
+                    release.map(&:values)
+                end
+            end.flatten.uniq
+        end
+
+        def entry_matches?(entry, identifiers)
+            !(entry.keys.first.split(',').map(&:strip) & identifiers).empty?
+        end
+
+        def resolved_entries
+            os_name, os_version = operating_system
+            os_version << 'default' unless os_version.include?('default')
+
+            distribution_filtered = definitions.select do |entry|
+                entry_matches?(entry, os_name)
+            end.map(&:values).flatten
+
+            release_filtered = distribution_filtered.select { |entry| entry_matches?(entry, os_version) }
+            release_filtered.map(&:values).flatten.uniq
+        end
+
+        # OS repos definitions must follow the format:
+        #
+        # - distribution:
+        #   - release:
+        #     - key1: value1
+        #       key2: value2
+        #
+        # The key, value pairs are OS dependent, and will be verified/parsed by the
+        # corresponding 'repository manager'. Thus, for a debian-like distribution
+        # one would have something like:
+        #
+        # - ubuntu:
+        #   - xenial:
+        #     - type: repo
+        #       repo: 'deb http://br.archive.ubuntu.com/ubuntu/ xenial main restricted'
+        def self.verify_definitions(array, path = [])
+            verify_type(array, Array, path)
+            array.each do |entry|
+                verify_type(entry, Hash, path)
+                verify_os(entry, (path + [entry]))
+            end
+        end
+
+        def self.verify_os(hash, path = [])
+            verify_type(hash, Hash, path)
+            hash.each do |key, value|
+                verify_type(key, String, path)
+                verify_release(value, (path + [value]))
+            end
+        end
+
+        def self.verify_release(array, path = [])
+            verify_type(array, Array, path)
+            array.each do |releases|
+                verify_type(releases, Hash, path)
+
+                releases.values.each do |entries|
+                    verify_type(entries, Array, path)
+                    verify_entries(entries, path + entries)
+                end
+            end
+        end
+
+        def self.verify_entries(array, path = [])
+            verify_type(array, Array, path)
+            array.each do |entry|
+                verify_type(entry, Hash, path)
+            end
+        end
+
+        def self.verify_type(obj, type, path = [])
+            return if obj.is_a?(type)
+
+            raise ArgumentError, "invalid osrepos definition in #{path.join('/')}: "\
+                "expected a #{type}, found a #{obj.class}"
+        end
+    end
+end

data/lib/autoproj/package_managers/apt_dpkg_manager.rb

@@ -1,3 +1,5 @@
+require 'autoproj/package_managers/debian_version'
+
 module Autoproj
     module PackageManagers
         # Package manager interface for systems that use APT and dpkg for
@@ -8,15 +10,36 @@ class AptDpkgManager < ShellScriptManager
             def initialize(ws, status_file = "/var/lib/dpkg/status")
                 @status_file = status_file
                 @installed_packages = nil
+                @installed_versions = nil
                 super(ws, true,
                       %w{apt-get install},
                       %w{DEBIAN_FRONTEND=noninteractive apt-get install -y})
             end
 
-            def self.parse_package_status(installed_packages, paragraph)
+            def configure_manager
+                super
+                ws.config.declare 'apt_dpkg_update', 'boolean',
+                    default: 'yes',
+                    doc: ['Would you like autoproj to keep apt packages up-to-date?']
+                keep_uptodate?
+            end
+
+            def keep_uptodate?
+                ws.config.get('apt_dpkg_update')
+            end
+
+            def keep_uptodate=(flag)
+                ws.config.set('apt_dpkg_update', flag, true)
+            end
+
+            def self.parse_package_status(installed_packages, installed_versions, paragraph)
                 if paragraph =~ /^Status: install ok installed$/
                     if paragraph =~ /^Package: (.*)$/
-                        installed_packages << $1
+                        package_name = $1
+                        installed_packages << package_name
+                        if paragraph =~ /^Version: (.*)$/
+                            installed_versions[package_name] = DebianVersion.new($1)
+                        end
                     end
                     if paragraph =~ /^Provides: (.*)$/
                         installed_packages.merge($1.split(',').map(&:strip))
@@ -26,6 +49,7 @@ def self.parse_package_status(installed_packages, paragraph)
 
             def self.parse_dpkg_status(status_file)
                 installed_packages = Set.new
+                installed_versions = {}
                 dpkg_status = File.read(status_file)
                 dpkg_status << "\n"
 
@@ -36,17 +60,53 @@ def self.parse_dpkg_status(status_file)
 
                 while paragraph_end = dpkg_status.scan_until(/Package: /)
                     paragraph = "Package: #{paragraph_end[0..-10]}"
-                    parse_package_status(installed_packages, paragraph)
+                    parse_package_status(installed_packages, installed_versions, paragraph)
+                end
+                parse_package_status(installed_packages, installed_versions, "Package: #{dpkg_status.rest}")
+                [installed_packages, installed_versions]
+            end
+
+            def self.parse_apt_cache_paragraph(paragraph)
+                version = '0'
+                if paragraph =~ /^Package: (.*)$/
+                    package_name = $1
+                    if paragraph =~ /^Version: (.*)$/
+                        version = $1
+                    end
+                end
+                [package_name, version]
+            end
+
+            def self.parse_packages_versions(packages)
+                packages_versions = {}
+                apt_cache_show = `apt-cache show --no-all-versions #{packages.join(' ')}`
+                apt_cache_show = StringScanner.new(apt_cache_show)
+                if !apt_cache_show.scan(/Package: /)
+                    return packages_versions
+                end
+
+                while paragraph_end = apt_cache_show.scan_until(/Package: /)
+                    paragraph = "Package: #{paragraph_end[0..-10]}"
+                    package_name, version = parse_apt_cache_paragraph(paragraph)
+                    packages_versions[package_name] = DebianVersion.new(version)
                 end
-                parse_package_status(installed_packages, "Package: #{dpkg_status.rest}")
-                installed_packages
+                package_name, version = parse_apt_cache_paragraph("Package: #{apt_cache_show.rest}")
+                packages_versions[package_name] = DebianVersion.new(version)
+                packages_versions
+            end
+
+            def updated?(package, available_version)
+                # Consider up-to-date if the package is provided by another package (purely virtual)
+                # Ideally, we should check the version of the package that provides it
+                return true unless available_version && @installed_versions[package]
+
+                (available_version <= @installed_versions[package])
             end
 
             # On a dpkg-enabled system, checks if the provided package is installed
             # and returns true if it is the case
             def installed?(package_name, filter_uptodate_packages: false, install_only: false)
-                @installed_packages ||= AptDpkgManager.parse_dpkg_status(status_file)
-                
+                @installed_packages, @installed_versions = self.class.parse_dpkg_status(status_file) unless @installed_packages && @installed_versions
                 if package_name =~ /^(\w[a-z0-9+-.]+)/
                     @installed_packages.include?($1)
                 else
@@ -54,11 +114,12 @@ def installed?(package_name, filter_uptodate_packages: false, install_only: fals
                     false
                 end
             end
-            
+
             def install(packages, filter_uptodate_packages: false, install_only: false)
+                packages_versions = self.class.parse_packages_versions(packages)
                 if filter_uptodate_packages || install_only
                     packages = packages.find_all do |package_name|
-                        !installed?(package_name)
+                        !installed?(package_name) || (keep_uptodate? && !updated?(package_name, packages_versions[package_name]))
                     end
                 end
 
@@ -71,4 +132,3 @@ def install(packages, filter_uptodate_packages: false, install_only: false)
         end
     end
 end
-

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -213,7 +213,7 @@ def self.run_bundler(ws, *commandline, gem_home: nil, gemfile: nil)
                         'GEM_PATH' => nil,
                         'BUNDLE_GEMFILE' => gemfile,
                         'RUBYOPT' => nil,
-                        'RUBYLIB' => nil
+                        'RUBYLIB' => rubylib_for_bundler
                     ]
                     ws.run 'autoproj', 'osdeps',
                         Autobuild.tool('bundle'), *commandline,
@@ -388,6 +388,11 @@ def discover_rubylib
                 end
             end
 
+            def self.rubylib_for_bundler
+                rx = Regexp.new("/gems/#{Regexp.quote("bundler-#{Bundler::VERSION}")}/")
+                $LOAD_PATH.grep(rx).join(File::PATH_SEPARATOR)
+            end
+
             def discover_bundle_rubylib(silent_errors: false)
                 require 'bundler'
                 gemfile = File.join(ws.prefix_dir, 'gems', 'Gemfile')
@@ -398,7 +403,9 @@ def discover_bundle_rubylib(silent_errors: false)
                 env = ws.env.resolved_env
                 Tempfile.open 'autoproj-rubylib' do |io|
                     result = Bundler.clean_system(
-                        Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'], 'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil, 'RUBYLIB' => nil],
+                        Hash['GEM_HOME' => env['GEM_HOME'], 'GEM_PATH' => env['GEM_PATH'],
+                             'BUNDLE_GEMFILE' => gemfile, 'RUBYOPT' => nil,
+                             'RUBYLIB' => self.class.rubylib_for_bundler],
                         Autobuild.tool('ruby'), '-rbundler/setup', '-e', 'puts $LOAD_PATH',
                         out: io, **silent_redirect)
                         

data/lib/autoproj/package_managers/debian_version.rb

@@ -0,0 +1,123 @@
+module Autoproj
+    module PackageManagers
+        class DebianVersion
+            attr_reader :version
+            attr_reader :epoch
+            attr_reader :upstream_version
+            attr_reader :debian_revision
+
+            include Comparable
+
+            def initialize(version)
+                @version = version
+                parse_version
+            end
+
+            def split
+                [epoch, upstream_version, debian_revision]
+            end
+
+            def <=>(b)
+                (0..2).inject(0) do |result, i|
+                    return result unless result == 0
+                    normalize(compare_fragments(self.split[i], b.split[i]))
+                end
+            end
+
+            def self.compare(a, b)
+                new(a)<=>new(b)
+            end
+        
+            private
+
+            def normalize(value)
+                return -1 if value < 0
+                return 1 if value > 0
+                return 0 if value == 0
+            end
+
+            # Reference: https://www.debian.org/doc/debian-policy/ch-controlfields.html#version
+            def parse_version
+                @epoch = '0'
+                @debian_revision = '0'
+
+                @upstream_version = @version.split(':')
+                if @upstream_version.size > 1
+                    @epoch = @upstream_version.first
+                    @upstream_version = @upstream_version[1..-1].join(':')
+                else
+                    @upstream_version = @upstream_version.first
+                end
+
+                @upstream_version = @upstream_version.split('-')
+                if @upstream_version.size > 1
+                    @debian_revision = @upstream_version.last
+                    @upstream_version = @upstream_version[0..-2].join('-')
+                else
+                    @upstream_version = @upstream_version.first
+                end
+            end
+
+            def alpha?(look_ahead)
+                look_ahead =~ /[[:alpha:]]/
+            end
+
+            def digit?(look_ahead)
+                look_ahead =~ /[[:digit:]]/
+            end
+
+            def order(c)
+                if digit?(c)
+                    return 0
+                elsif alpha?(c)
+                    return c.ord
+                elsif c == '~'
+                    return -1
+                elsif c
+                    return c.ord + 256
+                else
+                    return 0
+                end
+            end
+
+            # Ported from https://github.com/Debian/apt/blob/master/apt-pkg/deb/debversion.cc
+            def compare_fragments(a, b)
+                i = 0
+                j = 0
+                while i != a.size && j != b.size
+                    first_diff = 0
+                    while i != a.size && j != b.size && (!digit?(a[i]) || !digit?(b[j]))
+                        vc = order(a[i])
+                        rc = order(b[j])
+                        return vc-rc if vc != rc
+                        i += 1
+                        j += 1
+                    end
+
+                    i += 1 while a[i] == '0'
+                    j += 1 while b[j] == '0'
+                    while digit?(a[i]) && digit?(b[j])
+                        first_diff = a[i].ord - b[j].ord if first_diff == 0
+                        i += 1
+                        j += 1
+                    end
+
+                    return 1 if digit?(a[i])
+                    return -1 if digit?(b[j])
+                    return first_diff if first_diff != 0
+                end
+
+                return 0 if i == a.size && j == b.size
+
+                if i == a.size
+                    return 1 if b[j] == '~'
+                    return -1
+                end
+                if j == b.size
+                    return -1 if a[i] == '~'
+                    return 1
+                end
+            end
+        end
+    end
+end

data/lib/autoproj/package_managers/manager.rb

@@ -41,6 +41,11 @@ def strict?
                 false
             end
 
+            # If this package manager depends on OS packages, they should be added here
+            def os_dependencies
+                []
+            end
+
             # Create a package manager
             #
             # @param [Workspace] ws the underlying workspace
@@ -56,6 +61,10 @@ def initialize(ws)
             # This is e.g. needed for python pip or rubygems
             def initialize_environment
             end
+
+            # Perform additional configuration required for the package manager
+            def configure_manager
+            end
         end
     end
 end

data/lib/autoproj/package_managers/pip_manager.rb

@@ -20,6 +20,10 @@ def initialize(ws)
                 @installed_pips = Set.new
             end
 
+            def os_dependencies
+                super + ['pip']
+            end
+
             def guess_pip_program
                 if Autobuild.programs['pip']
                     return Autobuild.programs['pip']