autoproj 2.3.1 → 2.4.0

data/lib/autoproj/manifest.rb

@@ -240,7 +240,7 @@ def validate_package_name_argument(package, require_existing: true)
         #
         # Validate that the given package object is defined in self
         def validate_package_set_in_self(package_set)
-            if find_package_set(package.name) != package_set
+            if find_package_set(package_set.name) != package_set
                 raise UnregisteredPackageSet, "#{package_set.name} is not registered on #{self}"
             end
         end

data/lib/autoproj/ops/cache.rb

@@ -47,7 +47,7 @@ def cache_git(pkg, options = Hash.new)
                     Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'remote', 'update', 'autobuild')
                 end
                 with_retry(10) do
-                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', '--tags')
+                    Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'fetch', 'autobuild', '--tags')
                 end
                 Autobuild::Subprocess.run('autoproj-cache', :import, Autobuild.tool('git'), '--git-dir', pkg.importdir, 'gc', '--prune=all')
             end
@@ -63,17 +63,28 @@ def cache_archive(pkg)
                 end
             end
 
-            def create_or_update(options = Hash.new)
-                options = Kernel.validate_options options,
-                    keep_going: false,
-                    checkout_only: false
-                keep_going = options[:keep_going]
-                checkout_only = options[:checkout_only]
-
+            def create_or_update(*package_names, all: true, keep_going: false, checkout_only: false)
                 FileUtils.mkdir_p cache_dir
 
-                packages = manifest.each_autobuild_package.
-                    sort_by(&:name)
+                if package_names.empty?
+                    packages =
+                        if all
+                            manifest.each_autobuild_package
+                        else
+                            manifest.all_selected_source_packages.map(&:autobuild)
+                        end
+                else
+                    packages = package_names.map do |name|
+                        if pkg = manifest.find_autobuild_package(name)
+                            pkg
+                        else
+                            raise PackageNotFound, "no package named #{name}"
+                        end
+                    end
+                end
+
+                packages = packages.sort_by(&:name)
+
                 total = packages.size
                 Autoproj.message "Handling #{total} packages"
                 packages.each_with_index do |pkg, i|

data/lib/autoproj/ops/main_config_switcher.rb

@@ -73,15 +73,12 @@ def validate_autoproj_current_root(reuse)
 
             MAIN_CONFIGURATION_TEMPLATE = File.expand_path(File.join("..", "..", "..", "samples", 'autoproj'), File.dirname(__FILE__))
 
-            def bootstrap(buildconf_info, options = Hash.new)
-                options = validate_options options,
-                    reuse: Array.new
-
+            def bootstrap(buildconf_info, reuse: Array.new)
                 check_root_dir_empty
-                validate_autoproj_current_root(options[:reuse])
+                validate_autoproj_current_root(reuse)
 
                 ws.config.validate_ruby_executable
-                ws.config.set 'reused_autoproj_installations', options[:reuse], true
+                ws.config.set 'reused_autoproj_installations', reuse, true
                 ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
 
                 # If we are not getting the installation setup from a VCS, copy the template
@@ -101,14 +98,14 @@ def bootstrap(buildconf_info, options = Hash.new)
                             raise ConfigError.new, "cannot read file / URL #{manifest_url}, did you mean 'autoproj bootstrap VCSTYPE #{manifest_url}' ?"
                         end
 
-                    File.open(File.join(Autoproj.config_dir, "manifest"), "w") do |io|
+                    File.open(File.join(ws.config_dir, "manifest"), "w") do |io|
                         io.write(manifest_data)
                     end
 
                 elsif buildconf_info.size >= 2 # is a VCS definition for the manifest itself ...
-                    type, url, *options = *buildconf_info
+                    type, url, *vcs_options = *buildconf_info
                     url = VCSDefinition.to_absolute_url(url, ws.root_dir)
-                    do_switch_config(false, type, url, *options)
+                    do_switch_config(false, type, url, *vcs_options)
                 end
                 ws.env.export_env_sh(shell_helpers: ws.config.shell_helpers?)
                 ws.config.save

data/lib/autoproj/os_package_query.rb

@@ -0,0 +1,99 @@
+module Autoproj
+    # Match class to query OS packages
+    #
+    # This class allows to create a query object based on a textual
+    # representation, and then match osdeps packages using this query object.
+    #
+    # The queries are of the form
+    #
+    #   FIELD=VALUE:FIELD~VALUE:FIELD=VALUE
+    #
+    # The F=V form requires an exact match while F~V allows partial
+    # matches. The different matches are combined with AND (i.e. only packages
+    # matching all criterias will be returned)
+    #
+    # The following fields are allowed:
+    #   * name: the osdep name
+    #   * real_package: a regexp that matches the name of the underlying package
+    #   * package_manager: a regexp that matches the underlying package manager
+    #
+    class OSPackageQuery < QueryBase
+        ALLOWED_FIELDS = [
+            'name',
+            'real_package',
+            'package_manager'
+        ]
+        DEFAULT_FIELDS = {
+        }
+
+        def initialize(fields, value, partial, os_package_resolver)
+            super(fields, value, partial)
+            @os_package_resolver = os_package_resolver
+        end
+
+        class Adapter
+            def initialize(pkg, os_package_resolver)
+                @pkg = pkg
+                @os_package_resolver = os_package_resolver
+            end
+
+            def name
+                [@pkg]
+            end
+
+            def real_package
+                packages = @os_package_resolver.resolve_os_packages([@pkg])
+                packages.flat_map do |handler, handler_packages|
+                    handler_packages
+                end.uniq
+            end
+
+            def package_manager
+                packages = @os_package_resolver.resolve_os_packages([@pkg])
+                packages.flat_map do |handler, handler_packages|
+                    handler
+                end.uniq
+            end
+        end
+
+        # Checks if a package matches against the query
+        #
+        # @param [String] pkg the osdep package name
+        # @return [Boolean] true if it does, false otherwise
+        #
+        # If the package matches, the returned value can be one of:
+        #
+        # EXACT:: this is an exact match
+        # PARTIAL::
+        #   the expected value can be found in the package field. The
+        #   match is done in a case-insensitive way
+        #
+        # If partial? is not set (i.e. if FIELD=VALUE was used), then only EXACT
+        # or false can be returned.
+        def match(pkg)
+            pkg = Adapter.new(pkg, @os_package_resolver)
+            pkg_value = fields.inject(pkg) do |v, field_name|
+                v.send(field_name)
+            end
+
+            if pkg_value.include?(value)
+                return EXACT
+            end
+
+            if !partial?
+                return
+            end
+
+            if pkg_value.any? { |v| @value_rx === v }
+                return PARTIAL
+            end
+        end
+
+        # Parse a single field in a query (i.e. a FIELD[=~]VALUE string)
+        def self.parse(str, os_package_resolver)
+            fields, value, partial =
+                super(str, allowed_fields: ALLOWED_FIELDS)
+            OSPackageQuery.new(fields, value, partial, os_package_resolver)
+        end
+    end
+end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -231,7 +231,14 @@ def merge_gemfiles(*path, unlock: [])
                     end
                 end
                 path.each do |gemfile|
-                    bundler_def = Bundler::Dsl.evaluate(gemfile, nil, [])
+                    bundler_def =
+                        begin Bundler::Dsl.evaluate(gemfile, nil, [])
+                        rescue Exception => e
+                            cleaned_message = e.message.
+                                gsub(/There was an error parsing([^:]+)/, "Error in gem definitions").
+                                gsub(/#  from.*/, '')
+                            raise ConfigError, cleaned_message
+                        end
                     gems_remotes |= bundler_def.send(:sources).rubygems_remotes.to_set
                     bundler_def.dependencies.each do |d|
                         d.groups.each do |group_name|

data/lib/autoproj/package_managers/yum_manager.rb

@@ -42,8 +42,8 @@ def install(packages, filter_uptodate_packages: false, install_only: false)
                 result = false
                 if !patterns.empty?
                     result |= super(patterns,
-                                    auto_install_cmd: "yum groupinstall -y '%s'",
-                                    user_install_cmd: "yum groupinstall '%s'")
+                                    auto_install_cmd: %w{yum groupinstall -y},
+                                    user_install_cmd: %w{yum groupinstall})
                 end
                 if !packages.empty?
                     result |= super(packages)

data/lib/autoproj/package_managers/zypper_manager.rb

@@ -29,8 +29,8 @@ def install(packages, filter_uptodate_packages: false, install_only: false)
                 result = false
                 if !patterns.empty?
                     result |= super(patterns,
-                                    auto_install_cmd: "zypper --non-interactive install --type pattern '%s'",
-                                    user_install_cmd: "zypper install --type pattern '%s'")
+                                    auto_install_cmd: %w{zypper --non-interactive install --type pattern},
+                                    user_install_cmd: %w{zypper install --type pattern})
                 end
                 if !packages.empty?
                     result |= super(packages)

data/lib/autoproj/query_base.rb

@@ -0,0 +1,128 @@
+module Autoproj
+    # Base implementation for the query classes {SourcePackageQuery} and {OSPackageQuery}
+    class QueryBase
+        # Match priorities
+        EXACT = 4
+        PARTIAL = 3
+
+        # The call chain to be matched (i.e. autobuild.name becomes
+        # ['autobuild', 'name']
+        attr_reader :fields
+        # The expected value
+        attr_reader :value
+        # Whether the match can be partial
+        attr_predicate :partial?, true
+
+        # @api private
+        #
+        # Match class that matches anything
+        #
+        # Use {.all}
+        class All
+            def match(pkg); true end
+        end
+
+        # Get a query that matches anything
+        #
+        # @return [All]
+        def self.all
+            All.new
+        end
+
+        def initialize(fields, value, partial)
+            @fields = fields
+            @value  = value
+            @value_rx = Regexp.new(Regexp.quote(value), true)
+            @partial = partial
+        end
+
+        # Checks if a package matches against the query
+        #
+        # @param [String] pkg the osdep package name
+        # @return [Boolean] true if it does, false otherwise
+        #
+        # If the package matches, the returned value can be one of:
+        #
+        # EXACT:: this is an exact match
+        # PARTIAL::
+        #   the expected value can be found in the package field. The
+        #   match is done in a case-insensitive way
+        #
+        # If partial? is not set (i.e. if FIELD=VALUE was used), then only EXACT
+        # or false can be returned.
+        def match(pkg)
+            raise NotImplementedError
+        end
+
+        # @api private
+        #
+        # Parse a single field in a query (i.e. a FIELD[=~]VALUE string)
+        # 
+        # This is NOT meant to be used directly. Subclasses are supposed to
+        # redefine .parse to create the relevant match object.
+        def self.parse(str, allowed_fields: [], default_fields: Hash.new)
+            if parsed = /[=~]/.match(str)
+                field, value = parsed.pre_match, parsed.post_match
+                partial = (parsed[0] == '~')
+            else
+                raise ArgumentError, "invalid query string '#{str}', expected FIELD and VALUE separated by either = or ~"
+            end
+
+            field = default_fields[field] || field
+
+            # Validate the query key
+            if !allowed_fields.include?(field)
+                raise ArgumentError, "'#{field}' is not a known query key"
+            end
+
+            fields = field.split('.')
+            return fields, value, partial
+        end
+
+        # Parse a complete query
+        def self.parse_query(query, *args)
+            query = query.split(':')
+            query = query.map do |str|
+                parse(str, *args)
+            end
+            if query.size == 1
+                query.first
+            else
+                And.new(query)
+            end
+        end
+
+        # Match object that combines multiple matches using a logical OR
+        class Or
+            def initialize(submatches)
+                @submatches = submatches
+            end
+            def each_subquery(&block)
+                @submatches.each(&block)
+            end
+            def match(pkg)
+                @submatches.map { |m| m.match(pkg) }.compact.max
+            end
+        end
+
+        # Match object that combines multiple matches using a logical AND
+        class And
+            def initialize(submatches)
+                @submatches = submatches
+            end
+            def each_subquery(&block)
+                @submatches.each(&block)
+            end
+            def match(pkg)
+                matches = @submatches.map do |m|
+                    if p = m.match(pkg)
+                        p
+                    else return
+                    end
+                end
+                matches.min
+            end
+        end
+    end
+end
+

data/lib/autoproj/reporter.rb

@@ -88,6 +88,18 @@ def self.report(root_dir: nil, silent: nil, debug: Autobuild.debug,
                 package_failures, on_package_failures: on_package_failures, interrupted_by: interrupted)
         end
 
+    rescue CLI::CLIException, InvalidWorkspace, ConfigError => e
+        if silent_errors
+            return [e]
+        elsif on_package_failures == :raise
+            raise e
+        elsif on_package_failures == :report
+            Autoproj.error e.message
+        elsif on_package_failures == :exit
+            Autoproj.error e.message
+            exit 1
+        end
+
     rescue SystemExit
         raise
     ensure

data/lib/autoproj/{query.rb → source_package_query.rb}

@@ -27,7 +27,7 @@ module Autoproj
     #   * 'vcs' can be used instead of 'vcs.url'
     #   * 'package_set' can be used instead of 'package_set.name'
     #
-    class Query
+    class SourcePackageQuery < QueryBase
         ALLOWED_FIELDS = [
             'autobuild.name',
             'autobuild.srcdir',
@@ -44,29 +44,13 @@ class Query
         }
 
         # Match priorities
-        EXACT = 4
-        PARTIAL = 3
         DIR_PREFIX_STRONG = 2
         DIR_PREFIX_WEAK = 1
 
-        attr_reader :fields
-        attr_reader :value
         attr_predicate :use_dir_prefix?
-        attr_predicate :partial?
-
-        class All
-            def match(pkg); true end
-        end
-
-        def self.all
-            All.new
-        end
 
         def initialize(fields, value, partial)
-            @fields = fields
-            @value = value
-            @value_rx = Regexp.new(Regexp.quote(value), true)
-            @partial = partial
+            super(fields, value, partial)
 
             directories = value.split('/')
             if !directories.empty?
@@ -119,15 +103,15 @@ def match(pkg)
                 return
             end
 
-            if pkg_value =~ @value_rx
+            if @value_rx === pkg_value
                 return PARTIAL
             end
 
             # Special match for directories: match directory prefixes
             if use_dir_prefix?
-                if pkg_value =~ @dir_prefix_strong_rx
+                if @dir_prefix_strong_rx === pkg_value
                     return DIR_PREFIX_STRONG
-                elsif pkg_value =~ @dir_prefix_weak_rx
+                elsif @dir_prefix_weak_rx === pkg_value
                     return DIR_PREFIX_WEAK
                 end
             end
@@ -135,69 +119,19 @@ def match(pkg)
 
         # Parse a single field in a query (i.e. a FIELD[=~]VALUE string)
         def self.parse(str)
-            field, value = str.split('=')
-            if !value
-                partial = true
-                field, value = str.split('~')
-            end
-
-            if DEFAULT_FIELDS[field]
-                field = DEFAULT_FIELDS[field]
-            end
-
-            # Validate the query key
-            if !ALLOWED_FIELDS.include?(field)
-                raise ArgumentError, "#{field} is not a known query key"
-            end
-
-            fields = field.split('.')
-            new(fields, value, partial)
-        end
-
-        # Parse a complete query
-        def self.parse_query(query)
-            query = query.split(':')
-            query = query.map do |str|
-                if str !~ /[=~]/
-                    match_name = Query.parse("autobuild.name~#{str}")
-                    match_dir  = Query.parse("autobuild.srcdir~#{str}")
-                    Or.new([match_name, match_dir])
-                else
-                    Query.parse(str)
-                end
-            end
-            if query.size == 1
-                query.first
+            if str !~ /[=~]/
+                match_name = parse("autobuild.name~#{str}")
+                match_dir  = parse("autobuild.srcdir~#{str}")
+                return Or.new([match_name, match_dir])
             else
-                And.new(query)
-            end
-        end
-
-        # Match object that combines multiple matches using a logical OR
-        class Or
-            def initialize(submatches)
-                @submatches = submatches
-            end
-            def match(pkg)
-                @submatches.map { |m| m.match(pkg) }.compact.max
-            end
-        end
-
-        # Match object that combines multiple matches using a logical AND
-        class And
-            def initialize(submatches)
-                @submatches = submatches
-            end
-            def match(pkg)
-                matches = @submatches.map do |m|
-                    if p = m.match(pkg)
-                        p
-                    else return
-                    end
-                end
-                matches.min
+                fields, value, partial =
+                    super(str, default_fields: DEFAULT_FIELDS, allowed_fields: ALLOWED_FIELDS)
+                return new(fields, value, partial)
             end
         end
     end
+
+    # For backward compatibility
+    Query = SourcePackageQuery
 end