autoproj 2.0.0.rc37 → 2.0.0.rc38

data/lib/autoproj/package_selection.rb

@@ -55,6 +55,16 @@ def each(&block)
                 each_source_package_name(&block)
             end
 
+            def each_package_name(&block)
+                return enum_for(__method__) if !block
+                each_source_package_name(&block)
+                each_osdep_package_name(&block)
+            end
+
+            def selected_source_package?(pkg)
+                source_packages.include?(pkg.name)
+            end
+
             def each_source_package_name(&block)
                 source_packages.each(&block)
             end
@@ -106,6 +116,10 @@ def has_match_for?(sel)
                 matches.has_key?(sel)
             end
 
+            def match_for(sel)
+                matches[sel]
+            end
+
             # Remove packages that are explicitely excluded and/or ignored
             #
             # Raise an error if an explicit selection expands only to an
@@ -130,9 +144,9 @@ def filter_excluded_and_ignored_packages(manifest)
                                 raise ExcludedSelection.new(sel), "#{base_msg}, but its dependency #{exclusions.map(&:first).join(", ")} is excluded from the build: #{reason}"
                             end
                         elsif weak_dependencies[sel]
-                            raise ExcludedSelection.new(sel), "#{base_msg}, but expands to #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
+                            raise ExcludedSelection.new(sel), "#{base_msg}, but expands to #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |e_name, e_reason| "#{e_name}: #{e_reason}" }.join("\n  ")}"
                         else
-                            raise ExcludedSelection.new(sel), "#{base_msg}, but it requires #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
+                            raise ExcludedSelection.new(sel), "#{base_msg}, but it requires #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |e_name, e_reason| "#{e_name}: #{e_reason}" }.join("\n  ")}"
                         end
                     else
                         self.exclusions[sel] |= excluded.to_set.dup

data/lib/autoproj/package_set.rb

@@ -27,8 +27,29 @@ def add_source_file(name)
             end
         end
 
-        # @return [Manifest] the manifest this package set is being used by
-        attr_reader :manifest
+        # The underlying workspace
+        #
+        # @return [Workspace]
+        attr_reader :ws
+
+        # The manifest this package set is registered on
+        #
+        # @return [Manifest] 
+        def manifest
+            ws.manifest
+        end
+
+        # The minimum autoproj version this package set requires
+        #
+        # It defaults to 0
+        #
+        # @return [String]
+        attr_accessor :required_autoproj_version
+
+        # The package set name
+        #
+        # @return [String]
+        attr_accessor :name
 
         # The VCSDefinition object that defines the version control holding
         # information for this source. Local package sets (i.e. the ones that are not
@@ -38,10 +59,13 @@ def add_source_file(name)
 
         # The set of OSPackageResolver object that represent the osdeps files
         # available in this package set
+        #
+        # @return [Array<(String,OSPackageResolver)>] the list of osdep files
+        #   and the corresponding OSPackageResolver object
         attr_reader :all_osdeps
 
         # The OSPackageResolver which is a merged version of all OSdeps in
-        # #all_osdeps
+        # {#all_osdeps}
         attr_reader :os_package_resolver
 
         # If this package set has been imported from another package set, this
@@ -53,9 +77,22 @@ def add_source_file(name)
         def explicit?; !!@explicit end
         attr_writer :explicit
 
-        attr_reader :source_definition
+        # Definition of key => value mappings used to resolve e.g. $KEY values
+        # in the version control sections
         attr_reader :constants_definitions
 
+        # The version control information defined in this package set
+        attr_reader :version_control
+
+        # Cached results of {#importer_definition_for}
+        attr_reader :importer_definitions_cache
+
+        # The importer that should be used for packages that have no explicit
+        # entry
+        #
+        # @return [VCSDefinition]
+        attr_accessor :default_importer
+
         # The set of overrides defined in this package set
         attr_reader :overrides
 
@@ -83,18 +120,43 @@ def metapackage
         # List of the packages that are built if the package set is selected in
         # the layout
         def default_packages
-            metapackage.packages
+            metapackage.each_package
         end
 
+        # Remote sources can be accessed through a hidden directory in
+        # {Workspace#remotes_dir}, or through a symbolic link in
+        # autoproj/remotes/
+        #
+        # This returns the former. See #user_local_dir for the latter.
+        #
+        # For local sources, is simply returns the path to the source directory.
+        attr_reader :raw_local_dir
+
         # Create this source from a VCSDefinition object
-        def initialize(manifest, vcs)
-            @manifest = manifest
+        def initialize(
+            ws, vcs,
+            name: self.class.name_of(ws, vcs),
+            raw_local_dir: self.class.raw_local_dir_of(ws, vcs))
+
+            @ws = ws
             @vcs = vcs
-            @os_package_resolver = OSPackageResolver.new
+            if !vcs
+                raise ArgumentError, "cannot create a package set with a nil vcs, create a null VCS using VCSDefinition.none"
+            end
+            @name = name
+            @os_package_resolver = OSPackageResolver.new(
+                operating_system: ws.os_package_resolver.operating_system,
+                package_managers: ws.os_package_resolver.package_managers,
+                os_package_manager: ws.os_package_resolver.os_package_manager)
+            @importer_definitions_cache = Hash.new
             @all_osdeps = []
+            @constants_definitions = Hash.new
+            @required_autoproj_version = '0'
+            @version_control = Array.new
             @overrides = Array.new
+            @raw_local_dir = raw_local_dir
+            @default_importer = VCSDefinition.from_raw(type: 'none')
 
-            @provides = Set.new
             @imports  = Set.new
             @imports_vcs  = Array.new
             @imported_from = Array.new
@@ -103,8 +165,8 @@ def initialize(manifest, vcs)
         end
 
         # Load a new osdeps file for this package set
-        def load_osdeps(file)
-            new_osdeps = OSPackageResolver.load(file)
+        def load_osdeps(file, **options)
+            new_osdeps = OSPackageResolver.load(file, **options)
             all_osdeps << new_osdeps
             os_package_resolver.merge(all_osdeps.last)
             new_osdeps
@@ -112,7 +174,7 @@ def load_osdeps(file)
 
         # Enumerate all osdeps package names from this package set
         def each_osdep(&block)
-            os_package_resolver.definitions.each_key(&block)
+            os_package_resolver.all_package_names.each(&block)
         end
 
         # True if this source has already been checked out on the local autoproj
@@ -125,7 +187,7 @@ def main?; false end
         def local?; vcs.local? end
         # True if this source defines nothing
         def empty?
-            !source_definition['version_control'] && overrides.empty?
+            version_control.empty? && overrides.empty?
                 !each_package.find { true } &&
                 !File.exist?(File.join(raw_local_dir, "overrides.rb")) &&
                 !File.exist?(File.join(raw_local_dir, "init.rb"))
@@ -136,6 +198,8 @@ def autobuild
             create_autobuild_package
         end
 
+        # Create a stub autobuild package to handle the import of this package
+        # set
         def create_autobuild_package
             Ops::Tools.create_autobuild_package(vcs, name, raw_local_dir)
         end
@@ -158,12 +222,12 @@ def snapshot(target_dir, options = Hash.new)
         # checked out, and the vcs (as a string) otherwise
         #
         # @return [String]
-        def self.name_of(manifest, vcs)
-            pkg_set = PackageSet.new(manifest, vcs)
-            if pkg_set.present?
-                name = pkg_set.raw_description_file['name']
+        def self.name_of(ws, vcs, raw_local_dir: raw_local_dir_of(ws, vcs))
+            if File.directory?(raw_local_dir)
+                raw_description_file(raw_local_dir, package_set_name: "#{vcs.type}:#{vcs.url}")['name']
+            else
+                vcs.to_s
             end
-            name || vcs.to_s
         end
 
         # Returns the local directory in which the given package set should be
@@ -172,11 +236,13 @@ def self.name_of(manifest, vcs)
         # @param [VCSDefinition] vcs the version control information for the
         #   package set
         # @return [String]
-        def self.raw_local_dir_of(vcs)
-            if vcs.local?
+        def self.raw_local_dir_of(ws, vcs)
+            if vcs.needs_import?
+                repository_id = vcs.create_autobuild_importer.repository_id
+                path = File.join(ws.remotes_dir, repository_id.gsub(/[^\w]/, '_'))
+                File.expand_path(path)
+            elsif !vcs.none?
                 File.expand_path(vcs.url)
-            else
-                File.expand_path(File.join(Autoproj.workspace.remotes_dir, vcs.create_autobuild_importer.repository_id.gsub(/[^\w]/, '_')))
             end
         end
 
@@ -185,21 +251,12 @@ def self.raw_local_dir_of(vcs)
         # This parses the information stored in the package_sets section of
         # autoproj/manifest, or the imports section of the source.yml files and
         # returns the corresponding VCSDefinition object
-        def self.resolve_definition(manifest, raw_spec)
-            if raw_spec.respond_to?(:to_str)
-                local_path = File.join(Autoproj.workspace.config_dir, raw_spec)
-                if File.directory?(local_path)
-                    raw_spec = { :type => 'local', :url => local_path }
-                end
-            end
-            spec = VCSDefinition.vcs_definition_to_hash(raw_spec)
-            options, vcs_spec = Kernel.filter_options spec, :auto_imports => true
+        def self.resolve_definition(ws, raw_spec, from: nil, raw: Array.new)
+            spec = VCSDefinition.normalize_vcs_hash(raw_spec, base_dir: ws.config_dir)
+            options, vcs_spec = Kernel.filter_options spec, auto_imports: true
 
-            # Look up for short notation (i.e. not an explicit hash). It is
-            # either vcs_type:url or just url. In the latter case, we expect
-            # 'url' to be a path to a local directory
-            vcs_spec = Autoproj.expand(vcs_spec, manifest.constant_definitions)
-            return VCSDefinition.from_raw(vcs_spec, raw: [[nil, raw_spec]]), options
+            vcs_spec = Autoproj.expand(vcs_spec, ws.manifest.constant_definitions)
+            return VCSDefinition.from_raw(vcs_spec, from: from, raw: raw), options
         end
 
         # Returns a string that uniquely represents the version control
@@ -210,7 +267,7 @@ def self.resolve_definition(manifest, raw_spec)
         # from exactly the same source.
         def repository_id
             if local?
-                local_dir
+                raw_local_dir
             else
                 importer = vcs.create_autobuild_importer
                 if importer.respond_to?(:repository_id)
@@ -221,17 +278,6 @@ def repository_id
             end
         end
 
-        # Remote sources can be accessed through a hidden directory in
-        # {Workspace#remotes_dir}, or through a symbolic link in
-        # autoproj/remotes/
-        #
-        # This returns the former. See #user_local_dir for the latter.
-        #
-        # For local sources, is simply returns the path to the source directory.
-        def raw_local_dir
-            self.class.raw_local_dir_of(vcs)
-        end
-
         # Remote sources can be accessed through a hidden directory in
         # {Workspace#remotes_dir}, or through a symbolic link in
         # autoproj/remotes/
@@ -243,7 +289,7 @@ def user_local_dir
             if local?
                 return vcs.url 
             else
-                File.join(Autoproj.workspace.config_dir, 'remotes', name)
+                File.join(ws.config_dir, 'remotes', name)
             end
         end
 
@@ -251,35 +297,26 @@ def user_local_dir
         def local_dir
             ugly_dir   = raw_local_dir
             pretty_dir = user_local_dir
-            if File.symlink?(pretty_dir) && File.readlink(pretty_dir) == ugly_dir
+            if ugly_dir == pretty_dir
+                pretty_dir
+            elsif File.symlink?(pretty_dir) && File.readlink(pretty_dir) == ugly_dir
                 pretty_dir
             else
                 ugly_dir
             end
         end
 
-        def required_autoproj_version
-            definition = @source_definition || raw_description_file
-            definition['required_autoproj_version'] || '0'
-        end
-
-        # Returns the source name
-        def name
-            @name || self.class.name_of(manifest, vcs)
-        end
-
-        # Loads the source.yml file, validates it and returns it as a hash
+        # @api private
         #
-        # Raises InternalError if the source has not been checked out yet (it
-        # should have), and ConfigError if the source.yml file is not valid.
-        def raw_description_file
-            if !present?
-                raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
-            end
-
+        # Read the description information for a package set in a given
+        # directory
+        #
+        # @param [String] raw_local_dir the package set's directory
+        # @return [Hash] the raw description information
+        def self.raw_description_file(raw_local_dir, package_set_name: nil)
             master_source_file = File.join(raw_local_dir, PackageSet.master_source_file)
             if !File.exist?(master_source_file)
-                raise ConfigError.new, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
+                raise ConfigError.new, "package set #{package_set_name} present in #{raw_local_dir} should have a source.yml file, but does not"
             end
 
             source_definition = Hash.new
@@ -297,13 +334,24 @@ def raw_description_file
                     end
                 end
             end
-
             if !source_definition['name']
-                raise ConfigError.new(master_source_file), "in #{master_source_file}: missing a 'name' field"
+                raise ConfigError.new(master_source_file), "#{master_source_file} does not have a 'name' field"
             end
             source_definition
         end
 
+        # Loads the source.yml file, validates it and returns it as a hash
+        #
+        # Raises InternalError if the source has not been checked out yet (it
+        # should have), and ConfigError if the source.yml file is not valid.
+        def raw_description_file
+            if !present?
+                raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
+            end
+
+            self.class.raw_description_file(raw_local_dir, package_set_name: name)
+        end
+
         # Yields the package sets imported by this package set
         #
         # This information is available only after the whole configuration has
@@ -314,23 +362,53 @@ def each_imported_set(&block)
             @imports.each(&block)
         end
 
+        # Add a new constant to be used to resolve e.g. version control entries
+        def add_constant_definition(key, value)
+            constants_definitions[key] = value
+        end
+
+        # Add a new VCS import to the list of imports
+        #
+        # @param [VCSDefinition] vcs the VCS specification for the import
+        # @return [void]
+        def add_raw_imported_set(vcs, auto_imports: true)
+            imports_vcs << [vcs, Hash[auto_imports: auto_imports]]
+        end
+
         # Yields the imports raw information
         #
         # @yieldparam [VCSDefinition] vcs the import VCS information
         # @yieldparam [Hash] options import options
         def each_raw_imported_set(&block)
-            @imports_vcs.each(&block)
+            imports_vcs.each(&block)
+        end
+
+        # Add a new entry in the list of version control resolutions
+        def add_version_control_entry(matcher, vcs_definition)
+            invalidate_importer_definitions_cache
+            version_control << [matcher, vcs_definition]
+        end
+
+        # Add a new entry in the list of version control resolutions
+        def add_overrides_entry(matcher, vcs_definition, file: '#add_overrides_entry')
+            if (last_entry = overrides.last) && last_entry[0] == file
+                last_entry[1] << [matcher, vcs_definition]
+            else
+                overrides << [file, [[matcher, vcs_definition]]]
+            end
         end
 
         # Path to the source.yml file
         def source_file
-            File.join(local_dir, 'source.yml')
+            if local_dir
+                File.join(local_dir, 'source.yml')
+            end
         end
 
         # Load the source.yml file and resolves all information it contains.
         def load_description_file
-            @source_definition = raw_description_file
-            name = @source_definition['name']
+            source_definition = raw_description_file
+            name = source_definition['name']
             if name !~ /^[\w\.-]+$/
                 raise ConfigError.new(source_file),
                     "in #{source_file}: invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
@@ -339,198 +417,251 @@ def load_description_file
                     "in #{source_file}: the name 'local' is a reserved name"
             end
 
-            parse_source_definition
-            @overrides = load_overrides
+            parse_source_definition(source_definition)
         end
 
-        def load_overrides
+        def load_overrides(source_definition)
             if data = source_definition['overrides']
                 [[source_file, data]]
-            else
-                []
             end
         end
 
-        def parse_source_definition
-            @name = source_definition['name']
-            @provides = (source_definition['provides'] || Set.new).to_set
-            @imports_vcs  = Array(source_definition['imports'] || Array.new).map do |set_def|
-                if !set_def.kind_of?(Hash) && !set_def.respond_to?(:to_str)
-                    raise ConfigError.new(source_file),
-                        "in #{source_file}: wrong format for 'imports' section. Expected an array of maps or strings (e.g. - github: my/url)."
+        def parse_source_definition(source_definition)
+            @name = source_definition['name'] || self.name
+            @required_autoproj_version = source_definition.fetch('required_autoproj_version', self.required_autoproj_version)
+            if new_imports = source_definition['imports']
+                @imports_vcs  = Array(new_imports).map do |set_def|
+                    if !set_def.kind_of?(Hash) && !set_def.respond_to?(:to_str)
+                        raise ConfigError.new(source_file),
+                            "in #{source_file}: wrong format for 'imports' section. Expected an array of maps or strings (e.g. - github: my/url)."
+                    end
+
+                    Autoproj.in_file(source_file) do
+                        PackageSet.resolve_definition(ws, set_def, from: self, raw: [VCSDefinition::RawEntry.new(self, source_file, set_def)])
+                    end
                 end
+            end
 
+            # Compute the definition of constants
+            if new_constants = source_definition['constants']
                 Autoproj.in_file(source_file) do
-                    PackageSet.resolve_definition(manifest, set_def)
+                    variables = inject_constants_and_config_for_expansion(Hash.new)
+                    @constants_definitions = Autoproj.resolve_constant_definitions(new_constants, variables)
                 end
             end
 
-            # Compute the definition of constants
-            Autoproj.in_file(source_file) do
-                constants = source_definition['constants'] || Hash.new
-                @constants_definitions = Autoproj.resolve_constant_definitions(constants)
+            if new_version_control = source_definition['version_control']
+                invalidate_importer_definitions_cache
+                @version_control = normalize_vcs_list('version_control', source_file, new_version_control)
+
+                Autoproj.in_file(source_file) do
+                    default_vcs_spec, raw = version_control_field('default', version_control, file: source_file)
+                    if default_vcs_spec
+                        @default_importer = VCSDefinition.from_raw(default_vcs_spec, raw: raw, from: self)
+                    end
+                end
             end
+            if new_overrides = load_overrides(source_definition)
+                @overrides = new_overrides.map do |file, entries|
+                    [file, normalize_vcs_list('overrides', file, entries)]
+                end
+            end
+        end
+
+        # @api private
+        #
+        # Injects the values of {#constants_definitions} and
+        # {#manifest}.constant_definitions, as well as the available
+        # configuration variables, into a hash suitable to be used for variable
+        # expansion using {Autoproj.expand} and {Autoproj.single_expansion}
+        def inject_constants_and_config_for_expansion(additional_expansions)
+            defs = Hash[
+                "AUTOPROJ_ROOT" => ws.root_dir,
+                "AUTOPROJ_CONFIG" => ws.config_dir,
+                "AUTOPROJ_SOURCE_DIR" => local_dir].
+                merge(manifest.constant_definitions).
+                merge(constants_definitions).
+                merge(additional_expansions)
+
+            config = ws.config
+            Hash.new do |h, k|
+                if config.has_value_for?(k) || config.declared?(k)
+                    config.get(k)
+                end
+            end.merge(defs)
         end
 
         def single_expansion(data, additional_expansions = Hash.new)
-            if !source_definition
-                raise NotLoaded.new(self), "you must load the package set information with #load_description_file before you can call #single_expansion"
-            end
-            Autoproj.single_expansion(data, additional_expansions.merge(constants_definitions))
+            defs = inject_constants_and_config_for_expansion(additional_expansions)
+            Autoproj.single_expansion(data, defs)
         end
 
         # Expands the given string as much as possible using the expansions
         # listed in the source.yml file, and returns it. Raises if not all
         # variables can be expanded.
         def expand(data, additional_expansions = Hash.new)
-            if !source_definition
-                load_description_file
-            end
-            Autoproj.expand(data, additional_expansions.merge(constants_definitions))
-        end
-
-        # Returns the default importer definition for this package set, as a
-        # VCSDefinition instance
-        def default_importer
-            importer_definition_for('default')
+            defs = inject_constants_and_config_for_expansion(additional_expansions)
+            Autoproj.expand(data, defs)
         end
 
-        # Returns an importer definition for the given package, if one is
-        # available. Otherwise returns nil.
+        # @api private
         #
-        # The returned value is a VCSDefinition object.
-        def version_control_field(package_name, section_name, validate = true)
-            vcs_field( source_definition, package_name, section_name, validate )
+        # Converts a number to an ordinal string representation (i.e. 1st, 25th)
+        def number_to_nth(number)
+            Hash[1 => '1st', 2 => '2nd', 3 => '3rd'].fetch(number, "#{number}th")
         end
 
-        def vcs_field( source_definition, package_name, section_name, validate )
-            urls = source_definition['urls'] || Hash.new
-            urls['HOME'] = ENV['HOME']
-
-            all_vcs     = source_definition[section_name]
-            if all_vcs
-                if all_vcs.kind_of?(Hash)
-                    raise ConfigError.new, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
-                elsif !all_vcs.kind_of?(Array)
-                    raise ConfigError.new, "wrong format for the #{section_name} section"
-                end
+        # @api private
+        #
+        # Validate the format of a VCS list field (formatted in array-of-hashes)
+        def normalize_vcs_list(section_name, file, list)
+            if list.kind_of?(Hash)
+                raise InvalidYAMLFormatting, "wrong format for the #{section_name} section of #{file}, you forgot the '-' in front of the package names"
+            elsif !list.kind_of?(Array)
+                raise InvalidYAMLFormatting, "wrong format for the #{section_name} section of #{file}"
             end
 
-            raw = []
-            vcs_spec = Hash.new
+            list.each_with_index.map do |spec, spec_idx|
+                spec_nth = number_to_nth(spec_idx + 1)
+                if !spec.kind_of?(Hash)
+                    raise InvalidYAMLFormatting, "wrong format for the #{spec_nth} entry (#{spec.inspect}) of the #{section_name} section of #{file}, expected a package name, followed by a colon, and one importer option per following line"
+                end
 
-            if all_vcs
-                all_vcs.each do |spec|
-                    spec = spec.dup
-                    if !spec.kind_of?(Hash)
-                        raise ConfigError.new, "wrong format for the #{spec} entry, expected #{spec} followed by a colon and one importer option per following line"
+                spec = spec.dup
+                if spec.values.size != 1
+                    # Maybe the user wrote the spec like
+                    #   - package_name:
+                    #     type: git
+                    #     url: blah
+                    #
+                    # In that case, we should have the package name as
+                    # "name => nil". Check that.
+                    name, _ = spec.find { |n, v| v.nil? }
+                    if name
+                        spec.delete(name)
+                    else
+                        raise InvalidYAMLFormatting, "cannot make sense of the #{spec_nth} entry in the #{section_name} section of #{file}: #{spec}"
                     end
-
-                    if spec.values.size != 1
-                        # Maybe the user wrote the spec like
-                        #   - package_name:
-                        #     type: git
-                        #     url: blah
-                        #
-                        # or as
-                        #   - package_name
-                        #     type: git
-                        #     url: blah
-                        #
-                        # In that case, we should have the package name as
-                        # "name => nil". Check that.
-                        name, _ = spec.find { |n, v| v.nil? }
-                        if name
-                            spec.delete(name)
+                else
+                    name, spec = spec.to_a.first
+                    if spec.respond_to?(:to_str)
+                        if spec == "none"
+                            spec = Hash['type' => "none"]
                         else
-                            name, _ = spec.find { |n, v| n =~ / \w+$/ }
-                            name =~ / (\w+)$/
-                            spec[$1] = spec.delete(name)
-                            name = name.gsub(/ \w+$/, '')
-                        end
-                    else
-                        name, spec = spec.to_a.first
-                        if name =~ / (\w+)/
-                            spec = { $1 => spec }
-                            name = name.gsub(/ \w+$/, '')
-                        end
-
-                        if spec.respond_to?(:to_str)
-                            if spec == "none"
-                                spec = { :type => "none" }
-                            else
-                                raise ConfigError.new, "invalid VCS specification in the #{section_name} section '#{name}: #{spec}'"
-                            end
+                            raise ConfigError.new, "invalid VCS specification in the #{section_name} section of #{file}: '#{name}: #{spec}'. One can only use this shorthand to declare the absence of a VCS with the 'none' keyword"
                         end
+                    elsif !spec.kind_of?(Hash)
+                        raise InvalidYAMLFormatting, "expected '#{name}:' followed by version control options, but got nothing, in the #{spec_nth} entry of the #{section_name} section of #{file}"
                     end
+                end
 
-                    name_match = name
-                    if name_match =~ /[^\w\/-]/
-                        name_match = Regexp.new("^" + name_match)
-                    end
-                    if name_match === package_name
-                        raw << [self, spec]
-                        vcs_spec =
-                            begin
-                                VCSDefinition.update_raw_vcs_spec(vcs_spec, spec)
-                            rescue ConfigError => e
-                                raise ConfigError.new, "invalid VCS definition in the #{section_name} section for '#{name}': #{e.message}", e.backtrace
-                            end
-                    end
+                name_match = name
+                if name_match =~ /[^\w\/-]/
+                    name_match = Regexp.new("^" + name_match)
                 end
+
+                [name_match, spec]
             end
+        end
 
-            if !vcs_spec.empty?
-                expansions = Hash["PACKAGE" => package_name,
-                    "PACKAGE_BASENAME" => File.basename(package_name),
-                    "AUTOPROJ_ROOT" => Autoproj.workspace.root_dir,
-                    "AUTOPROJ_CONFIG" => Autoproj.workspace.config_dir,
-                    "AUTOPROJ_SOURCE_DIR" => local_dir]
+        # Returns a VCS definition for the given package, if one is
+        # available. Otherwise returns nil.
+        #
+        # @return [[(Hash,nil),Array]] the resolved VCS definition, as well
+        #   as the "history" of it, that is the list of entries that matched the
+        #   package in the form (PackageSet,Hash), where PackageSet is self.
+        #   The Hash part is nil if there are no matching entries. Hash keys are
+        #   normalized to symbols
+        def version_control_field(package_name, entry_list, validate: true, file: source_file)
+            raw = []
+            vcs_spec = Hash.new
 
-                vcs_spec = expand(vcs_spec, expansions)
-                vcs_spec.dup.each do |name, value|
-                    vcs_spec[name] = expand(value, expansions)
+            entry_list.each do |name_match, spec|
+                if name_match === package_name
+                    raw << VCSDefinition::RawEntry.new(self, file, spec)
+                    vcs_spec =
+                        begin
+                            VCSDefinition.update_raw_vcs_spec(vcs_spec, spec)
+                        rescue ConfigError => e
+                            raise ConfigError.new, "invalid VCS definition in the #{section_name} section for '#{name}': #{e.message}", e.backtrace
+                        end
                 end
+            end
 
-                # If required, verify that the configuration is a valid VCS
-                # configuration
-                if validate
-                    begin
-                        VCSDefinition.from_raw(vcs_spec)
-                    rescue ConfigError => e
-                        raise ConfigError.new, "invalid resulting VCS definition for package #{package_name}: #{e.message}", e.backtrace
-                    end
-                end
-                return vcs_spec, raw
-            else
+            if vcs_spec.empty?
                 return nil, []
             end
+
+            expansions = Hash["PACKAGE" => package_name,
+                "PACKAGE_BASENAME" => File.basename(package_name)]
+
+            vcs_spec = expand(vcs_spec, expansions)
+            vcs_spec.dup.each do |name, value|
+                vcs_spec[name] = expand(value, expansions)
+            end
+
+            # Resolve relative paths w.r.t. the workspace root dir
+            if url = (vcs_spec.delete('url') || vcs_spec.delete(:url))
+                vcs_spec[:url] = VCSDefinition.to_absolute_url(url, ws.root_dir)
+            end
+
+            # If required, verify that the configuration is a valid VCS
+            # configuration
+            if validate
+                begin
+                    VCSDefinition.from_raw(vcs_spec)
+                rescue ConfigError => e
+                    raise ConfigError.new, "invalid resulting VCS definition for package #{package_name}: #{e.message}", e.backtrace
+                end
+            end
+            return vcs_spec, raw
+        end
+
+        # @api private
+        #
+        # Invalidate {#importer_definitions_cache}
+        def invalidate_importer_definitions_cache
+            @importer_definitions_cache.clear
         end
 
         # Returns the VCS definition for +package_name+ as defined in this
         # source, or nil if the source does not have any.
         #
-        # The definition is an instance of VCSDefinition
-        def importer_definition_for(package_name)
-            Autoproj.in_file source_file do
-                vcs_spec, raw = version_control_field(package_name, 'version_control')
+        # @param [PackageDefinition] package
+        # @return [VCSDefinition] the importer definition, or nil if none
+        #   could be found
+        def importer_definition_for(package, default: default_importer, require_existing: true)
+            package_name = manifest.validate_package_name_argument(package, require_existing: require_existing)
+            importer_definitions_cache[package_name] ||= Autoproj.in_file source_file do
+                vcs_spec, raw = version_control_field(package_name, version_control, file: source_file)
                 if vcs_spec
                     VCSDefinition.from_raw(vcs_spec, raw: raw, from: self)
+                else
+                    default
                 end
             end
         end
 
         # Update a VCS object using the overrides defined in this package set
         #
-        # @param [String] package_name the package name
+        # @param [PackageDefinition] package the package
         # @param [VCSDefinition] the vcs to be updated
         # @return [VCSDefinition] the new, updated vcs object
-        def overrides_for(package_name, vcs)
-            overrides.each do |file, override|
+        def overrides_for(package, vcs, require_existing: true)
+            package_name = manifest.validate_package_name_argument(package, require_existing: require_existing)
+            resolve_overrides(package_name, vcs)
+        end
+
+        # @api private
+        #
+        # Apply overrides on a VCS object from its (string) key
+        #
+        # This is a helper for {#overrides_for}
+        def resolve_overrides(key, vcs)
+            overrides.each do |file, file_overrides|
                 new_spec, new_raw_entry = 
                     Autoproj.in_file file do
-                        vcs_field(Hash['overrides' => override],
-                                              package_name, 'overrides', false)
+                        version_control_field(key, file_overrides, validate: false, file: file)
                     end
 
                 if new_spec
@@ -538,7 +669,7 @@ def overrides_for(package_name, vcs)
                         begin
                             vcs = vcs.update(new_spec, raw: new_raw_entry, from: self)
                         rescue ConfigError => e
-                            raise ConfigError.new, "invalid resulting VCS specification in the overrides section for package #{package_name}: #{e.message}"
+                            raise ConfigError.new, "invalid resulting VCS specification in the overrides section for #{key}: #{e.message}"
                         end
                     end
                 end
@@ -553,117 +684,29 @@ def each_package
                 return enum_for(:each_package)
             end
 
-            manifest.packages.each_value do |pkg|
-                if pkg.package_set.name == name
+            manifest.each_package_definition do |pkg|
+                if pkg.package_set == self
                     yield(pkg.autobuild)
                 end
             end
         end
 
-        # True if this package set provides the given package set name. I.e. if
-        # it has this name or the name is listed in the "replaces" field of
-        # source.yml
-        def provides?(name)
-            name == self.name ||
-                provides.include?(name)
-        end
-    end
-
-    # Specialization of the PackageSet class for the overrides listed in autoproj/
-    class LocalPackageSet < PackageSet
-        def initialize(manifest, local_dir = nil)
-            super(manifest, manifest.vcs)
-            @local_dir = local_dir
-        end
-
-        def name
-            'main configuration'
-        end
-
-        def vcs
-            manifest.vcs
-        end
-
-        def main?
-            true
-        end
-
-        def local?
-            true
-        end
-
-        def local_dir
-            @local_dir || (File.dirname(manifest.file) if manifest.file)
-        end
-
-        def raw_local_dir; local_dir end
-
-        def manifest_path
-            manifest.file
-        end
-
-        def overrides_file_path
-            if d = local_dir
-                File.join(d, "overrides.yml")
-            end
-        end
-
-        def source_file
-            manifest_path
-        end
-
-        # Returns the default importer for this package set
-        def default_importer
-            importer_definition_for('default') ||
-                VCSDefinition.from_raw(:type => 'none')
-        end
-
-        def load_description_file
-            @source_definition = raw_description_file
-            parse_source_definition
-            @overrides = load_overrides
-        end
-
-        def load_overrides
-            files = Dir.glob(File.join( Autoproj.workspace.overrides_dir, "*.yml" ) ).sort
-            overrides = files.map do |file|
-                source_data = Autoproj.in_file(file, Autoproj::YAML_LOAD_ERROR) do
-                    YAML.load(File.read(file)) || Array.new
-                end
-                source_data =
-                    if source_data.respond_to?(:to_ary)
-                        source_data
-                    else source_data['overrides'] || Array.new
-                    end
-                [file, source_data]
+        # List the autobuild files that are part of this package set
+        def each_autobuild_file
+            return enum_for(__method__) if !block_given?
+            Dir.glob(File.join(local_dir, "*.autobuild")).sort.each do |file|
+                yield(file)
             end
-            overrides + super
         end
 
-        def raw_description_file
-            description = Hash[
-                'imports' => Array.new,
-                'version_control' => Array.new,
-                'overrides' => Array.new]
-            if File.file?(overrides_file_path)
-                overrides_data = Autoproj.in_file(overrides_file_path, Autoproj::YAML_LOAD_ERROR) do
-                    YAML.load(File.read(overrides_file_path)) || Hash.new
-                end
-                description = description.merge(overrides_data)
-            end
-
-            manifest_data = Autoproj.in_file(manifest_path, Autoproj::YAML_LOAD_ERROR) do
-                YAML.load(File.read(manifest_path)) || Hash.new
+        # Yields each osdeps definition files that are present in this package
+        # set
+        def each_osdeps_file
+            return enum_for(__method__) if !block_given?
+            Dir.glob(File.join(local_dir, "*.osdeps")).each do |file|
+                yield(file)
             end
-            description['imports'] = description['imports'].
-                concat(manifest_data['package_sets'] || Array.new)
-            description['name'] = name
-            description
         end
     end
-
-    # DEPRECATED. For backward-compatibility only.
-    Source = PackageSet
-    # DEPRECATED. For backward-compatibility only.
-    LocalSource = LocalPackageSet
 end
+