autoproj 1.6.2 → 1.7.0.b1

data/lib/autoproj/manifest.rb

@@ -86,7 +86,7 @@ module Autoproj
         def initialize(type, url, options)
             @type, @url, @options = type, url, options
             if type != "none" && type != "local" && !Autobuild.respond_to?(type)
-                raise ConfigError, "version control #{type} is unknown to autoproj"
+                raise ConfigError.new, "version control #{type} is unknown to autoproj"
             end
         end
 
@@ -94,6 +94,17 @@ module Autoproj
             @type == 'local'
         end
 
+        def ==(other_vcs)
+            return false if !other_vcs.kind_of?(VCSDefinition)
+            if local?
+                other_vcs.local? && url == other.url
+            elsif !other_vcs.local?
+                this_importer = create_autobuild_importer
+                other_importer = other_vcs.create_autobuild_importer
+                this_importer.repository_id == other_importer.repository_id
+            end
+        end
+
         def self.to_absolute_url(url, root_dir = nil)
             # NOTE: we MUST use nil as default argument of root_dir as we don't
             # want to call Autoproj.root_dir unless completely necessary
@@ -119,7 +130,7 @@ module Autoproj
             else
                 desc = "#{type}:#{url}"
                 if !options.empty?
-                    desc = "#{desc} #{options.map { |key, value| "#{key}=#{value}" }.join(" ")}"
+                    desc = "#{desc} #{options.to_a.sort_by { |key, _| key.to_s }.map { |key, value| "#{key}=#{value}" }.join(" ")}"
                 end
                 desc
             end
@@ -127,12 +138,21 @@ module Autoproj
     end
 
     def self.vcs_definition_to_hash(spec)
+        options = Hash.new
+        if spec.size == 1 && spec.keys.first =~ /auto_imports$/
+            # The user probably wrote
+            #   - string
+            #     auto_imports: false
+            options['auto_imports'] = spec.values.first
+            spec = spec.keys.first.split(" ").first
+        end
+
         if spec.respond_to?(:to_str)
             vcs, *url = spec.to_str.split ':'
             spec = if url.empty?
                        source_dir = File.expand_path(File.join(Autoproj.config_dir, spec))
                        if !File.directory?(source_dir)
-                           raise ConfigError, "'#{spec.inspect}' is neither a remote source specification, nor a local source definition"
+                           raise ConfigError.new, "'#{spec.inspect}' is neither a remote source specification, nor a local source definition"
                        end
 
                        Hash[:type => 'local', :url => source_dir]
@@ -143,7 +163,7 @@ module Autoproj
 
         spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
 
-        return spec.merge(vcs_options)
+        return spec.merge(vcs_options).merge(options)
     end
 
     # Autoproj configuration files accept VCS definitions in three forms:
@@ -155,7 +175,7 @@ module Autoproj
     def self.normalize_vcs_definition(spec)
         spec = vcs_definition_to_hash(spec)
         if !(spec[:type] && (spec[:type] == 'none' || spec[:url]))
-            raise ConfigError, "the source specification #{spec.inspect} misses either the VCS type or an URL"
+            raise ConfigError.new, "the source specification #{spec.inspect} misses either the VCS type or an URL"
         end
 
         spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
@@ -191,7 +211,7 @@ module Autoproj
         else
             value = single_expansion(value, definitions)
             if contains_expansion?(value)
-                raise ConfigError, "some expansions are not defined in #{value.inspect}"
+                raise ConfigError.new, "some expansions are not defined in #{value.inspect}"
             end
             value
         end
@@ -219,22 +239,43 @@ module Autoproj
         result
     end
 
-    # A source is a version control repository which contains general source
+    # A package set is a version control repository which contains general
     # information with package version control information (source.yml file),
     # package definitions (.autobuild files), and finally definition of
     # dependencies that are provided by the operating system (.osdeps file).
-    class Source
+    class PackageSet
+        attr_reader :manifest
         # The VCSDefinition object that defines the version control holding
         # information for this source. Local package sets (i.e. the ones that are not
         # under version control) use the 'local' version control name. For them,
         # local? returns true.
         attr_accessor :vcs
+
+        # If this package set has been imported from another package set, this
+        # is the other package set object
+        attr_accessor :imported_from
+
+        # If true, this package set has been loaded because another set imports
+        # it. If false, it is loaded explicitely by the user
+        def explicit?; !@imported_from end
+
         attr_reader :source_definition
         attr_reader :constants_definitions
 
+        # Sets the auto_imports flag. See #auto_imports?
+        attr_writer :auto_imports
+        # If true (the default), imports listed in this package set will be
+        # automatically loaded by autoproj
+        def auto_imports?; !!@auto_imports end
+
         # Create this source from a VCSDefinition object
-        def initialize(vcs)
+        def initialize(manifest, vcs)
+            @manifest = manifest
             @vcs = vcs
+
+            @provides = Set.new
+            @imports  = Array.new
+            @auto_imports = true
         end
 
         # True if this source has already been checked out on the local autoproj
@@ -250,6 +291,47 @@ module Autoproj
                 !File.exists?(File.join(raw_local_dir, "init.rb"))
         end
 
+        # Create a PackageSet instance from its description as found in YAML
+        # configuration files
+        def self.from_spec(manifest, spec, load_description)
+            spec = Autoproj.vcs_definition_to_hash(spec)
+            options, vcs_spec = Kernel.filter_options spec, :auto_imports => true
+
+            # Look up for short notation (i.e. not an explicit hash). It is
+            # either vcs_type:url or just url. In the latter case, we expect
+            # 'url' to be a path to a local directory
+            vcs_spec = Autoproj.expand(vcs_spec, manifest.constant_definitions)
+            vcs_def  = Autoproj.normalize_vcs_definition(vcs_spec)
+
+            source = PackageSet.new(manifest, vcs_def)
+            source.auto_imports = options[:auto_imports]
+            if load_description
+                if source.present?
+                    source.load_description_file
+                else
+                    raise InternalError, "cannot load description file as it has not been checked out yet"
+                end
+            else
+                # Try to load just the name from the source.yml file
+                source.load_minimal
+            end
+
+            source
+        end
+
+        def repository_id
+            if local?
+                local_dir
+            else
+                importer = vcs.create_autobuild_importer
+                if importer.respond_to?(:repository_id)
+                    importer.repository_id
+                else
+                    vcs.to_s
+                end
+            end
+        end
+
         # Remote sources can be accessed through a hidden directory in
         # $AUTOPROJ_ROOT/.remotes, or through a symbolic link in
         # autoproj/remotes/
@@ -259,9 +341,9 @@ module Autoproj
         # For local sources, is simply returns the path to the source directory.
         def raw_local_dir
             if local?
-                return vcs.url 
+                File.expand_path(vcs.url)
             else
-                File.join(Autoproj.remotes_dir, automatic_name)
+                File.expand_path(File.join(Autoproj.remotes_dir, vcs.to_s.gsub(/[^\w]/, '_')))
             end
         end
 
@@ -291,19 +373,12 @@ module Autoproj
             end
         end
 
-        # A name generated from the VCS url
-        def automatic_name
-            vcs.to_s.gsub(/[^\w]/, '_')
-        end
-
         # Returns the source name
         def name
-            if @source_definition then
-                @source_definition['name'] || automatic_name
-            elsif @name
+            if @name
                 @name
             else
-                automatic_name
+                vcs.to_s
             end
         end
 
@@ -318,34 +393,54 @@ module Autoproj
 
             source_file = File.join(raw_local_dir, "source.yml")
             if !File.exists?(source_file)
-                raise ConfigError, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
+                raise ConfigError.new, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
             end
 
-            begin
-                source_definition = YAML.load(File.read(source_file))
-            rescue ArgumentError => e
-                raise ConfigError, "error in #{source_file}: #{e.message}"
+            source_definition = Autoproj.in_file(source_file, ArgumentError) do
+                YAML.load(File.read(source_file))
             end
 
             if !source_definition || !source_definition['name']
-                raise ConfigError, "#{source_file} does not have a 'name' field"
+                raise ConfigError.new(source_file), "in #{source_file}: missing a 'name' field"
             end
 
             source_definition
         end
 
-        # Load and validate the name from the YAML hash
-        def load_name
+        # Load and validate the self-contained information from the YAML hash
+        def load_minimal
+            # If @source_definition is set, it means that load_description_file
+            # has been called and that therefore all information has already
+            # been parsed
             definition = @source_definition || raw_description_file
             @name = definition['name']
 
             if @name !~ /^[\w_\.-]+$/
-                raise ConfigError, "invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
+                raise ConfigError.new(source_file),
+                    "in #{source_file}: invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
             elsif @name == "local"
-                raise ConfigError, "source #{self} is named 'local', but this is a reserved name"
+                raise ConfigError.new(source_file),
+                    "in #{source_file}: the name 'local' is a reserved name"
+            end
+
+            @provides = (definition['provides'] || Set.new).to_set
+            @imports  = (definition['imports'] || Array.new).map do |set_def|
+                pkg_set = Autoproj.in_file(source_file) do
+                    PackageSet.from_spec(manifest, set_def, false)
+                end
+
+                pkg_set.imported_from = self
+                pkg_set
             end
 
         rescue InternalError
+            # This ignores raw_description_file error if the package set is not
+            # checked out yet
+        end
+
+        # Yields the imports this package set declares, as PackageSet instances
+        def each_imported_set(&block)
+            @imports.each(&block)
         end
 
         # Path to the source.yml file
@@ -353,21 +448,22 @@ module Autoproj
             File.join(local_dir, 'source.yml')
         end
 
-        # Load the source.yml file that describes this source, and resolve the
-        # $BLABLA values that are in there. Use #raw_description_file to avoid
-        # resolving those values
+        # Load the source.yml file and resolves all information it contains.
+        #
+        # This for instance requires configuration options to be defined. Use
+        # PackageSet#load_minimal to load only self-contained information
         def load_description_file
+            if @source_definition
+                return
+            end
+
             @source_definition = raw_description_file
-            load_name
+            load_minimal
 
-            
             # Compute the definition of constants
-            begin
+            Autoproj.in_file(source_file) do
                 constants = source_definition['constants'] || Hash.new
                 @constants_definitions = Autoproj.resolve_constant_definitions(constants)
-
-            rescue ConfigError => e
-                raise ConfigError, "#{File.join(local_dir, "source.yml")}: #{e.message}", e.backtrace
             end
         end
 
@@ -399,15 +495,19 @@ module Autoproj
         #
         # The returned value is a VCSDefinition object.
         def version_control_field(package_name, section_name, validate = true)
+            if !source_definition
+                raise InternalError, "source #{name} has not been loaded"
+            end
+
             urls = source_definition['urls'] || Hash.new
             urls['HOME'] = ENV['HOME']
 
             all_vcs     = source_definition[section_name]
             if all_vcs
                 if all_vcs.kind_of?(Hash)
-                    raise ConfigError, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
+                    raise ConfigError.new, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
                 elsif !all_vcs.kind_of?(Array)
-                    raise ConfigError, "wrong format for the #{section_name} section"
+                    raise ConfigError.new, "wrong format for the #{section_name} section"
                 end
             end
 
@@ -449,7 +549,7 @@ module Autoproj
                             if spec == "none"
                                 spec = { :type => "none" }
                             else
-                                raise ConfigError, "invalid VCS specification '#{name}: #{spec}'"
+                                raise ConfigError.new, "invalid VCS specification '#{name}: #{spec}'"
                             end
                         end
                     end
@@ -480,8 +580,6 @@ module Autoproj
                 end
                 vcs_spec
             end
-        rescue ConfigError => e
-            raise ConfigError, "#{e.message} in #{source_file}", e.backtrace
         end
 
         # Returns the VCS definition for +package_name+ as defined in this
@@ -508,18 +606,29 @@ module Autoproj
                 end
             end
         end
+
+        # True if this package set provides the given package set name. I.e. if
+        # it has this name or the name is listed in the "replaces" field of
+        # source.yml
+        def provides?(name)
+            name == self.name ||
+                provides.include?(name)
+        end
     end
 
-    # Specialization of the Source class for the overrides listed in autoproj/
-    class LocalSource < Source
-        def initialize
-            super(Autoproj.normalize_vcs_definition(:type => 'local', :url => Autoproj.config_dir))
+    # Specialization of the PackageSet class for the overrides listed in autoproj/
+    class LocalPackageSet < PackageSet
+        def initialize(manifest)
+            super(manifest, Autoproj.normalize_vcs_definition(:type => 'local', :url => Autoproj.config_dir))
         end
 
         def name
             'local'
         end
-        def load_name
+        def load_minimal
+        end
+        def repository_id
+            'local'
         end
 
         def source_file
@@ -535,10 +644,8 @@ module Autoproj
         def raw_description_file
             path = source_file
             if File.file?(path)
-                begin
-                    data = YAML.load(File.read(path)) || Hash.new
-                rescue ArgumentError => e
-                    raise ConfigError, "error in #{source_file}: #{e.message}"
+                data = Autoproj.in_file(path, ArgumentError) do
+                    YAML.load(File.read(path)) || Hash.new
                 end
                 data['name'] = 'local'
                 data
@@ -548,12 +655,16 @@ module Autoproj
         end
     end
 
+    # DEPRECATED. For backward-compatibility only.
+    Source = PackageSet
+    # DEPRECATED. For backward-compatibility only.
+    LocalSource = LocalPackageSet
+
     PackageDefinition = Struct.new :autobuild, :user_block, :package_set, :file
 
     # The Manifest class represents the information included in the main
     # manifest file, and allows to manipulate it
     class Manifest
-        FakePackage = Struct.new :text_name, :name, :srcdir, :importer, :updated
 
         # Data structure used to use autobuild importers without a package, to
         # import configuration data.
@@ -561,7 +672,21 @@ module Autoproj
         # It has to match the interface of Autobuild::Package that is relevant
         # for importers
         class FakePackage
-            def autoproj_name; name end
+            attr_reader :text_name
+            attr_reader :name
+            attr_reader :srcdir
+            attr_reader :importer
+
+            # Used by the autobuild importers
+            attr_accessor :updated
+
+            def initialize(text_name, srcdir, importer = nil)
+                @text_name = text_name
+                @name = text_name.gsub /[^\w]/, '_'
+                @srcdir = srcdir
+                @importer = importer
+            end
+
             def import
                 importer.import(self)
             end
@@ -593,21 +718,34 @@ module Autoproj
         # Loads the manifest file located at +file+ and returns the Manifest
         # instance that represents it
 	def self.load(file)
-            begin
-                data = YAML.load(File.read(file))
-            rescue Errno::ENOENT
-                raise ConfigError, "expected an autoproj configuration in #{File.expand_path(File.dirname(file))}, but #{file} does not exist"
-            rescue ArgumentError => e
-                raise ConfigError, "error in #{file}: #{e.message}"
-            end
-	    Manifest.new(file, data)
+	    manifest = Manifest.new
+            manifest.load(file)
+            manifest
 	end
 
+        # Load the manifest data contained in +file+
+        def load(file)
+            if !File.exists?(file)
+                raise ConfigError.new(dirname), "expected an autoproj configuration in #{dirname}, but #{file} does not exist"
+            end
+
+            data = Autoproj.in_file(file, ArgumentError) do
+                YAML.load(File.read(file))
+            end
+
+            @file = file
+            @data = data
+
+            if data['constants']
+                @constant_definitions = Autoproj.resolve_constant_definitions(data['constants'])
+            end
+        end
+
         # The manifest data as a Hash
         attr_reader :data
 
         # The set of packages defined so far as a mapping from package name to 
-        # [Autobuild::Package, source, file] tuple
+        # [Autobuild::Package, package_set, file] tuple
         attr_reader :packages
 
         # A mapping from package names into PackageManifest objects
@@ -633,22 +771,19 @@ module Autoproj
 
         attr_reader :constant_definitions
 
-	def initialize(file, data)
-            @file = file
-	    @data = data
+	def initialize
+            @file = nil
+	    @data = nil
             @packages = Hash.new
             @package_manifests = Hash.new
             @automatic_exclusions = Hash.new
             @constants_definitions = Hash.new
+            @disabled_imports = Set.new
 
+            @constant_definitions = Hash.new
             if Autoproj.has_config_key?('manifest_source')
                 @vcs = Autoproj.normalize_vcs_definition(Autoproj.user_config('manifest_source'))
             end
-            if data['constants']
-                @constant_definitions = Autoproj.resolve_constant_definitions(data['constants'])
-            else
-                @constant_definitions = Hash.new
-            end
 	end
 
         # True if the given package should not be built, with the packages that
@@ -730,7 +865,7 @@ module Autoproj
             end
 
             if source_name && !done_something
-                raise ConfigError, "in #{file}: source '#{source_name}' does not exist"
+                raise ConfigError.new(file), "in #{file}: source '#{source_name}' does not exist"
             end
 	end
 
@@ -767,11 +902,11 @@ module Autoproj
         end
 
         # Like #each_source, but filters out local package sets
-        def each_remote_source(load_description = true)
+        def each_remote_package_set(load_description = true)
             if !block_given?
-                enum_for(:each_remote_source, load_description)
+                enum_for(:each_remote_package_set, load_description)
             else
-                each_source(load_description) do |source|
+                each_package_set(load_description) do |source|
                     if !source.local?
                         yield(source)
                     end
@@ -779,73 +914,95 @@ module Autoproj
             end
         end
 
-        def source_from_spec(spec, load_description) # :nodoc:
-            # Look up for short notation (i.e. not an explicit hash). It is
-            # either vcs_type:url or just url. In the latter case, we expect
-            # 'url' to be a path to a local directory
-            vcs_def = begin
-                          spec = Autoproj.expand(spec, constant_definitions)
-                          Autoproj.normalize_vcs_definition(spec)
-                      rescue ConfigError => e
-                          raise ConfigError, "in #{file}: #{e.message}"
-                      end
-
-            source = Source.new(vcs_def)
-            if load_description
-                if source.present?
-                    source.load_description_file
-                else
-                    raise InternalError, "cannot load description file as it has not been checked out yet"
-                end
-            else
-                # Try to load just the name from the source.yml file
-                source.load_name
+        def each_remote_source(*args, &block)
+            each_remote_package_set(*args, &block)
+        end
+
+        # Helper method for #each_package_set
+        def enumerate_package_set(pkg_set, explicit_sets, all_sets) # :nodoc:
+            if @disabled_imports.include?(pkg_set.name)
+                pkg_set.auto_imports = false
             end
 
-            source
-        end
+            result = []
+            if pkg_set.auto_imports?
+                pkg_set.each_imported_set do |imported_set|
+                    if explicit_sets.any? { |src| src.vcs == imported_set.vcs } ||
+                        all_sets.any? { |src| src.vcs == imported_set.vcs }
+                        next
+                    end
 
+                    all_sets << imported_set
+                    result.concat(enumerate_package_set(imported_set, explicit_sets, all_sets))
+                end
+            end
+            result << pkg_set
+            result
+        end
 
         # call-seq:
-        #   each_source { |source_description| ... }
+        #   each_package_set { |pkg_set| ... }
         #
-        # Lists all package sets defined in this manifest, by yielding a Source
-        # object that describes it.
-        def each_source(load_description = true, &block)
+        # Lists all package sets defined in this manifest, by yielding a
+        # PackageSet object that describes it.
+        def each_package_set(load_description = true, &block)
             if !block_given?
-                return enum_for(:each_source, load_description)
+                return enum_for(:each_package_set, load_description)
             end
 
-            if @sources
+            if @package_sets
                 if load_description
-                    @sources.each do |src|
+                    @package_sets.each do |src|
                         if !src.source_definition
                             src.load_description_file
                         end
                     end
                 end
-                return @sources.each(&block)
+                return @package_sets.each(&block)
             end
 
-            all_sources = []
+	    explicit_sets = (data['package_sets'] || []).map do |spec|
+                Autoproj.in_file(self.file) do
+                    PackageSet.from_spec(self, spec, load_description)
+                end
+            end
 
-	    (data['package_sets'] || []).each do |spec|
-                all_sources << source_from_spec(spec, load_description)
+            all_sets = Array.new
+            explicit_sets.each do |pkg_set|
+                all_sets.concat(enumerate_package_set(pkg_set, explicit_sets, all_sets + [pkg_set]))
             end
 
             # Now load the local source 
-            local = LocalSource.new
+            local = LocalPackageSet.new(self)
             if load_description
                 local.load_description_file
             else
-                local.load_name
+                local.load_minimal
             end
             if !load_description || !local.empty?
-                all_sources << local
+                all_sets << local
+            end
+            
+            if load_description
+                all_sets.each(&:load_description_file)
+            else
+                all_sets.each(&:load_minimal)
             end
+            all_sets.each(&block)
+        end
+
+        # DEPRECATED. For backward-compatibility only
+        #
+        # use #each_package_set instead
+        def each_source(*args, &block)
+            each_package_set(*args, &block)
+        end
 
-            all_sources.each(&block)
-            @sources = all_sources
+        # Save the currently known package sets. After this call,
+        # #each_package_set will always return the same set regardless of
+        # changes on the manifest's data structures
+        def cache_package_sets
+            @package_sets = each_package_set(false).to_a
         end
 
         # Register a new package
@@ -860,6 +1017,10 @@ module Autoproj
             @packages[package_name].file
         end
 
+        def package(name)
+            packages[name]
+        end
+
         # Lists all defined packages and where they have been defined
         def each_package
             if !block_given?
@@ -877,11 +1038,11 @@ module Autoproj
             end
 
             if vcs
-                yield(vcs, "autoproj main configuration", "autoproj_config", Autoproj.config_dir)
+                yield(vcs, "autoproj main configuration", Autoproj.config_dir)
             end
 
             each_remote_source(false) do |source|
-                yield(source.vcs, source.name || source.vcs.url, source.automatic_name, source.local_dir)
+                yield(source.vcs, source.name || source.vcs.url, source.local_dir)
             end
             self
         end
@@ -893,81 +1054,108 @@ module Autoproj
         # the name used when displaying the import progress, +pkg_name+ the
         # internal name used to represent the package and +into+ the directory
         # in which the package should be checked out.
-        def self.create_autobuild_package(vcs, text_name, pkg_name, into)
+        def self.create_autobuild_package(vcs, text_name, into)
+            if !into
+                raise
+            end
             importer     = vcs.create_autobuild_importer
             return if !importer # updates have been disabled by using the 'none' type
 
-            fake_package = FakePackage.new(text_name, pkg_name, into)
-            fake_package.importer = importer
-            fake_package
+            FakePackage.new(text_name, into, importer)
 
         rescue Autobuild::ConfigException => e
-            raise ConfigError, "cannot import #{name}: #{e.message}", e.backtrace
+            raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
         end
 
         # Imports or updates a source (remote or otherwise).
         #
         # See create_autobuild_package for informations about the arguments.
-        def self.update_source(vcs, text_name, pkg_name, into)
-            fake_package = create_autobuild_package(vcs, text_name, pkg_name, into)
+        def self.update_package_set(vcs, text_name, into)
+            if !into
+                raise
+            end
+            fake_package = create_autobuild_package(vcs, text_name, into)
             fake_package.import
 
         rescue Autobuild::ConfigException => e
-            raise ConfigError, "cannot import #{name}: #{e.message}", e.backtrace
+            raise ConfigError.new, "cannot import #{name}: #{e.message}", e.backtrace
         end
 
         # Updates the main autoproj configuration
         def update_yourself
-            Manifest.update_source(vcs, "autoproj main configuration", "autoproj_conf", Autoproj.config_dir)
+            Manifest.update_package_set(vcs, "autoproj main configuration", Autoproj.config_dir)
         end
 
-        # Updates all the remote sources in ROOT_DIR/.remotes, as well as the
-        # symbolic links in ROOT_DIR/autoproj/remotes
-        def update_remote_sources
-            # Iterate on the remote sources, without loading the source.yml
-            # file (we're not ready for that yet)
-            sources = []
-            each_remote_source(false) do |source|
-                name = if source.present? then source.name
-                       else source.vcs.url
-                       end
-                Manifest.update_source(source.vcs, name, source.automatic_name, source.raw_local_dir)
-                sources << source
-            end
+        def update_remote_set(pkg_set, remotes_symlinks_dir = nil)
+            Manifest.update_package_set(
+                pkg_set.vcs,
+                pkg_set.name,
+                pkg_set.raw_local_dir)
 
-            # Check for directories in ROOT_DIR/.remotes that do not map to a
-            # source repository, and remove them
-            Dir.glob(File.join(Autoproj.remotes_dir, '*')).each do |dir|
-                dir = File.expand_path(dir)
-                if File.directory?(dir) && !sources.any? { |s| s.raw_local_dir == dir }
-                    FileUtils.rm_rf dir
-                end
-            end
-
-            remotes_symlinks_dir = File.join(Autoproj.config_dir, 'remotes')
-            FileUtils.mkdir_p remotes_symlinks_dir
-            known_remotes = []
-
-            # Create symbolic links from .remotes/weird_url to
-            # autoproj/remotes/name. Explicitely load the source name first
-            each_remote_source(false) do |source|
-                source.load_name
-                symlink_dest = File.join(remotes_symlinks_dir, source.name)
+            if remotes_symlinks_dir
+                pkg_set.load_minimal
+                symlink_dest = File.join(remotes_symlinks_dir, pkg_set.name)
 
                 # Check if the current symlink is valid, and recreate it if it
                 # is not
                 if File.symlink?(symlink_dest)
                     dest = File.readlink(symlink_dest)
-                    if dest != source.raw_local_dir
+                    if dest != pkg_set.raw_local_dir
                         FileUtils.rm_f symlink_dest
-                        FileUtils.ln_sf source.raw_local_dir, symlink_dest
+                        FileUtils.ln_sf pkg_set.raw_local_dir, symlink_dest
                     end
                 else
                     FileUtils.rm_f symlink_dest
-                    FileUtils.ln_sf source.raw_local_dir, symlink_dest
+                    FileUtils.ln_sf pkg_set.raw_local_dir, symlink_dest
+                end
+
+                symlink_dest
+            end
+        end
+
+        # Updates all the remote sources in ROOT_DIR/.remotes, as well as the
+        # symbolic links in ROOT_DIR/autoproj/remotes
+        def update_remote_package_sets
+            remotes_symlinks_dir = File.join(Autoproj.config_dir, 'remotes')
+            FileUtils.mkdir_p remotes_symlinks_dir
+
+            # Iterate on the remote sources, without loading the source.yml
+            # file (we're not ready for that yet)
+            #
+            # Do it iteratively to properly take imports into account, but we
+            # first unconditionally update all the existing sets to properly
+            # handle imports that have been removed
+            updated_sets     = Hash.new
+            known_remotes = []
+            each_remote_package_set(false) do |pkg_set|
+                next if !pkg_set.explicit?
+                if pkg_set.present?
+                    known_remotes << update_remote_set(pkg_set, remotes_symlinks_dir)
+                    updated_sets[pkg_set.repository_id] = pkg_set
+                end
+            end
+
+            old_updated_sets = nil
+            while old_updated_sets != updated_sets
+                old_updated_sets = updated_sets.dup
+                each_remote_package_set(false) do |pkg_set|
+                    next if updated_sets.has_key?(pkg_set.repository_id)
+
+                    if !pkg_set.explicit?
+                        Autoproj.progress "  #{pkg_set.imported_from.name}: auto-importing #{pkg_set.name}"
+                    end
+                    known_remotes << update_remote_set(pkg_set, remotes_symlinks_dir)
+                    updated_sets[pkg_set.repository_id] = pkg_set
                 end
+            end
 
-                known_remotes << symlink_dest
+            # Check for directories in ROOT_DIR/.remotes that do not map to a
+            # source repository, and remove them
+            Dir.glob(File.join(Autoproj.remotes_dir, '*')).each do |dir|
+                dir = File.expand_path(dir)
+                if File.directory?(dir) && !updated_sets.values.find { |pkg| pkg.raw_local_dir == dir }
+                    FileUtils.rm_rf dir
+                end
             end
 
             # Now remove obsolete symlinks
@@ -978,6 +1166,11 @@ module Autoproj
             end
         end
 
+        # DEPRECATED. For backward-compatibility only
+        def update_remote_sources(*args, &block)
+            update_remote_package_sets(*args, &block)
+        end
+
         def importer_definition_for(package_name, package_source = nil)
             if !package_source
                 package_source = packages.values.
@@ -985,13 +1178,16 @@ module Autoproj
                     package_set
             end
 
-            sources = each_source.to_a
+            sources = each_source.to_a.dup
 
             # Remove sources listed before the package source
             while !sources.empty? && sources[0].name != package_source.name
                 sources.shift
             end
             package_source = sources.shift
+            if !package_source
+                raise InternalError, "cannot find the package set that defines #{package_name}"
+            end
 
             # Get the version control information from the package source. There
             # must be one
@@ -1032,7 +1228,7 @@ module Autoproj
                     Autoproj.add_build_system_dependency vcs.type
                     pkg.autobuild.importer = vcs.create_autobuild_importer
                 else
-                    raise ConfigError, "source #{pkg.package_set.name} defines #{pkg.autobuild.name}, but does not provide a version control definition for it"
+                    raise ConfigError.new, "source #{pkg.package_set.name} defines #{pkg.autobuild.name}, but does not provide a version control definition for it"
                 end
             end
         end
@@ -1044,12 +1240,12 @@ module Autoproj
             if Autobuild::Package[name]
                 [name]
             else
-                source = each_source.find { |source| source.name == name }
-                if !source
-                    raise ConfigError, "in #{file}: #{name} is neither a package nor a source"
+                pkg_set = each_package_set(false).find { |set| set.name == name }
+                if !pkg_set
+                    raise ConfigError.new, "#{name} is neither a package nor a source"
                 end
                 packages.values.
-                    find_all { |pkg| pkg.package_set.name == source.name }.
+                    find_all { |pkg| pkg.package_set.name == pkg_set.name }.
                     map { |pkg| pkg.autobuild.name }.
                     find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
             end
@@ -1059,11 +1255,15 @@ module Autoproj
         #
         # If recursive is false, yields only the packages at this level.
         # Otherwise, return all packages.
-        def layout_packages(layout_def, recursive)
+        def layout_packages(layout_def, recursive, validate = true)
             result = []
             layout_def.each do |value|
                 if !value.kind_of?(Hash) # sublayout
-                    result.concat(resolve_package_set(value))
+                    begin
+                        result.concat(resolve_package_set(value))
+                    rescue ConfigError
+                        raise if validate
+                    end
                 end
             end
 
@@ -1088,7 +1288,7 @@ module Autoproj
 
         # Looks into the layout setup in the manifest, and yields each layout
         # and sublayout in order
-        def each_package_set(selection = nil, layout_name = '/', layout_def = data['layout'], &block)
+        def each_layout_level(selection = nil, layout_name = '/', layout_def = data['layout'], &block)
             if !layout_def
                 yield(layout_name, default_packages, default_packages)
                 return nil
@@ -1112,14 +1312,14 @@ module Autoproj
 
             # Now, enumerate the sublayouts
             each_sublayout(layout_def) do |subname, sublayout|
-                each_package_set(selection, "#{layout_name}#{subname}/", sublayout, &block)
+                each_layout_level(selection, "#{layout_name}#{subname}/", sublayout, &block)
             end
         end
 
         # Returns the set of package names that are explicitely listed in the
         # layout, minus the excluded and ignored ones
-        def all_layout_packages
-            default_packages
+        def all_layout_packages(validate = true)
+            default_packages(validate)
         end
 
         # Returns all defined package names, minus the excluded and ignored ones
@@ -1134,11 +1334,30 @@ module Autoproj
                 find_all { |pkg_name| !Autoproj.osdeps || !Autoproj.osdeps.has?(pkg_name) }
         end
 
+        # Returns true if +name+ is a valid package and is included in the build
+        #
+        # If +validate+ is true, the layout will have to be well-defined. It can
+        # therefore be used only after all sources have been successfully
+        # loaded.
+        #
+        # If it is false, non-defined packages/package sets in the layout will
+        # simply be ignored
+        def package_enabled?(name, validate = true)
+            if !Autobuild::Package[name]
+                if validate
+                    raise ArgumentError, "package #{name} does not exist"
+                end
+                return false
+            end
+
+            !excluded?(name)
+        end
+
         # Returns the set of packages that should be built if the user does not
         # specify any on the command line
-        def default_packages
+        def default_packages(validate = true)
             names = if layout = data['layout']
-                        layout_packages(layout, true)
+                        layout_packages(layout, true, validate)
                     else
                         # No layout, all packages are selected
                         all_packages
@@ -1150,12 +1369,20 @@ module Autoproj
 
         # Returns the package directory for the given package name
         def whereis(package_name)
-            each_package_set do |layout_name, packages, _|
-                if packages.include?(package_name)
-                    return layout_name
+            Autoproj.in_file(self.file) do
+                each_layout_level do |layout_name, packages, _|
+                    if packages.include?(package_name)
+                        return layout_name
+                    end
                 end
+                raise ArgumentError, "cannot find #{package_name} in the layout section"
+            end
+        end
+
+        def resolve_optional_dependencies
+            packages.each_value do |pkg|
+                pkg.autobuild.resolve_optional_dependencies
             end
-            raise ArgumentError, "cannot find #{package_name} in the current layout"
         end
 
         # Loads the package's manifest.xml file for the current package
@@ -1180,13 +1407,19 @@ module Autoproj
             manifest = PackageManifest.load(package, manifest_path)
             package_manifests[package.name] = manifest
 
-            manifest.each_dependency do |name|
+            manifest.each_dependency do |name, is_optional|
                 begin
-                    package.depends_on name
+                    if is_optional
+                        package.optional_dependency name
+                    else
+                        package.depends_on name
+                    end
                 rescue Autobuild::ConfigException => e
-                    raise ConfigError, "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, which is listed in the layout of #{file} but has no autobuild definition", e.backtrace
+                    raise ConfigError.new(manifest_path),
+                        "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, which is listed in the layout of #{file} but has no autobuild definition", e.backtrace
                 rescue ConfigError => e
-                    raise ConfigError, "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, but it is neither a normal package nor an osdeps package. osdeps reports: #{e.message}", e.backtrace
+                    raise ConfigError.new(manifest_path),
+                        "manifest #{manifest_path} of #{package.name} from #{source.name} lists '#{name}' as dependency, but it is neither a normal package nor an osdeps package. osdeps reports: #{e.message}", e.backtrace
                 end
             end
         end
@@ -1198,6 +1431,11 @@ module Autoproj
             selected_packages.each(&:load_package_manifest)
         end
 
+        # Disable all automatic imports from the given package set name
+        def disable_imports_from(pkg_set_name)
+            @disabled_imports << pkg_set_name
+        end
+
         # call-seq:
         #   list_os_dependencies(packages) => required_packages, ospkg_to_pkg
         #
@@ -1264,7 +1502,7 @@ module Autoproj
             end
 
             # Now, search for layout names
-            each_package_set(nil) do |layout_name, packages, _|
+            each_layout_level(nil) do |layout_name, packages, _|
                 selected_packages.each do |sel|
                     if layout_name[0..-1] =~ Regexp.new("#{sel}\/?$")
                         expanded_packages |= packages.to_set
@@ -1325,6 +1563,15 @@ module Autoproj
         # The raw XML data as a Nokogiri document
         attr_reader :xml
 
+        def short_documentation
+            xml.xpath('//description').each do |node|
+                if doc = node['brief']
+                    return doc.to_s
+                end
+            end
+            nil
+        end
+
         def initialize(package, doc)
             @package = package
             @xml = doc
@@ -1342,10 +1589,10 @@ module Autoproj
         def each_os_dependency
             if block_given?
                 xml.xpath('//rosdep').each do |node|
-                    yield(node['name'])
+                    yield(node['name'], false)
                 end
                 package.os_packages.each do |name|
-                    yield(name)
+                    yield(name, false)
                 end
             else
                 enum_for :each_os_dependency
@@ -1356,10 +1603,16 @@ module Autoproj
             if block_given?
                 xml.xpath('//depend').each do |node|
                     dependency = node['package']
+                    optional   =
+                        if node['optional'].to_s == '1'
+                            true
+                        else false
+                        end
+
                     if dependency
-                        yield(dependency)
+                        yield(dependency, optional)
                     else
-                        raise ConfigError, "manifest of #{package.name} has a <depend> tag without a 'package' attribute"
+                        raise ConfigError.new, "manifest of #{package.name} has a <depend> tag without a 'package' attribute"
                     end
                 end
             else