autoproj 1.9.6 → 1.9.7.rc1

data/lib/autoproj/environment.rb

@@ -0,0 +1,78 @@
+module Autoproj
+    # Sets an environment variable
+    #
+    # This sets (or resets) the environment variable +name+ to the given value.
+    # If multiple values are given, they are joined with ':'
+    #
+    # The values can contain configuration parameters using the
+    # $CONF_VARIABLE_NAME syntax.
+    def self.env_set(name, *value)
+        Autobuild.env_clear(name)
+        env_add(name, *value)
+    end
+
+    # Adds new values to a given environment variable
+    #
+    # Adds the given value(s) to the environment variable named +name+. The
+    # values are added using the ':' marker.
+    #
+    # The values can contain configuration parameters using the
+    # $CONF_VARIABLE_NAME syntax.
+    def self.env_add(name, *value)
+        value = value.map { |v| expand_environment(v) }
+        Autobuild.env_add(name, *value)
+    end
+
+    # Sets an environment variable which is a path search variable (such as
+    # PATH, RUBYLIB, PYTHONPATH)
+    #
+    # This sets (or resets) the environment variable +name+ to the given value.
+    # If multiple values are given, they are joined with ':'. Unlike env_set,
+    # duplicate values will be removed.
+    #
+    # The values can contain configuration parameters using the
+    # $CONF_VARIABLE_NAME syntax.
+    def self.env_set_path(name, *value)
+        Autobuild.env_clear(name)
+        env_add_path(name, *value)
+    end
+
+    # Adds new values to a given environment variable, which is a path search
+    # variable (such as PATH, RUBYLIB, PYTHONPATH)
+    #
+    # Adds the given value(s) to the environment variable named +name+. The
+    # values are added using the ':' marker. Unlike env_set, duplicate values
+    # will be removed.
+    #
+    # The values can contain configuration parameters using the
+    # $CONF_VARIABLE_NAME syntax.
+    #
+    # This is usually used in package configuration blocks to add paths
+    # dependent on the place of install, such as
+    #
+    #   cmake_package 'test' do |pkg|
+    #     Autoproj.env_add_path 'RUBYLIB', File.join(pkg.srcdir, 'bindings', 'ruby')
+    #   end
+    def self.env_add_path(name, *value)
+        value = value.map { |v| expand_environment(v) }
+        Autobuild.env_add_path(name, *value)
+    end
+
+    # Requests that autoproj source the given shell script in its own env.sh
+    # script
+    def self.env_source_file(file)
+        Autobuild.env_source_file(file)
+    end
+
+    # Requests that autoproj source the given shell script in its own env.sh
+    # script
+    def self.env_source_after(file)
+        Autobuild.env_source_after(file)
+    end
+
+    # Requests that autoproj source the given shell script in its own env.sh
+    # script
+    def self.env_source_before(file)
+        Autobuild.env_source_before(file)
+    end
+end

data/lib/autoproj/installation_manifest.rb

@@ -0,0 +1,36 @@
+module Autoproj
+    # Manifest of installed packages imported from another autoproj installation
+    class InstallationManifest
+        Package = Struct.new :name, :srcdir, :prefix
+
+        attr_reader :path
+        attr_reader :packages
+        def initialize(path)
+            @path = path
+        end
+
+        def load(path)
+            @packages = CSV.read(path).map do |row|
+                Package.new(*row)
+            end
+        end
+
+        def each(&block)
+            packages.each(&block)
+        end
+
+        def [](name)
+            packages.find { |pkg| pkg.name == name }
+        end
+
+        def self.from_root(root_dir)
+            manifest = InstallationManifest.new(root_dir)
+            manifest_file = File.join(root_dir,  ".autoproj-installation-manifest")
+            if !File.file?(manifest_file)
+                raise ConfigError.new, "no .autoproj-installation-manifest file exists in #{root_dir}. You should probably rerun autoproj envsh in that folder first"
+            end
+            manifest.load(manifest_file)
+            manifest
+        end
+    end
+end

data/lib/autoproj/manifest.rb

@@ -16,6 +16,7 @@ module Autoproj
     def self.add_build_system_dependency(*names)
         @build_system_dependencies |= names.to_set
     end
+
     class << self
         # Returns the set of OS packages that are needed to build and/or import
         # the packages
@@ -24,970 +25,6 @@ module Autoproj
         attr_reader :build_system_dependencies
     end
 
-    # Expand build options in +value+.
-    #
-    # The method will expand in +value+ patterns of the form $NAME, replacing
-    # them with the corresponding build option.
-    def self.expand_environment(value)
-        # Perform constant expansion on the defined environment variables,
-        # including the option set
-        options = Autoproj.option_set
-        options.each_key do |k|
-            options[k] = options[k].to_s
-        end
-
-        loop do
-            new_value = Autoproj.single_expansion(value, options)
-            if new_value == value
-                break
-            else
-                value = new_value
-            end
-        end
-        value
-    end
-
-    # Sets an environment variable
-    #
-    # This sets (or resets) the environment variable +name+ to the given value.
-    # If multiple values are given, they are joined with ':'
-    #
-    # The values can contain configuration parameters using the
-    # $CONF_VARIABLE_NAME syntax.
-    def self.env_set(name, *value)
-        Autobuild.env_clear(name)
-        env_add(name, *value)
-    end
-
-    # Adds new values to a given environment variable
-    #
-    # Adds the given value(s) to the environment variable named +name+. The
-    # values are added using the ':' marker.
-    #
-    # The values can contain configuration parameters using the
-    # $CONF_VARIABLE_NAME syntax.
-    def self.env_add(name, *value)
-        value = value.map { |v| expand_environment(v) }
-        Autobuild.env_add(name, *value)
-    end
-
-    # Sets an environment variable which is a path search variable (such as
-    # PATH, RUBYLIB, PYTHONPATH)
-    #
-    # This sets (or resets) the environment variable +name+ to the given value.
-    # If multiple values are given, they are joined with ':'. Unlike env_set,
-    # duplicate values will be removed.
-    #
-    # The values can contain configuration parameters using the
-    # $CONF_VARIABLE_NAME syntax.
-    def self.env_set_path(name, *value)
-        Autobuild.env_clear(name)
-        env_add_path(name, *value)
-    end
-
-    # Adds new values to a given environment variable, which is a path search
-    # variable (such as PATH, RUBYLIB, PYTHONPATH)
-    #
-    # Adds the given value(s) to the environment variable named +name+. The
-    # values are added using the ':' marker. Unlike env_set, duplicate values
-    # will be removed.
-    #
-    # The values can contain configuration parameters using the
-    # $CONF_VARIABLE_NAME syntax.
-    #
-    # This is usually used in package configuration blocks to add paths
-    # dependent on the place of install, such as
-    #
-    #   cmake_package 'test' do |pkg|
-    #     Autoproj.env_add_path 'RUBYLIB', File.join(pkg.srcdir, 'bindings', 'ruby')
-    #   end
-    def self.env_add_path(name, *value)
-        value = value.map { |v| expand_environment(v) }
-        Autobuild.env_add_path(name, *value)
-    end
-
-    # Requests that autoproj source the given shell script in its own env.sh
-    # script
-    def self.env_source_file(file)
-        Autobuild.env_source_file(file)
-    end
-
-    # Requests that autoproj source the given shell script in its own env.sh
-    # script
-    def self.env_source_after(file)
-        Autobuild.env_source_after(file)
-    end
-
-    # Requests that autoproj source the given shell script in its own env.sh
-    # script
-    def self.env_source_before(file)
-        Autobuild.env_source_before(file)
-    end
-
-    # Representation of a VCS definition contained in a source.yml file or in
-    # autoproj/manifest
-    class VCSDefinition
-        attr_reader :type
-        attr_reader :url
-        attr_reader :options
-
-        # The original spec in hash form. Set if this VCSDefinition object has
-        # been created using VCSDefinition.from_raw
-        attr_reader :raw
-
-        def initialize(type, url, options, raw = nil)
-            if raw && !raw.respond_to?(:to_ary)
-                raise ArgumentError, "wrong format for the raw field (#{raw.inspect})"
-            end
-
-            @type, @url, @options = type, url, options
-            if type != "none" && type != "local" && !Autobuild.respond_to?(type)
-                raise ConfigError.new, "version control #{type} is unknown to autoproj"
-            end
-            @raw = raw
-        end
-
-        def local?
-            @type == 'local'
-        end
-
-        # Updates the VCS specification +old+ by the information contained in
-        # +new+
-        #
-        # Both +old+ and +new+ are supposed to be in hash form. It is assumed
-        # that +old+ has already been normalized by a call to
-        # Autoproj.vcs_definition_to_hash. +new+ can be in "raw" form.
-        def self.update_raw_vcs_spec(old, new)
-            new = vcs_definition_to_hash(new)
-            if new.has_key?(:type) && (old[:type] != new[:type])
-                # The type changed. We replace the old definition by the new one
-                # completely, and we make sure that the new definition is valid
-                from_raw(new)
-                new
-            else
-                old.merge(new)
-            end
-        end
-
-        # Normalizes a VCS definition contained in a YAML file into a hash
-        #
-        # It handles custom source handler expansion, as well as the bad habit
-        # of forgetting a ':' at the end of a line:
-        #
-        #   - package_name
-        #     branch: value
-        def self.vcs_definition_to_hash(spec)
-            options = Hash.new
-
-            plain = Array.new
-            filtered_spec = Hash.new
-            spec.each do |key, value|
-                keys = key.to_s.split(/\s+/)
-                plain.concat(keys[0..-2])
-                filtered_spec[keys[-1].to_sym] = value
-            end
-            spec = filtered_spec
-
-            if plain.size > 1
-                raise ConfigError.new, "invalid syntax"
-            elsif plain.size == 1
-                short_url = plain.first
-                vcs, *url = short_url.split(':')
-
-                # Check if VCS is a known version control system or source handler
-                # shortcut. If it is not, look for a local directory called
-                # short_url
-                if Autobuild.respond_to?(vcs)
-                    spec.merge!(:type => vcs, :url => url.join(':'))
-                elsif Autoproj.has_source_handler?(vcs)
-                    spec = Autoproj.call_source_handler(vcs, url.join(':'), spec)
-                else
-                    source_dir = File.expand_path(File.join(Autoproj.config_dir, short_url))
-                    if !File.directory?(source_dir)
-                        raise ConfigError.new, "'#{spec.inspect}' is neither a remote source specification, nor a local source definition"
-                    end
-                    spec.merge!(:type => 'local', :url => source_dir)
-                end
-            end
-
-            spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
-            spec.merge!(vcs_options)
-            if !spec[:url]
-                # Verify that none of the keys are source handlers. If it is the
-                # case, convert
-                filtered_spec = Hash.new
-                spec.dup.each do |key, value|
-                    if Autoproj.has_source_handler?(key)
-                        spec.delete(key)
-                        spec = Autoproj.call_source_handler(key, value, spec)
-                        break
-                    end
-                end
-            end
-
-            spec
-        end
-
-        # Autoproj configuration files accept VCS definitions in three forms:
-        #  * as a plain string, which is a relative/absolute path
-        #  * as a plain string, which is a vcs_type:url string
-        #  * as a hash
-        #
-        # This method returns the VCSDefinition object matching one of these
-        # specs. It raises ConfigError if there is no type and/or url
-        def self.from_raw(spec, raw_spec = [[nil, spec]])
-            spec = vcs_definition_to_hash(spec)
-            if !(spec[:type] && (spec[:type] == 'none' || spec[:url]))
-                raise ConfigError.new, "the source specification #{spec.inspect} misses either the VCS type or an URL"
-            end
-
-            spec, vcs_options = Kernel.filter_options spec, :type => nil, :url => nil
-            return VCSDefinition.new(spec[:type], spec[:url], vcs_options, raw_spec)
-        end
-
-        def ==(other_vcs)
-            return false if !other_vcs.kind_of?(VCSDefinition)
-            if local?
-                other_vcs.local? && url == other.url
-            elsif !other_vcs.local?
-                this_importer = create_autobuild_importer
-                other_importer = other_vcs.create_autobuild_importer
-                this_importer.repository_id == other_importer.repository_id
-            end
-        end
-
-        def self.to_absolute_url(url, root_dir = nil)
-            # NOTE: we MUST use nil as default argument of root_dir as we don't
-            # want to call Autoproj.root_dir unless completely necessary
-            # (to_absolute_url might be called on installations that are being
-            # bootstrapped, and as such don't have a root dir yet).
-            url = Autoproj.single_expansion(url, 'HOME' => ENV['HOME'])
-            if url && url !~ /^(\w+:\/)?\/|^[:\w]+\@|^(\w+\@)?[\w\.-]+:/
-                url = File.expand_path(url, root_dir || Autoproj.root_dir)
-            end
-            url
-        end
-
-        # Returns a properly configured instance of a subclass of
-        # Autobuild::Importer that match this VCS definition
-        #
-        # Returns nil if the VCS type is 'none'
-        def create_autobuild_importer
-            return if type == "none"
-
-            url = VCSDefinition.to_absolute_url(self.url)
-            Autobuild.send(type, url, options)
-        end
-
-        # Returns a pretty representation of this VCS definition
-        def to_s 
-            if type == "none"
-                "none"
-            else
-                desc = "#{type}:#{url}"
-                if !options.empty?
-                    desc = "#{desc} #{options.to_a.sort_by { |key, _| key.to_s }.map { |key, value| "#{key}=#{value}" }.join(" ")}"
-                end
-                desc
-            end
-        end
-    end
-
-    @custom_source_handlers = Hash.new
-
-    # Returns true if +vcs+ refers to a source handler name added by
-    # #add_source_handler
-    def self.has_source_handler?(vcs)
-        @custom_source_handlers.has_key?(vcs.to_s)
-    end
-
-    # Returns the source handlers associated with +vcs+
-    #
-    # Source handlers are added by Autoproj.add_source_handler. The returned
-    # value is an object that responds to #call(url, options) and return a VCS
-    # definition as a hash
-    def self.call_source_handler(vcs, url, options)
-        handler = @custom_source_handlers[vcs.to_s]
-        if !handler
-            raise ArgumentError, "there is no source handler for #{vcs}"
-        else
-            return handler.call(url, options)
-        end
-    end
-
-    # call-seq:
-    #   Autoproj.add_source_handler name do |url, options|
-    #     # build a hash that represent source configuration
-    #     # and return it
-    #   end
-    #
-    # Add a custom source handler named +name+
-    #
-    # Custom source handlers are shortcuts that can be used to represent VCS
-    # information. For instance, the gitorious_server_configuration method
-    # defines a source handler that allows to easily add new gitorious packages:
-    #
-    #   gitorious_server_configuration 'GITORIOUS', 'gitorious.org'
-    #
-    # defines the "gitorious" source handler, which allows people to write
-    #
-    #
-    #   version_control:
-    #       - tools/orocos.rb
-    #         gitorious: rock-toolchain/orocos-rb
-    #         branch: test
-    #
-    # 
-    def self.add_source_handler(name, &handler)
-        @custom_source_handlers[name.to_s] = lambda(&handler)
-    end
-
-    # Does a non-recursive expansion in +data+ of configuration variables
-    # ($VAR_NAME) listed in +definitions+
-    #
-    # If the values listed in +definitions+ also contain configuration
-    # variables, they do not get expanded
-    def self.single_expansion(data, definitions)
-        if !data.respond_to?(:to_str)
-            return data
-        end
-        definitions = { 'HOME' => ENV['HOME'] }.merge(definitions)
-
-        data = data.gsub /(.|^)\$(\w+)/ do |constant_name|
-            prefix = constant_name[0, 1]
-            if prefix == "\\"
-                next(constant_name[1..-1])
-            end
-            if prefix == "$"
-                prefix, constant_name = "", constant_name[1..-1]
-            else
-                constant_name = constant_name[2..-1]
-            end
-
-            if !(value = definitions[constant_name])
-                if !(value = Autoproj.user_config(constant_name))
-                    if !block_given? || !(value = yield(constant_name))
-                        raise ArgumentError, "cannot find a definition for $#{constant_name}"
-                    end
-                end
-            end
-            "#{prefix}#{value}"
-        end
-        data
-    end
-
-    # Expand constants within +value+
-    #
-    # The list of constants is given in +definitions+. It raises ConfigError if
-    # some values are not found
-    def self.expand(value, definitions = Hash.new)
-        if value.respond_to?(:to_hash)
-            value.dup.each do |name, definition|
-                value[name] = expand(definition, definitions)
-            end
-            value
-        elsif value.respond_to?(:to_ary)
-            value.map { |val| expand(val, definitions) }
-        else
-            value = single_expansion(value, definitions)
-            if contains_expansion?(value)
-                raise ConfigError.new, "some expansions are not defined in #{value.inspect}"
-            end
-            value
-        end
-    end
-
-    # True if the given string contains expansions
-    def self.contains_expansion?(string); string =~ /\$/ end
-
-    def self.resolve_one_constant(name, value, result, definitions)
-        result[name] = single_expansion(value, result) do |missing_name|
-            result[missing_name] = resolve_one_constant(missing_name, definitions.delete(missing_name), result, definitions)
-        end
-    end
-
-    # Resolves all possible variable references from +constants+
-    #
-    # I.e. replaces variables by their values, so that no value in +constants+
-    # refers to variables defined in +constants+
-    def self.resolve_constant_definitions(constants)
-        constants = constants.dup
-        constants['HOME'] = ENV['HOME']
-        
-        result = Hash.new
-        while !constants.empty?
-            name  = constants.keys.first
-            value = constants.delete(name)
-            resolve_one_constant(name, value, result, constants)
-        end
-        result
-    end
-
-    # A package set is a version control repository which contains general
-    # information with package version control information (source.yml file),
-    # package definitions (.autobuild files), and finally definition of
-    # dependencies that are provided by the operating system (.osdeps file).
-    class PackageSet
-        attr_reader :manifest
-        # The VCSDefinition object that defines the version control holding
-        # information for this source. Local package sets (i.e. the ones that are not
-        # under version control) use the 'local' version control name. For them,
-        # local? returns true.
-        attr_accessor :vcs
-
-        # The set of OSDependencies object that represent the osdeps files
-        # available in this package set
-        attr_reader :all_osdeps
-
-        # The OSDependencies which is a merged version of all OSdeps in
-        # #all_osdeps
-        attr_reader :osdeps
-
-        # If this package set has been imported from another package set, this
-        # is the other package set object
-        attr_accessor :imported_from
-
-        # If true, this package set has been loaded because another set imports
-        # it. If false, it is loaded explicitely by the user
-        def explicit?; !@imported_from end
-
-        attr_reader :source_definition
-        attr_reader :constants_definitions
-
-        # Sets the auto_imports flag. See #auto_imports?
-        attr_writer :auto_imports
-        # If true (the default), imports listed in this package set will be
-        # automatically loaded by autoproj
-        def auto_imports?; !!@auto_imports end
-
-        # Returns the Metapackage object that has the same name than this
-        # package set
-        def metapackage
-            manifest.metapackage(name)
-        end
-
-        # List of the packages that are built if the package set is selected in
-        # the layout
-        def default_packages
-            metapackage.packages
-        end
-
-        # Create this source from a VCSDefinition object
-        def initialize(manifest, vcs)
-            @manifest = manifest
-            @vcs = vcs
-            @osdeps = OSDependencies.new
-            @all_osdeps = []
-
-            @provides = Set.new
-            @imports  = Array.new
-            @auto_imports = true
-        end
-
-        # Load a new osdeps file for this package set
-        def load_osdeps(file)
-            new_osdeps = OSDependencies.load(file)
-            @all_osdeps << new_osdeps
-            @osdeps.merge(@all_osdeps.last)
-            new_osdeps
-        end
-
-        # Enumerate all osdeps package names from this package set
-        def each_osdep(&block)
-            @osdeps.definitions.each_key(&block)
-        end
-
-        # True if this source has already been checked out on the local autoproj
-        # installation
-        def present?; File.directory?(raw_local_dir) end
-        # True if this source is local, i.e. is not under a version control
-        def local?; vcs.local? end
-        # True if this source defines nothing
-        def empty?
-            !source_definition['version_control'] && !source_definition['overrides']
-                !each_package.find { true } &&
-                !File.exists?(File.join(raw_local_dir, "overrides.rb")) &&
-                !File.exists?(File.join(raw_local_dir, "init.rb"))
-        end
-
-        # Create a PackageSet instance from its description as found in YAML
-        # configuration files
-        def self.from_spec(manifest, raw_spec, load_description)
-            if raw_spec.respond_to?(:to_str)
-                local_path = File.join(Autoproj.config_dir, raw_spec)
-                if File.directory?(local_path)
-                    raw_spec = { :type => 'local', :url => local_path }
-                end
-            end
-            spec = VCSDefinition.vcs_definition_to_hash(raw_spec)
-            options, vcs_spec = Kernel.filter_options spec, :auto_imports => true
-
-            # Look up for short notation (i.e. not an explicit hash). It is
-            # either vcs_type:url or just url. In the latter case, we expect
-            # 'url' to be a path to a local directory
-            vcs_spec = Autoproj.expand(vcs_spec, manifest.constant_definitions)
-            vcs_def  = VCSDefinition.from_raw(vcs_spec, [[nil, raw_spec]])
-
-            source = PackageSet.new(manifest, vcs_def)
-            source.auto_imports = options[:auto_imports]
-            if load_description
-                if source.present?
-                    source.load_description_file
-                else
-                    raise InternalError, "cannot load description file as it has not been checked out yet"
-                end
-            else
-                # Try to load just the name from the source.yml file
-                source.load_minimal
-            end
-
-            source
-        end
-
-        # Returns a string that uniquely represents the version control
-        # information for this package set.
-        #
-        # I.e. for two package sets set1 and set2, if set1.repository_id ==
-        # set2.repository_id, it means that both package sets are checked out
-        # from exactly the same source.
-        def repository_id
-            if local?
-                local_dir
-            else
-                importer = vcs.create_autobuild_importer
-                if importer.respond_to?(:repository_id)
-                    importer.repository_id
-                else
-                    vcs.to_s
-                end
-            end
-        end
-
-        # Remote sources can be accessed through a hidden directory in
-        # $AUTOPROJ_ROOT/.remotes, or through a symbolic link in
-        # autoproj/remotes/
-        #
-        # This returns the former. See #user_local_dir for the latter.
-        #
-        # For local sources, is simply returns the path to the source directory.
-        def raw_local_dir
-            if local?
-                File.expand_path(vcs.url)
-            else
-                File.expand_path(File.join(Autoproj.remotes_dir, vcs.create_autobuild_importer.repository_id.gsub(/[^\w]/, '_')))
-            end
-        end
-
-        # Remote sources can be accessed through a hidden directory in
-        # $AUTOPROJ_ROOT/.remotes, or through a symbolic link in
-        # autoproj/remotes/
-        #
-        # This returns the latter. See #raw_local_dir for the former.
-        #
-        # For local sources, is simply returns the path to the source directory.
-        def user_local_dir
-            if local?
-                return vcs.url 
-            else
-                File.join(Autoproj.config_dir, 'remotes', name)
-            end
-        end
-
-        # The directory in which data for this source will be checked out
-        def local_dir
-            ugly_dir   = raw_local_dir
-            pretty_dir = user_local_dir
-            if File.symlink?(pretty_dir) && File.readlink(pretty_dir) == ugly_dir
-                pretty_dir
-            else
-                ugly_dir
-            end
-        end
-
-        def required_autoproj_version
-            definition = @source_definition || raw_description_file
-            definition['required_autoproj_version'] || '0'
-        end
-
-        # Returns the source name
-        def name
-            if @name
-                @name
-            else
-                vcs.to_s
-            end
-        end
-
-        # Loads the source.yml file, validates it and returns it as a hash
-        #
-        # Raises InternalError if the source has not been checked out yet (it
-        # should have), and ConfigError if the source.yml file is not valid.
-        def raw_description_file
-            if !present?
-                raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
-            end
-
-            source_file = File.join(raw_local_dir, "source.yml")
-            if !File.exists?(source_file)
-                raise ConfigError.new, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
-            end
-
-            source_definition = Autoproj.in_file(source_file, Autoproj::YAML_LOAD_ERROR) do
-                YAML.load(File.read(source_file))
-            end
-
-            if !source_definition || !source_definition['name']
-                raise ConfigError.new(source_file), "in #{source_file}: missing a 'name' field"
-            end
-
-            source_definition
-        end
-
-        # Load and validate the self-contained information from the YAML hash
-        def load_minimal
-            # If @source_definition is set, it means that load_description_file
-            # has been called and that therefore all information has already
-            # been parsed
-            definition = @source_definition || raw_description_file
-            @name = definition['name']
-
-            if @name !~ /^[\w_\.-]+$/
-                raise ConfigError.new(source_file),
-                    "in #{source_file}: invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
-            elsif @name == "local"
-                raise ConfigError.new(source_file),
-                    "in #{source_file}: the name 'local' is a reserved name"
-            end
-
-            @provides = (definition['provides'] || Set.new).to_set
-            @imports  = (definition['imports'] || Array.new).map do |set_def|
-                pkg_set = Autoproj.in_file(source_file) do
-                    PackageSet.from_spec(manifest, set_def, false)
-                end
-
-                pkg_set.imported_from = self
-                pkg_set
-            end
-
-        rescue InternalError
-            # This ignores raw_description_file error if the package set is not
-            # checked out yet
-        end
-
-        # Yields the imports this package set declares, as PackageSet instances
-        def each_imported_set(&block)
-            @imports.each(&block)
-        end
-
-        # Path to the source.yml file
-        def source_file
-            File.join(local_dir, 'source.yml')
-        end
-
-        # Load the source.yml file and resolves all information it contains.
-        #
-        # This for instance requires configuration options to be defined. Use
-        # PackageSet#load_minimal to load only self-contained information
-        def load_description_file
-            if @source_definition
-                return
-            end
-
-            @source_definition = raw_description_file
-            load_minimal
-
-            # Compute the definition of constants
-            Autoproj.in_file(source_file) do
-                constants = source_definition['constants'] || Hash.new
-                @constants_definitions = Autoproj.resolve_constant_definitions(constants)
-            end
-        end
-
-        def single_expansion(data, additional_expansions = Hash.new)
-            if !source_definition
-                load_description_file
-            end
-            Autoproj.single_expansion(data, additional_expansions.merge(constants_definitions))
-        end
-
-        # Expands the given string as much as possible using the expansions
-        # listed in the source.yml file, and returns it. Raises if not all
-        # variables can be expanded.
-        def expand(data, additional_expansions = Hash.new)
-            if !source_definition
-                load_description_file
-            end
-            Autoproj.expand(data, additional_expansions.merge(constants_definitions))
-        end
-
-        # Returns the default importer definition for this package set, as a
-        # VCSDefinition instance
-        def default_importer
-            importer_definition_for('default')
-        end
-
-        # Returns an importer definition for the given package, if one is
-        # available. Otherwise returns nil.
-        #
-        # The returned value is a VCSDefinition object.
-        def version_control_field(package_name, section_name, validate = true)
-            urls = source_definition['urls'] || Hash.new
-            urls['HOME'] = ENV['HOME']
-
-            all_vcs     = source_definition[section_name]
-            if all_vcs
-                if all_vcs.kind_of?(Hash)
-                    raise ConfigError.new, "wrong format for the #{section_name} section, you forgot the '-' in front of the package names"
-                elsif !all_vcs.kind_of?(Array)
-                    raise ConfigError.new, "wrong format for the #{section_name} section"
-                end
-            end
-
-            raw = []
-            vcs_spec = Hash.new
-
-            if all_vcs
-                all_vcs.each do |spec|
-                    spec = spec.dup
-                    if spec.values.size != 1
-                        # Maybe the user wrote the spec like
-                        #   - package_name:
-                        #     type: git
-                        #     url: blah
-                        #
-                        # or as
-                        #   - package_name
-                        #     type: git
-                        #     url: blah
-                        #
-                        # In that case, we should have the package name as
-                        # "name => nil". Check that.
-                        name, _ = spec.find { |n, v| v.nil? }
-                        if name
-                            spec.delete(name)
-                        else
-                            name, _ = spec.find { |n, v| n =~ / \w+$/ }
-                            name =~ / (\w+)$/
-                            spec[$1] = spec.delete(name)
-                            name = name.gsub(/ \w+$/, '')
-                        end
-                    else
-                        name, spec = spec.to_a.first
-                        if name =~ / (\w+)/
-                            spec = { $1 => spec }
-                            name = name.gsub(/ \w+$/, '')
-                        end
-
-                        if spec.respond_to?(:to_str)
-                            if spec == "none"
-                                spec = { :type => "none" }
-                            else
-                                raise ConfigError.new, "invalid VCS specification in the #{section_name} section '#{name}: #{spec}'"
-                            end
-                        end
-                    end
-
-                    name_match = name
-                    if name_match =~ /[^\w\/_-]/
-                        name_match = Regexp.new("^" + name_match)
-                    end
-                    if name_match === package_name
-                        raw << [self.name, spec]
-                        vcs_spec =
-                            begin
-                                VCSDefinition.update_raw_vcs_spec(vcs_spec, spec)
-                            rescue ConfigError => e
-                                raise ConfigError.new, "invalid VCS definition in the #{section_name} section for '#{name}': #{e.message}", e.backtrace
-                            end
-                    end
-                end
-            end
-
-            if !vcs_spec.empty?
-                expansions = Hash["PACKAGE" => package_name,
-                    "PACKAGE_BASENAME" => File.basename(package_name),
-                    "AUTOPROJ_ROOT" => Autoproj.root_dir,
-                    "AUTOPROJ_CONFIG" => Autoproj.config_dir,
-                    "AUTOPROJ_SOURCE_DIR" => local_dir]
-
-                vcs_spec = expand(vcs_spec, expansions)
-                vcs_spec.dup.each do |name, value|
-                    vcs_spec[name] = expand(value, expansions)
-                end
-
-                # If required, verify that the configuration is a valid VCS
-                # configuration
-                if validate
-                    begin
-                        VCSDefinition.from_raw(vcs_spec)
-                    rescue ConfigError => e
-                        raise ConfigError.new, "invalid resulting VCS definition for package #{package_name}: #{e.message}", e.backtrace
-                    end
-                end
-                return vcs_spec, raw
-            else
-                return nil, []
-            end
-        end
-
-        # Returns the VCS definition for +package_name+ as defined in this
-        # source, or nil if the source does not have any.
-        #
-        # The definition is an instance of VCSDefinition
-        def importer_definition_for(package_name)
-            Autoproj.in_file source_file do
-                vcs_spec, raw = version_control_field(package_name, 'version_control')
-                if vcs_spec
-                    VCSDefinition.from_raw(vcs_spec, raw)
-                end
-            end
-        end
-
-        # Enumerates the Autobuild::Package instances that are defined in this
-        # source
-        def each_package
-            if !block_given?
-                return enum_for(:each_package)
-            end
-
-            Autoproj.manifest.packages.each_value do |pkg|
-                if pkg.package_set.name == name
-                    yield(pkg.autobuild)
-                end
-            end
-        end
-
-        # True if this package set provides the given package set name. I.e. if
-        # it has this name or the name is listed in the "replaces" field of
-        # source.yml
-        def provides?(name)
-            name == self.name ||
-                provides.include?(name)
-        end
-    end
-
-    # Specialization of the PackageSet class for the overrides listed in autoproj/
-    class LocalPackageSet < PackageSet
-        def initialize(manifest)
-            super(manifest, VCSDefinition.from_raw(:type => 'local', :url => Autoproj.config_dir))
-        end
-
-        def name
-            'local'
-        end
-        def load_minimal
-        end
-        def repository_id
-            'local'
-        end
-
-        def source_file
-            File.join(Autoproj.config_dir, "overrides.yml")
-        end
-
-        # Returns the default importer for this package set
-        def default_importer
-            importer_definition_for('default') ||
-                VCSDefinition.from_raw(:type => 'none')
-        end
-
-        def raw_description_file
-            path = source_file
-            if File.file?(path)
-                data = Autoproj.in_file(path, Autoproj::YAML_LOAD_ERROR) do
-                    YAML.load(File.read(path)) || Hash.new
-                end
-                data['name'] = 'local'
-                data
-            else
-                { 'name' => 'local' }
-            end
-        end
-    end
-
-    # DEPRECATED. For backward-compatibility only.
-    Source = PackageSet
-    # DEPRECATED. For backward-compatibility only.
-    LocalSource = LocalPackageSet
-
-    # Class used to store information about a package definition
-    class PackageDefinition
-        attr_reader :autobuild
-        attr_reader :user_blocks
-        attr_reader :package_set
-        attr_reader :file
-        def setup?; !!@setup end
-        attr_writer :setup
-        attr_accessor :vcs
-
-        def initialize(autobuild, package_set, file)
-            @autobuild, @package_set, @file =
-                autobuild, package_set, file
-            @user_blocks = []
-        end
-
-        def name
-            autobuild.name
-        end
-
-        def add_setup_block(block)
-            user_blocks << block
-            if setup?
-                block.call(autobuild)
-            end
-        end
-    end
-
-    # A set of packages that can be referred to by name
-    class Metapackage
-        # The metapackage name
-        attr_reader :name
-        # The packages listed in this metapackage
-        attr_reader :packages
-        # The normal dependency handling behaviour is to generate an error if a
-        # metapackage is selected for the build but some of its dependencies
-        # cannot be built. This modifies the behaviour to simply ignore the
-        # problematic packages.
-        attr_writer :weak_dependencies
-
-        # @return [Boolean] whether the dependencies from this metapackage are
-        #   weak or not
-        # @see #weak_dependencies
-        def weak_dependencies?
-            !!@weak_dependencies
-        end
-
-        def initialize(name)
-            @name = name
-            @packages = []
-            @weak_dependencies = false
-        end
-
-        # Adds a package to this metapackage
-        #
-        # @param [Autobuild::Package] pkg
-        def add(pkg)
-            @packages << pkg
-        end
-
-        # Lists the packages contained in this metapackage
-        #
-        # @yieldparam [Autobuild::Package] pkg
-        def each_package(&block)
-            @packages.each(&block)
-        end
-
-        # Tests if the given package is included in this metapackage
-        #
-        # @param [String,#name] pkg the package or package name
-        def include?(pkg)
-            if !pkg.respond_to?(:to_str)
-                pkg = pkg.name
-            end
-            @packages.any? { |p| p.name == pkg }
-        end
-    end
-
     # The Manifest class represents the information included in the main
     # manifest file, and allows to manipulate it
     class Manifest
@@ -1011,7 +48,7 @@ module Autoproj
             end
 
             def import
-                importer.import(self)
+                importer.import(self,Manifest.only_local_updates)
             end
 
             def add_stat(*args)
@@ -1093,6 +130,11 @@ module Autoproj
 
         attr_reader :metapackages
 
+        class << self
+            #Set to true if this manifest should only do local updates through the Autobuild::Importer
+            attr_accessor :only_local_updates
+        end
+
 	def initialize
             @file = nil
 	    @data = Hash.new
@@ -1107,6 +149,7 @@ module Autoproj
             @ignored_os_dependencies = Set.new
             @reused_installations = Array.new
             @ignored_packages = Set.new
+            Manifest::only_local_updates = false
 
             @constant_definitions = Hash.new
             if Autoproj.has_config_key?('manifest_source')
@@ -1387,11 +430,15 @@ module Autoproj
             @metapackages["#{pkg.package_set.name}.all"].add(pkg.autobuild)
         end
 
-        def definition_source(package_name)
+        def definition_package_set(package_name)
             if pkg_def = @packages[package_name]
                 pkg_def.package_set
             end
         end
+
+        def definition_source(package_name)
+            definition_package_set(package_name)
+        end
         def definition_file(package_name)
             if pkg_def = @packages[package_name]
                 pkg_def.file
@@ -1689,9 +736,6 @@ module Autoproj
         # The returned array can be empty if +name+ is an ignored package
         def resolve_package_name(name, options = Hash.new)
             if pkg_set = find_metapackage(name)
-                if !pkg_set
-                    raise ConfigError.new, "#{name} is neither a package nor a package set name. Packages in autoproj must be declared in an autobuild file."
-                end
                 pkg_names = pkg_set.each_package.map(&:name)
             else
                 pkg_names = [name]
@@ -2152,16 +1196,6 @@ module Autoproj
             @osdeps_overrides.delete(osdeps_name.to_s)
         end
 
-        # Exception raised by
-        # PackageSelection#filter_excluded_and_ignored_packages when a given
-        # selection is completely excluded
-        class ExcludedSelection < ConfigError
-            attr_reader :selection
-            def initialize(selection)
-                @selection = selection
-            end
-        end
-
         # Exception raised when an unknown package is encountered
         class UnknownPackage < ConfigError
             attr_reader :name
@@ -2170,132 +1204,7 @@ module Autoproj
             end
         end
 
-        # Class holding information about which packages have been selected, and
-        # why. It is used to decide whether some non-availability of packages
-        # are errors or simply warnings (i.e. if the user really wants a given
-        # package, or merely might be adding it by accident)
-        class PackageSelection
-            # The set of matches, i.e. a mapping from a user-provided string to
-            # the set of packages it selected
-            attr_reader :matches
-            # The set of selected packages, as a hash of the package name to the
-            # set of user-provided strings that caused that package to be
-            # selected
-            attr_reader :selection
-            # A flag that tells #filter_excluded_and_ignored_packages whether
-            # the a given package selection is weak or not.
-            #
-            # If true, a selection that have some excluded packages will not
-            # generate an error. Otherwise (the default), an error is generated
-            attr_reader :weak_dependencies
-            # After a call to #filter_excluded_and_ignored_packages, this
-            # contains the set of package exclusions that have been ignored
-            # because the corresponding metapackage has a weak dependency policy
-            attr_reader :exclusions
-            # After a call to #filter_excluded_and_ignored_packages, this
-            # contains the set of package ignores that have been ignored because
-            # the corresponding metapackage has a weak dependency policy
-            attr_reader :ignores
-
-            def initialize
-                @selection = Hash.new { |h, k| h[k] = Set.new }
-                @matches = Hash.new { |h, k| h[k] = Set.new }
-                @weak_dependencies = Hash.new
-                @ignores = Hash.new { |h, k| h[k] = Set.new }
-                @exclusions = Hash.new { |h, k| h[k] = Set.new }
-            end
-
-            # The set of packages that have been selected
-            def packages
-                selection.keys
-            end
-
-            def include?(pkg_name)
-                selection.has_key?(pkg_name)
-            end
-
-            def empty?
-                selection.empty?
-            end
-
-            def each(&block)
-                selection.each_key(&block)
-            end
-
-            def select(sel, packages, weak = false)
-                packages = Array(packages)
-                matches[sel] |= packages.to_set.dup
-                packages.each do |pkg_name|
-                    selection[pkg_name] << sel
-                end
-                weak_dependencies[sel] = weak
-            end
-
-            def initialize_copy(old)
-                old.selection.each do |pkg_name, set|
-                    @selection[pkg_name] = set.dup
-                end
-                old.matches.each do |sel, set|
-                    @matches[sel] = set.dup
-                end
-            end
-
-            def has_match_for?(sel)
-                matches.has_key?(sel)
-            end
-
-            # Remove packages that are explicitely excluded and/or ignored
-            #
-            # Raise an error if an explicit selection expands only to an
-            # excluded package, and display a warning for ignored packages
-            def filter_excluded_and_ignored_packages(manifest)
-                matches.each do |sel, expansion|
-                    excluded, other = expansion.partition { |pkg_name| manifest.excluded?(pkg_name) }
-                    ignored,  ok    = other.partition { |pkg_name| manifest.ignored?(pkg_name) }
-
-                    if !excluded.empty? && (!weak_dependencies[sel] || (ok.empty? && ignored.empty?))
-                        exclusions = excluded.map do |pkg_name|
-                            [pkg_name, manifest.exclusion_reason(pkg_name)]
-                        end
-                        base_msg = "#{sel} is selected in the manifest or on the command line"
-                        if exclusions.size == 1
-                            reason = exclusions[0][1]
-                            if sel == exclusions[0][0]
-                                raise ExcludedSelection.new(sel), "#{base_msg}, but it is excluded from the build: #{reason}"
-                            elsif weak_dependencies[sel]
-                                raise ExcludedSelection.new(sel), "#{base_msg}, but it expands to #{exclusions.map(&:first).join(", ")}, which is excluded from the build: #{reason}"
-                            else
-                                raise ExcludedSelection.new(sel), "#{base_msg}, but its dependency #{exclusions.map(&:first).join(", ")} is excluded from the build: #{reason}"
-                            end
-                        elsif weak_dependencies[sel]
-                            raise ExcludedSelection.new(sel), "#{base_msg}, but expands to #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
-                        else
-                            raise ExcludedSelection.new(sel), "#{base_msg}, but it requires #{exclusions.map(&:first).join(", ")}, and all these packages are excluded from the build:\n  #{exclusions.map { |name, reason| "#{name}: #{reason}" }.join("\n  ")}"
-                        end
-                    else
-                        self.exclusions[sel] |= excluded.to_set.dup
-                        self.ignores[sel] |= ignored.to_set.dup
-                    end
-
-                    excluded = excluded.to_set
-                    ignored = ignored.to_set
-                    expansion.delete_if do |pkg_name|
-                        ignored.include?(pkg_name) || excluded.include?(pkg_name)
-                    end
-                end
-
-                selection.keys.sort.each do |pkg_name|
-                    if manifest.excluded?(pkg_name)
-                        selection.delete(pkg_name)
-                    elsif manifest.ignored?(pkg_name)
-                        selection.delete(pkg_name)
-                    end
-                end
-                matches.delete_if do |key, sel|
-                    sel.empty?
-                end
-            end
-        end
+        PackageSelection = Autoproj::PackageSelection
 
         # Package selection can be done in three ways:
         #  * as a subdirectory in the layout
@@ -2426,211 +1335,39 @@ module Autoproj
                 ignore_package pkg.name
             end
         end
+
+        # Load OS dependency information contained in our registered package
+        # sets into the provided osdep object
+        #
+        # @param [OSDependencies] osdeps the osdep handling object
+        # @return [void]
+        def load_osdeps_from_package_sets(osdeps)
+            each_osdeps_file do |source, file|
+                osdeps.merge(source.load_osdeps(file))
+            end
+        end
     end
 
     class << self
-        # The singleton manifest object on which the current run works
+        # The singleton manifest object that represents the current build
+        # configuration
+        #
+        # @return [Manifest]
         attr_accessor :manifest
 
-        # The operating system package definitions
+        # The known osdeps definitions
+        #
+        # @return [OSDependencies]
+        # @see load_osdeps_from_package_sets
         attr_accessor :osdeps
     end
 
+    # Load the osdeps files contained in {manifest} into {osdeps}
     def self.load_osdeps_from_package_sets
-        manifest.each_osdeps_file do |source, file|
-            osdeps.merge(source.load_osdeps(file))
-        end
+        manifest.load_osdeps_from_package_sets(osdeps)
         osdeps
     end
 
-    # Manifest of installed packages imported from another autoproj installation
-    class InstallationManifest
-        Package = Struct.new :name, :srcdir, :prefix
-
-        attr_reader :path
-        attr_reader :packages
-        def initialize(path)
-            @path = path
-        end
-
-        def load(path)
-            @packages = CSV.read(path).map do |row|
-                Package.new(*row)
-            end
-        end
-
-        def each(&block)
-            packages.each(&block)
-        end
-
-        def [](name)
-            packages.find { |pkg| pkg.name == name }
-        end
-
-        def self.from_root(root_dir)
-            manifest = InstallationManifest.new(root_dir)
-            manifest_file = File.join(root_dir,  ".autoproj-installation-manifest")
-            if !File.file?(manifest_file)
-                raise ConfigError.new, "no .autoproj-installation-manifest file exists in #{root_dir}. You should probably rerun autoproj envsh in that folder first"
-            end
-            manifest.load(manifest_file)
-            manifest
-        end
-    end
-
-    # Access to the information contained in a package's manifest.xml file
-    #
-    # Use PackageManifest.load to create
-    class PackageManifest
-        # Load a manifest.xml file and returns the corresponding
-        # PackageManifest object
-        def self.load(package, file)
-            doc =
-                begin REXML::Document.new(File.read(file))
-                rescue REXML::ParseException => e
-                    raise Autobuild::PackageException.new(package.name, 'prepare'), "invalid #{file}: #{e.message}"
-                end
-
-            PackageManifest.new(package, doc)
-        end
-
-        # The Autobuild::Package instance this manifest applies on
-        attr_reader :package
-        # The raw XML data as a Nokogiri document
-        attr_reader :xml
-
-        # The list of tags defined for this package
-        #
-        # Tags are defined as multiple <tags></tags> blocks, each of which can
-        # contain multiple comma-separated tags 
-        def tags
-            result = []
-            xml.elements.each('package/tags') do |node|
-                result.concat((node.text || "").strip.split(','))
-            end
-            result
-        end
-
-        def documentation
-            xml.elements.each('package/description') do |node|
-                doc = (node.text || "").strip
-                if !doc.empty?
-                    return doc
-                end
-            end
-            return short_documentation
-        end
-
-        def short_documentation
-            xml.elements.each('package/description') do |node|
-                doc = node.attributes['brief']
-                if doc
-                    doc = doc.to_s.strip
-                end
-                if doc && !doc.empty?
-                    return doc.to_s
-                end
-            end
-            "no documentation available for #{package.name} in its manifest.xml file"
-        end
-
-        def initialize(package, doc = REXML::Document.new)
-            @package = package
-            @xml = doc
-        end
-
-        def each_dependency(&block)
-            if block_given?
-                each_os_dependency(&block)
-                each_package_dependency(&block)
-            else
-                enum_for(:each_dependency, &block)
-            end
-        end
-
-        def each_os_dependency
-            if block_given?
-                xml.elements.each('package/rosdep') do |node|
-                    yield(node.attributes['name'], false)
-                end
-                package.os_packages.each do |name|
-                    yield(name, false)
-                end
-            else
-                enum_for :each_os_dependency
-            end
-        end
-
-        def each_package_dependency
-            if block_given?
-                depend_nodes = xml.elements.to_a('package/depend') +
-                    xml.elements.to_a('package/depend_optional')
-
-                depend_nodes.each do |node|
-                    dependency = node.attributes['package']
-                    optional = (node.attributes['optional'].to_s == '1' || node.name == "depend_optional")
-
-                    if dependency
-                        yield(dependency, optional)
-                    else
-                        raise ConfigError.new, "manifest of #{package.name} has a <depend> tag without a 'package' attribute"
-                    end
-                end
-            else
-                enum_for :each_package_dependency
-            end
-        end
-
-        # Enumerates the name and email of each author. If no email is present,
-        # yields (name, nil)
-        def each_author
-            if !block_given?
-                return enum_for(:each_author)
-            end
-
-            xml.elements.each('package/author') do |author|
-                (author.text || "").strip.split(',').each do |str|
-                    name, email = str.split('/').map(&:strip)
-                    email = nil if email && email.empty?
-                    yield(name, email)
-                end
-            end
-        end
-
-        # If +name+ points to a text element in the XML document, returns the
-        # content of that element. If no element matches +name+, or if the
-        # content is empty, returns nil
-        def text_node(name)
-            xml.elements.each(name) do |str|
-                str = (str.text || "").strip
-                if !str.empty?
-                    return str
-                end
-            end
-            nil
-        end
-
-        # The package associated URL, usually meant to direct to a website
-        #
-        # Returns nil if there is none
-        def url
-            return text_node('package/url')
-        end
-
-        # The package license name
-        #
-        # Returns nil if there is none
-        def license
-            return text_node('package/license')
-        end
-
-        # The package version number
-        #
-        # Returns 0 if none is declared
-        def version
-            return text_node("version")
-        end
-    end
     def self.add_osdeps_overrides(*args, &block)
         manifest.add_osdeps_overrides(*args, &block)
     end