autoproj 1.4.4 → 1.5.0

data/lib/autoproj/default.osdeps

@@ -12,7 +12,17 @@ ruby18:
         - dev-lang/ruby:1.8
 
 ruby19:
-    debian,ubuntu:
+    debian:
+        squeeze,sid:
+            - ruby1.9.1
+            - ruby1.9.1-dev
+            - rubygems1.9.1
+        stable:
+            - ruby1.9.1
+            - ruby1.9.1-dev
+            - rubygems1.9.1
+
+    ubuntu:
         - ruby1.9.1
         - ruby1.9.1-dev
         - rubygems1.9.1
@@ -46,7 +56,7 @@ autoproj: gem
 # The following definitions are for the VCS and build systems
 git:
     debian,ubuntu: git-core
-    gentoo: dev-util/git
+    gentoo: dev-vcs/git
     arch: git
 svn:
     debian,ubuntu: subversion

data/lib/autoproj/manifest.rb

@@ -3,18 +3,6 @@ require 'utilrb/kernel/options'
 require 'nokogiri'
 require 'set'
 
-module Autobuild
-    class Package
-        def os_packages
-            @os_packages || Array.new
-        end
-        def depends_on_os_package(name)
-            @os_packages ||= Array.new
-            @os_packages << name
-        end
-    end
-end
-
 module Autoproj
     @build_system_dependencies = Set.new
 
@@ -106,13 +94,22 @@ module Autoproj
             @type == 'local'
         end
 
+        def self.to_absolute_url(url, root_dir = nil)
+            # NOTE: we MUST use nil as default argument of root_dir as we don't
+            # want to call Autoproj.root_dir unless completely necessary
+            # (to_absolute_url might be called on installations that are being
+            # bootstrapped, and as such don't have a root dir yet).
+            url = Autoproj.single_expansion(url, 'HOME' => ENV['HOME'])
+            if url && url !~ /^(\w+:\/)?\/|^\w+\@|^(\w+\@)?[\w\.-]+:/
+                url = File.expand_path(url, root_dir || Autoproj.root_dir)
+            end
+            url
+        end
+
         def create_autobuild_importer
             return if type == "none"
 
-            url = Autoproj.single_expansion(self.url, 'HOME' => ENV['HOME'])
-            if url && url !~ /^(\w+:\/)?\/|^\w+\@|^(\w+\@)?[\w\.-]+:/
-                url = File.expand_path(url, Autoproj.root_dir)
-            end
+            url = VCSDefinition.to_absolute_url(self.url)
             Autobuild.send(type, url, options)
         end
 
@@ -196,26 +193,44 @@ module Autoproj
 
         # True if this source has already been checked out on the local autoproj
         # installation
-        def present?; File.directory?(local_dir) end
+        def present?; File.directory?(raw_local_dir) end
         # True if this source is local, i.e. is not under a version control
         def local?; vcs.local? end
         # True if this source defines nothing
         def empty?
             !source_definition['version_control'] &&
                 !each_package.find { true } &&
-                !File.exists?(File.join(local_dir, "overrides.rb")) &&
-                !File.exists?(File.join(local_dir, "init.rb"))
+                !File.exists?(File.join(raw_local_dir, "overrides.rb")) &&
+                !File.exists?(File.join(raw_local_dir, "init.rb"))
         end
 
-        # The directory in which data for this source will be checked out
-        def local_dir
+        def raw_local_dir
             if local?
-                vcs.url
+                return vcs.url 
             else
                 File.join(Autoproj.remotes_dir, automatic_name)
             end
         end
 
+        def user_local_dir
+            if local?
+                return vcs.url 
+            else
+                File.join(Autoproj.config_dir, 'remotes', name)
+            end
+        end
+
+        # The directory in which data for this source will be checked out
+        def local_dir
+            ugly_dir   = raw_local_dir
+            pretty_dir = user_local_dir
+            if File.symlink?(pretty_dir) && File.readlink(pretty_dir) == ugly_dir
+                pretty_dir
+            else
+                ugly_dir
+            end
+        end
+
         # A name generated from the VCS url
         def automatic_name
             vcs.to_s.gsub(/[^\w]/, '_')
@@ -237,7 +252,7 @@ module Autoproj
                 raise InternalError, "source #{vcs} has not been fetched yet, cannot load description for it"
             end
 
-            source_file = File.join(local_dir, "source.yml")
+            source_file = File.join(raw_local_dir, "source.yml")
             if !File.exists?(source_file)
                 raise ConfigError, "source #{vcs.type}:#{vcs.url} should have a source.yml file, but does not"
             end
@@ -255,21 +270,30 @@ module Autoproj
             source_definition
         end
 
+        # Load and validate the name from the YAML hash
         def load_name
-            definition = raw_description_file
+            definition = @source_definition || raw_description_file
             @name = definition['name']
-            if @name == "local"
+
+            if @name !~ /^[\w_\.-]+$/
+                raise ConfigError, "invalid source name '#{@name}': source names can only contain alphanumeric characters, and .-_"
+            elsif @name == "local"
                 raise ConfigError, "source #{self} is named 'local', but this is a reserved name"
             end
 
         rescue InternalError
         end
 
+        def source_file
+            File.join(local_dir, 'source.yml')
+        end
+
         # Load the source.yml file that describes this source, and resolve the
         # $BLABLA values that are in there. Use #raw_description_file to avoid
         # resolving those values
         def load_description_file
             @source_definition = raw_description_file
+            load_name
 
             # Compute the definition of constants
             begin
@@ -342,8 +366,12 @@ module Autoproj
             urls['HOME'] = ENV['HOME']
 
             all_vcs     = source_definition['version_control']
-            if all_vcs && !all_vcs.kind_of?(Array)
-                raise ConfigError, "wrong format for the version_control field"
+            if all_vcs
+                if all_vcs.kind_of?(Hash)
+                    raise ConfigError, "wrong format for the version_control field, you forgot the '-' in front of the package names"
+                elsif !all_vcs.kind_of?(Array)
+                    raise ConfigError, "wrong format for the version_control field"
+                end
             end
 
             vcs_spec = Hash.new
@@ -412,7 +440,7 @@ module Autoproj
                 Autoproj.normalize_vcs_definition(vcs_spec)
             end
         rescue ConfigError => e
-            raise ConfigError, "#{e.message} in the source.yml file of #{name} (#{File.join(local_dir, "source.yml")})", e.backtrace
+            raise ConfigError, "#{e.message} in #{source_file}", e.backtrace
         end
 
         def each_package
@@ -439,8 +467,12 @@ module Autoproj
         def load_name
         end
 
+        def source_file
+            File.join(Autoproj.config_dir, "overrides.yml")
+        end
+
         def raw_description_file
-            path = File.join(Autoproj.config_dir, "overrides.yml")
+            path = source_file
             if File.file?(path)
                 begin
                     data = YAML.load(File.read(path)) || Hash.new
@@ -458,7 +490,7 @@ module Autoproj
     PackageDefinition = Struct.new :autobuild, :user_block, :package_set, :file
 
     class Manifest
-        FakePackage = Struct.new :text_name, :name, :srcdir, :importer
+        FakePackage = Struct.new :text_name, :name, :srcdir, :importer, :updated
         class FakePackage
             def autoproj_name; name end
             def import
@@ -503,6 +535,10 @@ module Autoproj
 
         attr_reader :file
 
+        def auto_update?
+            !!data['auto_update']
+        end
+
 	def initialize(file, data)
             @file = file
 	    @data = data
@@ -633,36 +669,26 @@ module Autoproj
                 return @sources.each(&block)
             end
 
-            # Load the local source first ...
+            all_sources = []
+
+	    (data['package_sets'] || []).each do |spec|
+                all_sources << source_from_spec(spec, load_description)
+            end
+
+            # Now load the local source 
             local = LocalSource.new
             if load_description
                 local.load_description_file
             else
                 local.load_name
             end
-            if load_description
-                if !local.empty?
-                    @sources = [local]
-                else
-                    @sources = []
-                end
-            else
-                yield(local)
-            end
-
-            return if !data['package_sets']
-
-	    data['package_sets'].each do |spec|
-                source = source_from_spec(spec, load_description)
-                if load_description
-                    @sources << source
-                else
-                    yield(source)
-                end
+            if !load_description || !local.empty?
+                all_sources << local
             end
 
+            all_sources.each(&block)
             if load_description
-                @sources.each(&block)
+                @sources = all_sources
             end
         end
 
@@ -704,6 +730,13 @@ module Autoproj
             self
         end
 
+        # Creates an autobuild package whose job is to allow the import of a
+        # specific repository into a given directory.
+        #
+        # +vcs+ is the VCSDefinition file describing the repository, +text_name+
+        # the name used when displaying the import progress, +pkg_name+ the
+        # internal name used to represent the package and +into+ the directory
+        # in which the package should be checked out.
         def self.create_autobuild_package(vcs, text_name, pkg_name, into)
             importer     = vcs.create_autobuild_importer
             return if !importer # updates have been disabled by using the 'none' type
@@ -716,6 +749,9 @@ module Autoproj
             raise ConfigError, "cannot import #{name}: #{e.message}", e.backtrace
         end
 
+        # Imports or updates a source (remote or otherwise).
+        #
+        # See create_autobuild_package for informations about the arguments.
         def self.update_source(vcs, text_name, pkg_name, into)
             fake_package = create_autobuild_package(vcs, text_name, pkg_name, into)
             fake_package.import
@@ -724,15 +760,38 @@ module Autoproj
             raise ConfigError, "cannot import #{name}: #{e.message}", e.backtrace
         end
 
+        # Updates the main autoproj configuration
         def update_yourself
             Manifest.update_source(vcs, "autoproj main configuration", "autoproj_conf", Autoproj.config_dir)
         end
 
+        # Updates all the remote sources in ROOT_DIR/.remotes, as well as the
+        # symbolic links in ROOT_DIR/autoproj/remotes
         def update_remote_sources
             # Iterate on the remote sources, without loading the source.yml
             # file (we're not ready for that yet)
+            sources = []
             each_remote_source(false) do |source|
-                Manifest.update_source(source.vcs, source.name || source.vcs.url, source.automatic_name, source.local_dir)
+                Manifest.update_source(source.vcs, source.name || source.vcs.url, source.automatic_name, source.raw_local_dir)
+                sources << source
+            end
+
+            # Check for directories in ROOT_DIR/.remotes that do not map to a
+            # source repository, and remove them
+            Dir.glob(File.join(Autoproj.remotes_dir, '*')).each do |dir|
+                dir = File.expand_path(dir)
+                if File.directory?(dir) && !sources.any? { |s| s.raw_local_dir == dir }
+                    FileUtils.rm_rf dir
+                end
+            end
+
+            remotes_symlinks_dir = File.join(Autoproj.config_dir, 'remotes')
+            FileUtils.rm_rf remotes_symlinks_dir
+            FileUtils.mkdir remotes_symlinks_dir
+            # Create symbolic links from .remotes/weird_url to
+            # autoproj/remotes/name
+            each_remote_source(false) do |source|
+                FileUtils.ln_sf source.raw_local_dir, File.join(remotes_symlinks_dir, source.name)
             end
         end
 
@@ -772,7 +831,8 @@ module Autoproj
         #    by S1
         def load_importers
             packages.each_value do |pkg|
-                vcs = importer_definition_for(pkg.autobuild.name, pkg.package_set)
+                vcs = importer_definition_for(pkg.autobuild.name, pkg.package_set) ||
+                    importer_definition_for("default", pkg.package_set)
 
                 if vcs
                     Autoproj.add_build_system_dependency vcs.type
@@ -876,21 +936,6 @@ module Autoproj
             yield
         end
 
-        def handle_enabled_packages(selected_packages)
-            handled_packages = Set.new
-            each_package_set(selected_packages) do |name, packages, enabled_packages|
-                packages         -= handled_packages
-                enabled_packages -= handled_packages
-
-                in_sublayout(name, packages) do
-                    if !packages.empty?
-                        yield(name, packages, enabled_packages)
-                    end
-                end
-                handled_packages |= packages
-            end
-        end
-
         def default_packages
             names = if layout = data['layout']
                         layout_packages(layout, true)
@@ -925,10 +970,7 @@ module Autoproj
             manifest = PackageManifest.load(package, manifest_path)
             package_manifests[package.name] = manifest
 
-            manifest.each_package_dependency do |name|
-                if Autoproj.verbose
-                    STDERR.puts "  #{package.name} depends on #{name}"
-                end
+            manifest.each_dependency do |name|
                 begin
                     package.depends_on name
                 rescue Autobuild::ConfigException => e
@@ -959,18 +1001,14 @@ module Autoproj
             required_os_packages = Set.new
             package_os_deps = Hash.new { |h, k| h[k] = Array.new }
             packages.each do |pkg_name|
-                if manifest = package_manifests[pkg_name]
-                    manifest.each_os_dependency.each do |osdep_name|
-                        package_os_deps[osdep_name] << pkg_name
-                        required_os_packages << osdep_name
-                    end
+                pkg = Autobuild::Package[pkg_name]
+                if !pkg
+                    raise InternalError, "internal error: #{pkg_name} is not a package"
                 end
 
-                if pkg = Autobuild::Package[pkg_name]
-                    pkg.os_packages.each do |osdep_name|
-                        package_os_deps[osdep_name] << pkg_name
-                        required_os_packages << osdep_name
-                    end
+                pkg.os_packages.each do |osdep_name|
+                    package_os_deps[osdep_name] << pkg_name
+                    required_os_packages << osdep_name
                 end
             end
 
@@ -1020,11 +1058,12 @@ module Autoproj
                     if layout_name[0..-1] =~ Regexp.new("#{sel}\/?$")
                         expanded_packages.concat(packages.to_a)
                     else
-                        packages = packages.find_all do |pkg_name|
-                            sel =~ Regexp.new(Regexp.quote(layout_name + pkg_name))
+                        match = Regexp.new("^#{Regexp.quote(sel)}")
+                        Autobuild::Package.each(true) do |name, pkg|
+                            if pkg.srcdir =~ match
+                                expanded_packages << name
+                            end
                         end
-                        expanded_packages.concat(packages)
-                        !packages.empty?
                     end
                 end
             end
@@ -1032,14 +1071,19 @@ module Autoproj
         end
     end
 
-    # The singleton manifest object on which the current run works
     class << self
+        # The singleton manifest object on which the current run works
         attr_accessor :manifest
+
+        # The operating system package definitions
+        attr_accessor :osdeps
     end
 
     class PackageManifest
         def self.load(package, file)
-            doc = Nokogiri::XML(File.read(file))
+            doc = Nokogiri::XML(File.read(file)) do |c|
+                c.noblanks
+            end
             PackageManifest.new(package, doc)
         end
 
@@ -1053,6 +1097,15 @@ module Autoproj
             @xml = doc
         end
 
+        def each_dependency(&block)
+            if block_given?
+                each_os_dependency(&block)
+                each_package_dependency(&block)
+            else
+                enum_for(:each_dependency, &block)
+            end
+        end
+
         def each_os_dependency
             if block_given?
                 xml.xpath('//rosdep').each do |node|

data/lib/autoproj/osdeps.rb

@@ -2,12 +2,12 @@ require 'tempfile'
 module Autoproj
     class OSDependencies
         def self.load(file)
-            data =
-                begin
-                    YAML.load(File.read(file))
-                rescue ArgumentError => e
-                    raise ConfigError, "error in #{file}: #{e.message}"
-                end
+            begin
+                data = YAML.load(File.read(file))
+                verify_definitions(data)
+            rescue ArgumentError => e
+                raise ConfigError, "error in #{file}: #{e.message}"
+            end
 
             OSDependencies.new(data)
         end
@@ -47,6 +47,23 @@ module Autoproj
             @definitions = definitions.merge(info.definitions)
         end
 
+        def self.verify_definitions(hash = nil)
+            hash ||= definitions
+            hash.each do |key, value|
+                if !key.kind_of?(String)
+                    raise ArgumentError, "invalid osdeps definition: found an #{key.class}. Don't forget to put quotes around numbers"
+                end
+                next if !value
+                if value.kind_of?(Array) || value.kind_of?(Hash)
+                    verify_definitions(value)
+                else
+                    if !value.kind_of?(String)
+                        raise ArgumentError, "invalid osdeps definition: found an #{value.class}. Don't forget to put quotes around numbers"
+                    end
+                end
+            end
+        end
+
         def operating_system
             if @operating_system
                 return @operating_system
@@ -144,7 +161,9 @@ module Autoproj
                     version_entry = data.find do |version_list, data|
                         version_list.to_s.split(',').
                             map(&:downcase).
-                            any? { |v| os_version.include?(v) }
+                            any? do |v|
+                                os_version.any? { |osv| Regexp.new(v) =~ osv }
+                            end
                     end
 
                     if !version_entry
@@ -168,8 +187,24 @@ module Autoproj
                 "\n" + shell_snippets
         end
 
+        # Returns true if there is an operating-system package with that name,
+        # and false otherwise
+        def has?(name)
+            partition_packages([name].to_set)
+            true
+        rescue ConfigError
+            false
+        end
+
         # call-seq:
         #   partition_packages(package_names) => os_packages, gem_packages
+        #
+        # Resolves the package names listed in +package_set+, and returns a set
+        # of packages that have to be installed using the platform's native
+        # package manager, and the set of packages that have to be installed
+        # using Ruby's package manager, RubyGems.
+        #
+        # Raises ConfigError if no package can be found
         def partition_packages(package_set, package_osdeps = Hash.new)
             package_set = package_set.
                 map { |name| OSDependencies.aliases[name] || name }.