autoproj 1.7.14 → 1.7.15.rc1

data/History.txt

@@ -1,3 +1,22 @@
+= Version 1.7.15
+ * remove dependency on nokogiri. This is the first step in an effort to make
+   autoproj as straightforward to bootstrap as possible.
+ * gracefully handle multiple autoproj instances trying to access the OS
+   packaging system concurrently. A message is now issued that one has to wait
+   for the other to finish. This is critical for build servers.
+ * allow conditional loading of osdeps file. The system works by adding suffixes
+   to Autoproj::OSDependencies.suffixes which tune what files to load. As an
+   example, autoproj's handling of Ruby adds either the ruby18 or ruby19 suffix,
+   which leads to either loading
+
+     file.osdeps-ruby18
+
+   or
+
+     file.osdeps-ruby19
+
+   after a file.osdeps "normal" file gets loaded.
+
 = Version 1.7.14
  * fix support for local package sets, i.e. package sets that are saved in
    autoproj/directory_name and are part of the main configuration VCS.

data/Rakefile

@@ -16,14 +16,8 @@ Utilrb::Rake.hoe do
 
             extra_deps << 
                 ['autobuild',   '>= 1.5.33'] <<
-                ['rmail',   '>= 1.0.0'] <<
                 ['utilrb', '>= 1.3.3'] <<
-                ['nokogiri', '>= 1.3.3'] <<
                 ['highline', '>= 1.5.0']
-
-            extra_dev_deps <<
-                ['webgen', '>= 0.5.9'] <<
-                ['rdoc', '>= 2.4.0']
         end
     end
 end

data/bin/amake

@@ -49,6 +49,7 @@ if remaining.grep(/^-/).size == remaining.size && !build_all
 end
 
 remaining.unshift mode
+ORIGINAL_ARGV = ARGV.dup # This is used in case autoproj needs to update itself
 ARGV.clear
 ARGV.concat(remaining)
 load File.expand_path('autoproj', File.dirname(__FILE__))

data/bin/aup

@@ -41,6 +41,7 @@ if remaining.grep(/^-/).size == remaining.size && !build_all
 end
 remaining.unshift 'update'
 
+ORIGINAL_ARGV = ARGV.dup # This is used in case autoproj needs to update itself
 ARGV.clear
 ARGV.concat(remaining)
 load File.expand_path('autoproj', File.dirname(__FILE__))

data/bin/autoproj_bootstrap

@@ -100,16 +100,41 @@ end
 require 'tempfile'
 module Autoproj
     class OSDependencies
+	class << self
+	    # When requested to load a file called X.Y, the osdeps code will
+	    # also look for files called X-suffix.Y, where 'suffix' is an
+	    # element in +osdeps_suffixes+
+	    #
+	    # A usage of this functionality is to make loading conditional to
+	    # the available version of certain tools, namely Ruby. Autoproj for
+	    # instance adds ruby18 when started on Ruby 1.8 and ruby19 when
+	    # started on Ruby 1.9
+	    attr_reader :suffixes
+	end
+	@suffixes = []
+
         def self.load(file)
-            file = File.expand_path(file)
-            begin
-                data = YAML.load(File.read(file)) || Hash.new
-                verify_definitions(data)
-            rescue ArgumentError => e
-                raise ConfigError.new, "error in #{file}: #{e.message}"
-            end
+	    if !File.file?(file)
+		raise ArgumentError, "no such file or directory #{file}"
+	    end
+
+	    candidates = [file]
+	    candidates.concat(suffixes.map { |s| "#{file}-#{s}" })
+	    
+	    result = OSDependencies.new
+	    candidates.each do |file|
+                next if !File.file?(file)
+                file = File.expand_path(file)
+                begin
+                    data = YAML.load(File.read(file)) || Hash.new
+                    verify_definitions(data)
+                rescue ArgumentError => e
+                    raise ConfigError.new, "error in #{file}: #{e.message}", e.backtrace
+                end
 
-            OSDependencies.new(data, file)
+                result.merge(OSDependencies.new(data, file))
+	    end
+	    result
         end
 
         class << self
@@ -128,13 +153,16 @@ module Autoproj
             @aliases[new_name] = old_name
         end
 
+	def self.ruby_version_keyword
+            if RUBY_VERSION < "1.9.0" then "ruby18"
+            else "ruby19"
+            end
+        end
+
         def self.autodetect_ruby
-            ruby_package =
-                if RUBY_VERSION < "1.9.0" then "ruby18"
-                else "ruby19"
-                end
-            self.alias(ruby_package, "ruby")
+            self.alias(ruby_version_keyword, "ruby")
         end
+	self.suffixes << ruby_version_keyword
 
         AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
         def self.load_default
@@ -1030,12 +1058,23 @@ So, what do you want ? (all, ruby, os or none)
                         Autoproj.progress shell_script
                     end
 
-                    Tempfile.open('osdeps_sh') do |io|
-                        io.puts "#! /bin/bash"
-                        io.puts GAIN_ROOT_ACCESS
-                        io.write shell_script
-                        io.flush
-                        Autobuild::Subprocess.run 'autoproj', 'osdeps', '/bin/bash', io.path
+                    File.open('/tmp/autoproj_osdeps_lock', 'w') do |lock_io|
+                        begin
+                            while !lock_io.flock(File::LOCK_EX | File::LOCK_NB)
+                                Autoproj.progress "  waiting for other autoproj instances to finish their osdeps installation"
+                                sleep 5
+                            end
+
+                            Tempfile.open('osdeps_sh') do |io|
+                                io.puts "#! /bin/bash"
+                                io.puts GAIN_ROOT_ACCESS
+                                io.write shell_script
+                                io.flush
+                                Autobuild::Subprocess.run 'autoproj', 'osdeps', '/bin/bash', io.path
+                            end
+                        ensure
+                            lock_io.flock(File::LOCK_UN)
+                        end
                     end
                     did_something = true
                 end
@@ -1588,7 +1627,7 @@ rescue Autoproj::ConfigError => e
 end
 
 # Now try to find out the name of the gem binary
-PACKAGES      = %w{libxml2 libxslt zlib build-essential lsb_release}
+PACKAGES      = %w{lsb_release}
 
 ENV['RUBYOPT']  = "-rubygems"
 require 'rubygems'

data/lib/autoproj/cmdline.rb

@@ -165,7 +165,11 @@ module Autoproj
                 ENV['AUTOPROJ_RESTARTING'] = '1'
                 require 'rbconfig'
                 ruby = RbConfig::CONFIG['RUBY_INSTALL_NAME']
-                exec(ruby, $0, *ARGV)
+                if defined?(ORIGINAL_ARGV)
+                    exec(ruby, $0, *ORIGINAL_ARGV)
+                else
+                    exec(ruby, $0, *ARGV)
+                end
             end
         end
 
@@ -484,14 +488,10 @@ module Autoproj
             if selected_packages.empty?
                 return manifest.default_packages
             end
-            if selected_packages.empty? # no packages, terminate
-                Autoproj.progress
-                Autoproj.progress("autoproj: no packages defined", :red)
-                exit 0
-            end
             selected_packages = selected_packages.to_set
 
-            selected_packages, nonresolved = manifest.expand_package_selection(selected_packages)
+            selected_packages, nonresolved = manifest.
+                expand_package_selection(selected_packages)
 
             # Try to auto-add stuff if nonresolved
             nonresolved.delete_if do |sel|

data/lib/autoproj/manifest.rb

@@ -1,7 +1,7 @@
 require 'yaml'
 require 'utilrb/kernel/options'
-require 'nokogiri'
 require 'set'
+require 'rexml/document'
 
 module Autoproj
     @build_system_dependencies = Set.new
@@ -1631,6 +1631,24 @@ module Autoproj
             result
         end
 
+        # Resolves all the source-based dependencies of this package (excluding
+        # the OS dependencies). The result is returned as a set of package
+        # names.
+        def resolve_packages_dependencies(*root_names)
+            result = Set.new
+            queue = root_names.dup
+            while pkg_name = queue.shift
+                next if result.include?(pkg_name)
+                result << pkg_name
+
+                pkg = Autobuild::Package[pkg_name]
+                pkg.dependencies.each do |dep_name|
+                    queue << dep_name
+                end
+            end
+            result
+        end
+
         # Resolves a package name, where +name+ cannot be resolved as a
         # metapackage
         #
@@ -2050,8 +2068,10 @@ module Autoproj
                 packages = all_layout_packages.
                     find_all { |pkg_name| pkg_name =~ match_pkg_name }.
                     to_set
-                matches[sel] = packages
-                expanded_packages |= packages
+                if !packages.empty?
+                    matches[sel] = packages
+                    expanded_packages |= packages
+                end
 
                 each_metapackage do |pkg|
                     if pkg.name =~ match_pkg_name
@@ -2141,11 +2161,19 @@ module Autoproj
         osdeps
     end
 
+    # Access to the information contained in a package's manifest.xml file
+    #
+    # Use PackageManifest.load to create
     class PackageManifest
+        # Load a manifest.xml file and returns the corresponding
+        # PackageManifest object
         def self.load(package, file)
-            doc = Nokogiri::XML(File.read(file)) do |c|
-                c.noblanks
-            end
+            doc =
+                begin REXML::Document.new(File.read(file))
+                rescue REXML::ParseException => e
+                    raise ConfigError, "invalid #{file}: #{e.message}"
+                end
+
             PackageManifest.new(package, doc)
         end
 
@@ -2160,30 +2188,25 @@ module Autoproj
         # contain multiple comma-separated tags 
         def tags
             result = []
-            xml.xpath('//tags').each do |node|
-                result.concat(node.content.strip.split(','))
+            xml.elements.each('package/tags') do |node|
+                result.concat(node.text.strip.split(','))
             end
             result
         end
 
         def documentation
-            xml.xpath('//description').each do |node|
-                doc = node.content.strip
-                if doc.empty?
-                    if doc = short_documentation
-                        return doc
-                    end
-                    return "no documentation available for #{package.name} in its manifest.xml file"
-                else
+            xml.elements.each('package/description') do |node|
+                doc = node.text.strip
+                if !doc.empty?
                     return doc
                 end
             end
-            nil
+            return short_documentation
         end
 
         def short_documentation
-            xml.xpath('//description').each do |node|
-                doc = node['brief']
+            xml.elements.each('package/description') do |node|
+                doc = node.attributes['brief']
                 if doc
                     doc = doc.to_s.strip
                 end
@@ -2191,7 +2214,7 @@ module Autoproj
                     return doc.to_s
                 end
             end
-            nil
+            "no documentation available for #{package.name} in its manifest.xml file"
         end
 
         def initialize(package, doc)
@@ -2210,8 +2233,8 @@ module Autoproj
 
         def each_os_dependency
             if block_given?
-                xml.xpath('//rosdep').each do |node|
-                    yield(node['name'], false)
+                xml.elements.each('package/rosdep') do |node|
+                    yield(node.attributes['name'], false)
                 end
                 package.os_packages.each do |name|
                     yield(name, false)
@@ -2223,12 +2246,12 @@ module Autoproj
 
         def each_package_dependency
             if block_given?
-                depend_nodes = xml.xpath('//depend').to_a +
-                    xml.xpath('//depend_optional').to_a
+                depend_nodes = xml.elements.to_a('package/depend') +
+                    xml.elements.to_a('package/depend_optional')
 
                 depend_nodes.each do |node|
-                    dependency = node['package']
-                    optional = (node['optional'].to_s == '1' || node.name == "depend_optional")
+                    dependency = node.attributes['package']
+                    optional = (node.attributes['optional'].to_s == '1' || node.name == "depend_optional")
 
                     if dependency
                         yield(dependency, optional)
@@ -2240,6 +2263,49 @@ module Autoproj
                 enum_for :each_package_dependency
             end
         end
+
+        # Enumerates the name and email of each author. If no email is present,
+        # yields (name, nil)
+        def each_author
+            if !block_given?
+                return enum_for(:each_author)
+            end
+
+            xml.elements.each('package/author') do |author|
+                author.text.strip.split(',').each do |str|
+                    name, email = str.split('/').map(&:strip)
+                    email = nil if email && email.empty?
+                    yield(name, email)
+                end
+            end
+        end
+
+        # If +name+ points to a text element in the XML document, returns the
+        # content of that element. If no element matches +name+, or if the
+        # content is empty, returns nil
+        def text_node(name)
+            xml.elements.each(name) do |str|
+                str = str.text.strip
+                if !str.empty?
+                    return str
+                end
+            end
+            nil
+        end
+
+        # The package associated URL, usually meant to direct to a website
+        #
+        # Returns nil if there is none
+        def url
+            return text_node('package/url')
+        end
+
+        # The package license name
+        #
+        # Returns nil if there is none
+        def license
+            return text_node('package/license')
+        end
     end
     def self.add_osdeps_overrides(*args, &block)
         manifest.add_osdeps_overrides(*args, &block)

data/lib/autoproj/osdeps.rb

@@ -1,16 +1,41 @@
 require 'tempfile'
 module Autoproj
     class OSDependencies
+	class << self
+	    # When requested to load a file called X.Y, the osdeps code will
+	    # also look for files called X-suffix.Y, where 'suffix' is an
+	    # element in +osdeps_suffixes+
+	    #
+	    # A usage of this functionality is to make loading conditional to
+	    # the available version of certain tools, namely Ruby. Autoproj for
+	    # instance adds ruby18 when started on Ruby 1.8 and ruby19 when
+	    # started on Ruby 1.9
+	    attr_reader :suffixes
+	end
+	@suffixes = []
+
         def self.load(file)
-            file = File.expand_path(file)
-            begin
-                data = YAML.load(File.read(file)) || Hash.new
-                verify_definitions(data)
-            rescue ArgumentError => e
-                raise ConfigError.new, "error in #{file}: #{e.message}"
-            end
+	    if !File.file?(file)
+		raise ArgumentError, "no such file or directory #{file}"
+	    end
+
+	    candidates = [file]
+	    candidates.concat(suffixes.map { |s| "#{file}-#{s}" })
+	    
+	    result = OSDependencies.new
+	    candidates.each do |file|
+                next if !File.file?(file)
+                file = File.expand_path(file)
+                begin
+                    data = YAML.load(File.read(file)) || Hash.new
+                    verify_definitions(data)
+                rescue ArgumentError => e
+                    raise ConfigError.new, "error in #{file}: #{e.message}", e.backtrace
+                end
 
-            OSDependencies.new(data, file)
+                result.merge(OSDependencies.new(data, file))
+	    end
+	    result
         end
 
         class << self
@@ -29,13 +54,16 @@ module Autoproj
             @aliases[new_name] = old_name
         end
 
+	def self.ruby_version_keyword
+            if RUBY_VERSION < "1.9.0" then "ruby18"
+            else "ruby19"
+            end
+        end
+
         def self.autodetect_ruby
-            ruby_package =
-                if RUBY_VERSION < "1.9.0" then "ruby18"
-                else "ruby19"
-                end
-            self.alias(ruby_package, "ruby")
+            self.alias(ruby_version_keyword, "ruby")
         end
+	self.suffixes << ruby_version_keyword
 
         AUTOPROJ_OSDEPS = File.join(File.expand_path(File.dirname(__FILE__)), 'default.osdeps')
         def self.load_default
@@ -931,12 +959,23 @@ So, what do you want ? (all, ruby, os or none)
                         Autoproj.progress shell_script
                     end
 
-                    Tempfile.open('osdeps_sh') do |io|
-                        io.puts "#! /bin/bash"
-                        io.puts GAIN_ROOT_ACCESS
-                        io.write shell_script
-                        io.flush
-                        Autobuild::Subprocess.run 'autoproj', 'osdeps', '/bin/bash', io.path
+                    File.open('/tmp/autoproj_osdeps_lock', 'w') do |lock_io|
+                        begin
+                            while !lock_io.flock(File::LOCK_EX | File::LOCK_NB)
+                                Autoproj.progress "  waiting for other autoproj instances to finish their osdeps installation"
+                                sleep 5
+                            end
+
+                            Tempfile.open('osdeps_sh') do |io|
+                                io.puts "#! /bin/bash"
+                                io.puts GAIN_ROOT_ACCESS
+                                io.write shell_script
+                                io.flush
+                                Autobuild::Subprocess.run 'autoproj', 'osdeps', '/bin/bash', io.path
+                            end
+                        ensure
+                            lock_io.flock(File::LOCK_UN)
+                        end
                     end
                     did_something = true
                 end

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "1.7.14"
+    VERSION = "1.7.15.rc1"
 end

data/test/test_package_manifest.rb

@@ -0,0 +1,65 @@
+require 'test/unit'
+require 'autoproj'
+
+class TC_PackageManifest < Test::Unit::TestCase
+
+    DATA_DIR = File.expand_path('data', File.dirname(__FILE__))
+
+    FakePackage = Struct.new :name, :os_packages
+
+    attr_reader :pkg
+
+    def setup
+        @pkg = FakePackage.new('test', [])
+    end
+
+    def test_complete_manifest
+        data = File.join(DATA_DIR, 'full_manifest.xml')
+        manifest = Autoproj::PackageManifest.load(pkg, data)
+
+        assert_equal %w{tag1 tag2 tag3}, manifest.tags
+        assert_equal "full_doc", manifest.documentation
+        assert_equal "short_doc", manifest.short_documentation
+
+        deps = [['dep1', false], ['dep2', false]]
+        opt_deps = [['opt_dep1', true], ['opt_dep2', true]]
+        osdeps = [['osdep1', false], ['osdep2', false]]
+        pkg.os_packages << "osdep2"
+
+        assert_equal(osdeps, manifest.each_os_dependency.to_a)
+        assert_equal(deps + opt_deps, manifest.each_package_dependency.to_a)
+        assert_equal((deps + opt_deps + osdeps).to_set, manifest.each_dependency.to_set)
+
+        authors = [
+            ["Author1", "author1@email"],
+            ["Author2", "author2@email"],
+            ["Author3", nil]]
+        assert_equal(authors.to_set, manifest.each_author.to_set)
+
+        assert_equal('the_url', manifest.url)
+        assert_equal('BSD', manifest.license)
+    end
+
+    def test_empty_manifest
+        data = File.join(DATA_DIR, 'empty_manifest.xml')
+        manifest = Autoproj::PackageManifest.load(pkg, data)
+
+        assert_equal [], manifest.tags
+        assert(manifest.documentation =~ /no documentation available/)
+        assert(manifest.short_documentation =~ /no documentation available/)
+
+        assert(manifest.each_os_dependency.to_a.empty?)
+        assert(manifest.each_package_dependency.to_a.empty?)
+        assert(manifest.each_dependency.to_a.empty?)
+        assert(manifest.each_author.to_a.empty?)
+
+        assert_equal(nil, manifest.url)
+        assert_equal(nil, manifest.license)
+    end
+
+    def test_failure_on_wrong_document
+        data = File.join(DATA_DIR, 'invalid_manifest.xml')
+        assert_raises(Autoproj::ConfigError) { Autoproj::PackageManifest.load(pkg, data) }
+    end
+end
+

metadata

@@ -1,13 +1,15 @@
 --- !ruby/object:Gem::Specification 
 name: autoproj
 version: !ruby/object:Gem::Version 
-  hash: 23
-  prerelease: 
+  hash: 15424063
+  prerelease: 7
   segments: 
   - 1
   - 7
-  - 14
-  version: 1.7.14
+  - 15
+  - rc
+  - 1
+  version: 1.7.15.rc1
 platform: ruby
 authors: 
 - Sylvain Joyeux
@@ -15,7 +17,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-11-03 00:00:00 Z
+date: 2011-12-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: autobuild
@@ -33,42 +35,10 @@ dependencies:
         version: 1.5.33
   type: :runtime
   version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: rmail
-  prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 23
-        segments: 
-        - 1
-        - 0
-        - 0
-        version: 1.0.0
-  type: :runtime
-  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: utilrb
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 1
-        - 3
-        - 3
-        version: 1.3.3
-  type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
-  prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id002 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -80,11 +50,11 @@ dependencies:
         - 3
         version: 1.3.3
   type: :runtime
-  version_requirements: *id004
+  version_requirements: *id002
 - !ruby/object:Gem::Dependency 
   name: highline
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -96,43 +66,11 @@ dependencies:
         - 0
         version: 1.5.0
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: webgen
-  prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 25
-        segments: 
-        - 0
-        - 5
-        - 9
-        version: 0.5.9
-  type: :development
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: rdoc
-  prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 2
-        - 4
-        - 0
-        version: 2.4.0
-  type: :development
-  version_requirements: *id007
+  version_requirements: *id003
 - !ruby/object:Gem::Dependency 
   name: hoe
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -143,7 +81,7 @@ dependencies:
         - 12
         version: "2.12"
   type: :development
-  version_requirements: *id008
+  version_requirements: *id004
 description: |-
   What is Autoproj
   ----------------
@@ -225,6 +163,7 @@ files:
 - test/data/test_manifest/autoproj/manifest
 - test/test_debian.rb
 - test/test_manifest.rb
+- test/test_package_manifest.rb
 - .gemtest
 homepage: http://rock-robotics.org/autoproj
 licenses: []
@@ -247,12 +186,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version 
-      hash: 3
+      hash: 25
       segments: 
-      - 0
-      version: "0"
+      - 1
+      - 3
+      - 1
+      version: 1.3.1
 requirements: []
 
 rubyforge_project: autobuild
@@ -263,3 +204,4 @@ summary: Easy installation and management of software packages
 test_files: 
 - test/test_manifest.rb
 - test/test_debian.rb
+- test/test_package_manifest.rb