autoproj 2.0.0.rc37 → 2.0.0.rc38

data/lib/autoproj/package_definition.rb

@@ -34,6 +34,7 @@ def initialize(autobuild, package_set, file)
             @user_blocks = []
             @modes = ['import', 'build']
             @setup = false
+            @vcs = VCSDefinition.none
         end
 
         # The modes in which this package will be used

data/lib/autoproj/package_managers/apt_dpkg_manager.rb

@@ -13,36 +13,39 @@ def initialize(ws, status_file = "/var/lib/dpkg/status")
                       %w{DEBIAN_FRONTEND=noninteractive apt-get install -y})
             end
 
+            def self.parse_package_status(installed_packages, paragraph)
+                if paragraph =~ /^Status: install ok installed$/
+                    if paragraph =~ /^Package: (.*)$/
+                        installed_packages << $1
+                    end
+                    if paragraph =~ /^Provides: (.*)$/
+                        installed_packages.merge($1.split(',').map(&:strip))
+                    end
+                end
+            end
+
+            def self.parse_dpkg_status(status_file)
+                installed_packages = Set.new
+                dpkg_status = File.read(status_file)
+                dpkg_status << "\n"
+
+                dpkg_status = StringScanner.new(dpkg_status)
+                if !dpkg_status.scan(/Package: /)
+                    raise ArgumentError, "expected #{status_file} to have Package: lines but found none"
+                end
+
+                while paragraph_end = dpkg_status.scan_until(/Package: /)
+                    paragraph = "Package: #{paragraph_end[0..-10]}"
+                    parse_package_status(installed_packages, paragraph)
+                end
+                parse_package_status(installed_packages, "Package: #{dpkg_status.rest}")
+                installed_packages
+            end
+
             # On a dpkg-enabled system, checks if the provided package is installed
             # and returns true if it is the case
             def installed?(package_name, filter_uptodate_packages: false, install_only: false)
-                if !@installed_packages
-                    @installed_packages = Set.new
-                    dpkg_status = File.readlines(status_file)
-                    dpkg_status << ""
-
-                    current_packages = []
-                    is_installed = false
-                    dpkg_status.each do |line|
-                        line = line.chomp
-                        line = line.encode( "UTF-8", "binary", :invalid => :replace, :undef => :replace)
-                        if line == ""
-                            if is_installed
-                                current_packages.each do |pkg|
-                                    @installed_packages << pkg
-                                end
-                                is_installed = false
-                            end
-                            current_packages.clear
-                        elsif line =~ /Package: (.*)$/
-                            current_packages << $1
-                        elsif line =~ /Provides: (.*)$/
-                            current_packages.concat($1.split(',').map(&:strip))
-                        elsif line == "Status: install ok installed"
-                            is_installed = true
-                        end
-                    end
-                end
+                @installed_packages ||= AptDpkgManager.parse_dpkg_status(status_file)
                 
                 if package_name =~ /^(\w[a-z0-9+-.]+)/
                     @installed_packages.include?($1)

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -24,13 +24,18 @@ def self.with_prerelease(*value)
                 end
             end
 
-            def initialize(ws)
-                super
-                self.call_while_empty = true
+            # (see Manager#call_while_empty?)
+            def call_while_empty?
+                !workspace_configuration_gemfiles.empty?
             end
 
-            # Filters all paths that come from other autoproj installations out
-            # of GEM_PATH
+            # (see Manager#strict?)
+            def strict?
+                true
+            end
+
+            # Set up the workspace environment to work with the bundler-managed
+            # gems
             def initialize_environment
                 env = ws.env
 
@@ -87,6 +92,15 @@ def initialize_environment
                 end
             end
 
+            # @api private
+            #
+            # Update RUBYLIB to add the gems that are part of the bundler
+            # install
+            #
+            # @param [Array<String>] bundle_rubylib the rubylib entries reported
+            #   by bundler
+            # @param [Array<String>] system_rubylib the rubylib entries that are
+            #   set by the underlying ruby interpreter itself
             def update_env_rubylib(bundle_rubylib, system_rubylib = discover_rubylib)
                 current = (ws.env.resolved_env['RUBYLIB'] || '').split(File::PATH_SEPARATOR) + system_rubylib
                 (bundle_rubylib - current).each do |p|
@@ -94,6 +108,16 @@ def update_env_rubylib(bundle_rubylib, system_rubylib = discover_rubylib)
                 end
             end
 
+            # @api private
+            #
+            # Parse an osdep entry into a gem name and gem version
+            #
+            # The 'gem' entries in the osdep files can contain a version
+            # specification. This method parses the two parts and return them
+            #
+            # @param [String] entry the osdep entry
+            # @return [(String,String),(String,nil)] the gem name, and an
+            #   optional version specification
             def parse_package_entry(entry)
                 if entry =~ /^([^><=~]*)([><=~]+.*)$/
                     [$1.strip, $2.strip]
@@ -103,7 +127,14 @@ def parse_package_entry(entry)
             end
 
             class NotCleanState < RuntimeError; end
-
+            
+            # @api private
+            #
+            # Create backup files matching a certain file mapping
+            #
+            # @param [Hash<String,String>] mapping a mapping from the original
+            #   file to the file into which it should be backed up. The source
+            #   file might not exist.
             def backup_files(mapping)
                 mapping.each do |file, backup_file|
                     if File.file?(file)
@@ -112,6 +143,11 @@ def backup_files(mapping)
                 end
             end
 
+            # @api private
+            #
+            # Restore backups saved by {#backup_file}
+            #
+            # @param (see #backup_file)
             def backup_restore(mapping)
                 mapping.each do |file, backup_file|
                     if File.file?(backup_file)
@@ -120,6 +156,11 @@ def backup_restore(mapping)
                 end
             end
 
+            # @api private
+            #
+            # Remove backups created by {#backup_files}
+            #
+            # @param (see #backup_file)
             def backup_clean(mapping)
                 mapping.each do |file, backup_file|
                     if File.file?(backup_file)
@@ -183,8 +224,8 @@ def self.run_bundler(ws, *commandline, gemfile: nil)
             def merge_gemfiles(*path, unlock: [])
                 gems_remotes = Set.new
                 dependencies = Hash.new do |h, k|
-                    h[k] = Hash.new do |h, k|
-                        h[k] = Hash.new do |a, b|
+                    h[k] = Hash.new do |i, j|
+                        i[j] = Hash.new do |a, b|
                             a[b] = Array.new
                         end
                     end
@@ -239,6 +280,20 @@ def merge_gemfiles(*path, unlock: [])
                 contents.join("\n")
             end
 
+            def workspace_configuration_gemfiles
+                gemfiles = []
+                ws.manifest.each_package_set do |source|
+                    if source.local_dir && File.file?(pkg_set_gemfile = File.join(source.local_dir, 'Gemfile'))
+                        gemfiles << pkg_set_gemfile
+                    end
+                end
+                # In addition, look into overrides.d
+                Dir.glob(File.join(ws.overrides_dir, "*.gemfile")) do |overrides_gemfile_path|
+                    gemfiles << overrides_gemfile_path
+                end
+                gemfiles
+            end
+
             def install(gems, filter_uptodate_packages: false, install_only: false)
                 root_dir     = File.join(ws.prefix_dir, 'gems')
                 gemfile_path = File.join(root_dir, 'Gemfile')
@@ -257,16 +312,7 @@ def install(gems, filter_uptodate_packages: false, install_only: false)
                     end
                 end
 
-                gemfiles = []
-                ws.manifest.each_package_set do |source|
-                    if source.local_dir && File.file?(pkg_set_gemfile = File.join(source.local_dir, 'Gemfile'))
-                        gemfiles << pkg_set_gemfile
-                    end
-                end
-                # In addition, look into overrides.d
-                Dir.glob(File.join(ws.overrides_dir, "*.gemfile")) do |gemfile_path|
-                    gemfiles << gemfile_path
-                end
+                gemfiles = workspace_configuration_gemfiles
                 gemfiles << File.join(ws.dot_autoproj_dir, 'Gemfile')
 
                 # Save the osdeps entries in a temporary gemfile and finally

data/lib/autoproj/package_managers/gem_manager.rb

@@ -90,6 +90,8 @@ def self.use_cache_dir
             def self.gem_home
                 @gem_home
             end
+
+            @gem_home = nil
             
             # Returns the set of default options that are added to gem
             #
@@ -158,7 +160,7 @@ def build_gem_cmdlines(gems)
 
             def pristine(gems)
                 guess_gem_program
-                base_cmdline = [Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('gem')]
+                base_cmdline = [Autobuild.tool_in_path('ruby', env: ws.env), '-S', Autobuild.tool('gem')]
                 cmdlines = [
                     [*base_cmdline, 'clean'],
                 ]
@@ -176,7 +178,7 @@ def pristine(gems)
             def install(gems)
                 guess_gem_program
 
-                base_cmdline = [Autobuild.tool_in_path('ruby'), '-S', Autobuild.tool('gem'), 'install', *GemManager.default_install_options]
+                base_cmdline = [Autobuild.tool_in_path('ruby', env: ws.env), '-S', Autobuild.tool('gem'), 'install', *GemManager.default_install_options]
                 if !GemManager.with_doc
                     base_cmdline << '--no-rdoc' << '--no-ri'
                 end

data/lib/autoproj/package_managers/manager.rb

@@ -16,19 +16,38 @@ def enabled?; !!@enabled end
             attr_writer :silent
             def silent?; !!@silent end
 
-            attr_writer :call_while_empty
-            def call_while_empty?; !!@call_while_empty end
+            # Whether this package manager should be called even if no packages
+            # should be installed
+            #
+            # This is needed if the installer has ways to get specifications of
+            # packages to install through other means than the osdep system, as
+            # e.g. {BundlerManager} that would install gems listed in
+            # autoproj/Gemfile
+            def call_while_empty?
+                false
+            end
+
+            # Whether this package manager needs to maintain a list of all the
+            # packages that are needed for the whole installation (true), or
+            # needs only to be called with packages to install
+            #
+            # OS package managers are generally non-strict (once a package is
+            # installed, it's available to all). Package managers like
+            # {BundlerManager} are strict as they maintain a list of gems that
+            # are then made available to the whole installation
+            #
+            # The default is false, reimplement in subclasses to return true
+            def strict?
+                false
+            end
 
-            # Create a package manager registered with various names
+            # Create a package manager
             #
-            # @param [Array<String>] names the package manager names. It MUST be
-            #   different from the OS names that autoproj uses. See the comment
-            #   for OS_PACKAGE_HANDLERS for an explanation
+            # @param [Workspace] ws the underlying workspace
             def initialize(ws)
                 @ws = ws
                 @enabled = true
                 @silent = true
-                @call_while_empty = false
             end
 
             # Overload to perform initialization of environment variables in

data/lib/autoproj/package_managers/shell_script_manager.rb

@@ -3,7 +3,7 @@ module PackageManagers
         # Base class for all package managers that simply require the call of a
         # shell script to install packages (e.g. yum, apt, ...)
         class ShellScriptManager < Manager
-            def self.execute(command_line, with_locking, with_root)
+            def self.execute(command_line, with_locking, with_root, env: Autobuild.env)
                 if with_locking
                     File.open('/tmp/autoproj_osdeps_lock', 'w') do |lock_io|
                         begin
@@ -11,7 +11,7 @@ def self.execute(command_line, with_locking, with_root)
                                 Autoproj.message "  waiting for other autoproj instances to finish their osdeps installation"
                                 sleep 5
                             end
-                            return execute(command_line, false, with_root)
+                            return execute(command_line, false, with_root, env: env)
                         ensure
                             lock_io.flock(File::LOCK_UN)
                         end
@@ -19,7 +19,7 @@ def self.execute(command_line, with_locking, with_root)
                 end
                 
                 if with_root
-                    sudo = Autobuild.tool_in_path('sudo')
+                    sudo = Autobuild.tool_in_path('sudo', env: env)
                     command_line = [sudo, *command_line]
                 end
 
@@ -188,7 +188,7 @@ def install(packages, filter_uptodate_packages: false, install_only: false,
                         Autoproj.message shell_script
                     end
 
-                    ShellScriptManager.execute([*auto_install_cmd, *packages], needs_locking?, needs_root?)
+                    ShellScriptManager.execute([*auto_install_cmd, *packages], needs_locking?, needs_root?, env: ws.env)
                     return true
                 end
                 false

data/lib/autoproj/package_managers/zypper_manager.rb

@@ -9,7 +9,7 @@ def initialize(ws)
             end
 
             def filter_uptodate_packages(packages)
-                result = `LANG=C rpm -q --whatprovides '#{packages.join("' '")}'`
+                `LANG=C rpm -q --whatprovides '#{packages.join("' '")}'`
                 has_all_pkgs = $?.success?
 
                 if !has_all_pkgs

data/lib/autoproj/package_manifest.rb

@@ -3,201 +3,218 @@ module Autoproj
     #
     # Use PackageManifest.load to create
     class PackageManifest
-        # Load a manifest.xml file and returns the corresponding
-        # PackageManifest object
+        # Create a null manifest for the given package
+        def self.null(package)
+            new(package, null: true)
+        end
+
+        # Load a manifest.xml file and returns the corresponding PackageManifest
+        # object
+        #
+        # @param [PackageDescription] the package we're loading it for
+        # @param [String] file the path to the manifest.xml file
+        # @return [PackageManifest]
+        # @see parse
         def self.load(package, file)
-            doc =
-                begin REXML::Document.new(File.read(file))
-                rescue REXML::ParseException => e
-                    raise Autobuild::PackageException.new(package.name, 'prepare'), "invalid #{file}: #{e.message}"
-                end
+            parse(package, File.read(file), path: file)
+        end
 
-            PackageManifest.new(package, doc)
+        # Create a PackageManifest object from the XML content provided as a
+        # string
+        #
+        # @param [PackageDescription] the package we're loading it for
+        # @param [String] contents the manifest.xml contents as a string
+        # @param [String] path the file path, used for error reporting
+        # @return [PackageManifest]
+        # @see load
+        def self.parse(package, contents, path: '<loaded from string>')
+            manifest = PackageManifest.new(package)
+            loader = Loader.new(path, manifest)
+            begin
+                REXML::Document.parse_stream(contents, loader)
+            rescue REXML::ParseException => e
+                raise Autobuild::PackageException.new(package.name, 'prepare'), "invalid #{file}: #{e.message}"
+            end
+            manifest
         end
 
+        ContactInfo = Struct.new :name, :email
+        Dependency  = Struct.new :name, :optional, :modes
+
         # The Autobuild::Package instance this manifest applies on
         attr_reader :package
-        # The raw XML data as a Nokogiri document
-        attr_reader :xml
-
-        # The list of tags defined for this package
-        #
-        # Tags are defined as multiple <tags></tags> blocks, each of which can
-        # contain multiple comma-separated tags 
-        def tags
-            result = []
-            xml.elements.each('package/tags') do |node|
-                result.concat((node.text || "").strip.split(','))
-            end
-            result
+        attr_accessor :description
+        attr_accessor :brief_description
+        attr_reader :dependencies
+        attr_accessor :tags
+        attr_accessor :url
+        attr_accessor :license
+        attr_accessor :version
+        attr_accessor :authors
+        attr_accessor :maintainers
+        attr_accessor :rock_maintainers
+
+        # Add a declared dependency to this package
+        def add_dependency(name, optional: false, modes: [])
+            dependencies << Dependency.new(name, optional, modes)
         end
 
         def has_documentation?
-            xml.elements.each('package/description') do |node|
-                doc = (node.text || "").strip
-                if !doc.empty?
-                    return true
-                end
-            end
-            return false
+            !!description
         end
 
         def documentation
-            xml.elements.each('package/description') do |node|
-                doc = (node.text || "").strip
-                if !doc.empty?
-                    return doc
-                end
-            end
-            return short_documentation
+            description || short_documentation
         end
 
         def has_short_documentation?
-            xml.elements.each('package/description') do |node|
-                doc = node.attributes['brief']
-                if doc
-                    doc = doc.to_s.strip
-                end
-                if doc && !doc.empty?
-                    return true
-                end
-            end
-            false
+            !!brief_description
         end
 
         def short_documentation
-            xml.elements.each('package/description') do |node|
-                doc = node.attributes['brief']
-                if doc
-                    doc = doc.to_s.strip
-                end
-                if doc && !doc.empty?
-                    return doc.to_s
-                end
-            end
-            "no documentation available for #{package.name} in its manifest.xml file"
+            brief_description ||
+                "no documentation available for package '#{package.name}' in its manifest.xml file"
         end
 
-        def initialize(package, doc = REXML::Document.new)
+        def initialize(package, null: false)
             @package = package
-            @xml = doc
+            @dependencies = []
+            @authors = []
+            @maintainers = []
+            @rock_maintainers = []
+            @tags = []
+            @null = null
         end
 
-        def each_dependency(in_modes = Array.new, &block)
-            return enum_for(__method__) if !block_given?
-
-            depend_nodes = xml.elements.to_a('package/depend') +
-                xml.elements.to_a('package/depend_optional') +
-                xml.elements.to_a('package/rosdep')
-            in_modes.each do |m|
-                depend_nodes += xml.elements.to_a("package/#{m}_depend")
-            end
-
-            depend_nodes.each do |node|
-                dependency = node.attributes['package'] || node.attributes['name']
-                optional   = (node.attributes['optional'].to_s == '1' || node.name == "depend_optional")
-                modes      = node.attributes['modes'].to_s.split(',')
-                if node.name =~ /^(\w+)_depend$/
-                    modes << $1
-                end
-                if !modes.empty? && modes.none? { |m| in_modes.include?(m) }
-                    next
-                end
+        # Whether this is a null manifest (used for packages that have actually
+        # no manifest) or not
+        def null?
+            !!@null
+        end
 
-                if dependency
-                    yield(dependency, optional)
-                elsif node.name == 'rosdep'
-                    raise ConfigError.new, "manifest of #{package.name} has a <rosdep> tag without a 'name' attribute"
-                else
-                    raise ConfigError.new, "manifest of #{package.name} has a <#{node.name}> tag without a 'package' attribute"
+        def each_dependency(in_modes = Array.new, &block)
+            return enum_for(__method__, in_modes) if !block_given?
+            dependencies.each do |dep|
+                if dep.modes.empty? || in_modes.any? { |m| dep.modes.include?(m) }
+                    yield(dep.name, dep.optional)
                 end
             end
-
-            package.os_packages.each do |name|
-                yield(name, false)
-            end
         end
 
         def each_os_dependency(modes = Array.new, &block)
-            Autoproj.warn "PackageManifest#each_os_dependency called, fix your code"
+            Autoproj.warn_deprecated "#{self.class}##{__method__}", "call #each_dependency instead"
             return each_dependency(modes, &block)
         end
 
         def each_package_dependency(modes = Array.new, &block)
-            Autoproj.warn "PackageManifest#each_os_dependency called, fix your code"
+            Autoproj.warn_deprecated "#{self.class}##{__method__}", "call #each_dependency instead"
             return each_dependency(modes, &block)
         end
 
         def each_rock_maintainer
             return enum_for(__method__) if !block_given?
-            xml.elements.each('package/rock_maintainer') do |maintainer|
-                (maintainer.text || "").strip.split(',').each do |str|
-                    name, email = str.split('/').map(&:strip)
-                    email = nil if email && email.empty?
-                    yield(name, email)
-                end
+            rock_maintainers.each do |m|
+                yield(m.name, m.email)
             end
         end
 
         def each_maintainer
             return enum_for(__method__) if !block_given?
-            xml.elements.each('package/maintainer') do |maintainer|
-                (maintainer.text || "").strip.split(',').each do |str|
-                    name, email = str.split('/').map(&:strip)
-                    email = nil if email && email.empty?
-                    yield(name, email)
-                end
+            maintainers.each do |m|
+                yield(m.name, m.email)
             end
         end
 
         # Enumerates the name and email of each author. If no email is present,
         # yields (name, nil)
         def each_author
-            if !block_given?
-                return enum_for(:each_author)
+            return enum_for(__method__) if !block_given?
+            authors.each do |m|
+                yield(m.name, m.email)
             end
+        end
+
+        # @api private
+        #
+        # REXML stream parser object used to load the XML contents into a
+        # {PackageManifest} object
+        class Loader
+            include REXML::StreamListener
 
-            xml.elements.each('package/author') do |author|
-                (author.text || "").strip.split(',').each do |str|
+            attr_reader :path, :manifest
+
+            def initialize(path, manifest)
+                @path = path
+                @manifest = manifest
+            end
+
+            def parse_depend_tag(tag_name, attributes, modes: [], optional: false)
+                package = attributes['package'] || attributes['name']
+                if !package
+                    raise InvalidPackageManifest, "found '#{tag_name}' tag in #{path} without a 'package' attribute"
+                end
+
+                if tag_modes = attributes['modes']
+                    modes += tag_modes.split(',')
+                end
+
+                manifest.add_dependency(
+                    package,
+                    optional: optional || (attributes['optional'] == '1'),
+                    modes: modes)
+            end
+
+            def parse_contact_field(text)
+                text.strip.split(',').map do |str|
                     name, email = str.split('/').map(&:strip)
                     email = nil if email && email.empty?
-                    yield(name, email)
+                    ContactInfo.new(name, email)
                 end
             end
-        end
 
-        # If +name+ points to a text element in the XML document, returns the
-        # content of that element. If no element matches +name+, or if the
-        # content is empty, returns nil
-        def text_node(name)
-            xml.elements.each(name) do |str|
-                str = (str.text || "").strip
-                if !str.empty?
-                    return str
+            TEXT_FIELDS = Set['url', 'license', 'version', 'description']
+            AUTHOR_FIELDS = Set['author', 'maintainer', 'rock_maintainer']
+
+            def tag_start(name, attributes)
+                if name == 'depend'
+                    parse_depend_tag(name, attributes)
+                elsif name == 'depend_optional'
+                    parse_depend_tag(name, attributes, optional: true)
+                elsif name == 'rosdep'
+                    parse_depend_tag(name, attributes)
+                elsif name =~ /^(\w+)_depend$/
+                    parse_depend_tag(name, attributes, modes: [$1])
+                elsif name == 'description'
+                    if brief = attributes['brief']
+                        manifest.brief_description = brief
+                    end
+                    @tag_text = ''
+                elsif TEXT_FIELDS.include?(name) || AUTHOR_FIELDS.include?(name)
+                    @tag_text = ''
+                elsif name == 'tags'
+                    @tag_text = ''
+                else
+                    @tag_text = nil
                 end
             end
-            nil
-        end
-
-        # The package associated URL, usually meant to direct to a website
-        #
-        # Returns nil if there is none
-        def url
-            return text_node('package/url')
-        end
-
-        # The package license name
-        #
-        # Returns nil if there is none
-        def license
-            return text_node('package/license')
-        end
-
-        # The package version number
-        #
-        # Returns 0 if none is declared
-        def version
-            return text_node("version")
+            def text(text)
+                if @tag_text
+                    @tag_text << text
+                end
+            end
+            def tag_end(name)
+                if AUTHOR_FIELDS.include?(name)
+                    manifest.send("#{name}s").concat(parse_contact_field(@tag_text))
+                elsif TEXT_FIELDS.include?(name)
+                    field = @tag_text.strip
+                    if !field.empty?
+                        manifest.send("#{name}=", field)
+                    end
+                elsif name == 'tags'
+                    manifest.tags.concat(@tag_text.strip.split(',').map(&:strip))
+                end
+                @tag_text = nil
+            end
         end
     end
 end