autoproj 2.15.0 → 2.15.3

data/lib/autoproj/package_managers/apt_dpkg_manager.rb

@@ -9,6 +9,12 @@ module Autoproj
         class AptDpkgManager < ShellScriptManager
             attr_accessor :status_file
 
+            class << self
+                def inherit
+                    @inherit ||= Set.new
+                end
+            end
+
             def initialize(ws, status_file = "/var/lib/dpkg/status")
                 @status_file = status_file
                 @installed_packages = nil
@@ -129,6 +135,7 @@ module Autoproj
                 end
             end
 
+            # rubocop:disable Metrics/AbcSize
             def install(packages, filter_uptodate_packages: false, install_only: false)
                 if filter_uptodate_packages || install_only
                     already_installed, missing = packages.partition do |package_name|
@@ -144,12 +151,13 @@ module Autoproj
                     packages = missing + (need_update || [])
                 end
 
-                if super(packages)
+                if super(packages, inherit: self.class.inherit)
                     # Invalidate caching of installed packages, as we just
                     # installed new packages !
                     @installed_packages = nil
                 end
             end
+            # rubocop:enable Metrics/AbcSize
         end
     end
 end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -370,7 +370,7 @@ module Autoproj
             end
 
             # Parse the contents of a gemfile into a set of
-            def merge_gemfiles(*path, unlock: [])
+            def merge_gemfiles(*path, ruby_version: nil, unlock: [])
                 gems_remotes = Set.new
                 dependencies = Hash.new do |h, k|
                     h[k] = Hash.new do |i, j|
@@ -410,6 +410,9 @@ module Autoproj
                     g = g[0..-2] if g.end_with?("/")
                     contents << "source '#{g}'"
                 end
+                if ruby_version
+                    contents << "ruby \"#{ruby_version}\" if respond_to?(:ruby)"
+                end
                 valid_keys = %w[group groups git path glob name branch ref tag
                                 require submodules platform platforms type
                                 source install_if]
@@ -526,34 +529,32 @@ module Autoproj
                     end
                 end
 
-                gemfiles = workspace_configuration_gemfiles
-                gemfiles << File.join(ws.dot_autoproj_dir, "Gemfile")
-
-                # Save the osdeps entries in a temporary gemfile and finally
-                # merge the whole lot of it
-                gemfile_contents = Tempfile.open "autoproj-gemfile" do |io|
-                    gems.map { |entry| GemEntry.parse(entry) }
-                        .sort_by(&:name)
-                        .each { |entry| io.puts entry.to_gemfile_line }
+                gemfiles = []
 
-                    io.flush
-                    gemfiles.unshift io.path
-                    # The autoproj gemfile needs to be last, we really don't
-                    # want to mess it up
-                    merge_gemfiles(*gemfiles)
+                unless gems.empty?
+                    osdeps_gemfile_io = prepare_osdeps_gemfile(gems)
+                    gemfiles << osdeps_gemfile_io.path
                 end
 
+                gemfiles += workspace_configuration_gemfiles
+                # The autoproj gemfile needs to be last, we really don't
+                # want to mess it up
+                gemfiles << File.join(ws.dot_autoproj_dir, "Gemfile")
+
+                ruby_version = RUBY_VERSION.gsub(/\.\d+$/, ".0")
+                gemfile_contents =
+                    merge_gemfiles(*gemfiles, ruby_version: "~> #{ruby_version}")
+
                 FileUtils.mkdir_p root_dir
                 updated = (!File.exist?(gemfile_path) ||
                            File.read(gemfile_path) != gemfile_contents)
                 if updated
                     Ops.atomic_write(gemfile_path) do |io|
-                        io.puts "ruby \"#{RUBY_VERSION}\" if respond_to?(:ruby)"
                         io.puts gemfile_contents
                     end
                 end
 
-                options = Array.new
+                options = []
                 binstubs_path = File.join(root_dir, "bin")
                 if updated || !install_only || !File.file?("#{gemfile_path}.lock")
                     self.class.run_bundler_install(ws, gemfile_path, *options,
@@ -580,6 +581,25 @@ module Autoproj
                 backup_clean(backups)
             end
 
+            # Prepare a Gemfile that contains osdeps gem declarations
+            #
+            # @param [Array<String,Hash>] gems osdeps declarations as understood
+            #   by {GemEntry.parse}
+            # @return [File]
+            def prepare_osdeps_gemfile(gems)
+                io = Tempfile.open "autoproj-gemfile"
+                io.puts "source \"https://rubygems.org\""
+                gems.map { |entry| GemEntry.parse(entry) }
+                    .sort_by(&:name)
+                    .each { |entry| io.puts entry.to_gemfile_line }
+
+                io.flush
+                io
+            rescue Exception
+                io&.close
+                raise
+            end
+
             def discover_rubylib
                 require "bundler"
                 Tempfile.open "autoproj-rubylib" do |io|

data/lib/autoproj/package_managers/pip_manager.rb

@@ -27,24 +27,16 @@ module Autoproj
             end
 
             def guess_pip_program
-                unless ws.config.has_value_for?("USE_PYTHON")
-                    Autoproj::Python.setup_python_configuration_options(ws: ws)
-                end
-                unless ws.config.get("USE_PYTHON")
-                    raise ConfigError,
-                          "Your current package selection requires the use of pip, but" \
-                          " the use of python is either unspecified or has been denied,"\
-                          " see setting of USE_PYTHON in your workspace configuration." \
-                          " Either remove all packages depending on pip packages " \
-                          " from the workspace layout (manifest) or " \
-                          " call 'autoproj reconfigure' to change the setting."
-
-                end
-
+                activate_python
                 Autobuild.programs["pip"] = "pip" unless Autobuild.programs["pip"]
                 Autobuild.programs["pip"]
             end
 
+            def activate_python
+                Autoproj::Python.setup_python_configuration_options(ws: ws)
+                Autoproj::Python.assert_python_activated(ws: ws)
+            end
+
             def install(pips, filter_uptodate_packages: false, install_only: false)
                 guess_pip_program
                 pips = [pips] if pips.is_a?(String)

data/lib/autoproj/package_managers/shell_script_manager.rb

@@ -3,7 +3,8 @@ module Autoproj
         # Base class for all package managers that simply require the call of a
         # shell script to install packages (e.g. yum, apt, ...)
         class ShellScriptManager < Manager
-            def self.execute(command_line, with_locking, with_root, env: Autobuild.env)
+            def self.execute(command_line, with_locking, with_root,
+                env: Autoproj.workspace.env, inherit: Set.new)
                 if with_locking
                     File.open("/tmp/autoproj_osdeps_lock", "w") do |lock_io|
                         until lock_io.flock(File::LOCK_EX | File::LOCK_NB)
@@ -12,18 +13,28 @@ module Autoproj
                                              "installation"
                             sleep 5
                         end
-                        return execute(command_line, false, with_root, env: env)
+                        return execute(command_line, false, with_root,
+                                       env: env, inherit: inherit)
                     ensure
                         lock_io.flock(File::LOCK_UN)
                     end
                 end
 
+                process_env = env
                 if with_root
-                    sudo = Autobuild.tool_in_path("sudo", env: env)
-                    command_line = [sudo, *command_line]
+                    process_env = Autobuild::Environment.new
+                    process_env.isolate
+                    process_env.add_path("PATH", "/usr/local/sbin",
+                                         "/usr/sbin", "/sbin")
+
+                    inherit.each { |var| process_env.set(var, env[var]) }
+                    sudo = Autobuild.tool_in_path("sudo", env: process_env)
+                    command_line = [sudo, "--preserve-env", *command_line]
                 end
 
-                Autobuild::Subprocess.run "autoproj", "osdeps", *command_line
+                Autobuild::Subprocess.run "autoproj", "osdeps", *command_line,
+                                          env: process_env.resolved_env,
+                                          env_inherit: false
             end
 
             # Overrides the {#needs_locking?} flag
@@ -194,7 +205,8 @@ module Autoproj
             # @return [Boolean] true if packages got installed, false otherwise
             def install(packages, filter_uptodate_packages: false, install_only: false,
                 auto_install_cmd: self.auto_install_cmd,
-                user_install_cmd: self.user_install_cmd)
+                user_install_cmd: self.user_install_cmd,
+                inherit: Set.new)
                 return if packages.empty?
 
                 handled_os = ws.supported_operating_system?
@@ -218,7 +230,7 @@ module Autoproj
 
                     ShellScriptManager.execute(
                         [*auto_install_cmd, *packages], needs_locking?,
-                        needs_root?, env: ws.env
+                        needs_root?, env: ws.env, inherit: inherit
                     )
                     return true
                 end

data/lib/autoproj/package_manifest.rb

@@ -17,7 +17,7 @@ module Autoproj
         # @return [PackageManifest]
         # @see parse
         def self.load(package, file, ros_manifest: false)
-            loader_class = ros_manifest ? RosLoader : Loader
+            loader_class = ros_manifest ? RosPackageManifest::Loader : Loader
             parse(package, File.read(file), path: file, loader_class: loader_class)
         end
 
@@ -32,7 +32,7 @@ module Autoproj
         # @see load
         def self.parse(package, contents,
             path: "<loaded from string>", loader_class: Loader)
-            manifest = PackageManifest.new(package, path)
+            manifest = loader_class::MANIFEST_CLASS.new(package, path)
             loader = loader_class.new(path, manifest)
             begin
                 REXML::Document.parse_stream(contents, loader)
@@ -167,6 +167,7 @@ module Autoproj
             end
 
             def text(text)
+                @tag_text = @tag_text.dup
                 @tag_text << text if @tag_text
             end
         end
@@ -178,6 +179,8 @@ module Autoproj
         class Loader < BaseLoader
             attr_reader :path, :manifest
 
+            MANIFEST_CLASS = PackageManifest
+
             def initialize(path, manifest)
                 super()
                 @path = path
@@ -249,55 +252,5 @@ module Autoproj
                 @tag_text = nil
             end
         end
-
-        # @api private
-        #
-        # REXML stream parser object used to load the XML contents into a
-        # {PackageManifest} object
-        class RosLoader < Loader
-            SUPPORTED_MODES = %w[test doc].freeze
-            DEPEND_TAGS = %w[depend build_depend build_export_depend
-                             buildtool_depend buildtool_export_depend
-                             exec_depend test_depend run_depend doc_depend].to_set.freeze
-
-            def toplevel_tag_start(name, attributes)
-                if DEPEND_TAGS.include?(name)
-                    @tag_text = ""
-                elsif TEXT_FIELDS.include?(name)
-                    @tag_text = ""
-                elsif AUTHOR_FIELDS.include?(name)
-                    @author_email = attributes["email"]
-                    @tag_text = ""
-                else
-                    @tag_text = nil
-                end
-            end
-
-            def toplevel_tag_end(name)
-                if DEPEND_TAGS.include?(name)
-                    if @tag_text.strip.empty?
-                        raise InvalidPackageManifest, "found '#{name}' tag in #{path} "\
-                                                      "without content"
-                    end
-
-                    mode = []
-                    if (m = /^(\w+)_depend$/.match(name))
-                        mode = SUPPORTED_MODES & [m[1]]
-                    end
-
-                    manifest.add_dependency(@tag_text, modes: mode)
-                elsif AUTHOR_FIELDS.include?(name)
-                    author_name = @tag_text.strip
-                    email = @author_email ? @author_email.strip : nil
-                    email = nil if email&.empty?
-                    contact = ContactInfo.new(author_name, email)
-                    manifest.send("#{name}s").concat([contact])
-                elsif TEXT_FIELDS.include?(name)
-                    field = @tag_text.strip
-                    manifest.send("#{name}=", field) unless field.empty?
-                end
-                @tag_text = nil
-            end
-        end
     end
 end

data/lib/autoproj/python.rb

@@ -168,18 +168,18 @@ module Autoproj
             end
         end
 
-        def self.remove_python_shims(root_dir)
-            shim_path = File.join(root_dir, "install", "bin", "python")
+        def self.remove_python_shims(prefix_dir)
+            shim_path = File.join(prefix_dir, "bin", "python")
             FileUtils.rm shim_path if File.exist?(shim_path)
         end
 
-        def self.remove_pip_shims(root_dir)
-            shim_path = File.join(root_dir, "install", "bin", "pip")
+        def self.remove_pip_shims(prefix_dir)
+            shim_path = File.join(prefix_dir, "bin", "pip")
             FileUtils.rm shim_path if File.exist?(shim_path)
         end
 
-        def self.rewrite_python_shims(python_executable, root_dir)
-            shim_path = File.join(root_dir, "install", "bin")
+        def self.rewrite_python_shims(python_executable, prefix_dir)
+            shim_path = File.join(prefix_dir, "bin")
             unless File.exist?(shim_path)
                 FileUtils.mkdir_p shim_path
                 Autoproj.warn "Autoproj::Python.rewrite_python_shims: creating "\
@@ -196,8 +196,8 @@ module Autoproj
             python_path
         end
 
-        def self.rewrite_pip_shims(python_executable, root_dir)
-            shim_path = File.join(root_dir, "install", "bin")
+        def self.rewrite_pip_shims(python_executable, prefix_dir)
+            shim_path = File.join(prefix_dir, "bin")
             unless File.exist?(shim_path)
                 FileUtils.mkdir_p shim_path
                 Autoproj.warn "Autoproj::Python.rewrite_pip_shims: creating "\
@@ -224,14 +224,14 @@ module Autoproj
 
             ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./
 
-            rewrite_python_shims(bin, ws.root_dir)
-            rewrite_pip_shims(bin, ws.root_dir)
+            rewrite_python_shims(bin, ws.dot_autoproj_dir)
+            rewrite_pip_shims(bin, ws.dot_autoproj_dir)
             [bin, version]
         end
 
         def self.deactivate_python(ws: Autoproj.workspace)
-            remove_python_shims(ws.root_dir)
-            remove_pip_shims(ws.root_dir)
+            remove_python_shims(ws.dot_autoproj_dir)
+            remove_pip_shims(ws.dot_autoproj_dir)
             ws.config.reset("python_executable")
             ws.config.reset("python_version")
         end
@@ -260,6 +260,18 @@ module Autoproj
             [bin, version, path]
         end
 
+        def self.assert_python_activated(ws: Autoproj.workspace)
+            return true if ws.config.get("USE_PYTHON")
+
+            raise ConfigError,
+                  "Your current package selection requires the use of python," \
+                  " but this is either unspecified or has been denied,"\
+                  " see setting of USE_PYTHON in your workspace configuration." \
+                  " Either remove all packages depending on pip packages " \
+                  " from the workspace layout (manifest) or " \
+                  " call 'autoproj reconfigure' to change the setting."
+        end
+
         def self.setup_python_configuration_options(ws: Autoproj.workspace)
             ws.config.declare "USE_PYTHON", "boolean",
                               default: "no",
@@ -267,8 +279,8 @@ module Autoproj
 
             if ws.config.get("USE_PYTHON")
                 unless ws.config.has_value_for?("python_executable")
-                    remove_python_shims(ws.root_dir)
-                    remove_pip_shims(ws.root_dir)
+                    remove_python_shims(ws.dot_autoproj_dir)
+                    remove_pip_shims(ws.dot_autoproj_dir)
                     python_bin, = auto_resolve_python(ws: ws)
                 end
 

data/lib/autoproj/repository_managers/apt.rb

@@ -173,7 +173,7 @@ module Autoproj
                         id
                     )
                 else
-                    open(origin) do |io|
+                    URI(origin).open do |io|
                         Autobuild::Subprocess.run(
                             "autoproj",
                             "osrepos",

data/lib/autoproj/ros_condition_parser.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "parslet"
+require "autoproj/exceptions"
+require "autoproj/variable_expansion"
+
+module Autoproj
+    # Parses a conditional expression
+    # Syntax and rules as defined in https://www.ros.org/reps/rep-0149.html#id20
+    class RosConditionParser < Parslet::Parser
+        def initialize(context)
+            @context = context
+            super()
+        end
+
+        def expand(var)
+            Autoproj.expand(var, @context)
+        rescue StandardError
+            ""
+        end
+
+        # Evaluates the Abstract Syntax Tree generated by the parser
+        class Transform < Parslet::Transform
+            rule(literal: simple(:x)) { String(x) }
+            rule(variable: simple(:x)) { expander.call(String(x)) }
+            rule(op: simple(:o), lhs: simple(:l), rhs: simple(:r)) { l.send(o, r) }
+            rule(op: "and", lhs: simple(:l), rhs: simple(:r)) { l && r }
+            rule(op: "or", lhs: simple(:l), rhs: simple(:r)) { l || r }
+        end
+
+        def evaluate(expr)
+            Transform.new.apply(parse(expr.strip), expander: method(:expand))
+        rescue Parslet::ParseFailed => e
+            raise Autoproj::ConfigError, e.message
+        end
+
+        def any_of(strings)
+            strings = strings.dup
+            strings.reduce(str(strings.shift)) { |acc, s| acc | str(s) }
+        end
+
+        def quoted_literal(quote)
+            str(quote) >> (
+                str(quote).absent? >> any
+            ).repeat.maybe.as(:literal) >> str(quote)
+        end
+
+        def chain(lhs, operator, operation)
+            (lhs.as(:lhs) >> operator >> operation.as(:rhs)) | lhs
+        end
+
+        RESERVED = %w[and or].freeze
+
+        rule(:space) { match['\s\t'].repeat(1) }
+        rule(:space?) { space.maybe }
+        rule(:lparen) { str("(") >> space? }
+        rule(:rparen) { str(")") >> space? }
+        rule(:comp_operator) { any_of(%w[== != >= <= > <]).as(:op) >> space? }
+        rule(:and_operator) { str("and").as(:op) >> space? }
+        rule(:or_operator)  { str("or").as(:op) >> space? }
+        rule(:literal) do
+            (any_of(RESERVED) >> space).absent? >>
+                (match('\w') | match["_-"]).repeat(1)
+                                           .as(:literal)
+        end
+        rule(:double_quote) { quoted_literal('"') }
+        rule(:single_quote) { quoted_literal("'") }
+        rule(:variable) do
+            (
+                str("$") >> match('\d').absent? >> (
+                    match('\w') | match["_"]
+                ).repeat(1)
+            ).as(:variable)
+        end
+
+        rule(:term) { (literal | single_quote | double_quote | variable) >> space? }
+        rule(:primary) { (lparen >> or_operation >> rparen) | term }
+        rule(:comp_operation) { chain(primary, comp_operator, comp_operation) }
+        rule(:and_operation) { chain(comp_operation, and_operator, and_operation) }
+        rule(:or_operation) { chain(and_operation, or_operator, or_operation) }
+
+        root(:or_operation)
+    end
+end

data/lib/autoproj/ros_package_manifest.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Autoproj
+    # Access to the information contained in a package's package.xml file
+    #
+    # Use PackageManifest.load to create
+    class RosPackageManifest < PackageManifest
+        attr_accessor :name
+        attr_writer :build_type
+
+        def build_type
+            @build_type || "catkin"
+        end
+
+        # @api private
+        #
+        # REXML stream parser object used to load the XML contents into a
+        # {PackageManifest} object
+        class Loader < PackageManifest::Loader
+            MANIFEST_CLASS = RosPackageManifest
+            SUPPORTED_MODES = %w[test doc].freeze
+            DEPEND_TAGS = %w[depend build_depend build_export_depend
+                             buildtool_depend buildtool_export_depend
+                             exec_depend test_depend run_depend doc_depend].to_set.freeze
+
+            def initialize(path, manifest)
+                super
+                @env = manifest.package.ws.env
+                @condition_parser = RosConditionParser.new(@env)
+            end
+
+            def tag_start(name, attributes)
+                super
+                exportlevel_tag_start(name, attributes) if @export_level
+            end
+
+            def tag_end(name)
+                super
+                exportlevel_tag_end(name) if @export_level
+                if @tag_level == 0 && name == "package" &&
+                   (!manifest.name || manifest.name.empty?)
+                    raise InvalidPackageManifest, "Package name missiing in #{path}"
+                end
+            end
+
+            def exportlevel_tag_start(name, attributes)
+                return unless name == "build_type"
+
+                @build_type_condition = attributes["condition"]
+                @tag_text = ""
+            end
+
+            def exportlevel_tag_end(name)
+                return unless name == "build_type"
+                return unless handle_condition(@build_type_condition)
+
+                manifest.build_type = @tag_text.strip
+            end
+
+            def toplevel_tag_start(name, attributes)
+                if DEPEND_TAGS.include?(name)
+                    @depend_condition = attributes["condition"]
+                    @tag_text = ""
+                elsif TEXT_FIELDS.include?(name)
+                    @tag_text = ""
+                elsif AUTHOR_FIELDS.include?(name)
+                    @author_email = attributes["email"]
+                    @tag_text = ""
+                elsif name == "name"
+                    @tag_text = ""
+                elsif name == "export"
+                    @export_level = true
+                else
+                    @tag_text = nil
+                end
+            end
+
+            def handle_condition(expr)
+                return true unless expr && !expr.empty?
+
+                @condition_parser.evaluate(expr)
+            end
+
+            def depend_tag_end(name)
+                return unless handle_condition(@depend_condition)
+
+                if @tag_text.strip.empty?
+                    raise InvalidPackageManifest, "found '#{name}' tag in #{path} "\
+                                                  "without content"
+                end
+
+                mode = []
+                if (m = /^(\w+)_depend$/.match(name))
+                    mode = SUPPORTED_MODES & [m[1]]
+                end
+
+                manifest.add_dependency(@tag_text, modes: mode)
+            end
+
+            def author_tag_end(name)
+                author_name = @tag_text.strip
+                email = @author_email ? @author_email.strip : nil
+                email = nil if email&.empty?
+                contact = PackageManifest::ContactInfo.new(author_name, email)
+                manifest.send("#{name}s").concat([contact])
+            end
+
+            def toplevel_tag_end(name)
+                if DEPEND_TAGS.include?(name)
+                    depend_tag_end(name)
+                elsif AUTHOR_FIELDS.include?(name)
+                    author_tag_end(name)
+                elsif TEXT_FIELDS.include?(name)
+                    field = @tag_text.strip
+                    manifest.send("#{name}=", field) unless field.empty?
+                elsif name == "name"
+                    manifest.name = @tag_text.strip
+                elsif name == "export"
+                    @export_level = false
+                end
+                @tag_text = nil
+            end
+        end
+    end
+end

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "2.15.0"
+    VERSION = "2.15.3"
 end

data/lib/autoproj.rb

@@ -27,7 +27,9 @@ require "autoproj/package_definition"
 require "autoproj/package_selection"
 require "autoproj/metapackage"
 require "autoproj/manifest"
+require "autoproj/ros_condition_parser"
 require "autoproj/package_manifest"
+require "autoproj/ros_package_manifest"
 require "autoproj/installation_manifest"
 require "autoproj/os_package_installer"
 require "autoproj/os_package_resolver"
@@ -61,7 +63,7 @@ require "utilrb/logger"
 
 module Autoproj
     class << self
-        attr_reader :logger
+        attr_accessor :logger
     end
     @logger = Logger.new(STDOUT)
     logger.level = Logger::WARN