autoproj 2.11.0 → 2.14.0

data/lib/autoproj/package_set.rb

@@ -168,7 +168,7 @@ module Autoproj
             @version_control = Array.new
             @overrides = Array.new
             @raw_local_dir = raw_local_dir
-            @default_importer = VCSDefinition.from_raw(type: 'none')
+            @default_importer = VCSDefinition.from_raw({ type: 'none' })
 
             @imports  = Set.new
             @imports_vcs  = Array.new
@@ -239,7 +239,7 @@ module Autoproj
             else
                 package = create_autobuild_package
                 if package.importer.respond_to?(:snapshot)
-                    package.importer.snapshot(package, target_dir, options)
+                    package.importer.snapshot(package, target_dir, **options)
                 end
             end
         end

data/lib/autoproj/python.rb

@@ -0,0 +1,285 @@
+require 'open3'
+require 'rubygems'
+
+module Autoproj
+    module Python
+        # Get the python version for a given python executable
+        # @return [String] The python version as <major>.<minor>
+        def self.get_python_version(python_bin)
+            unless File.exist?(python_bin)
+                raise ArgumentError, "Autoproj::Python.get_python_version executable "\
+                            "'#{python_bin}' does not exist"
+            end
+
+            cmd = "#{python_bin} -c \"import sys;"\
+                "version=sys.version_info[:3]; "\
+                "print('{0}.{1}'.format(*version))\"".strip
+
+            msg, status = Open3.capture2e(cmd)
+            if status.success?
+                msg.strip
+
+            else
+                raise "Autoproj::Python.get_python_version identification"\
+                    " of python version for '#{python_bin}' failed: #{msg}"
+            end
+        end
+
+        def self.get_pip_version(pip_bin)
+            unless File.exist?(pip_bin)
+                raise ArgumentError, "Autoproj::Python.get_pip_version executable "\
+                            "'#{pip_bin}' does not exist"
+            end
+
+            cmd = "#{pip_bin} --version"
+
+            msg, status = Open3.capture2e(cmd)
+            if status.success?
+                msg.split(" ")[1]
+
+            else
+                raise "Autoproj::Python.get_pip_version identification"\
+                    " of pip version for '#{pip_bin}' failed: #{msg}"
+            end
+        end
+
+        def self.validate_version(version, version_constraint)
+            if !version_constraint
+                true
+            else
+                dependency = Gem::Dependency.new("python", version_constraint)
+                dependency.match?("python", version)
+            end
+        end
+
+        # Validate that a given python executable's version fulfills
+        # a given version constraint
+        # @param [String] python_bin the python executable
+        # @param [String] version_constraint version constraint, e.g., <3.8, >= 3.7, 3.6
+        # @return [String,Bool] Version and validation result, i.e.,
+        #         True if binary fulfills the version constraint, false otherwise
+        def self.validate_python_version(python_bin, version_constraint)
+            version = get_python_version(python_bin)
+            [version, validate_version(version, version_constraint)]
+        end
+
+        # Find python given a version constraint
+        # @return [String,String] path to python executable and python version
+        def self.find_python(ws: Autoproj.workspace,
+                             version: ws.config.get('python_version', nil))
+            finders = [
+                -> { Autobuild.programs['python'] },
+                -> { `which python3`.strip },
+                -> { `which python`.strip }
+            ]
+
+            finders.each do |finder|
+                python_bin = finder.call
+                if python_bin && !python_bin.empty?
+                    python_version, valid = validate_python_version(python_bin, version)
+                    return python_bin, python_version if valid
+                end
+            end
+            raise "Autoproj::Python.find_python_bin: failed to find python" \
+                " for version '#{version}'"
+        end
+
+        # Get information about the python executable from autoproj config,
+        # but ensure the version constraint matches
+        #
+        # @return [String, String] Return path and version if the constraints
+        #      are fulfilled nil otherwise
+
+        def self.get_python_from_config(ws: Autoproj.workspace, version: nil)
+            config_bin = ws.config.get('python_executable', nil)
+            return unless config_bin
+
+            config_version = ws.config.get('python_version', nil)
+            config_version ||= get_python_version(config_bin)
+
+            # If a version constraint is given, ensure fulfillment
+            if validate_version(config_version, version)
+                [config_bin, config_version]
+            else
+                raise "python_executable in autoproj config with " \
+                  "version '#{config_version}' does not match "\
+                  "version constraints '#{version}'"
+            end
+        end
+
+        def self.custom_resolve_python(bin: nil,
+                                       version: nil)
+            version, valid = validate_python_version(bin, version)
+            if valid
+                [bin, version]
+            else
+                raise "Autoproj::Python.resolve_python: requested python"\
+                    "executable '#{bin}' does not satisfy version"\
+                    "constraints '#{version}'"
+            end
+        end
+
+        def self.auto_resolve_python(ws: Autoproj.workspace,
+                                     version: nil)
+            version_constraint = version
+            resolvers = [
+                -> { get_python_from_config(ws: ws, version: version_constraint) },
+                -> { find_python(ws: ws, version: version_constraint) }
+            ]
+
+            bin = nil
+            resolvers.each do |resolver|
+                    bin, version = resolver.call
+                    if bin && File.exist?(bin) && version
+                        Autoproj.debug "Autoproj::Python.resolve_python: " \
+                          "found python '#{bin}' version '#{version}'"
+                        break
+                    end
+            rescue RuntimeError => e
+                    Autoproj.debug "Autoproj::Python.resolve_python: " \
+                      "resolver failed: #{e}"
+            end
+
+            unless bin
+                msg = "Autoproj::Python.resolve_python: " \
+                      "failed to find a python executable"
+                if version_constraint
+                    msg += " satisfying version constraint '#{version_constraint}'"
+                end
+                raise msg
+            end
+            [bin, version]
+        end
+
+        # Resolve the python executable according to a given version constraint
+        # @param [Autoproj.workspace] ws Autoproj workspace
+        # @param [String] bin Path to the python executable that shall be used,
+        #   first fallback is the python_executable set in Autoproj's configuration,
+        #   second fallback is a full search
+        # @param [String] version version constraint
+        # @return [String,String] python path and python version
+        def self.resolve_python(ws: Autoproj.workspace,
+                                bin: nil,
+                                version: nil)
+            if bin
+                custom_resolve_python(bin: bin, version: version)
+            else
+                auto_resolve_python(ws: ws, version: version)
+            end
+        end
+
+        def self.remove_python_shims(root_dir)
+            shim_path = File.join(root_dir, "install", "bin", "python")
+            FileUtils.rm shim_path if File.exist?(shim_path)
+        end
+
+        def self.remove_pip_shims(root_dir)
+            shim_path = File.join(root_dir, "install", "bin", "pip")
+            FileUtils.rm shim_path if File.exist?(shim_path)
+        end
+
+        def self.rewrite_python_shims(python_executable, root_dir)
+            shim_path = File.join(root_dir, "install", "bin")
+            unless File.exist?(shim_path)
+                FileUtils.mkdir_p shim_path
+                Autoproj.warn "Autoproj::Python.rewrite_python_shims: creating "\
+                    "#{shim_path} - "\
+                    "are you operating on a valid autoproj workspace?"
+            end
+
+            python_path = File.join(shim_path, 'python')
+            File.open(python_path, 'w') do |io|
+                io.puts "#! /bin/sh"
+                io.puts "exec #{python_executable} \"$@\""
+            end
+            FileUtils.chmod 0o755, python_path
+            python_path
+        end
+
+        def self.rewrite_pip_shims(python_executable, root_dir)
+            shim_path = File.join(root_dir, "install", "bin")
+            unless File.exist?(shim_path)
+                FileUtils.mkdir_p shim_path
+                Autoproj.warn "Autoproj::Python.rewrite_pip_shims: creating "\
+                    "#{shim_path} - "\
+                    "are you operating on a valid autoproj workspace?"
+            end
+            pip_path = File.join(shim_path, 'pip')
+            File.open(pip_path, 'w') do |io|
+                io.puts "#! /bin/sh"
+                io.puts "exec #{python_executable} -m pip \"$@\""
+            end
+            FileUtils.chmod 0o755, pip_path
+            pip_path
+        end
+
+        # Activate configuration for python in the autoproj configuration
+        # @return [String,String] python path and python version
+        def self.activate_python(ws: Autoproj.workspace,
+                                 bin: nil,
+                                 version: nil)
+            bin, version = resolve_python(ws: ws, bin: bin, version: version)
+            ws.config.set('python_executable', bin, true)
+            ws.config.set('python_version', version, true)
+
+            ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./
+
+            rewrite_python_shims(bin, ws.root_dir)
+            rewrite_pip_shims(bin, ws.root_dir)
+            [bin, version]
+        end
+
+        def self.deactivate_python(ws: Autoproj.workspace)
+            remove_python_shims(ws.root_dir)
+            remove_pip_shims(ws.root_dir)
+            ws.config.reset('python_executable')
+            ws.config.reset('python_version')
+        end
+
+        # Allow to update the PYTHONPATH for package if autoproj configuration
+        # USE_PYTHON is set to true.
+        # Then tries to guess the python binary from Autobuild.programs['python']
+        # and system's default setting
+        # @param [Autobuild::Package] pkg
+        # @param [Autoproj.workspace] ws Autoproj workspace
+        # @param [String] bin Path to a custom python version
+        # @param [String] version version constraint for python executable
+        # @return tuple of [executable, version, site-packages path] if set,
+        #    otherwise nil
+        def self.activate_python_path(pkg,
+                                 ws: Autoproj.workspace,
+                                 bin: nil,
+                                 version: nil)
+            return unless ws.config.get('USE_PYTHON', nil)
+
+            bin, version = resolve_python(ws: ws, bin: bin, version: version)
+            path = File.join(pkg.prefix, "lib",
+                             "python#{version}", "site-packages")
+            pkg.env_add_path 'PYTHONPATH', path
+
+            [bin, version, path]
+        end
+
+        def self.setup_python_configuration_options(ws: Autoproj.workspace)
+            ws.config.declare 'USE_PYTHON', 'boolean',
+                              default: 'no',
+                              doc: ["Do you want to activate python?"]
+
+            if ws.config.get("USE_PYTHON")
+                unless ws.config.has_value_for?('python_executable')
+                    remove_python_shims(ws.root_dir)
+                    remove_pip_shims(ws.root_dir)
+                    python_bin, = auto_resolve_python(ws: ws)
+                end
+
+                ws.config.declare 'python_executable', 'string',
+                                  default: python_bin.to_s,
+                                  doc: ["Select the path to the python executable"]
+
+                activate_python(ws: ws)
+            else
+                deactivate_python(ws: ws)
+            end
+        end
+    end
+end

data/lib/autoproj/test.rb

@@ -71,6 +71,7 @@ module Autoproj
             Autobuild.logdir = make_tmpdir
             ws_define_package_manager 'os'
             ws_define_package_manager 'os_indep'
+            Autobuild.progress_display_period = 0
 
             super
         end
@@ -80,6 +81,7 @@ module Autoproj
             @tmpdir.each do |dir|
                 FileUtils.remove_entry_secure dir
             end
+            Rake::Task.clear
             Autobuild::Package.clear
             Autoproj.silent = false
 
@@ -94,6 +96,11 @@ module Autoproj
             end
         end
 
+        def data_path(*args)
+            File.expand_path(File.join(*args),
+                             File.join(__dir__, '..', '..', 'test', 'data'))
+        end
+
         def create_bootstrap
             ws_create
         end
@@ -170,7 +177,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
                     'PACKAGE_BASE_DIR' => package_base_dir,
                     'RUBY' => Gem.ruby
                 ]
-                result = Bundler.clean_system(
+                result = Autoproj.bundler_unbundled_system(
                     default_env.merge(env), script, *arguments,
                     in: :close, **Hash[chdir: dir].merge(system_options))
             end
@@ -201,7 +208,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
             cached_bundler_gem = File.join(vendor, bundler_filename)
             unless File.file?(cached_bundler_gem)
                 FileUtils.mkdir_p vendor
-                Bundler.clean_system(
+                Autoproj.bundler_unbundled_system(
                     Ops::Install.guess_gem_program, 'fetch', '-v',
                     Bundler::VERSION, 'bundler', chdir: vendor)
 
@@ -211,7 +218,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
             end
 
             capture_subprocess_io do
-                Bundler.clean_system(
+                Autoproj.bundler_unbundled_system(
                     Hash['GEM_HOME' => fixture_gem_home, 'GEM_PATH' => nil],
                     Ops::Install.guess_gem_program, 'install', '--no-document',
                     cached_bundler_gem)
@@ -256,7 +263,7 @@ gem 'autobuild', path: '#{autobuild_dir}'
 
         def find_bundled_gem_path(bundler, gem_name, gemfile)
             out_r, out_w = IO.pipe
-            result = Bundler.clean_system(
+            result = Autoproj.bundler_unbundled_system(
                 bundler, 'show', gem_name,
                 out: out_w,
                 chdir: File.dirname(gemfile))
@@ -326,8 +333,9 @@ gem 'autobuild', path: '#{autobuild_dir}'
             ws.manifest.clear_layout
         end
 
-        def ws_define_package_set(name, vcs = VCSDefinition.from_raw(type: 'none'),
-                                  **options)
+        def ws_define_package_set(
+            name, vcs = VCSDefinition.from_raw({ type: 'none' }), **options
+        )
             package_set = PackageSet.new(ws, vcs, name: name, **options)
             ws.manifest.register_package_set(package_set)
             package_set
@@ -344,8 +352,9 @@ gem 'autobuild', path: '#{autobuild_dir}'
             package_set
         end
 
-        def ws_add_package_set_to_layout(name, vcs = VCSDefinition.from_raw(type: 'none'),
-                                         **options)
+        def ws_add_package_set_to_layout(
+            name, vcs = VCSDefinition.from_raw({ type: 'none' }), **options
+        )
             package_set = ws_define_package_set(name, vcs, **options)
             ws.manifest.add_package_set_to_layout(package_set)
             package_set
@@ -370,9 +379,16 @@ gem 'autobuild', path: '#{autobuild_dir}'
 
         def ws_define_package(package_type, package_name,
                               package_set: ws.manifest.main_package_set,
-                              create: true)
+                              create: true, &block)
             package = Autobuild.send(package_type, package_name)
-            package.srcdir = File.join(ws.root_dir, package_name.to_s)
+            ws_setup_package(
+                package, package_set: package_set, create: create, &block
+            )
+        end
+
+        def ws_setup_package(package, package_set: ws.manifest.main_package_set,
+                                      create: true)
+            package.srcdir = File.join(ws.root_dir, package.name.to_s)
             FileUtils.mkdir_p package.srcdir if create
             autoproj_package = ws.register_package(package, nil, package_set)
             yield(package) if block_given?

data/lib/autoproj/variable_expansion.rb

@@ -80,7 +80,9 @@ module Autoproj
     end
 
     # True if the given string contains expansions
-    def self.contains_expansion?(string); string =~ /\$/ end
+    def self.contains_expansion?(string)
+        string.respond_to?(:to_str) && string.to_str =~ /\$/
+    end
 
     def self.resolve_one_constant(name, value, result, definitions)
         result[name] ||= single_expansion(value, result) do |missing_name|

data/lib/autoproj/vcs_definition.rb

@@ -33,7 +33,7 @@ module Autoproj
         HistoryEntry  = Struct.new :package_set, :vcs
         RawEntry      = Struct.new :package_set, :file, :vcs
 
-        def initialize(type, url, vcs_options, from: nil, raw: Array.new, history: Array.new)
+        def initialize(type, url, vcs_options, from: nil, raw: [], history: [])
             if !raw.respond_to?(:to_ary)
                 raise ArgumentError, "wrong format for the raw field (#{raw.inspect})"
             end
@@ -49,7 +49,7 @@ module Autoproj
 
         # Create a null VCS object
         def self.none
-            from_raw(type: 'none')
+            from_raw({ type: 'none' })
         end
 
         # Whether there is actually a version control definition
@@ -151,10 +151,14 @@ module Autoproj
                         elsif base_dir
                             File.expand_path(short_url, base_dir)
                         else
-                            raise ArgumentError, "VCS path '#{short_url}' is relative and no base_dir was given"
+                            raise ArgumentError,
+                                  "VCS path '#{short_url}' is relative and no "\
+                                  "base_dir was given"
                         end
                     if !File.directory?(source_dir)
-                        raise ArgumentError, "'#{short_url}' is neither a remote source specification, nor an existing local directory"
+                        raise ArgumentError,
+                              "'#{short_url}' is neither a remote source "\
+                              'specification, nor an existing local directory'
                     end
                     spec.merge!(type: 'local', url: source_dir)
                 end
@@ -195,17 +199,26 @@ module Autoproj
         # @return [VCSDefinition]
         # @raise ArgumentError if the raw specification does not match an
         #   expected format
-        def self.from_raw(spec, from: nil, raw: Array.new, history: Array.new)
+        def self.from_raw(spec, from: nil, raw: [], history: [])
             normalized_spec = normalize_vcs_hash(spec)
             if !(type = normalized_spec.delete(:type))
-                raise ArgumentError, "the source specification #{raw_spec_to_s(spec)} normalizes into #{raw_spec_to_s(normalized_spec)}, which does not have a VCS type"
-            elsif !(url  = normalized_spec.delete(:url))
-                if type != 'none'
-                    raise ArgumentError, "the source specification #{raw_spec_to_s(spec)} normalizes into #{raw_spec_to_s(normalized_spec)}, which does not have a URL. Only VCS of type 'none' do not require one"
-                end
+                raise ArgumentError,
+                      "the source specification #{raw_spec_to_s(spec)} normalizes "\
+                      "into #{raw_spec_to_s(normalized_spec)}, "\
+                      'which does not have a VCS type'
+            end
+
+            if !(url  = normalized_spec.delete(:url)) && type != 'none'
+                raise ArgumentError,
+                      "the source specification #{raw_spec_to_s(spec)} normalizes "\
+                      "into #{raw_spec_to_s(normalized_spec)}, "\
+                      'which does not have a URL. '\
+                      'Only VCS of type \'none\' do not require one'
             end
 
-            VCSDefinition.new(type, url, normalized_spec, from: from, history: history, raw: raw)
+            VCSDefinition.new(
+                type, url, normalized_spec, from: from, history: history, raw: raw
+            )
         end
 
         def ==(other_vcs)
@@ -307,7 +320,7 @@ module Autoproj
         if !handler
             raise ArgumentError, "there is no source handler for #{vcs}"
         else
-            return handler.call(url, options)
+            handler.call(url, **options)
         end
     end