autoproj 2.13.0 → 2.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d0594abdc1c9523788d70185cef7fee0094e70aa0aad5a7c06432e78fd3f2324
-  data.tar.gz: 7fbde174feb043143cf9d64a23517858982216bd15e33c659954085f5168af64
+  metadata.gz: 3f10cbf09589b7ad19d1ac9c21d4630c1f31e285a4ee7ac2db544fb6c23059e8
+  data.tar.gz: 3d50fe0556577a1e5ca7601417e1b8d8ebeb73f36e9ce0c4ba76853c813833b0
 SHA512:
-  metadata.gz: 82529e41b820b90048063019025470390718393f7479dfbf9714d4898abdc6d8a327306f1b672191f48e925b1af8c2e05e94583a4f9dde8a418f6aac11a99523
-  data.tar.gz: 9eeec00ec5b9aaaa565e85a65b2a07a22ba0701044657434d2a99c837174e0d87551a389f73d3ee55eff0f05ee2221b61c7f35e2b057c2fe5bbfbbe568015723
+  metadata.gz: 7306758a6842ab1175f70deeb8b69d5268a2145681cb0aff459bbd119e00292b8cd14e661ebae12964963ff2af6a0007c217cba020422b0445b873dad5f70ffd
+  data.tar.gz: cfaa2391a0dae4d77d41281fb6ebc7f2174c728ec532e330515a9da6b4d9235c14eb13100f35d17a863a7701a7fad338e7a40c920b9c9cf8e37f0ee2ecb78fa8

data/lib/autoproj/default.osdeps

@@ -166,7 +166,12 @@ cvs:
   default: cvs
 
 pip:
-  debian,ubuntu: python-pip
+  debian:
+    '8,jessie,9,stretch': python-pip
+    default: python3-pip
+  ubuntu:
+    '16.04,18.04': python-pip
+    default: python3-pip
   arch: python2-pip
   opensuse: python-pip
   fedora: python-pip
@@ -175,7 +180,12 @@ pip:
 
 python:
   arch: python2
-  debian,ubuntu: python-dev
+  debian:
+    '8,jessie,9,stretch': python-dev
+    default: python3-dev
+  ubuntu:
+    '16.04,18.04': python-dev
+    default: python3-dev
   fedora: python-devel
   freebsd: python-devel
   gentoo: dev-lang/python
@@ -184,7 +194,12 @@ python:
 
 python-setuptools:
   arch: python2-distribute
-  debian,ubuntu: python-setuptools
+  debian:
+     '8,jessie,9,stretch': python-setuptools
+     default: python3-setuptools
+  ubuntu:
+    '16.04,18.04': python-setuptools
+    default: python3-setuptools
   fedora: python-setuptools
   gentoo: dev-python/setuptools
   default: ignore # will be installed manually by the user

data/lib/autoproj/environment.rb

@@ -17,6 +17,22 @@ module Autoproj
             Autoproj.filter_out_paths_in_workspace(env)
         end
 
+        def env_filename(shell, *subdir)
+            env_filename = if shell == 'sh'
+                               ENV_FILENAME
+                           else
+                               (Pathname(ENV_FILENAME).sub_ext '').to_s.concat(".#{shell}")
+                           end
+
+            File.join(root_dir, *subdir, env_filename)
+        end
+
+        def each_env_filename(*subdir)
+            (['sh'] + Autoproj.workspace.config.user_shells).to_set.each do |shell|
+                yield shell, env_filename(shell, *subdir)
+            end
+        end
+
         def export_env_sh(subdir = nil, options = Hash.new)
             if subdir.kind_of?(Hash)
                 subdir, options = nil, subdir
@@ -28,19 +44,7 @@ module Autoproj
             completion_dir = File.join(shell_dir, 'completion')
             env_updated = false
 
-            (['sh'] + Autoproj.workspace.config.user_shells).to_set.each do |shell|
-                env_filename = if shell == 'sh'
-                                   ENV_FILENAME
-                               else
-                                   (Pathname(ENV_FILENAME).sub_ext '').to_s.concat(".#{shell}")
-                               end
-
-                filename = if subdir
-                    File.join(root_dir, subdir, env_filename)
-                else
-                    File.join(root_dir, env_filename)
-                end
-
+            each_env_filename(*[subdir].compact) do |shell, filename|
                 helper = File.join(shell_dir, "autoproj_#{shell}")
                 if options[:shell_helpers]
                     source_after(helper, shell: shell) if File.file?(helper)

data/lib/autoproj/manifest.rb

@@ -60,6 +60,11 @@ module Autoproj
                 YAML.safe_load(File.read(file)) || {}
             end
 
+            if data["layout"]&.member?(nil)
+                Autoproj.warn "There is an empty entry in your layout in #{file}. All empty entries are ignored."
+                data["layout"] = data["layout"].compact
+            end
+
             @file = file
             initialize_from_hash(data)
         end

data/lib/autoproj/package_managers/bundler_manager.rb

@@ -123,13 +123,14 @@ module Autoproj
             end
 
             def create_cache_symlink(cache_dir, bundler_cache_dir)
-                valid = !File.exist?(bundler_cache_dir) ||
-                        File.symlink?(bundler_cache_dir)
-
-                unless valid
-                    Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
-                                  "#{bundler_cache_dir} already exists"
-                    return
+                if File.exist?(bundler_cache_dir)
+                    if !File.symlink?(bundler_cache_dir)
+                        Autoproj.warn "cannot use #{cache_dir} as gem cache as "\
+                                      "#{bundler_cache_dir} already exists"
+                        return
+                    elsif File.readlink(bundler_cache_dir) == cache_dir
+                        return
+                    end
                 end
 
                 FileUtils.rm_f bundler_cache_dir

data/lib/autoproj/package_managers/pip_manager.rb

@@ -1,3 +1,5 @@
+require 'autoproj/python'
+
 module Autoproj
     module PackageManagers
         # Using pip to install python packages
@@ -25,30 +27,40 @@ module Autoproj
             end
 
             def guess_pip_program
-                if Autobuild.programs['pip']
-                    return Autobuild.programs['pip']
+                unless ws.config.has_value_for?('USE_PYTHON')
+                  Autoproj::Python.setup_python_configuration_options(ws: ws)
+                end
+                unless ws.config.get('USE_PYTHON')
+                  raise ConfigError, "Your current package selection" \
+                    " requires the use of pip, but" \
+                    " the use of python is either unspecified or has been denied, see" \
+                    " setting of USE_PYTHON in your workspace configuration." \
+                    " Either remove all packages depending on pip packages " \
+                    " from the workspace layout (manifest) or " \
+                    " call 'autoproj reconfigure' to change the setting."
+
                 end
 
-                Autobuild.programs['pip'] = "pip"
+                Autobuild.programs['pip'] = "pip" unless Autobuild.programs['pip']
+                Autobuild.programs['pip']
             end
 
+            # rubocop:disable Lint/UnusedMethodArgument
             def install(pips, filter_uptodate_packages: false, install_only: false)
                 guess_pip_program
-                if pips.is_a?(String)
-                    pips = [pips]
-                end
+                pips = [pips] if pips.is_a?(String)
 
-                base_cmdline = [Autobuild.tool('pip'), 'install','--user']
+                base_cmdline = [Autobuild.tool('pip'), 'install', '--user']
 
                 cmdlines = [base_cmdline + pips]
 
-                if pips_interaction(pips, cmdlines)
-                    Autoproj.message "  installing/updating Python dependencies: "+
-                        "#{pips.sort.join(", ")}"
+                if pips_interaction(cmdlines)
+                    Autoproj.message "  installing/updating Python dependencies:" \
+                        " #{pips.sort.join(', ')}"
 
                     cmdlines.each do |c|
                         Autobuild::Subprocess.run 'autoproj', 'osdeps', *c,
-                            env: ws.env.resolved_env
+                                                  env: ws.env.resolved_env
                     end
 
                     pips.each do |p|
@@ -56,8 +68,9 @@ module Autoproj
                     end
                 end
             end
-            
-            def pips_interaction(pips, cmdlines)
+            # rubocop:enable Lint/UnusedMethodArgument
+
+            def pips_interaction(cmdlines)
                 if OSPackageInstaller.force_osdeps
                     return true
                 elsif enabled?
@@ -69,24 +82,23 @@ module Autoproj
                 # We're not supposed to install rubygem packages but silent is not
                 # set, so display information about them anyway
                 puts <<-EOMSG
-      #{Autoproj.color("The build process and/or the packages require some Python packages to be installed", :bold)}
-      #{Autoproj.color("and you required autoproj to not do it itself", :bold)}
+      #{Autoproj.color('The build process and/or the packages require some Python packages to be installed', :bold)}
+      #{Autoproj.color('and you required autoproj to not do it itself', :bold)}
         The following command line can be used to install them manually
-        
-          #{cmdlines.map { |c| c.join(" ") }.join("\n      ")}
-        
+#{'        '}
+          #{cmdlines.map { |c| c.join(' ') }.join("\n      ")}
+#{'        '}
         Autoproj expects these Python packages to be installed in #{pip_home} This can
         be overridden by setting the AUTOPROJ_PYTHONUSERBASE environment variable manually
 
                 EOMSG
-                print "    #{Autoproj.color("Press ENTER to continue ", :bold)}"
+                print "    #{Autoproj.color('Press ENTER to continue ', :bold)}"
 
-                STDOUT.flush
-                STDIN.readline
+                $stdout.flush
+                $stdin.readline
                 puts
                 false
             end
         end
     end
 end
-

data/lib/autoproj/python.rb

@@ -0,0 +1,285 @@
+require 'open3'
+require 'rubygems'
+
+module Autoproj
+    module Python
+        # Get the python version for a given python executable
+        # @return [String] The python version as <major>.<minor>
+        def self.get_python_version(python_bin)
+            unless File.exist?(python_bin)
+                raise ArgumentError, "Autoproj::Python.get_python_version executable "\
+                            "'#{python_bin}' does not exist"
+            end
+
+            cmd = "#{python_bin} -c \"import sys;"\
+                "version=sys.version_info[:3]; "\
+                "print('{0}.{1}'.format(*version))\"".strip
+
+            msg, status = Open3.capture2e(cmd)
+            if status.success?
+                msg.strip
+
+            else
+                raise "Autoproj::Python.get_python_version identification"\
+                    " of python version for '#{python_bin}' failed: #{msg}"
+            end
+        end
+
+        def self.get_pip_version(pip_bin)
+            unless File.exist?(pip_bin)
+                raise ArgumentError, "Autoproj::Python.get_pip_version executable "\
+                            "'#{pip_bin}' does not exist"
+            end
+
+            cmd = "#{pip_bin} --version"
+
+            msg, status = Open3.capture2e(cmd)
+            if status.success?
+                msg.split(" ")[1]
+
+            else
+                raise "Autoproj::Python.get_pip_version identification"\
+                    " of pip version for '#{pip_bin}' failed: #{msg}"
+            end
+        end
+
+        def self.validate_version(version, version_constraint)
+            if !version_constraint
+                true
+            else
+                dependency = Gem::Dependency.new("python", version_constraint)
+                dependency.match?("python", version)
+            end
+        end
+
+        # Validate that a given python executable's version fulfills
+        # a given version constraint
+        # @param [String] python_bin the python executable
+        # @param [String] version_constraint version constraint, e.g., <3.8, >= 3.7, 3.6
+        # @return [String,Bool] Version and validation result, i.e.,
+        #         True if binary fulfills the version constraint, false otherwise
+        def self.validate_python_version(python_bin, version_constraint)
+            version = get_python_version(python_bin)
+            [version, validate_version(version, version_constraint)]
+        end
+
+        # Find python given a version constraint
+        # @return [String,String] path to python executable and python version
+        def self.find_python(ws: Autoproj.workspace,
+                             version: ws.config.get('python_version', nil))
+            finders = [
+                -> { Autobuild.programs['python'] },
+                -> { `which python3`.strip },
+                -> { `which python`.strip }
+            ]
+
+            finders.each do |finder|
+                python_bin = finder.call
+                if python_bin && !python_bin.empty?
+                    python_version, valid = validate_python_version(python_bin, version)
+                    return python_bin, python_version if valid
+                end
+            end
+            raise "Autoproj::Python.find_python_bin: failed to find python" \
+                " for version '#{version}'"
+        end
+
+        # Get information about the python executable from autoproj config,
+        # but ensure the version constraint matches
+        #
+        # @return [String, String] Return path and version if the constraints
+        #      are fulfilled nil otherwise
+
+        def self.get_python_from_config(ws: Autoproj.workspace, version: nil)
+            config_bin = ws.config.get('python_executable', nil)
+            return unless config_bin
+
+            config_version = ws.config.get('python_version', nil)
+            config_version ||= get_python_version(config_bin)
+
+            # If a version constraint is given, ensure fulfillment
+            if validate_version(config_version, version)
+                [config_bin, config_version]
+            else
+                raise "python_executable in autoproj config with " \
+                  "version '#{config_version}' does not match "\
+                  "version constraints '#{version}'"
+            end
+        end
+
+        def self.custom_resolve_python(bin: nil,
+                                       version: nil)
+            version, valid = validate_python_version(bin, version)
+            if valid
+                [bin, version]
+            else
+                raise "Autoproj::Python.resolve_python: requested python"\
+                    "executable '#{bin}' does not satisfy version"\
+                    "constraints '#{version}'"
+            end
+        end
+
+        def self.auto_resolve_python(ws: Autoproj.workspace,
+                                     version: nil)
+            version_constraint = version
+            resolvers = [
+                -> { get_python_from_config(ws: ws, version: version_constraint) },
+                -> { find_python(ws: ws, version: version_constraint) }
+            ]
+
+            bin = nil
+            resolvers.each do |resolver|
+                    bin, version = resolver.call
+                    if bin && File.exist?(bin) && version
+                        Autoproj.debug "Autoproj::Python.resolve_python: " \
+                          "found python '#{bin}' version '#{version}'"
+                        break
+                    end
+            rescue RuntimeError => e
+                    Autoproj.debug "Autoproj::Python.resolve_python: " \
+                      "resolver failed: #{e}"
+            end
+
+            unless bin
+                msg = "Autoproj::Python.resolve_python: " \
+                      "failed to find a python executable"
+                if version_constraint
+                    msg += " satisfying version constraint '#{version_constraint}'"
+                end
+                raise msg
+            end
+            [bin, version]
+        end
+
+        # Resolve the python executable according to a given version constraint
+        # @param [Autoproj.workspace] ws Autoproj workspace
+        # @param [String] bin Path to the python executable that shall be used,
+        #   first fallback is the python_executable set in Autoproj's configuration,
+        #   second fallback is a full search
+        # @param [String] version version constraint
+        # @return [String,String] python path and python version
+        def self.resolve_python(ws: Autoproj.workspace,
+                                bin: nil,
+                                version: nil)
+            if bin
+                custom_resolve_python(bin: bin, version: version)
+            else
+                auto_resolve_python(ws: ws, version: version)
+            end
+        end
+
+        def self.remove_python_shims(root_dir)
+            shim_path = File.join(root_dir, "install", "bin", "python")
+            FileUtils.rm shim_path if File.exist?(shim_path)
+        end
+
+        def self.remove_pip_shims(root_dir)
+            shim_path = File.join(root_dir, "install", "bin", "pip")
+            FileUtils.rm shim_path if File.exist?(shim_path)
+        end
+
+        def self.rewrite_python_shims(python_executable, root_dir)
+            shim_path = File.join(root_dir, "install", "bin")
+            unless File.exist?(shim_path)
+                FileUtils.mkdir_p shim_path
+                Autoproj.warn "Autoproj::Python.rewrite_python_shims: creating "\
+                    "#{shim_path} - "\
+                    "are you operating on a valid autoproj workspace?"
+            end
+
+            python_path = File.join(shim_path, 'python')
+            File.open(python_path, 'w') do |io|
+                io.puts "#! /bin/sh"
+                io.puts "exec #{python_executable} \"$@\""
+            end
+            FileUtils.chmod 0o755, python_path
+            python_path
+        end
+
+        def self.rewrite_pip_shims(python_executable, root_dir)
+            shim_path = File.join(root_dir, "install", "bin")
+            unless File.exist?(shim_path)
+                FileUtils.mkdir_p shim_path
+                Autoproj.warn "Autoproj::Python.rewrite_pip_shims: creating "\
+                    "#{shim_path} - "\
+                    "are you operating on a valid autoproj workspace?"
+            end
+            pip_path = File.join(shim_path, 'pip')
+            File.open(pip_path, 'w') do |io|
+                io.puts "#! /bin/sh"
+                io.puts "exec #{python_executable} -m pip \"$@\""
+            end
+            FileUtils.chmod 0o755, pip_path
+            pip_path
+        end
+
+        # Activate configuration for python in the autoproj configuration
+        # @return [String,String] python path and python version
+        def self.activate_python(ws: Autoproj.workspace,
+                                 bin: nil,
+                                 version: nil)
+            bin, version = resolve_python(ws: ws, bin: bin, version: version)
+            ws.config.set('python_executable', bin, true)
+            ws.config.set('python_version', version, true)
+
+            ws.osdep_suffixes << "python#{$1}" if version =~ /^([0-9]+)\./
+
+            rewrite_python_shims(bin, ws.root_dir)
+            rewrite_pip_shims(bin, ws.root_dir)
+            [bin, version]
+        end
+
+        def self.deactivate_python(ws: Autoproj.workspace)
+            remove_python_shims(ws.root_dir)
+            remove_pip_shims(ws.root_dir)
+            ws.config.reset('python_executable')
+            ws.config.reset('python_version')
+        end
+
+        # Allow to update the PYTHONPATH for package if autoproj configuration
+        # USE_PYTHON is set to true.
+        # Then tries to guess the python binary from Autobuild.programs['python']
+        # and system's default setting
+        # @param [Autobuild::Package] pkg
+        # @param [Autoproj.workspace] ws Autoproj workspace
+        # @param [String] bin Path to a custom python version
+        # @param [String] version version constraint for python executable
+        # @return tuple of [executable, version, site-packages path] if set,
+        #    otherwise nil
+        def self.activate_python_path(pkg,
+                                 ws: Autoproj.workspace,
+                                 bin: nil,
+                                 version: nil)
+            return unless ws.config.get('USE_PYTHON', nil)
+
+            bin, version = resolve_python(ws: ws, bin: bin, version: version)
+            path = File.join(pkg.prefix, "lib",
+                             "python#{version}", "site-packages")
+            pkg.env_add_path 'PYTHONPATH', path
+
+            [bin, version, path]
+        end
+
+        def self.setup_python_configuration_options(ws: Autoproj.workspace)
+            ws.config.declare 'USE_PYTHON', 'boolean',
+                              default: 'no',
+                              doc: ["Do you want to activate python?"]
+
+            if ws.config.get("USE_PYTHON")
+                unless ws.config.has_value_for?('python_executable')
+                    remove_python_shims(ws.root_dir)
+                    remove_pip_shims(ws.root_dir)
+                    python_bin, = auto_resolve_python(ws: ws)
+                end
+
+                ws.config.declare 'python_executable', 'string',
+                                  default: python_bin.to_s,
+                                  doc: ["Select the path to the python executable"]
+
+                activate_python(ws: ws)
+            else
+                deactivate_python(ws: ws)
+            end
+        end
+    end
+end

data/lib/autoproj/version.rb

@@ -1,3 +1,3 @@
 module Autoproj
-    VERSION = "2.13.0"
+    VERSION = "2.14.0"
 end

data/lib/autoproj/workspace.rb

@@ -348,7 +348,11 @@ module Autoproj
             osdep_suffixes << ruby_version_keyword
         end
 
-        def setup
+        # Perform initial configuration load and workspace setup
+        #
+        # @param [Boolean] load_global_configuration if true, load the global
+        #   autoprojrc file if it exists. Otherwise, ignore it.
+        def setup(load_global_configuration: true)
             setup_ruby_version_handling
             migrate_bundler_and_autoproj_gem_layout
             load_config
@@ -358,7 +362,7 @@ module Autoproj
             config.validate_ruby_executable
             Autobuild.programs['ruby'] = config.ruby_executable
             config.apply_autobuild_configuration
-            load_autoprojrc
+            load_autoprojrc if load_global_configuration
             load_main_initrb
             config.each_reused_autoproj_installation do |p|
                 manifest.reuse(p)
@@ -803,6 +807,20 @@ module Autoproj
             full_env = self.full_env
             changed = save_cached_env(full_env)
             full_env.export_env_sh(shell_helpers: shell_helpers)
+
+            build_dir = Pathname(self.build_dir)
+            full_env.each_env_filename do |_, filename|
+                basename = File.basename(filename)
+                File.open(File.join(prefix_dir, basename), "w") do |io|
+                    io.puts "source \"#{filename}\""
+                end
+                if build_dir.absolute?
+                    build_dir.mkpath
+                    (build_dir + basename).open("w") do |io|
+                        io.puts "source \"#{filename}\""
+                    end
+                end
+            end
             changed
         end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: autoproj
 version: !ruby/object:Gem::Version
-  version: 2.13.0
+  version: 2.14.0
 platform: ruby
 authors:
 - Sylvain Joyeux
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-15 00:00:00.000000000 Z
+date: 2021-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -394,6 +394,7 @@ files:
 - lib/autoproj/package_manifest.rb
 - lib/autoproj/package_selection.rb
 - lib/autoproj/package_set.rb
+- lib/autoproj/python.rb
 - lib/autoproj/query_base.rb
 - lib/autoproj/reporter.rb
 - lib/autoproj/repository_managers/apt.rb