poise-python 1.0.0

data/lib/poise_python/python_providers/scl.rb

@@ -0,0 +1,77 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/resource'
+require 'poise_languages'
+
+require 'poise_python/error'
+require 'poise_python/python_providers/base'
+
+
+module PoisePython
+  module PythonProviders
+    class Scl < Base
+      include PoiseLanguages::Scl::Mixin
+      provides(:scl)
+      scl_package('3.4.2', 'rh-python34', {
+        ['redhat', 'centos'] => {
+          '~> 7.0' => 'https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-7-x86_64/download/rhscl-rh-python34-epel-7-x86_64.noarch.rpm',
+          '~> 6.0' => 'https://www.softwarecollections.org/en/scls/rhscl/rh-python34/epel-6-x86_64/download/rhscl-rh-python34-epel-6-x86_64.noarch.rpm',
+        },
+      })
+      scl_package('3.3.2', 'python33', {
+        ['redhat', 'centos'] => {
+          '~> 7.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python33/epel-7-x86_64/download/rhscl-python33-epel-7-x86_64.noarch.rpm',
+          '~> 6.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python33/epel-6-x86_64/download/rhscl-python33-epel-6-x86_64.noarch.rpm',
+        },
+        'fedora' => {
+          '~> 21.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python33/fedora-21-x86_64/download/rhscl-python33-fedora-21-x86_64.noarch.rpm',
+          '~> 20.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python33/fedora-20-x86_64/download/rhscl-python33-fedora-20-x86_64.noarch.rpm',
+        },
+      })
+      scl_package('2.7.8', 'python27', {
+        ['redhat', 'centos'] => {
+          '~> 7.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python27/epel-7-x86_64/download/rhscl-python27-epel-7-x86_64.noarch.rpm',
+          '~> 6.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python27/epel-6-x86_64/download/rhscl-python27-epel-6-x86_64.noarch.rpm',
+        },
+        'fedora' => {
+          '~> 21.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python27/fedora-21-x86_64/download/rhscl-python27-fedora-21-x86_64.noarch.rpm',
+          '~> 20.0' => 'https://www.softwarecollections.org/en/scls/rhscl/python27/fedora-20-x86_64/download/rhscl-python27-fedora-20-x86_64.noarch.rpm',
+        },
+      })
+
+      def python_binary
+        ::File.join(scl_folder, 'root', 'usr', 'bin', 'python')
+      end
+
+      def python_environment
+        scl_environment
+      end
+
+      private
+
+      def install_python
+        install_scl_package
+      end
+
+      def uninstall_python
+        uninstall_scl_package
+      end
+
+    end
+  end
+end
+

data/lib/poise_python/python_providers/system.rb

@@ -0,0 +1,86 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/resource'
+require 'poise_languages'
+
+require 'poise_python/error'
+require 'poise_python/python_providers/base'
+
+
+module PoisePython
+  module PythonProviders
+    class System < Base
+      include PoiseLanguages::System::Mixin
+      provides(:system)
+      packages('python', {
+        debian: {
+          '8' => %w{python3.4 python2.7},
+          '7' => %w{python3.2 python2.7 python2.6},
+          '6' => %w{python3.1 python2.6 python2.5},
+        },
+        ubuntu: {
+          '14.04' => %w{python3.4 python2.7},
+          '12.04' => %w{python3.2 python2.7},
+          '10.04' => %w{python3.1 python2.6},
+        },
+        rhel: {default: %w{python}},
+        centos: {default: %w{python}},
+        fedora: {default: %w{python3 python}},
+        amazon: {default: %w{python27 python26 python}},
+      })
+
+      # Output value for the Python binary we are installing. Seems to match
+      # package name on all platforms I've checked.
+      def python_binary
+        ::File.join('', 'usr', 'bin', system_package_name)
+      end
+
+      private
+
+      def install_python
+        install_system_packages
+      end
+
+      def uninstall_python
+        uninstall_system_packages
+      end
+
+      def system_package_candidates(version)
+        [].tap do |names|
+          # For two (or more) digit versions.
+          if match = version.match(/^(\d+\.\d+)/)
+            # Debian style pythonx.y
+            names << "python#{match[1]}"
+            # Amazon style pythonxy
+            names << "python#{match[1].gsub(/\./, '')}"
+          end
+          # Aliases for 2 and 3.
+          if version == '3' || version == ''
+            names.concat(%w{python3.5 python35 python3.4 python34 python3.3 python33 python3.2 python32 python3.1 python31 python3.0 python30 python3})
+          end
+          if version == '2' || version == ''
+            names.concat(%w{python2.7 python27 python2.6 python26 python2.5 python25})
+          end
+          # For RHEL and friends.
+          names << 'python'
+          names.uniq!
+        end
+      end
+
+    end
+  end
+end

data/lib/poise_python/resources.rb

@@ -0,0 +1,31 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'poise_python/resources/pip_requirements'
+require 'poise_python/resources/python_package'
+require 'poise_python/resources/python_runtime'
+require 'poise_python/resources/python_runtime_pip'
+require 'poise_python/resources/python_execute'
+require 'poise_python/resources/python_virtualenv'
+
+
+module PoisePython
+  # Chef resources and providers for poise-python.
+  #
+  # @since 1.0.0
+  module Resources
+  end
+end

data/lib/poise_python/resources/pip_requirements.rb

@@ -0,0 +1,102 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/provider'
+require 'chef/resource'
+require 'poise'
+
+require 'poise_python/python_command_mixin'
+
+
+module PoisePython
+  module Resources
+    # (see PipRequirements::Resource)
+    # @since 1.0.0
+    module PipRequirements
+      # A `pip_requirements` resource to manage Python virtual environments.
+      #
+      # @provides pip_requirements
+      # @action install
+      # @action upgrade
+      # @example
+      #   pip_requirements '/opt/myapp/requirements.txt'
+      class Resource < Chef::Resource
+        include PoisePython::PythonCommandMixin
+        provides(:pip_requirements)
+        actions(:install, :upgrade)
+
+        # @!attribute path
+        #   Path to the requirements file, or a folder containing the
+        #   requirements file.
+        #   @return [String]
+        attribute(:path, kind_of: String, name_attribute: true)
+      end
+
+      # The default provider for `pip_requirements`.
+      #
+      # @see Resource
+      # @provides pip_requirements
+      class Provider < Chef::Provider
+        include Poise
+        include PoisePython::PythonCommandMixin
+        provides(:pip_requirements)
+
+        # The `install` action for the `pip_requirements` resource.
+        #
+        # @return [void]
+        def action_install
+          install_requirements(upgrade: false)
+        end
+
+        # The `upgrade` action for the `pip_requirements` resource.
+        #
+        # @return [void]
+        def action_upgrade
+          install_requirements(upgrade: true)
+        end
+
+        private
+
+        # Run an install --requirements command and parse the output.
+        #
+        # @param upgrade [Boolean] If we should use the --upgrade flag.
+        # @return [void]
+        def install_requirements(upgrade: false)
+          cmd = %w{-m pip.__main__ install}
+          cmd << '--upgrade' if upgrade
+          cmd << '--requirement'
+          cmd << requirements_path
+          output = python_shell_out!(cmd).stdout
+          if output.include?('Successfully installed')
+            new_resource.updated_by_last_action(true)
+          end
+        end
+
+        # Find the true path to the requirements file.
+        #
+        # @return [String]
+        def requirements_path
+          if ::File.directory?(new_resource.path)
+            ::File.join(new_resource.path, 'requirements.txt')
+          else
+            new_resource.path
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/poise_python/resources/python_execute.rb

@@ -0,0 +1,83 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef/mixin/which'
+require 'chef/provider/execute'
+require 'chef/resource/execute'
+require 'poise'
+
+require 'poise_python/python_command_mixin'
+
+
+module PoisePython
+  module Resources
+    # (see PythonExecute::Resource)
+    # @since 1.0.0
+    module PythonExecute
+      # A `python_execute` resource to run Python scripts and commands.
+      #
+      # @provides python_execute
+      # @action run
+      # @example
+      #   python_execute 'myapp.py' do
+      #     user 'myuser'
+      #   end
+      class Resource < Chef::Resource::Execute
+        include PoisePython::PythonCommandMixin
+        provides(:python_execute)
+        actions(:run)
+      end
+
+      # The default provider for `python_execute`.
+      #
+      # @see Resource
+      # @provides python_execute
+      class Provider < Chef::Provider::Execute
+        include Chef::Mixin::Which
+        provides(:python_execute)
+
+        private
+
+        # Command to pass to shell_out.
+        #
+        # @return [String, Array<String>]
+        def command
+          if new_resource.command.is_a?(Array)
+            [new_resource.python] + new_resource.command
+          else
+            "#{new_resource.python} #{new_resource.command}"
+          end
+        end
+
+        # Environment variables to pass to shell_out.
+        #
+        # @return [Hash]
+        def environment
+          if new_resource.parent_python
+            environment = new_resource.parent_python.python_environment
+            if new_resource.environment
+              environment = environment.merge(new_resource.environment)
+            end
+            environment
+          else
+            new_resource.environment
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/poise_python/resources/python_package.rb

@@ -0,0 +1,322 @@
+#
+# Copyright 2015, Noah Kantrowitz
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'shellwords'
+
+require 'chef/mixin/which'
+require 'chef/provider/package'
+require 'chef/resource/package'
+require 'poise'
+
+require 'poise_python/python_command_mixin'
+
+
+module PoisePython
+  module Resources
+    # (see PythonPackage::Resource)
+    # @since 1.0.0
+    module PythonPackage
+      # A Python snippet to hack pip a bit so `pip list --outdated` will show
+      # only the things we want and will understand version requirements.
+      # @api private
+      PIP_HACK_SCRIPT = <<-EOH
+import sys
+
+import pip
+try:
+    # >= 6.0
+    from pip.utils import get_installed_distributions
+except ImportError:
+    # <= 1.5.6
+    from pip.util import get_installed_distributions
+
+def replacement(*args, **kwargs):
+    import copy, sys
+    from pip._vendor import pkg_resources
+    dists = []
+    for raw_req in sys.argv[3:]:
+        req = pkg_resources.Requirement.parse(raw_req)
+        dist = pkg_resources.working_set.by_key.get(req.key)
+        if dist:
+          # Don't mutate stuff from the global working set.
+          dist = copy.copy(dist)
+        else:
+          # Make a fake one.
+          dist = pkg_resources.Distribution(project_name=req.key, version='0')
+        # Fool the .key property into using our string.
+        dist._key = raw_req
+        dists.append(dist)
+    return dists
+try:
+    # For Python 2.
+    get_installed_distributions.func_code = replacement.func_code
+except AttributeError:
+    # For Python 3.
+    get_installed_distributions.__code__ = replacement.__code__
+
+sys.exit(pip.main())
+EOH
+
+      # A `python_package` resource to manage Python installations using pip.
+      #
+      # @provides python_package
+      # @action install
+      # @action upgrade
+      # @action uninstall
+      # @example
+      #   python_package 'django' do
+      #     python '2'
+      #     version '1.8.3'
+      #   end
+      class Resource < Chef::Resource::Package
+        include PoisePython::PythonCommandMixin
+        provides(:python_package)
+
+
+        # @!attribute group
+        #   System group to install the package.
+        #   @return [String, Integer, nil]
+        attribute(:group, kind_of: [String, Integer, NilClass])
+        # @!attribute user
+        #   System user to install the package.
+        #   @return [String, Integer, nil]
+        attribute(:user, kind_of: [String, Integer, NilClass])
+
+        def initialize(*args)
+          super
+          # For older Chef.
+          @resource_name = :python_package
+          # We don't have these actions.
+          @allowed_actions.delete(:purge)
+          @allowed_actions.delete(:reconfig)
+        end
+
+        # Upstream attribute we don't support. Sets are an error and gets always
+        # return nil.
+        #
+        # @api private
+        # @param arg [Object] Ignored
+        # @return [nil]
+        def response_file(arg=nil)
+          raise NoMethodError if arg
+        end
+
+        # (see #response_file)
+        def response_file_variables(arg=nil)
+          raise NoMethodError if arg
+        end
+
+        # (see #response_file)
+        def source(arg=nil)
+          raise NoMethodError if arg
+        end
+      end
+
+      # The default provider for the `python_package` resource.
+      #
+      # @see Resource
+      class Provider < Chef::Provider::Package
+        include PoisePython::PythonCommandMixin
+        provides(:python_package)
+
+        # Load current and candidate versions for all needed packages.
+        #
+        # @api private
+        # @return [Chef::Resource]
+        def load_current_resource
+          @current_resource = new_resource.class.new(new_resource.name, run_context)
+          current_resource.package_name(new_resource.package_name)
+          check_package_versions(current_resource)
+          current_resource
+        end
+
+        # Populate current and candidate versions for all needed packages.
+        #
+        # @api private
+        # @param resource [PoisePython::Resources::PythonPackage::Resource]
+        #   Resource to load for.
+        # @param version [String, Array<String>] Current version(s) of package(s).
+        # @return [void]
+        def check_package_versions(resource, version=new_resource.version)
+          version_data = Hash.new {|hash, key| hash[key] = {current: nil, candidate: nil} }
+          # Get the version for everything currently installed.
+          list = pip_command('list').stdout
+          parse_pip_list(list).each do |name, current|
+            # Merge current versions in to the data.
+            version_data[name][:current] = current
+          end
+          # Check for newer candidates.
+          outdated = pip_outdated(pip_requirements(resource.package_name, version)).stdout
+          parse_pip_outdated(outdated).each do |name, candidate|
+            # Merge candidates in to the existing versions.
+            version_data[name][:candidate] = candidate
+          end
+          # Populate the current resource and candidate versions. Youch this is
+          # a gross mix of data flow.
+          if(resource.package_name.is_a?(Array))
+            @candidate_version = []
+            versions = []
+            [resource.package_name].flatten.each do |name|
+              ver = version_data[name.downcase]
+              versions << ver[:current]
+              @candidate_version << ver[:candidate]
+            end
+            resource.version(versions)
+          else
+            ver = version_data[resource.package_name.downcase]
+            resource.version(ver[:current])
+            @candidate_version = ver[:candidate]
+          end
+        end
+
+        # Install package(s) using pip.
+        #
+        # @param name [String, Array<String>] Name(s) of package(s).
+        # @param version [String, Array<String>] Version(s) of package(s).
+        # @return [void]
+        def install_package(name, version)
+          pip_install(name, version, upgrade: false)
+        end
+
+        # Upgrade package(s) using pip.
+        #
+        # @param name [String, Array<String>] Name(s) of package(s).
+        # @param version [String, Array<String>] Version(s) of package(s).
+        # @return [void]
+        def upgrade_package(name, version)
+          pip_install(name, version, upgrade: true)
+        end
+
+        # Uninstall package(s) using pip.
+        #
+        # @param name [String, Array<String>] Name(s) of package(s).
+        # @param version [String, Array<String>] Version(s) of package(s).
+        # @return [void]
+        def remove_package(name, version)
+          pip_command('uninstall', %w{--yes} + [name].flatten)
+        end
+
+        private
+
+        # Convert name(s) and version(s) to an array of pkg_resources.Requirement
+        # compatible strings. These are strings like "django" or "django==1.0".
+        #
+        # @param name [String, Array<String>] Name or names for the packages.
+        # @param version [String, Array<String>] Version or versions for the
+        #   packages.
+        # @return [Array<String>]
+        def pip_requirements(name, version)
+          [name].flatten.zip([version].flatten).map do |n, v|
+            v = v.to_s.strip
+            if v.empty?
+              # No version requirement, send through unmodified.
+              n
+            elsif v =~ /^\d/
+              "#{n}==#{v}"
+            else
+              # If the first character isn't a digit, assume something fancy.
+              n + v
+            end
+          end
+        end
+
+        # Run a pip command.
+        #
+        # @param pip_command [String] The pip subcommand to run (eg. install).
+        # @param pip_options [Array<String>] Options for the pip command.
+        # @param opts [Hash] Mixlib::ShellOut options.
+        # @return [Mixlib::ShellOut]
+        def pip_command(pip_command, pip_options=[], opts={})
+          runner = opts.delete(:pip_runner) || %w{-m pip.__main__}
+          full_cmd = if new_resource.options
+            # We have to use a string for this case to be safe because the
+            # options are a string and I don't want to try and parse that.
+            "##{runner.join(' ')} #{pip_command} #{new_resource.options} #{Shellwords.join(pip_options)}"
+          else
+            # No special options, use an array to skip the extra /bin/sh.
+            runner + [pip_command] + pip_options
+          end
+          # Set user and group.
+          opts[:user] = new_resource.user if new_resource.user
+          opts[:group] = new_resource.group if new_resource.group
+
+          python_shell_out!(full_cmd, opts)
+        end
+
+        # Run `pip install` to install a package(s).
+        #
+        # @param name [String, Array<String>] Name(s) of package(s) to install.
+        # @param version [String, Array<String>] Version(s) of package(s) to
+        #   install.
+        # @param upgrade [Boolean] Use upgrade mode?
+        # @return [Mixlib::ShellOut]
+        def pip_install(name, version, upgrade: false)
+          cmd = pip_requirements(name, version)
+          # Prepend --upgrade if needed.
+          cmd = %w{--upgrade} + cmd if upgrade
+          pip_command('install', cmd)
+        end
+
+        # Run my hacked version of `pip list --outdated` with a specific set of
+        # package requirements.
+        #
+        # @see #pip_requirements
+        # @param requirements [Array<String>] Pip-formatted package requirements.
+        # @return [Mixlib::ShellOut]
+        def pip_outdated(requirements)
+          pip_command('list', %w{--outdated} + requirements, input: PIP_HACK_SCRIPT, pip_runner: %w{-})
+        end
+
+        # Parse the output from `pip list --outdate`. Returns a hash of package
+        # key to candidate version.
+        #
+        # @param text [String] Output to parse.
+        # @return [Hash<String, String>]
+        def parse_pip_outdated(text)
+          text.split(/\n/).inject({}) do |memo, line|
+            # Example of a line:
+            # boto (Current: 2.25.0 Latest: 2.38.0 [wheel])
+            if md = line.match(/^(\S+)\s+\(.*?latest:\s+([^\s,]+).*\)$/i)
+              memo[md[1].downcase] = md[2]
+            else
+              Chef::Log.debug("[#{new_resource}] Unparsable line in pip outdated: #{line}")
+            end
+            memo
+          end
+        end
+
+        # Parse the output from `pip list`. Returns a hash of package key to
+        # current version.
+        #
+        # @param text [String] Output to parse.
+        # @return [Hash<String, String>]
+        def parse_pip_list(text)
+          text.split(/\n/).inject({}) do |memo, line|
+            # Example of a line:
+            # boto (2.25.0)
+            if md = line.match(/^(\S+)\s+\(([^\s,]+).*\)$/i)
+              memo[md[1].downcase] = md[2]
+            else
+              Chef::Log.debug("[#{new_resource}] Unparsable line in pip list: #{line}")
+            end
+            memo
+          end
+        end
+
+      end
+    end
+  end
+end