dependabot-core 0.76.1

data/lib/dependabot/update_checkers/python/pip.rb

@@ -0,0 +1,227 @@
+# frozen_string_literal: true
+
+require "excon"
+require "toml-rb"
+
+require "python_requirement_parser"
+require "dependabot/update_checkers/base"
+require "dependabot/shared_helpers"
+require "dependabot/utils/python/requirement"
+
+module Dependabot
+  module UpdateCheckers
+    module Python
+      class Pip < Dependabot::UpdateCheckers::Base
+        require_relative "pip/poetry_version_resolver"
+        require_relative "pip/pipfile_version_resolver"
+        require_relative "pip/pip_compile_version_resolver"
+        require_relative "pip/requirements_updater"
+        require_relative "pip/latest_version_finder"
+
+        MAIN_PYPI_INDEXES = %w(
+          https://pypi.python.org/simple/
+          https://pypi.org/simple/
+        ).freeze
+
+        def latest_version
+          @latest_version ||= fetch_latest_version
+        end
+
+        def latest_resolvable_version
+          @latest_resolvable_version ||=
+            case resolver_type
+            when :pipfile
+              PipfileVersionResolver.new(
+                resolver_args.merge(unlock_requirement: true)
+              ).latest_resolvable_version
+            when :poetry
+              PoetryVersionResolver.new(
+                resolver_args.merge(unlock_requirement: true)
+              ).latest_resolvable_version
+            when :pip_compile
+              PipCompileVersionResolver.new(
+                resolver_args.merge(unlock_requirement: true)
+              ).latest_resolvable_version
+            when :requirements
+              # pip doesn't (yet) do any dependency resolution, so if we don't
+              # have a Pipfile or a pip-compile file, we just return the latest
+              # version.
+              latest_version
+            else raise "Unexpected resolver type #{resolver_type}"
+            end
+        end
+
+        def latest_resolvable_version_with_no_unlock
+          @latest_resolvable_version_with_no_unlock ||=
+            case resolver_type
+            when :pipfile
+              PipfileVersionResolver.new(
+                resolver_args.merge(unlock_requirement: false)
+              ).latest_resolvable_version
+            when :poetry
+              PoetryVersionResolver.new(
+                resolver_args.merge(unlock_requirement: false)
+              ).latest_resolvable_version
+            when :pip_compile
+              PipCompileVersionResolver.new(
+                resolver_args.merge(unlock_requirement: false)
+              ).latest_resolvable_version
+            when :requirements
+              latest_pip_version_with_no_unlock
+            else raise "Unexpected resolver type #{resolver_type}"
+            end
+        end
+
+        def updated_requirements
+          RequirementsUpdater.new(
+            requirements: dependency.requirements,
+            latest_version: latest_version&.to_s,
+            latest_resolvable_version: latest_resolvable_version&.to_s,
+            update_strategy: requirements_update_strategy,
+            has_lockfile: pipfile_lock || poetry_lock || pyproject_lock
+          ).updated_requirements
+        end
+
+        def requirements_update_strategy
+          # If passed in as an option (in the base class) honour that option
+          if @requirements_update_strategy
+            return @requirements_update_strategy.to_sym
+          end
+
+          # Otherwise, check if this is a poetry library or not
+          poetry_library? ? :widen_ranges : :bump_versions
+        end
+
+        private
+
+        def latest_version_resolvable_with_full_unlock?
+          # Full unlock checks aren't implemented for pip because they're not
+          # relevant (pip doesn't have a resolver). This method always returns
+          # false to ensure `updated_dependencies_after_full_unlock` is never
+          # called.
+          false
+        end
+
+        def updated_dependencies_after_full_unlock
+          raise NotImplementedError
+        end
+
+        # rubocop:disable Metrics/PerceivedComplexity
+        def resolver_type
+          reqs = dependency.requirements
+          req_files = reqs.map { |r| r.fetch(:file) }
+
+          # If there are no requirements then this is a sub-dependency. It
+          # must come from one of Pipenv, Poetry or pip-tools, and can't come
+          # from the first two unless they have a lockfile.
+          return subdependency_resolver if reqs.none?
+
+          # Otherwise, this is a top-level dependency, and we can figure out
+          # which resolver to use based on the filename of its requirements
+          return :pipfile if req_files.any? { |f| f == "Pipfile" }
+          return :poetry if req_files.any? { |f| f == "pyproject.toml" }
+          return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
+
+          if dependency.version && !exact_requirement?(reqs)
+            subdependency_resolver
+          else
+            :requirements
+          end
+        end
+        # rubocop:enable Metrics/PerceivedComplexity
+
+        def subdependency_resolver
+          return :pipfile if pipfile_lock
+          return :poetry if poetry_lock || pyproject_lock
+          return :pip_compile if pip_compile_files.any?
+
+          raise "Claimed to be a sub-dependency, but no lockfile exists!"
+        end
+
+        def exact_requirement?(reqs)
+          reqs = reqs.map { |r| r.fetch(:requirement) }
+          reqs = reqs.compact
+          reqs = reqs.flat_map { |r| r.split(",").map(&:strip) }
+          reqs.any? { |r| Utils::Python::Requirement.new(r).exact? }
+        end
+
+        def resolver_args
+          {
+            dependency: dependency,
+            dependency_files: dependency_files,
+            credentials: credentials,
+            latest_allowable_version: latest_version
+          }
+        end
+
+        def fetch_latest_version
+          latest_version_finder.latest_version
+        end
+
+        def latest_pip_version_with_no_unlock
+          latest_version_finder.latest_version_with_no_unlock
+        end
+
+        def latest_version_finder
+          @latest_version_finder ||= LatestVersionFinder.new(
+            dependency: dependency,
+            dependency_files: dependency_files,
+            credentials: credentials,
+            ignored_versions: ignored_versions
+          )
+        end
+
+        def poetry_library?
+          return false unless pyproject
+
+          # Hit PyPi and check whether there are details for a library with a
+          # matching name and description
+          details = TomlRB.parse(pyproject.content).dig("tool", "poetry")
+          return false unless details
+
+          index_response = Excon.get(
+            "https://pypi.org/pypi/#{normalised_name(details['name'])}/json",
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          return false unless index_response.status == 200
+
+          pypi_info = JSON.parse(index_response.body)["info"] || {}
+          pypi_info["summary"] == details["description"]
+        rescue URI::InvalidURIError
+          false
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def pipfile
+          dependency_files.find { |f| f.name == "Pipfile" }
+        end
+
+        def pipfile_lock
+          dependency_files.find { |f| f.name == "Pipfile.lock" }
+        end
+
+        def pyproject
+          dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def pyproject_lock
+          dependency_files.find { |f| f.name == "pyproject.lock" }
+        end
+
+        def poetry_lock
+          dependency_files.find { |f| f.name == "poetry.lock" }
+        end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/update_checkers/python/pip/latest_version_finder.rb

@@ -0,0 +1,252 @@
+# frozen_string_literal: true
+
+require "excon"
+
+require "dependabot/update_checkers/python/pip"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module UpdateCheckers
+    module Python
+      class Pip
+        class LatestVersionFinder
+          def initialize(dependency:, dependency_files:, credentials:,
+                         ignored_versions:)
+            @dependency       = dependency
+            @dependency_files = dependency_files
+            @credentials      = credentials
+            @ignored_versions = ignored_versions
+          end
+
+          def latest_version
+            @latest_version ||= fetch_latest_version
+          end
+
+          def latest_version_with_no_unlock
+            @latest_version_with_no_unlock ||=
+              fetch_latest_version_with_no_unlock
+          end
+
+          private
+
+          attr_reader :dependency, :dependency_files, :credentials,
+                      :ignored_versions
+
+          def fetch_latest_version
+            versions = available_versions
+            versions.reject! { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }
+            versions.reject!(&:prerelease?) unless wants_prerelease?
+            versions.max
+          end
+
+          def fetch_latest_version_with_no_unlock
+            versions = available_versions
+            reqs = dependency.requirements.map do |r|
+              reqs = (r.fetch(:requirement) || "").split(",").map(&:strip)
+              requirement_class.new(reqs)
+            end
+            versions.reject!(&:prerelease?) unless wants_prerelease?
+            versions.sort.reverse.
+              reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }.
+              find { |v| reqs.all? { |r| r.satisfied_by?(v) } }
+          end
+
+          def wants_prerelease?
+            if dependency.version
+              version = version_class.new(dependency.version.tr("+", "."))
+              return version.prerelease?
+            end
+
+            dependency.requirements.any? do |req|
+              reqs = (req.fetch(:requirement) || "").split(",").map(&:strip)
+              reqs.any? { |r| r.match?(/[A-Za-z]/) }
+            end
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/ for details of the
+          # Simple Repository API we use here.
+          def available_versions
+            index_urls.flat_map do |index_url|
+              sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+              index_response = registry_response_for_dependency(index_url)
+
+              if [401, 403].include?(index_response.status) &&
+                 [401, 403].include?(registry_index_response(index_url).status)
+                raise PrivateSourceAuthenticationFailure, sanitized_url
+              end
+
+              index_response.body.
+                scan(%r{<a\s.*?>(.*?)</a>}m).flatten.
+                select { |n| n.match?(name_regex) }.
+                map do |filename|
+                  version =
+                    filename.
+                    gsub(/#{name_regex}-/i, "").
+                    split(/-|(\.tar\.)/).
+                    first
+                  next unless version_class.correct?(version)
+
+                  version_class.new(version)
+                end.compact
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              next if MAIN_PYPI_INDEXES.include?(index_url)
+
+              raise PrivateSourceAuthenticationFailure, sanitized_url
+            end
+          end
+
+          def index_urls
+            main_index_url =
+              config_variable_index_urls[:main] ||
+              pipfile_index_urls[:main] ||
+              requirement_file_index_urls[:main] ||
+              pip_conf_index_urls[:main] ||
+              "https://pypi.python.org/simple/"
+
+            if main_index_url
+              main_index_url = main_index_url.strip.gsub(%r{/*$}, "") + "/"
+            end
+
+            extra_index_urls =
+              config_variable_index_urls[:extra] +
+              pipfile_index_urls[:extra] +
+              requirement_file_index_urls[:extra] +
+              pip_conf_index_urls[:extra]
+
+            extra_index_urls =
+              extra_index_urls.map { |url| url.strip.gsub(%r{/*$}, "") + "/" }
+
+            [main_index_url, *extra_index_urls].uniq
+          end
+
+          def registry_response_for_dependency(index_url)
+            Excon.get(
+              index_url + normalised_name + "/",
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+          end
+
+          def registry_index_response(index_url)
+            Excon.get(
+              index_url,
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+          end
+
+          def requirement_file_index_urls
+            urls = { main: nil, extra: [] }
+
+            requirements_files.each do |file|
+              if file.content.match?(/^--index-url\s(.+)/)
+                urls[:main] =
+                  file.content.match(/^--index-url\s(.+)/).captures.first
+              end
+              urls[:extra] += file.content.scan(/^--extra-index-url\s(.+)/).
+                              flatten
+            end
+
+            urls
+          end
+
+          def pip_conf_index_urls
+            urls = { main: nil, extra: [] }
+
+            return urls unless pip_conf
+
+            content = pip_conf.content
+
+            if content.match?(/^index-url\s*=/x)
+              urls[:main] = content.match(/^index-url\s*=\s*(.+)/).
+                            captures.first
+            end
+            urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
+
+            urls
+          end
+
+          def pipfile_index_urls
+            urls = { main: nil, extra: [] }
+
+            return urls unless pipfile
+
+            pipfile_object = TomlRB.parse(pipfile.content)
+
+            urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
+
+            pipfile_object["source"]&.each do |source|
+              urls[:extra] << source.fetch("url") if source["url"]
+            end
+            urls[:extra] = urls[:extra].uniq
+
+            urls
+          rescue TomlRB::ParseError
+            urls
+          end
+
+          def config_variable_index_urls
+            urls = { main: nil, extra: [] }
+
+            index_url_creds = credentials.
+                              select { |cred| cred["type"] == "python_index" }
+            urls[:main] =
+              index_url_creds.
+              find { |cred| cred["replaces-base"] }&.
+              fetch("index-url")
+            urls[:extra] =
+              index_url_creds.
+              reject { |cred| cred["replaces-base"] }.
+              map { |cred| cred["index-url"] }
+
+            urls
+          end
+
+          def ignore_reqs
+            ignored_versions.map { |req| requirement_class.new(req.split(",")) }
+          end
+
+          # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+          def normalised_name
+            dependency.name.downcase.gsub(/[-_.]+/, "-")
+          end
+
+          def name_regex
+            parts = dependency.name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
+            /#{parts.join("[\s_.-]")}/i
+          end
+
+          def pip_conf
+            dependency_files.find { |f| f.name == "pip.conf" }
+          end
+
+          def pipfile
+            dependency_files.find { |f| f.name == "Pipfile" }
+          end
+
+          def pyproject
+            dependency_files.find { |f| f.name == "pyproject.toml" }
+          end
+
+          def requirements_files
+            dependency_files.select { |f| f.name.match?(/requirements/x) }
+          end
+
+          def pip_compile_files
+            dependency_files.select { |f| f.name.end_with?(".in") }
+          end
+
+          def version_class
+            Utils.version_class_for_package_manager(dependency.package_manager)
+          end
+
+          def requirement_class
+            Utils.requirement_class_for_package_manager(
+              dependency.package_manager
+            )
+          end
+        end
+      end
+    end
+  end
+end