dependabot-python 0.79.0

data/lib/dependabot/python/requirement_parser.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module Python
+    class RequirementParser
+      NAME = /[a-zA-Z0-9\-_\.]+/.freeze
+      EXTRA = /[a-zA-Z0-9\-_\.]+/.freeze
+      COMPARISON = /===|==|>=|<=|<|>|~=|!=/.freeze
+      VERSION = /[0-9]+[a-zA-Z0-9\-_\.*]*(\+[0-9a-zA-Z]+(\.[0-9a-zA-Z]+)*)?/.
+                freeze
+      REQUIREMENT =
+        /(?<comparison>#{COMPARISON})\s*\\?\s*(?<version>#{VERSION})/.freeze
+      HASH = /--hash=(?<algorithm>.*?):(?<hash>.*?)(?=\s|$)/.freeze
+      REQUIREMENTS = /#{REQUIREMENT}(\s*,\s*\\?\s*#{REQUIREMENT})*/.freeze
+      HASHES = /#{HASH}(\s*\\?\s*#{HASH})*/.freeze
+
+      INSTALL_REQ_WITH_REQUIREMENT =
+        /\s*\\?\s*(?<name>#{NAME})
+          \s*\\?\s*(\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
+          \s*\\?\s*(?<requirements>#{REQUIREMENTS})
+          \s*\\?\s*(?<hashes>#{HASHES})?
+          \s*#*\s*(?<comment>.+)?
+        /x.freeze
+
+      INSTALL_REQ_WITHOUT_REQUIREMENT =
+        /^\s*\\?\s*(?<name>#{NAME})
+          \s*\\?\s*(\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
+          \s*\\?\s*(?<hashes>#{HASHES})?
+          \s*#*\s*(?<comment>.+)?$
+        /x.freeze
+
+      NAME_WITH_EXTRAS =
+        /\s*\\?\s*(?<name>#{NAME})
+          (\s*\\?\s*\[\s*(?<extras>#{EXTRA}(\s*,\s*#{EXTRA})*)\s*\])?
+        /x.freeze
+    end
+  end
+end

data/lib/dependabot/python/update_checker.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "excon"
+require "toml-rb"
+
+require "dependabot/update_checkers"
+require "dependabot/update_checkers/base"
+require "dependabot/shared_helpers"
+require "dependabot/python/requirement"
+require "dependabot/python/requirement_parser"
+
+module Dependabot
+  module Python
+    class UpdateChecker < Dependabot::UpdateCheckers::Base
+      require_relative "update_checker/poetry_version_resolver"
+      require_relative "update_checker/pipfile_version_resolver"
+      require_relative "update_checker/pip_compile_version_resolver"
+      require_relative "update_checker/requirements_updater"
+      require_relative "update_checker/latest_version_finder"
+
+      MAIN_PYPI_INDEXES = %w(
+        https://pypi.python.org/simple/
+        https://pypi.org/simple/
+      ).freeze
+
+      def latest_version
+        @latest_version ||= fetch_latest_version
+      end
+
+      def latest_resolvable_version
+        @latest_resolvable_version ||=
+          case resolver_type
+          when :pipfile
+            PipfileVersionResolver.new(
+              resolver_args.merge(unlock_requirement: true)
+            ).latest_resolvable_version
+          when :poetry
+            PoetryVersionResolver.new(
+              resolver_args.merge(unlock_requirement: true)
+            ).latest_resolvable_version
+          when :pip_compile
+            PipCompileVersionResolver.new(
+              resolver_args.merge(unlock_requirement: true)
+            ).latest_resolvable_version
+          when :requirements
+            # pip doesn't (yet) do any dependency resolution, so if we don't
+            # have a Pipfile or a pip-compile file, we just return the latest
+            # version.
+            latest_version
+          else raise "Unexpected resolver type #{resolver_type}"
+          end
+      end
+
+      def latest_resolvable_version_with_no_unlock
+        @latest_resolvable_version_with_no_unlock ||=
+          case resolver_type
+          when :pipfile
+            PipfileVersionResolver.new(
+              resolver_args.merge(unlock_requirement: false)
+            ).latest_resolvable_version
+          when :poetry
+            PoetryVersionResolver.new(
+              resolver_args.merge(unlock_requirement: false)
+            ).latest_resolvable_version
+          when :pip_compile
+            PipCompileVersionResolver.new(
+              resolver_args.merge(unlock_requirement: false)
+            ).latest_resolvable_version
+          when :requirements
+            latest_pip_version_with_no_unlock
+          else raise "Unexpected resolver type #{resolver_type}"
+          end
+      end
+
+      def updated_requirements
+        RequirementsUpdater.new(
+          requirements: dependency.requirements,
+          latest_version: latest_version&.to_s,
+          latest_resolvable_version: latest_resolvable_version&.to_s,
+          update_strategy: requirements_update_strategy,
+          has_lockfile: !(pipfile_lock || poetry_lock || pyproject_lock).nil?
+        ).updated_requirements
+      end
+
+      def requirements_update_strategy
+        # If passed in as an option (in the base class) honour that option
+        if @requirements_update_strategy
+          return @requirements_update_strategy.to_sym
+        end
+
+        # Otherwise, check if this is a poetry library or not
+        poetry_library? ? :widen_ranges : :bump_versions
+      end
+
+      private
+
+      def latest_version_resolvable_with_full_unlock?
+        # Full unlock checks aren't implemented for pip because they're not
+        # relevant (pip doesn't have a resolver). This method always returns
+        # false to ensure `updated_dependencies_after_full_unlock` is never
+        # called.
+        false
+      end
+
+      def updated_dependencies_after_full_unlock
+        raise NotImplementedError
+      end
+
+      # rubocop:disable Metrics/PerceivedComplexity
+      def resolver_type
+        reqs = dependency.requirements
+        req_files = reqs.map { |r| r.fetch(:file) }
+
+        # If there are no requirements then this is a sub-dependency. It
+        # must come from one of Pipenv, Poetry or pip-tools, and can't come
+        # from the first two unless they have a lockfile.
+        return subdependency_resolver if reqs.none?
+
+        # Otherwise, this is a top-level dependency, and we can figure out
+        # which resolver to use based on the filename of its requirements
+        return :pipfile if req_files.any? { |f| f == "Pipfile" }
+        return :poetry if req_files.any? { |f| f == "pyproject.toml" }
+        return :pip_compile if req_files.any? { |f| f.end_with?(".in") }
+
+        if dependency.version && !exact_requirement?(reqs)
+          subdependency_resolver
+        else
+          :requirements
+        end
+      end
+      # rubocop:enable Metrics/PerceivedComplexity
+
+      def subdependency_resolver
+        return :pipfile if pipfile_lock
+        return :poetry if poetry_lock || pyproject_lock
+        return :pip_compile if pip_compile_files.any?
+
+        raise "Claimed to be a sub-dependency, but no lockfile exists!"
+      end
+
+      def exact_requirement?(reqs)
+        reqs = reqs.map { |r| r.fetch(:requirement) }
+        reqs = reqs.compact
+        reqs = reqs.flat_map { |r| r.split(",").map(&:strip) }
+        reqs.any? { |r| Python::Requirement.new(r).exact? }
+      end
+
+      def resolver_args
+        {
+          dependency: dependency,
+          dependency_files: dependency_files,
+          credentials: credentials,
+          latest_allowable_version: latest_version
+        }
+      end
+
+      def fetch_latest_version
+        latest_version_finder.latest_version
+      end
+
+      def latest_pip_version_with_no_unlock
+        latest_version_finder.latest_version_with_no_unlock
+      end
+
+      def latest_version_finder
+        @latest_version_finder ||= LatestVersionFinder.new(
+          dependency: dependency,
+          dependency_files: dependency_files,
+          credentials: credentials,
+          ignored_versions: ignored_versions
+        )
+      end
+
+      def poetry_library?
+        return false unless pyproject
+
+        # Hit PyPi and check whether there are details for a library with a
+        # matching name and description
+        details = TomlRB.parse(pyproject.content).dig("tool", "poetry")
+        return false unless details
+
+        index_response = Excon.get(
+          "https://pypi.org/pypi/#{normalised_name(details['name'])}/json",
+          idempotent: true,
+          **SharedHelpers.excon_defaults
+        )
+
+        return false unless index_response.status == 200
+
+        pypi_info = JSON.parse(index_response.body)["info"] || {}
+        pypi_info["summary"] == details["description"]
+      rescue URI::InvalidURIError
+        false
+      end
+
+      # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+      def normalised_name(name)
+        name.downcase.gsub(/[-_.]+/, "-")
+      end
+
+      def pipfile
+        dependency_files.find { |f| f.name == "Pipfile" }
+      end
+
+      def pipfile_lock
+        dependency_files.find { |f| f.name == "Pipfile.lock" }
+      end
+
+      def pyproject
+        dependency_files.find { |f| f.name == "pyproject.toml" }
+      end
+
+      def pyproject_lock
+        dependency_files.find { |f| f.name == "pyproject.lock" }
+      end
+
+      def poetry_lock
+        dependency_files.find { |f| f.name == "poetry.lock" }
+      end
+
+      def pip_compile_files
+        dependency_files.select { |f| f.name.end_with?(".in") }
+      end
+    end
+  end
+end
+
+Dependabot::UpdateCheckers.
+  register("pip", Dependabot::Python::UpdateChecker)

data/lib/dependabot/python/update_checker/latest_version_finder.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+require "excon"
+
+require "dependabot/python/update_checker"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module Python
+    class UpdateChecker
+      class LatestVersionFinder
+        def initialize(dependency:, dependency_files:, credentials:,
+                       ignored_versions:)
+          @dependency       = dependency
+          @dependency_files = dependency_files
+          @credentials      = credentials
+          @ignored_versions = ignored_versions
+        end
+
+        def latest_version
+          @latest_version ||= fetch_latest_version
+        end
+
+        def latest_version_with_no_unlock
+          @latest_version_with_no_unlock ||=
+            fetch_latest_version_with_no_unlock
+        end
+
+        private
+
+        attr_reader :dependency, :dependency_files, :credentials,
+                    :ignored_versions
+
+        def fetch_latest_version
+          versions = available_versions
+          versions.reject! { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }
+          versions.reject!(&:prerelease?) unless wants_prerelease?
+          versions.max
+        end
+
+        def fetch_latest_version_with_no_unlock
+          versions = available_versions
+          reqs = dependency.requirements.map do |r|
+            reqs = (r.fetch(:requirement) || "").split(",").map(&:strip)
+            requirement_class.new(reqs)
+          end
+          versions.reject!(&:prerelease?) unless wants_prerelease?
+          versions.sort.reverse.
+            reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }.
+            find { |v| reqs.all? { |r| r.satisfied_by?(v) } }
+        end
+
+        def wants_prerelease?
+          if dependency.version
+            version = version_class.new(dependency.version.tr("+", "."))
+            return version.prerelease?
+          end
+
+          dependency.requirements.any? do |req|
+            reqs = (req.fetch(:requirement) || "").split(",").map(&:strip)
+            reqs.any? { |r| r.match?(/[A-Za-z]/) }
+          end
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/ for details of the
+        # Simple Repository API we use here.
+        def available_versions
+          index_urls.flat_map do |index_url|
+            sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+            index_response = registry_response_for_dependency(index_url)
+
+            if [401, 403].include?(index_response.status) &&
+               [401, 403].include?(registry_index_response(index_url).status)
+              raise PrivateSourceAuthenticationFailure, sanitized_url
+            end
+
+            index_response.body.
+              scan(%r{<a\s.*?>(.*?)</a>}m).flatten.
+              select { |n| n.match?(name_regex) }.
+              map do |filename|
+                version =
+                  filename.
+                  gsub(/#{name_regex}-/i, "").
+                  split(/-|(\.tar\.)/).
+                  first
+                next unless version_class.correct?(version)
+
+                version_class.new(version)
+              end.compact
+          rescue Excon::Error::Timeout, Excon::Error::Socket
+            next if MAIN_PYPI_INDEXES.include?(index_url)
+
+            raise PrivateSourceAuthenticationFailure, sanitized_url
+          end
+        end
+
+        def index_urls
+          main_index_url =
+            config_variable_index_urls[:main] ||
+            pipfile_index_urls[:main] ||
+            requirement_file_index_urls[:main] ||
+            pip_conf_index_urls[:main] ||
+            "https://pypi.python.org/simple/"
+
+          if main_index_url
+            main_index_url = main_index_url.strip.gsub(%r{/*$}, "") + "/"
+          end
+
+          extra_index_urls =
+            config_variable_index_urls[:extra] +
+            pipfile_index_urls[:extra] +
+            requirement_file_index_urls[:extra] +
+            pip_conf_index_urls[:extra]
+
+          extra_index_urls =
+            extra_index_urls.map { |url| url.strip.gsub(%r{/*$}, "") + "/" }
+
+          [main_index_url, *extra_index_urls].uniq
+        end
+
+        def registry_response_for_dependency(index_url)
+          Excon.get(
+            index_url + normalised_name + "/",
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+        end
+
+        def registry_index_response(index_url)
+          Excon.get(
+            index_url,
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+        end
+
+        def requirement_file_index_urls
+          urls = { main: nil, extra: [] }
+
+          requirements_files.each do |file|
+            if file.content.match?(/^--index-url\s(.+)/)
+              urls[:main] =
+                file.content.match(/^--index-url\s(.+)/).captures.first
+            end
+            urls[:extra] += file.content.scan(/^--extra-index-url\s(.+)/).
+                            flatten
+          end
+
+          urls
+        end
+
+        def pip_conf_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pip_conf
+
+          content = pip_conf.content
+
+          if content.match?(/^index-url\s*=/x)
+            urls[:main] = content.match(/^index-url\s*=\s*(.+)/).
+                          captures.first
+          end
+          urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
+
+          urls
+        end
+
+        def pipfile_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pipfile
+
+          pipfile_object = TomlRB.parse(pipfile.content)
+
+          urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
+
+          pipfile_object["source"]&.each do |source|
+            urls[:extra] << source.fetch("url") if source["url"]
+          end
+          urls[:extra] = urls[:extra].uniq
+
+          urls
+        rescue TomlRB::ParseError
+          urls
+        end
+
+        def config_variable_index_urls
+          urls = { main: nil, extra: [] }
+
+          index_url_creds = credentials.
+                            select { |cred| cred["type"] == "python_index" }
+          urls[:main] =
+            index_url_creds.
+            find { |cred| cred["replaces-base"] }&.
+            fetch("index-url")
+          urls[:extra] =
+            index_url_creds.
+            reject { |cred| cred["replaces-base"] }.
+            map { |cred| cred["index-url"] }
+
+          urls
+        end
+
+        def ignore_reqs
+          ignored_versions.map { |req| requirement_class.new(req.split(",")) }
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalised_name
+          dependency.name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def name_regex
+          parts = dependency.name.split(/[\s_.-]/).map { |n| Regexp.quote(n) }
+          /#{parts.join("[\s_.-]")}/i
+        end
+
+        def pip_conf
+          dependency_files.find { |f| f.name == "pip.conf" }
+        end
+
+        def pipfile
+          dependency_files.find { |f| f.name == "Pipfile" }
+        end
+
+        def pyproject
+          dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def requirements_files
+          dependency_files.select { |f| f.name.match?(/requirements/x) }
+        end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+
+        def version_class
+          Utils.version_class_for_package_manager(dependency.package_manager)
+        end
+
+        def requirement_class
+          Utils.requirement_class_for_package_manager(
+            dependency.package_manager
+          )
+        end
+      end
+    end
+  end
+end