dependabot-python 0.79.0

data/lib/dependabot/python/file_updater/requirement_replacer.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "dependabot/python/requirement_parser"
+require "dependabot/python/file_updater"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      class RequirementReplacer
+        attr_reader :content, :dependency_name, :old_requirement,
+                    :new_requirement
+
+        def initialize(content:, dependency_name:, old_requirement:,
+                       new_requirement:)
+          @content         = content
+          @dependency_name = dependency_name
+          @old_requirement = old_requirement
+          @new_requirement = new_requirement
+        end
+
+        def updated_content
+          updated_content =
+            content.gsub(original_declaration_replacement_regex) do |mtch|
+              # If the "declaration" is setting an option (e.g., no-binary)
+              # ignore it, since it isn't actually a declaration
+              next mtch if Regexp.last_match.pre_match.match?(/--.*\z/)
+
+              updated_dependency_declaration_string(
+                old_requirement,
+                new_requirement
+              )
+            end
+
+          raise "Expected content to change!" if content == updated_content
+
+          updated_content
+        end
+
+        private
+
+        def original_dependency_declaration_string(old_req)
+          matches = []
+
+          dec =
+            if old_req.nil?
+              regex = RequirementParser::INSTALL_REQ_WITHOUT_REQUIREMENT
+              content.scan(regex) { matches << Regexp.last_match }
+              matches.find { |m| normalise(m[:name]) == dependency_name }
+            else
+              regex = RequirementParser::INSTALL_REQ_WITH_REQUIREMENT
+              content.scan(regex) { matches << Regexp.last_match }
+              matches.
+                select { |m| normalise(m[:name]) == dependency_name }.
+                find { |m| requirements_match(m[:requirements], old_req) }
+            end
+
+          raise "Declaration not found for #{dependency_name}!" unless dec
+
+          dec.to_s.strip
+        end
+
+        def updated_dependency_declaration_string(old_req, new_req)
+          if old_req
+            original_dependency_declaration_string(old_req).
+              sub(RequirementParser::REQUIREMENTS, new_req)
+          else
+            original_dependency_declaration_string(old_req).
+              sub(RequirementParser::NAME_WITH_EXTRAS) do |nm|
+                nm + new_req
+              end
+          end
+        end
+
+        def original_declaration_replacement_regex
+          original_string =
+            original_dependency_declaration_string(old_requirement)
+          /(?<![\-\w\.])#{Regexp.escape(original_string)}(?![\-\w\.])/
+        end
+
+        # See https://www.python.org/dev/peps/pep-0503/#normalized-names
+        def normalise(name)
+          name.downcase.gsub(/[-_.]+/, "-")
+        end
+
+        def requirements_match(req1, req2)
+          req1&.split(",")&.map { |r| r.gsub(/\s/, "") }&.sort ==
+            req2&.split(",")&.map { |r| r.gsub(/\s/, "") }&.sort
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/file_updater/setup_file_sanitizer.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "dependabot/python/file_updater"
+require "dependabot/python/file_parser/setup_file_parser"
+
+module Dependabot
+  module Python
+    class FileUpdater
+      # Take a setup.py, parses it (carefully!) and then create a new, clean
+      # setup.py using only the information which will appear in the lockfile.
+      class SetupFileSanitizer
+        def initialize(setup_file:, setup_cfg:)
+          @setup_file = setup_file
+          @setup_cfg = setup_cfg
+        end
+
+        def sanitized_content
+          # The part of the setup.py that Pipenv cares about appears to be the
+          # install_requires. A name and version are required by don't end up
+          # in the lockfile.
+          content =
+            "from setuptools import setup\n\n"\
+            "setup(name=\"sanitized-package\",version=\"0.0.1\","\
+            "install_requires=#{install_requires_array.to_json},"\
+            "extras_require=#{extras_require_hash.to_json}"
+
+          content += ',setup_requires=["pbr"],pbr=True' if include_pbr?
+          content + ")"
+        end
+
+        private
+
+        attr_reader :setup_file, :setup_cfg
+
+        def include_pbr?
+          setup_requires_array.any? { |d| d.start_with?("pbr") }
+        end
+
+        def install_requires_array
+          @install_requires_array ||=
+            parsed_setup_file.dependencies.map do |dep|
+              next unless dep.requirements.first[:groups].
+                          include?("install_requires")
+
+              dep.name + dep.requirements.first[:requirement].to_s
+            end.compact
+        end
+
+        def setup_requires_array
+          @setup_requires_array ||=
+            parsed_setup_file.dependencies.map do |dep|
+              next unless dep.requirements.first[:groups].
+                          include?("setup_requires")
+
+              dep.name + dep.requirements.first[:requirement].to_s
+            end.compact
+        end
+
+        def extras_require_hash
+          @extras_require_hash ||=
+            begin
+              hash = {}
+              parsed_setup_file.dependencies.each do |dep|
+                dep.requirements.first[:groups].each do |group|
+                  next unless group.start_with?("extras_require:")
+
+                  hash[group.split(":").last] ||= []
+                  hash[group.split(":").last] <<
+                    dep.name + dep.requirements.first[:requirement].to_s
+                end
+              end
+
+              hash
+            end
+        end
+
+        def parsed_setup_file
+          @parsed_setup_file ||=
+            Python::FileParser::SetupFileParser.new(
+              dependency_files: [
+                setup_file&.dup&.tap { |f| f.name = "setup.py" },
+                setup_cfg&.dup&.tap { |f| f.name = "setup.cfg" }
+              ].compact
+            ).dependency_set
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/metadata_finder.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/metadata_finders"
+require "dependabot/metadata_finders/base"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module Python
+    class MetadataFinder < Dependabot::MetadataFinders::Base
+      MAIN_PYPI_URL = "https://pypi.org/pypi"
+
+      def homepage_url
+        pypi_listing.dig("info", "home_page") || super
+      end
+
+      private
+
+      def look_up_source
+        potential_source_urls = [
+          pypi_listing.dig("info", "home_page"),
+          pypi_listing.dig("info", "bugtrack_url"),
+          pypi_listing.dig("info", "download_url"),
+          pypi_listing.dig("info", "docs_url")
+        ].compact
+
+        source_url = potential_source_urls.find { |url| Source.from_url(url) }
+        source_url ||= source_from_description
+        source_url ||= source_from_homepage
+
+        Source.from_url(source_url)
+      end
+
+      def source_from_description
+        github_urls = []
+        desc = pypi_listing.dig("info", "description")
+        return unless desc
+
+        desc.scan(Source::SOURCE_REGEX) do
+          github_urls << Regexp.last_match.to_s
+        end
+
+        github_urls.find do |url|
+          repo = Source.from_url(url).repo
+          repo.downcase.end_with?(dependency.name)
+        end
+      end
+
+      def source_from_homepage
+        return unless homepage_body
+
+        github_urls = []
+        homepage_body.scan(Source::SOURCE_REGEX) do
+          github_urls << Regexp.last_match.to_s
+        end
+
+        github_urls.find do |url|
+          repo = Source.from_url(url).repo
+          repo.downcase.end_with?(dependency.name)
+        end
+      end
+
+      def homepage_body
+        homepage_url = pypi_listing.dig("info", "home_page")
+
+        return unless homepage_url
+        return if homepage_url.include?("pypi.python.org")
+        return if homepage_url.include?("pypi.org")
+
+        @homepage_response ||=
+          begin
+            Excon.get(
+              homepage_url,
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+          rescue Excon::Error::Timeout, Excon::Error::Socket, ArgumentError
+            nil
+          end
+
+        return unless @homepage_response&.status == 200
+
+        @homepage_response.body
+      end
+
+      def pypi_listing
+        return @pypi_listing unless @pypi_listing.nil?
+        return @pypi_listing = {} if dependency.version.include?("+")
+
+        possible_listing_urls.each do |url|
+          response = Excon.get(
+            url,
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+          next unless response.status == 200
+
+          @pypi_listing = JSON.parse(response.body)
+          return @pypi_listing
+        rescue JSON::ParserError
+          next
+        end
+
+        @pypi_listing = {} # No listing found
+      end
+
+      def possible_listing_urls
+        credential_urls =
+          credentials.
+          select { |cred| cred["type"] == "python_index" }.
+          map { |cred| cred["index-url"].gsub(%r{/$}, "") }
+
+        (credential_urls + [MAIN_PYPI_URL]).map do |base_url|
+          base_url + "/#{dependency.name}/json"
+        end
+      end
+    end
+  end
+end
+
+Dependabot::MetadataFinders.
+  register("pip", Dependabot::Python::MetadataFinder)

data/lib/dependabot/python/native_helpers.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module Python
+    module NativeHelpers
+      def self.python_helper_path
+        helpers_dir = File.join(native_helpers_root, "python/helpers")
+        Pathname.new(File.join(helpers_dir, "run.py")).cleanpath.to_path
+      end
+
+      def self.native_helpers_root
+        default_path = File.join(__dir__, "../../../..")
+        ENV.fetch("DEPENDABOT_NATIVE_HELPERS_PATH", default_path)
+      end
+    end
+  end
+end

data/lib/dependabot/python/python_versions.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module Python
+    module PythonVersions
+      # Poetry doesn't handle Python versions, so we have to do so manually
+      # (checking from a list of versions Poetry supports).
+      # This list gets iterated through to find a valid version, so we have
+      # the two pre-installed versions listed first.
+      PYTHON_VERSIONS = %w(
+        3.6.7 2.7.15
+        3.7.1 3.7.0
+        3.6.7 3.6.6 3.6.5 3.6.4 3.6.3 3.6.2 3.6.1 3.6.0
+        3.5.6 3.5.5 3.5.4 3.5.3 3.5.2 3.5.1 3.5.0
+        3.4.9 3.4.8 3.4.7 3.4.6 3.4.5 3.4.4 3.4.3 3.4.2 3.4.1 3.4.0
+        2.7.15 2.7.14 2.7.13 2.7.12 2.7.11 2.7.10 2.7.9 2.7.8 2.7.7 2.7.6 2.7.5
+        2.7.4 2.7.3 2.7.2 2.7.1 2.7
+      ).freeze
+
+      PRE_INSTALLED_PYTHON_VERSIONS = %w(
+        3.6.7 2.7.15
+      ).freeze
+    end
+  end
+end

data/lib/dependabot/python/requirement.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "dependabot/python/version"
+
+module Dependabot
+  module Python
+    class Requirement < Gem::Requirement
+      OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|+/.freeze
+
+      # Add equality and arbitrary-equality matchers
+      OPS["=="] = ->(v, r) { v == r }
+      OPS["==="] = ->(v, r) { v.to_s == r.to_s }
+
+      quoted = OPS.keys.sort_by(&:length).reverse.
+               map { |k| Regexp.quote(k) }.join("|")
+      version_pattern = Python::Version::VERSION_PATTERN
+
+      PATTERN_RAW = "\\s*(#{quoted})?\\s*(#{version_pattern})\\s*"
+      PATTERN = /\A#{PATTERN_RAW}\z/.freeze
+
+      def self.parse(obj)
+        return ["=", Python::Version.new(obj.to_s)] if obj.is_a?(Gem::Version)
+
+        unless (matches = PATTERN.match(obj.to_s))
+          msg = "Illformed requirement [#{obj.inspect}]"
+          raise BadRequirementError, msg
+        end
+
+        return DefaultRequirement if matches[1] == ">=" && matches[2] == "0"
+
+        [matches[1] || "=", Python::Version.new(matches[2])]
+      end
+
+      # Returns an array of requirements. At least one requirement from the
+      # returned array must be satisfied for a version to be valid.
+      #
+      # NOTE: Or requirements are only valid for Poetry.
+      def self.requirements_array(requirement_string)
+        return [new(nil)] if requirement_string.nil?
+
+        requirement_string.strip.split(OR_SEPARATOR).map do |req_string|
+          new(req_string.strip)
+        end
+      end
+
+      def initialize(*requirements)
+        requirements = requirements.flatten.flat_map do |req_string|
+          next if req_string.nil?
+
+          req_string.split(",").map do |r|
+            convert_python_constraint_to_ruby_constraint(r)
+          end
+        end
+
+        super(requirements)
+      end
+
+      def satisfied_by?(version)
+        version = Python::Version.new(version.to_s)
+        super
+      end
+
+      def exact?
+        return false unless @requirements.size == 1
+
+        %w(= == ===).include?(@requirements[0][0])
+      end
+
+      private
+
+      def convert_python_constraint_to_ruby_constraint(req_string)
+        return nil if req_string.nil?
+        return nil if req_string == "*"
+
+        req_string = req_string.gsub("~=", "~>")
+        req_string = req_string.gsub(/(?<=\d)[<=>].*/, "")
+
+        if req_string.match?(/~[^>]/) then convert_tilde_req(req_string)
+        elsif req_string.start_with?("^") then convert_caret_req(req_string)
+        elsif req_string.include?(".*") then convert_wildcard(req_string)
+        else req_string
+        end
+      end
+
+      # Poetry uses ~ requirements.
+      # https://github.com/sdispater/poetry#tilde-requirements
+      def convert_tilde_req(req_string)
+        version = req_string.gsub(/^~\>?/, "")
+        parts = version.split(".")
+        parts << "0" if parts.count < 3
+        "~> #{parts.join('.')}"
+      end
+
+      # Poetry uses ^ requirements
+      # https://github.com/sdispater/poetry#caret-requirement
+      def convert_caret_req(req_string)
+        version = req_string.gsub(/^\^/, "")
+        parts = version.split(".")
+        parts = parts.fill(0, parts.length...3)
+        first_non_zero = parts.find { |d| d != "0" }
+        first_non_zero_index =
+          first_non_zero ? parts.index(first_non_zero) : parts.count - 1
+        upper_bound = parts.map.with_index do |part, i|
+          if i < first_non_zero_index then part
+          elsif i == first_non_zero_index then (part.to_i + 1).to_s
+          elsif i > first_non_zero_index && i == 2 then "0.a"
+          else 0
+          end
+        end.join(".")
+
+        [">= #{version}", "< #{upper_bound}"]
+      end
+
+      def convert_wildcard(req_string)
+        # Note: This isn't perfect. It replaces the "!= 1.0.*" case with
+        # "!= 1.0.0". There's no way to model this correctly in Ruby :'(
+        req_string.
+          split(".").
+          first(req_string.split(".").index("*") + 1).
+          join(".").
+          tr("*", "0").
+          gsub(/^(?<!!)=*/, "~>")
+      end
+    end
+  end
+end
+
+Dependabot::Utils.
+  register_requirement_class("pip", Dependabot::Python::Requirement)