dependabot-python 0.104.0 → 0.104.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93762e9f3a2a0f2f376cc958c13b0d4ec07cd07bb6208d4859566bb38dc8c1bb
-  data.tar.gz: 1ee335372559b02cd5ea0a12541875eb3c389726a759beb68d3dd22cbc9b9cc6
+  metadata.gz: 1704546d9328a26bba38bd159150a5238797253f07538aa5ee731e307d1a5a86
+  data.tar.gz: 4e1a258ebb51cedb6cd6c801d2ac4a7f2f191658b163b56046a7ab526c24c287
 SHA512:
-  metadata.gz: d4a11b74a94191a3e2e102a937d3ea327e0b46d0fda1b74faafd0f7626578a7bc6d496918c65e44b7895aa0103b92026d0dad323a1d8b872ef0c13634212c116
-  data.tar.gz: 6517220833b3f29e267f010ec2cc5f94aa2342ee4bc6edc8af83a490667ac3d97b625ab0c8c3d51dcd2da0735c55a29ebd0189c87cf67e78031a96ec4868c7ba
+  metadata.gz: 1b994991d16cafe0ece709fdc52f5bbe5f25cb3dd635ad08fab6b87885926afe281231642d28cfe24e26724bccd4427750d99dad27e718882ecc04bf333e8ff5
+  data.tar.gz: 37ffc6c72e569f50b2c73999089237512b08742e10473fb02f5b7047751be623ffeb9d9d8298376e1748748274dfe22c64c6a15ee60647de831317f8caf71e33

data/lib/dependabot/python/update_checker/index_finder.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require "dependabot/python/update_checker"
+require "dependabot/python/authed_url_builder"
+require "dependabot/errors"
+
+module Dependabot
+  module Python
+    class UpdateChecker
+      class IndexFinder
+        PYPI_BASE_URL = "https://pypi.python.org/simple/"
+        ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/.freeze
+
+        def initialize(dependency_files:, credentials:)
+          @dependency_files = dependency_files
+          @credentials      = credentials
+        end
+
+        def index_urls
+          extra_index_urls =
+            config_variable_index_urls[:extra] +
+            pipfile_index_urls[:extra] +
+            requirement_file_index_urls[:extra] +
+            pip_conf_index_urls[:extra]
+
+          extra_index_urls = extra_index_urls.map do |url|
+            clean_check_and_remove_environment_variables(url)
+          end
+
+          # URL encode any `@` characters within registry URL creds.
+          # TODO: The test that fails if the `map` here is removed is likely a
+          # bug in Ruby's URI parser, and should be fixed there.
+          [main_index_url, *extra_index_urls].map do |url|
+            url.rpartition("@").tap { |a| a.first.gsub!("@", "%40") }.join
+          end.uniq
+        end
+
+        private
+
+        attr_reader :dependency_files, :credentials
+
+        def main_index_url
+          url =
+            config_variable_index_urls[:main] ||
+            pipfile_index_urls[:main] ||
+            requirement_file_index_urls[:main] ||
+            pip_conf_index_urls[:main] ||
+            PYPI_BASE_URL
+
+          return unless url
+
+          clean_check_and_remove_environment_variables(url)
+        end
+
+        def requirement_file_index_urls
+          urls = { main: nil, extra: [] }
+
+          requirements_files.each do |file|
+            if file.content.match?(/^--index-url\s(.+)/)
+              urls[:main] =
+                file.content.match(/^--index-url\s(.+)/).captures.first
+            end
+            urls[:extra] += file.content.scan(/^--extra-index-url\s(.+)/).
+                            flatten
+          end
+
+          urls
+        end
+
+        def pip_conf_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pip_conf
+
+          content = pip_conf.content
+
+          if content.match?(/^index-url\s*=/x)
+            urls[:main] = content.match(/^index-url\s*=\s*(.+)/).
+                          captures.first
+          end
+          urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
+
+          urls
+        end
+
+        def pipfile_index_urls
+          urls = { main: nil, extra: [] }
+
+          return urls unless pipfile
+
+          pipfile_object = TomlRB.parse(pipfile.content)
+
+          urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
+
+          pipfile_object["source"]&.each do |source|
+            urls[:extra] << source.fetch("url") if source["url"]
+          end
+          urls[:extra] = urls[:extra].uniq
+
+          urls
+        rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
+          urls
+        end
+
+        def config_variable_index_urls
+          urls = { main: nil, extra: [] }
+
+          index_url_creds = credentials.
+                            select { |cred| cred["type"] == "python_index" }
+
+          if (main_cred = index_url_creds.find { |cred| cred["replaces-base"] })
+            urls[:main] = AuthedUrlBuilder.authed_url(credential: main_cred)
+          end
+
+          urls[:extra] =
+            index_url_creds.
+            reject { |cred| cred["replaces-base"] }.
+            map { |cred| AuthedUrlBuilder.authed_url(credential: cred) }
+
+          urls
+        end
+
+        def clean_check_and_remove_environment_variables(url)
+          url = url.strip.gsub(%r{/*$}, "") + "/"
+
+          unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
+            return authed_base_url(url)
+          end
+
+          config_variable_urls =
+            [
+              config_variable_index_urls[:main],
+              *config_variable_index_urls[:extra]
+            ].
+            compact.
+            map { |u| u.strip.gsub(%r{/*$}, "") + "/" }
+
+          regexp = url.split(ENVIRONMENT_VARIABLE_REGEX).
+                   map { |part| Regexp.quote(part) }.
+                   join(".+")
+          authed_url = config_variable_urls.find { |u| u.match?(regexp) }
+          return authed_url if authed_url
+
+          cleaned_url = url.gsub(%r{#{ENVIRONMENT_VARIABLE_REGEX}/?}, "")
+          authed_url = authed_base_url(cleaned_url)
+          return authed_url if credential_for(cleaned_url)
+
+          raise PrivateSourceAuthenticationFailure, url
+        end
+
+        def authed_base_url(base_url)
+          cred = credential_for(base_url)
+          return base_url unless cred
+
+          AuthedUrlBuilder.authed_url(credential: cred).gsub(%r{/*$}, "") + "/"
+        end
+
+        def credential_for(url)
+          credentials.
+            select { |c| c["type"] == "python_index" }.
+            find do |c|
+              cred_url = c.fetch("index-url").gsub(%r{/*$}, "") + "/"
+              cred_url.include?(url)
+            end
+        end
+
+        def pip_conf
+          dependency_files.find { |f| f.name == "pip.conf" }
+        end
+
+        def pipfile
+          dependency_files.find { |f| f.name == "Pipfile" }
+        end
+
+        def pyproject
+          dependency_files.find { |f| f.name == "pyproject.toml" }
+        end
+
+        def requirements_files
+          dependency_files.select { |f| f.name.match?(/requirements/x) }
+        end
+
+        def pip_compile_files
+          dependency_files.select { |f| f.name.end_with?(".in") }
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/python/update_checker/latest_version_finder.rb

@@ -10,7 +10,7 @@ module Dependabot
   module Python
     class UpdateChecker
       class LatestVersionFinder
-        ENVIRONMENT_VARIABLE_REGEX = /\$\{.+\}/.freeze
+        require_relative "index_finder"
 
         def initialize(dependency:, dependency_files:, credentials:,
                        ignored_versions:)
@@ -36,21 +36,37 @@ module Dependabot
 
         def fetch_latest_version
           versions = available_versions
-          versions.reject! { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }
-          versions.reject!(&:prerelease?) unless wants_prerelease?
+          versions = filter_prerelease_versions(versions)
+          versions = filter_ignored_versions(versions)
           versions.max
         end
 
         def fetch_latest_version_with_no_unlock
           versions = available_versions
+          versions = filter_prerelease_versions(versions)
+          versions = filter_ignored_versions(versions)
+          versions = filter_out_of_range_versions(versions)
+          versions.max
+        end
+
+        def filter_prerelease_versions(versions_array)
+          return versions_array if wants_prerelease?
+
+          versions_array.reject(&:prerelease?)
+        end
+
+        def filter_ignored_versions(versions_array)
+          versions_array.
+            reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }
+        end
+
+        def filter_out_of_range_versions(versions_array)
           reqs = dependency.requirements.map do |r|
-            reqs = (r.fetch(:requirement) || "").split(",").map(&:strip)
-            requirement_class.new(reqs)
-          end
-          versions.reject!(&:prerelease?) unless wants_prerelease?
-          versions.sort.reverse.
-            reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }.
-            find { |v| reqs.all? { |r| r.satisfied_by?(v) } }
+            requirement_class.requirements_array(r.fetch(:requirement))
+          end.compact
+
+          versions_array.
+            select { |v| reqs.all? { |r| r.any? { |o| o.satisfied_by?(v) } } }
         end
 
         def wants_prerelease?
@@ -68,65 +84,42 @@ module Dependabot
         # See https://www.python.org/dev/peps/pep-0503/ for details of the
         # Simple Repository API we use here.
         def available_versions
-          index_urls.flat_map do |index_url|
-            sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
-            index_response = registry_response_for_dependency(index_url)
+          @available_versions ||=
+            index_urls.flat_map do |index_url|
+              sanitized_url = index_url.gsub(%r{(?<=//).*(?=@)}, "redacted")
+              index_response = registry_response_for_dependency(index_url)
+
+              if [401, 403].include?(index_response.status) &&
+                 [401, 403].include?(registry_index_response(index_url).status)
+                raise PrivateSourceAuthenticationFailure, sanitized_url
+              end
+
+              index_response.body.
+                scan(%r{<a\s.*?>(.*?)</a>}m).flatten.
+                select { |n| n.match?(name_regex) }.
+                map do |filename|
+                  version =
+                    filename.
+                    gsub(/#{name_regex}-/i, "").
+                    split(/-|\.tar\.|\.zip|\.whl/).
+                    first
+                  next unless version_class.correct?(version)
+
+                  version_class.new(version)
+                end.compact
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              raise if MAIN_PYPI_INDEXES.include?(index_url)
 
-            if [401, 403].include?(index_response.status) &&
-               [401, 403].include?(registry_index_response(index_url).status)
               raise PrivateSourceAuthenticationFailure, sanitized_url
             end
-
-            index_response.body.
-              scan(%r{<a\s.*?>(.*?)</a>}m).flatten.
-              select { |n| n.match?(name_regex) }.
-              map do |filename|
-                version =
-                  filename.
-                  gsub(/#{name_regex}-/i, "").
-                  split(/-|\.tar\.|\.zip|\.whl/).
-                  first
-                next unless version_class.correct?(version)
-
-                version_class.new(version)
-              end.compact
-          rescue Excon::Error::Timeout, Excon::Error::Socket
-            next if MAIN_PYPI_INDEXES.include?(index_url)
-
-            raise PrivateSourceAuthenticationFailure, sanitized_url
-          end
         end
 
         def index_urls
-          extra_index_urls =
-            config_variable_index_urls[:extra] +
-            pipfile_index_urls[:extra] +
-            requirement_file_index_urls[:extra] +
-            pip_conf_index_urls[:extra]
-
-          extra_index_urls = extra_index_urls.map do |url|
-            clean_check_and_remove_environment_variables(url)
-          end
-
-          # URL encode any `@` characters within registry URL creds.
-          # TODO: The test that fails if the `map` here is removed is likely a
-          # bug in Ruby's URI parser, and should be fixed there.
-          [main_index_url, *extra_index_urls].map do |url|
-            url.rpartition("@").tap { |a| a.first.gsub!("@", "%40") }.join
-          end.uniq
-        end
-
-        def main_index_url
-          url =
-            config_variable_index_urls[:main] ||
-            pipfile_index_urls[:main] ||
-            requirement_file_index_urls[:main] ||
-            pip_conf_index_urls[:main] ||
-            "https://pypi.python.org/simple/"
-
-          return unless url
-
-          clean_check_and_remove_environment_variables(url)
+          @index_urls ||=
+            IndexFinder.new(
+              dependency_files: dependency_files,
+              credentials: credentials
+            ).index_urls
         end
 
         def registry_response_for_dependency(index_url)
@@ -145,118 +138,6 @@ module Dependabot
           )
         end
 
-        def requirement_file_index_urls
-          urls = { main: nil, extra: [] }
-
-          requirements_files.each do |file|
-            if file.content.match?(/^--index-url\s(.+)/)
-              urls[:main] =
-                file.content.match(/^--index-url\s(.+)/).captures.first
-            end
-            urls[:extra] += file.content.scan(/^--extra-index-url\s(.+)/).
-                            flatten
-          end
-
-          urls
-        end
-
-        def pip_conf_index_urls
-          urls = { main: nil, extra: [] }
-
-          return urls unless pip_conf
-
-          content = pip_conf.content
-
-          if content.match?(/^index-url\s*=/x)
-            urls[:main] = content.match(/^index-url\s*=\s*(.+)/).
-                          captures.first
-          end
-          urls[:extra] += content.scan(/^extra-index-url\s*=(.+)/).flatten
-
-          urls
-        end
-
-        def pipfile_index_urls
-          urls = { main: nil, extra: [] }
-
-          return urls unless pipfile
-
-          pipfile_object = TomlRB.parse(pipfile.content)
-
-          urls[:main] = pipfile_object["source"]&.first&.fetch("url", nil)
-
-          pipfile_object["source"]&.each do |source|
-            urls[:extra] << source.fetch("url") if source["url"]
-          end
-          urls[:extra] = urls[:extra].uniq
-
-          urls
-        rescue TomlRB::ParseError, TomlRB::ValueOverwriteError
-          urls
-        end
-
-        def config_variable_index_urls
-          urls = { main: nil, extra: [] }
-
-          index_url_creds = credentials.
-                            select { |cred| cred["type"] == "python_index" }
-
-          if (main_cred = index_url_creds.find { |cred| cred["replaces-base"] })
-            urls[:main] = AuthedUrlBuilder.authed_url(credential: main_cred)
-          end
-
-          urls[:extra] =
-            index_url_creds.
-            reject { |cred| cred["replaces-base"] }.
-            map { |cred| AuthedUrlBuilder.authed_url(credential: cred) }
-
-          urls
-        end
-
-        def clean_check_and_remove_environment_variables(url)
-          url = url.strip.gsub(%r{/*$}, "") + "/"
-
-          unless url.match?(ENVIRONMENT_VARIABLE_REGEX)
-            return authed_base_url(url)
-          end
-
-          config_variable_urls =
-            [
-              config_variable_index_urls[:main],
-              *config_variable_index_urls[:extra]
-            ].
-            compact.
-            map { |u| u.strip.gsub(%r{/*$}, "") + "/" }
-
-          regexp = url.split(ENVIRONMENT_VARIABLE_REGEX).
-                   map { |part| Regexp.quote(part) }.
-                   join(".+")
-          authed_url = config_variable_urls.find { |u| u.match?(regexp) }
-          return authed_url if authed_url
-
-          cleaned_url = url.gsub(%r{#{ENVIRONMENT_VARIABLE_REGEX}/?}, "")
-          authed_url = authed_base_url(cleaned_url)
-          return authed_url if credential_for(cleaned_url)
-
-          raise PrivateSourceAuthenticationFailure, url
-        end
-
-        def authed_base_url(base_url)
-          cred = credential_for(base_url)
-          return base_url unless cred
-
-          AuthedUrlBuilder.authed_url(credential: cred).gsub(%r{/*$}, "") + "/"
-        end
-
-        def credential_for(url)
-          credentials.
-            select { |c| c["type"] == "python_index" }.
-            find do |c|
-              cred_url = c.fetch("index-url").gsub(%r{/*$}, "") + "/"
-              cred_url.include?(url)
-            end
-        end
-
         def ignore_reqs
           ignored_versions.map { |req| requirement_class.new(req.split(",")) }
         end
@@ -271,26 +152,6 @@ module Dependabot
           /#{parts.join("[\s_.-]")}/i
         end
 
-        def pip_conf
-          dependency_files.find { |f| f.name == "pip.conf" }
-        end
-
-        def pipfile
-          dependency_files.find { |f| f.name == "Pipfile" }
-        end
-
-        def pyproject
-          dependency_files.find { |f| f.name == "pyproject.toml" }
-        end
-
-        def requirements_files
-          dependency_files.select { |f| f.name.match?(/requirements/x) }
-        end
-
-        def pip_compile_files
-          dependency_files.select { |f| f.name.end_with?(".in") }
-        end
-
         def version_class
           Utils.version_class_for_package_manager(dependency.package_manager)
         end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-python
 version: !ruby/object:Gem::Version
-  version: 0.104.0
+  version: 0.104.1
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-04-16 00:00:00.000000000 Z
+date: 2019-04-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.104.0
+        version: 0.104.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.104.0
+        version: 0.104.1
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -172,6 +172,7 @@ files:
 - lib/dependabot/python/requirement.rb
 - lib/dependabot/python/requirement_parser.rb
 - lib/dependabot/python/update_checker.rb
+- lib/dependabot/python/update_checker/index_finder.rb
 - lib/dependabot/python/update_checker/latest_version_finder.rb
 - lib/dependabot/python/update_checker/pip_compile_version_resolver.rb
 - lib/dependabot/python/update_checker/pipfile_version_resolver.rb