dependabot-npm_and_yarn 0.91.0

data/lib/dependabot/npm_and_yarn/update_checker/latest_version_finder.rb

@@ -0,0 +1,340 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/npm_and_yarn/update_checker"
+require "dependabot/npm_and_yarn/update_checker/registry_finder"
+require "dependabot/npm_and_yarn/version"
+require "dependabot/npm_and_yarn/requirement"
+require "dependabot/shared_helpers"
+require "dependabot/errors"
+
+module Dependabot
+  module NpmAndYarn
+    class UpdateChecker
+      class LatestVersionFinder
+        class RegistryError < StandardError; end
+
+        def initialize(dependency:, credentials:, dependency_files:,
+                       ignored_versions:)
+          @dependency       = dependency
+          @credentials      = credentials
+          @dependency_files = dependency_files
+          @ignored_versions = ignored_versions
+        end
+
+        def latest_version_details_from_registry
+          return nil unless npm_details&.fetch("dist-tags", nil)
+
+          dist_tag_version = version_from_dist_tags(npm_details)
+          return { version: dist_tag_version } if dist_tag_version
+          return nil if specified_dist_tag_requirement?
+
+          { version: version_from_versions_array }
+        rescue Excon::Error::Socket, Excon::Error::Timeout
+          raise if dependency_registry == "registry.npmjs.org"
+          # Custom registries can be flaky. We don't want to make that
+          # our problem, so we quietly return `nil` here.
+        end
+
+        def latest_resolvable_version_with_no_unlock
+          return unless npm_details
+
+          if specified_dist_tag_requirement?
+            return version_from_dist_tags(npm_details)
+          end
+
+          reqs = dependency.requirements.map do |r|
+            NpmAndYarn::Requirement.
+              requirements_array(r.fetch(:requirement))
+          end.compact
+
+          possible_versions.
+            find do |version|
+              reqs.all? { |r| r.any? { |opt| opt.satisfied_by?(version) } } &&
+                !yanked?(version)
+            end
+        rescue Excon::Error::Socket, Excon::Error::Timeout
+          raise if dependency_registry == "registry.npmjs.org"
+          # Sometimes custom registries are flaky. We don't want to make that
+          # our problem, so we quietly return `nil` here.
+        end
+
+        def possible_versions
+          npm_details.fetch("versions", {}).
+            reject { |_, details| details["deprecated"] }.
+            keys.map { |v| version_class.new(v) }.
+            reject { |v| v.prerelease? && !related_to_current_pre?(v) }.
+            reject { |v| ignore_reqs.any? { |r| r.satisfied_by?(v) } }.
+            sort.reverse
+        end
+
+        def possible_versions_with_details
+          npm_details.fetch("versions", {}).
+            reject { |_, details| details["deprecated"] }.
+            transform_keys { |k| version_class.new(k) }.
+            reject { |k, _| k.prerelease? && !related_to_current_pre?(k) }.
+            reject { |k, _| ignore_reqs.any? { |r| r.satisfied_by?(k) } }.
+            sort_by(&:first).reverse
+        end
+
+        private
+
+        attr_reader :dependency, :credentials, :dependency_files,
+                    :ignored_versions
+
+        def version_from_dist_tags(npm_details)
+          dist_tags = npm_details["dist-tags"].keys
+
+          # Check if a dist tag was specified as a requirement. If it was, and
+          # it exists, use it.
+          dist_tag_req = dependency.requirements.
+                         find { |r| dist_tags.include?(r[:requirement]) }&.
+                         fetch(:requirement)
+
+          if dist_tag_req
+            tag_vers =
+              version_class.new(npm_details["dist-tags"][dist_tag_req])
+            return tag_vers unless yanked?(tag_vers)
+          end
+
+          # Use the latest dist tag unless there's a reason not to
+          return nil unless npm_details["dist-tags"]["latest"]
+
+          latest = version_class.new(npm_details["dist-tags"]["latest"])
+
+          wants_latest_dist_tag?(latest) ? latest : nil
+        end
+
+        def related_to_current_pre?(version)
+          current_version = dependency.version
+          if current_version &&
+             version_class.correct?(current_version) &&
+             version_class.new(current_version).prerelease? &&
+             version_class.new(current_version).release == version.release
+            return true
+          end
+
+          dependency.requirements.any? do |req|
+            next unless req[:requirement]&.match?(/\d-[A-Za-z]/)
+
+            NpmAndYarn::Requirement.
+              requirements_array(req.fetch(:requirement)).
+              any? do |r|
+                r.requirements.any? { |a| a.last.release == version.release }
+              end
+          rescue Gem::Requirement::BadRequirementError
+            false
+          end
+        end
+
+        def specified_dist_tag_requirement?
+          dependency.requirements.any? do |req|
+            next false if req[:requirement].nil?
+
+            req[:requirement].match?(/^[A-Za-z]/)
+          end
+        end
+
+        def wants_latest_dist_tag?(latest_version)
+          ver = latest_version
+          return false if related_to_current_pre?(ver) ^ ver.prerelease?
+          return false if current_version_greater_than?(ver)
+          return false if current_requirement_greater_than?(ver)
+          return false if ignore_reqs.any? { |r| r.satisfied_by?(ver) }
+          return false if yanked?(ver)
+
+          true
+        end
+
+        def current_version_greater_than?(version)
+          return false unless dependency.version
+          return false unless version_class.correct?(dependency.version)
+
+          version_class.new(dependency.version) > version
+        end
+
+        def current_requirement_greater_than?(version)
+          dependency.requirements.any? do |req|
+            next false unless req[:requirement]
+
+            req_version = req[:requirement].sub(/^\^|~|>=?/, "")
+            next false unless version_class.correct?(req_version)
+
+            version_class.new(req_version) > version
+          end
+        end
+
+        def version_from_versions_array
+          possible_versions.find { |version| !yanked?(version) }
+        end
+
+        def yanked?(version)
+          @yanked ||= {}
+          return @yanked[version] if @yanked.key?(version)
+
+          @yanked[version] =
+            begin
+              version_not_found =
+                Excon.get(
+                  dependency_url + "/#{version}",
+                  SharedHelpers.excon_defaults.merge(
+                    headers: registry_auth_headers,
+                    idempotent: true
+                  )
+                ).status == 404
+              version_not_found && version_endpoint_working?
+            rescue Excon::Error::Timeout
+              # Give the benefit of the doubt if the registry is playing up
+              false
+            end
+        end
+
+        def version_endpoint_working?
+          return true if dependency_registry == "registry.npmjs.org"
+
+          if defined?(@version_endpoint_working)
+            return @version_endpoint_working
+          end
+
+          @version_endpoint_working =
+            begin
+              Excon.get(
+                dependency_url + "/latest",
+                SharedHelpers.excon_defaults.merge(
+                  headers: registry_auth_headers,
+                  idempotent: true
+                )
+              ).status < 400
+            rescue Excon::Error::Timeout
+              # Give the benefit of the doubt if the registry is playing up
+              true
+            end
+        end
+
+        def npm_details
+          return @npm_details if @npm_details_lookup_attempted
+
+          @npm_details_lookup_attempted = true
+          @npm_details ||=
+            begin
+              npm_response = fetch_npm_response
+
+              check_npm_response(npm_response)
+              JSON.parse(npm_response.body)
+            rescue JSON::ParserError, Excon::Error::Timeout,
+                   RegistryError => error
+              retry_count ||= 0
+              retry_count += 1
+              raise_npm_details_error(error) if retry_count > 2
+              sleep(rand(3.0..10.0)) && retry
+            end
+        end
+
+        def fetch_npm_response
+          response = Excon.get(
+            dependency_url,
+            SharedHelpers.excon_defaults.merge(
+              headers: registry_auth_headers,
+              idempotent: true
+            )
+          )
+
+          return response unless response.status == 500
+          return response unless registry_auth_headers["Authorization"]
+
+          auth = registry_auth_headers["Authorization"]
+          return response unless auth.start_with?("Basic")
+
+          decoded_token = Base64.decode64(auth.gsub("Basic ", ""))
+          return unless decoded_token.include?(":")
+
+          username, password = decoded_token.split(":")
+          Excon.get(
+            dependency_url,
+            SharedHelpers.excon_defaults.merge(
+              user: username,
+              password: password,
+              idempotent: true
+            )
+          )
+        end
+
+        def check_npm_response(npm_response)
+          return if git_dependency?
+
+          if private_dependency_not_reachable?(npm_response)
+            raise PrivateSourceAuthenticationFailure, dependency_registry
+          end
+
+          status = npm_response.status
+          return if status.to_s.start_with?("2")
+
+          # Ignore 404s from the registry for updates where a lockfile doesn't
+          # need to be generated. The 404 won't cause problems later.
+          return if status == 404 && dependency.version.nil?
+
+          msg = "Got #{status} response with body #{npm_response.body}"
+          raise RegistryError, msg
+        end
+
+        def raise_npm_details_error(error)
+          raise if dependency_registry == "registry.npmjs.org"
+          raise unless error.is_a?(Excon::Error::Timeout)
+
+          raise PrivateSourceTimedOut, dependency_registry
+        end
+
+        def private_dependency_not_reachable?(npm_response)
+          # Check whether this dependency is (likely to be) private
+          if dependency_registry == "registry.npmjs.org" &&
+             !dependency.name.start_with?("@")
+            return false
+          end
+
+          [401, 402, 403, 404].include?(npm_response.status)
+        end
+
+        def dependency_url
+          registry_finder.dependency_url
+        end
+
+        def dependency_registry
+          registry_finder.registry
+        end
+
+        def registry_auth_headers
+          registry_finder.auth_headers
+        end
+
+        def registry_finder
+          @registry_finder ||=
+            RegistryFinder.new(
+              dependency: dependency,
+              credentials: credentials,
+              npmrc_file: dependency_files.
+                          find { |f| f.name.end_with?(".npmrc") },
+              yarnrc_file: dependency_files.
+                           find { |f| f.name.end_with?(".yarnrc") }
+            )
+        end
+
+        def ignore_reqs
+          ignored_versions.
+            map { |req| NpmAndYarn::Requirement.new(req.split(",")) }
+        end
+
+        def version_class
+          NpmAndYarn::Version
+        end
+
+        # TODO: Remove need for me
+        def git_dependency?
+          GitCommitChecker.new(
+            dependency: dependency,
+            credentials: credentials
+          ).git_dependency?
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/npm_and_yarn/update_checker/library_detector.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/npm_and_yarn/update_checker"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module NpmAndYarn
+    class UpdateChecker
+      class LibraryDetector
+        def initialize(package_json_file:)
+          @package_json_file = package_json_file
+        end
+
+        def library?
+          return false unless package_json_may_be_for_library?
+
+          npm_response_matches_package_json?
+        end
+
+        private
+
+        attr_reader :package_json_file
+
+        def package_json_may_be_for_library?
+          return false unless project_name
+          return false if project_name.match?(/\{\{.*\}\}/)
+          return false unless parsed_package_json["version"]
+          return false if parsed_package_json["private"]
+
+          true
+        end
+
+        def npm_response_matches_package_json?
+          project_description = parsed_package_json["description"]
+          return false unless project_description
+
+          # Check if the project is listed on npm. If it is, it's a library
+          @project_npm_response ||= Excon.get(
+            "https://registry.npmjs.org/#{escaped_project_name}",
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          return false unless @project_npm_response.status == 200
+
+          @project_npm_response.body.force_encoding("UTF-8").encode.
+            include?(project_description)
+        rescue Excon::Error::Socket, Excon::Error::Timeout
+          false
+        end
+
+        def project_name
+          parsed_package_json.fetch("name", nil)
+        end
+
+        def escaped_project_name
+          project_name&.gsub("/", "%2F")
+        end
+
+        def parsed_package_json
+          @parsed_package_json ||= JSON.parse(package_json_file.content)
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/npm_and_yarn/update_checker/registry_finder.rb

@@ -0,0 +1,224 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/npm_and_yarn/update_checker"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module NpmAndYarn
+    class UpdateChecker
+      class RegistryFinder
+        NPM_AUTH_TOKEN_REGEX =
+          %r{//(?<registry>.*)/:_authToken=(?<token>.*)$}.freeze
+        NPM_GLOBAL_REGISTRY_REGEX =
+          /^registry\s*=\s*(?<registry>.*)$/.freeze
+        YARN_GLOBAL_REGISTRY_REGEX =
+          /^registry\s+['"](?<registry>.*)['"]/.freeze
+
+        def initialize(dependency:, credentials:, npmrc_file: nil,
+                       yarnrc_file: nil)
+          @dependency = dependency
+          @credentials = credentials
+          @npmrc_file = npmrc_file
+          @yarnrc_file = yarnrc_file
+        end
+
+        def registry
+          locked_registry || first_registry_with_dependency_details
+        end
+
+        def auth_headers
+          auth_header_for(auth_token)
+        end
+
+        def dependency_url
+          "#{registry_url.gsub(%r{/+$}, '')}/#{escaped_dependency_name}"
+        end
+
+        private
+
+        attr_reader :dependency, :credentials, :npmrc_file, :yarnrc_file
+
+        def first_registry_with_dependency_details
+          @first_registry_with_dependency_details ||=
+            known_registries.find do |details|
+              Excon.get(
+                "https://#{details['registry'].gsub(%r{/+$}, '')}/"\
+                "#{escaped_dependency_name}",
+                headers: auth_header_for(details["token"]),
+                idempotent: true,
+                **SharedHelpers.excon_defaults
+              ).status < 400
+            rescue Excon::Error::Timeout, Excon::Error::Socket
+              nil
+            end&.fetch("registry")
+
+          @first_registry_with_dependency_details ||= global_registry
+        end
+
+        def registry_url
+          protocol =
+            if private_registry_source_url
+              private_registry_source_url.split("://").first
+            else
+              "https"
+            end
+
+          "#{protocol}://#{registry}"
+        end
+
+        def auth_header_for(token)
+          return {} unless token
+
+          if token.include?(":")
+            encoded_token = Base64.encode64(token).delete("\n")
+            { "Authorization" => "Basic #{encoded_token}" }
+          elsif Base64.decode64(token).ascii_only? &&
+                Base64.decode64(token).include?(":")
+            { "Authorization" => "Basic #{token.delete("\n")}" }
+          else
+            { "Authorization" => "Bearer #{token}" }
+          end
+        end
+
+        def auth_token
+          known_registries.
+            find { |cred| cred["registry"] == registry }&.
+            fetch("token")
+        end
+
+        def locked_registry
+          return unless private_registry_source_url
+
+          lockfile_registry =
+            private_registry_source_url.
+            gsub("https://", "").
+            gsub("http://", "")
+          detailed_registry =
+            known_registries.
+            find { |h| h["registry"].include?(lockfile_registry) }&.
+            fetch("registry")
+
+          detailed_registry || lockfile_registry
+        end
+
+        def known_registries
+          @known_registries ||=
+            begin
+              registries = []
+              registries += credentials.
+                            select { |cred| cred["type"] == "npm_registry" }
+              registries += npmrc_registries
+              registries += yarnrc_registries
+
+              unique_registries(registries)
+            end
+        end
+
+        def npmrc_registries
+          return [] unless npmrc_file
+
+          registries = []
+          npmrc_file.content.scan(NPM_AUTH_TOKEN_REGEX) do
+            next if Regexp.last_match[:registry].include?("${")
+
+            registries << {
+              "type" => "npm_registry",
+              "registry" => Regexp.last_match[:registry],
+              "token" => Regexp.last_match[:token]
+            }
+          end
+
+          npmrc_file.content.scan(NPM_GLOBAL_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${")
+
+            registry = Regexp.last_match[:registry].strip.
+                       sub(%r{/+$}, "").
+                       sub(%r{^.*?//}, "")
+            next if registries.map { |r| r["registry"] }.include?(registry)
+
+            registries << {
+              "type" => "npm_registry",
+              "registry" => registry,
+              "token" => nil
+            }
+          end
+
+          registries
+        end
+
+        def yarnrc_registries
+          return [] unless yarnrc_file
+
+          registries = []
+          yarnrc_file.content.scan(YARN_GLOBAL_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${")
+
+            registry = Regexp.last_match[:registry].strip.
+                       sub(%r{/+$}, "").
+                       sub(%r{^.*?//}, "")
+            registries << {
+              "type" => "npm_registry",
+              "registry" => registry,
+              "token" => nil
+            }
+          end
+
+          registries
+        end
+
+        def unique_registries(registries)
+          registries.uniq.reject do |registry|
+            next if registry["token"]
+
+            # Reject this entry if an identical one with a token exists
+            registries.any? do |r|
+              r["token"] && r["registry"] == registry["registry"]
+            end
+          end
+        end
+
+        def global_registry
+          npmrc_file&.content.to_s.scan(NPM_GLOBAL_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${")
+
+            registry = Regexp.last_match[:registry].strip.
+                       sub(%r{/+$}, "").
+                       sub(%r{^.*?//}, "")
+            return registry
+          end
+
+          yarnrc_file&.content.to_s.scan(YARN_GLOBAL_REGISTRY_REGEX) do
+            next if Regexp.last_match[:registry].include?("${")
+
+            registry = Regexp.last_match[:registry].strip.
+                       sub(%r{/+$}, "").
+                       sub(%r{^.*?//}, "")
+            return registry
+          end
+
+          "registry.npmjs.org"
+        end
+
+        # npm registries expect slashes to be escaped
+        def escaped_dependency_name
+          dependency.name.gsub("/", "%2F")
+        end
+
+        def private_registry_source_url
+          sources = dependency.requirements.
+                    map { |r| r.fetch(:source) }.uniq.compact
+
+          # If there are multiple source types, or multiple source URLs, then
+          # it's unclear how we should proceed
+          if sources.map { |s| [s[:type], s[:url]] }.uniq.count > 1
+            raise "Multiple sources! #{sources.join(', ')}"
+          end
+
+          # Otherwise we just take the URL of the first private registry
+          sources.find { |s| s[:type] == "private_registry" }&.fetch(:url)
+        end
+      end
+    end
+  end
+end