dependabot-core 0.76.1

data/lib/dependabot/metadata_finders/java_script/npm_and_yarn.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require "excon"
+require "time"
+
+require "dependabot/metadata_finders/base"
+require "dependabot/shared_helpers"
+require "dependabot/utils"
+
+module Dependabot
+  module MetadataFinders
+    module JavaScript
+      class NpmAndYarn < Dependabot::MetadataFinders::Base
+        def homepage_url
+          # Attempt to use version_listing first, as fetching the entire listing
+          # array can be slow (if it's large)
+          if latest_version_listing["homepage"]
+            return latest_version_listing["homepage"]
+          end
+
+          listing = all_version_listings.find { |_, l| l["homepage"] }
+          listing&.last&.fetch("homepage", nil) || super
+        end
+
+        def maintainer_changes
+          return unless npm_releaser
+          return unless npm_listing.dig("time", dependency.version)
+          return if previous_releasers.include?(npm_releaser)
+
+          "This version was pushed to npm by "\
+          "[#{npm_releaser}](https://www.npmjs.com/~#{npm_releaser}), a new "\
+          "releaser for #{dependency.name} since your current version."
+        end
+
+        private
+
+        def look_up_source
+          return find_source_from_registry if new_source.nil?
+
+          source_type = new_source[:type] || new_source.fetch("type")
+
+          case source_type
+          when "git" then find_source_from_git_url
+          when "private_registry" then find_source_from_registry
+          else raise "Unexpected source type: #{source_type}"
+          end
+        end
+
+        def npm_releaser
+          all_version_listings.
+            find { |v, _| v == dependency.version }&.
+            last&.fetch("_npmUser", nil)&.fetch("name", nil)
+        end
+
+        def previous_releasers
+          times = npm_listing.fetch("time")
+
+          cutoff =
+            if dependency.previous_version && times[dependency.previous_version]
+              Time.parse(times[dependency.previous_version])
+            elsif times[dependency.version]
+              Time.parse(times[dependency.version]) - 1
+            end
+          return unless cutoff
+
+          all_version_listings.
+            reject { |v, _| Time.parse(times[v]) > cutoff }.
+            map { |_, d| d.fetch("_npmUser", nil)&.fetch("name", nil) }.compact
+        end
+
+        def find_source_from_registry
+          # Attempt to use version_listing first, as fetching the entire listing
+          # array can be slow (if it's large)
+          potential_source_urls =
+            [
+              get_url(latest_version_listing["repository"]),
+              get_url(latest_version_listing["homepage"]),
+              get_url(latest_version_listing["bugs"])
+            ].compact
+
+          source_url = potential_source_urls.find { |url| Source.from_url(url) }
+          return Source.from_url(source_url) if Source.from_url(source_url)
+
+          potential_source_urls =
+            all_version_listings.flat_map do |_, listing|
+              [
+                get_url(listing["repository"]),
+                get_url(listing["homepage"]),
+                get_url(listing["bugs"])
+              ]
+            end.compact
+
+          source_url = potential_source_urls.find { |url| Source.from_url(url) }
+          Source.from_url(source_url)
+        end
+
+        def new_source
+          sources = dependency.requirements.
+                    map { |r| r.fetch(:source) }.uniq.compact
+
+          raise "Multiple sources! #{sources.join(', ')}" if sources.count > 1
+
+          sources.first
+        end
+
+        def get_url(details)
+          case details
+          when String then details
+          when Hash then details.fetch("url", nil)
+          end
+        end
+
+        def find_source_from_git_url
+          url = new_source[:url] || new_source.fetch("url")
+          Source.from_url(url)
+        end
+
+        def latest_version_listing
+          return @latest_version_listing if defined?(@latest_version_listing)
+
+          response = Excon.get(
+            "#{dependency_url}/latest",
+            headers: registry_auth_headers,
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          if response.status == 200
+            return @latest_version_listing = JSON.parse(response.body)
+          end
+
+          @latest_version_listing = {}
+        rescue JSON::ParserError, Excon::Error::Timeout
+          @latest_version_listing = {}
+        end
+
+        def all_version_listings
+          return [] if npm_listing["versions"].nil?
+
+          npm_listing["versions"].
+            reject { |_, details| details["deprecated"] }.
+            sort_by { |version, _| Utils::JavaScript::Version.new(version) }.
+            reverse
+        end
+
+        def npm_listing
+          return @npm_listing unless @npm_listing.nil?
+
+          response = Excon.get(
+            dependency_url,
+            headers: registry_auth_headers,
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          return @npm_listing = {} if response.status >= 500
+
+          begin
+            @npm_listing = JSON.parse(response.body)
+          rescue JSON::ParserError
+            raise unless non_standard_registry?
+
+            @npm_listing = {}
+          end
+        rescue Excon::Error::Timeout
+          @npm_listing = {}
+        end
+
+        def dependency_url
+          registry_url =
+            if new_source.nil? then "https://registry.npmjs.org"
+            else new_source.fetch(:url)
+            end
+
+          # NPM registries expect slashes to be escaped
+          escaped_dependency_name = dependency.name.gsub("/", "%2F")
+          "#{registry_url}/#{escaped_dependency_name}"
+        end
+
+        def registry_auth_headers
+          return {} unless auth_token
+
+          { "Authorization" => "Bearer #{auth_token}" }
+        end
+
+        def dependency_registry
+          if new_source.nil? then "registry.npmjs.org"
+          else new_source.fetch(:url).gsub("https://", "").gsub("http://", "")
+          end
+        end
+
+        def auth_token
+          credentials.
+            select { |cred| cred["type"] == "npm_registry" }.
+            find { |cred| cred["registry"] == dependency_registry }&.
+            fetch("token")
+        end
+
+        def private_dependency_not_reachable?(npm_response)
+          # Check whether this dependency is (likely to be) private
+          if dependency_registry == "registry.npmjs.org" &&
+             !dependency.name.start_with?("@")
+            return false
+          end
+
+          [401, 403, 404].include?(npm_response.status)
+        end
+
+        def non_standard_registry?
+          dependency_registry != "registry.npmjs.org"
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/metadata_finders/php/composer.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/metadata_finders/base"
+require "dependabot/shared_helpers"
+require "dependabot/utils"
+
+module Dependabot
+  module MetadataFinders
+    module Php
+      class Composer < Dependabot::MetadataFinders::Base
+        private
+
+        def look_up_source
+          source_from_dependency || look_up_source_from_packagist
+        end
+
+        def source_from_dependency
+          source_url =
+            dependency.requirements.
+            map { |r| r.fetch(:source) }.compact.
+            first&.fetch(:url, nil)
+
+          Source.from_url(source_url)
+        end
+
+        def look_up_source_from_packagist
+          return nil if packagist_listing&.fetch("packages", nil) == []
+          unless packagist_listing&.dig("packages", dependency.name.downcase)
+            return nil
+          end
+
+          version_listings =
+            packagist_listing["packages"][dependency.name.downcase].
+            select { |version, _| Utils::Php::Version.correct?(version) }.
+            sort_by { |version, _| Utils::Php::Version.new(version) }.
+            map { |_, listing| listing }.
+            reverse
+
+          potential_source_urls =
+            version_listings.
+            flat_map { |info| [info["homepage"], info.dig("source", "url")] }.
+            compact
+
+          source_url = potential_source_urls.find { |url| Source.from_url(url) }
+
+          Source.from_url(source_url)
+        end
+
+        def packagist_listing
+          return @packagist_listing unless @packagist_listing.nil?
+
+          response = Excon.get(
+            "https://packagist.org/p/#{dependency.name.downcase}.json",
+            idempotent: true,
+            **SharedHelpers.excon_defaults
+          )
+
+          return nil unless response.status == 200
+
+          @packagist_listing = JSON.parse(response.body)
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/metadata_finders/python/pip.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/metadata_finders/base"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module MetadataFinders
+    module Python
+      class Pip < Dependabot::MetadataFinders::Base
+        MAIN_PYPI_URL = "https://pypi.org/pypi"
+
+        def homepage_url
+          pypi_listing.dig("info", "home_page") || super
+        end
+
+        private
+
+        def look_up_source
+          potential_source_urls = [
+            pypi_listing.dig("info", "home_page"),
+            pypi_listing.dig("info", "bugtrack_url"),
+            pypi_listing.dig("info", "download_url"),
+            pypi_listing.dig("info", "docs_url")
+          ].compact
+
+          source_url = potential_source_urls.find { |url| Source.from_url(url) }
+          source_url ||= source_from_description
+          source_url ||= source_from_homepage
+
+          Source.from_url(source_url)
+        end
+
+        def source_from_description
+          github_urls = []
+          desc = pypi_listing.dig("info", "description")
+          return unless desc
+
+          desc.scan(Source::SOURCE_REGEX) do
+            github_urls << Regexp.last_match.to_s
+          end
+
+          github_urls.find do |url|
+            repo = Source.from_url(url).repo
+            repo.downcase.end_with?(dependency.name)
+          end
+        end
+
+        def source_from_homepage
+          return unless homepage_body
+
+          github_urls = []
+          homepage_body.scan(Source::SOURCE_REGEX) do
+            github_urls << Regexp.last_match.to_s
+          end
+
+          github_urls.find do |url|
+            repo = Source.from_url(url).repo
+            repo.downcase.end_with?(dependency.name)
+          end
+        end
+
+        def homepage_body
+          homepage_url = pypi_listing.dig("info", "home_page")
+
+          return unless homepage_url
+          return if homepage_url.include?("pypi.python.org")
+          return if homepage_url.include?("pypi.org")
+
+          @homepage_response ||=
+            begin
+              Excon.get(
+                homepage_url,
+                idempotent: true,
+                **SharedHelpers.excon_defaults
+              )
+            rescue Excon::Error::Timeout, Excon::Error::Socket, ArgumentError
+              nil
+            end
+
+          return unless @homepage_response&.status == 200
+
+          @homepage_response.body
+        end
+
+        def pypi_listing
+          return @pypi_listing unless @pypi_listing.nil?
+          return @pypi_listing = {} if dependency.version.include?("+")
+
+          possible_listing_urls.each do |url|
+            response = Excon.get(
+              url,
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+            next unless response.status == 200
+
+            @pypi_listing = JSON.parse(response.body)
+            return @pypi_listing
+          rescue JSON::ParserError
+            next
+          end
+
+          @pypi_listing = {} # No listing found
+        end
+
+        def possible_listing_urls
+          credential_urls =
+            credentials.
+            select { |cred| cred["type"] == "python_index" }.
+            map { |cred| cred["index-url"].gsub(%r{/$}, "") }
+
+          (credential_urls + [MAIN_PYPI_URL]).map do |base_url|
+            base_url + "/#{dependency.name}/json"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/metadata_finders/ruby/bundler.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "excon"
+require "dependabot/metadata_finders/base"
+
+module Dependabot
+  module MetadataFinders
+    module Ruby
+      class Bundler < Dependabot::MetadataFinders::Base
+        SOURCE_KEYS = %w(
+          source_code_uri
+          homepage_uri
+          wiki_uri
+          bug_tracker_uri
+          documentation_uri
+          changelog_uri
+          mailing_list_uri
+          download_uri
+        ).freeze
+
+        def homepage_url
+          if new_source_type == "default" || new_source_type == "rubygems"
+            if rubygems_listing["homepage_uri"]
+              return rubygems_listing["homepage_uri"]
+            end
+          end
+
+          super
+        end
+
+        private
+
+        def look_up_source
+          case new_source_type
+          when "default", "rubygems" then find_source_from_rubygems_listing
+          when "git" then find_source_from_git_url
+          else raise "Unexpected source type: #{new_source_type}"
+          end
+        end
+
+        def new_source_type
+          sources =
+            dependency.requirements.map { |r| r.fetch(:source) }.uniq.compact
+
+          return "default" if sources.empty?
+          raise "Multiple sources! #{sources.join(', ')}" if sources.count > 1
+
+          sources.first[:type] || sources.first.fetch("type")
+        end
+
+        def find_source_from_rubygems_listing
+          source_url = rubygems_listing.
+                       values_at(*SOURCE_KEYS).
+                       compact.
+                       find { |url| Source.from_url(url) }
+
+          Source.from_url(source_url)
+        end
+
+        def find_source_from_git_url
+          info = dependency.requirements.map { |r| r[:source] }.compact.first
+
+          url = info[:url] || info.fetch("url")
+          Source.from_url(url)
+        end
+
+        def rubygems_listing
+          return @rubygems_listing unless @rubygems_listing.nil?
+
+          response =
+            Excon.get(
+              "#{registry_url}api/v1/gems/#{dependency.name}.json",
+              headers: registry_auth_headers,
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+          response_body = response.body
+          response_body = augment_private_response_if_appropriate(response_body)
+
+          @rubygems_listing = JSON.parse(response_body)
+          append_slash_to_source_code_uri(@rubygems_listing)
+        rescue JSON::ParserError, Excon::Error::Timeout
+          @rubygems_listing = {}
+        end
+
+        def append_slash_to_source_code_uri(listing)
+          # We have to do this so that `Source.from_url(...)` doesn't prune the
+          # last line off of the directory.
+          return listing unless listing&.fetch("source_code_uri", nil)
+          return listing if listing.fetch("source_code_uri").end_with?("/")
+
+          listing["source_code_uri"] = listing["source_code_uri"] + "/"
+          listing
+        end
+
+        def augment_private_response_if_appropriate(response_body)
+          return response_body if new_source_type == "default"
+
+          parsed_body = JSON.parse(response_body)
+          return response_body if (SOURCE_KEYS - parsed_body.keys).none?
+
+          digest = parsed_body.values_at("version", "authors", "info").hash
+
+          source_url = parsed_body.
+                       values_at(*SOURCE_KEYS).
+                       compact.
+                       find { |url| Source.from_url(url) }
+          return response_body if source_url
+
+          rubygems_response =
+            Excon.get(
+              "https://rubygems.org/api/v1/gems/#{dependency.name}.json",
+              idempotent: true,
+              **SharedHelpers.excon_defaults
+            )
+          parsed_rubygems_body = JSON.parse(rubygems_response.body)
+          rubygems_digest =
+            parsed_rubygems_body.values_at("version", "authors", "info").hash
+
+          digest == rubygems_digest ? rubygems_response.body : response_body
+        rescue JSON::ParserError, Excon::Error::Socket, Excon::Error::Timeout
+          response_body
+        end
+
+        def registry_url
+          return "https://rubygems.org/" if new_source_type == "default"
+
+          info = dependency.requirements.map { |r| r[:source] }.compact.first
+          info[:url] || info.fetch("url")
+        end
+
+        def registry_auth_headers
+          return {} unless new_source_type == "rubygems"
+
+          token =
+            credentials.
+            select { |cred| cred["type"] == "rubygems_server" }.
+            find { |cred| registry_url.include?(cred["host"]) }&.
+            fetch("token")
+
+          return {} unless token
+
+          token += ":" unless token.include?(":")
+          encoded_token = Base64.encode64(token).delete("\n")
+          { "Authorization" => "Basic #{encoded_token}" }
+        end
+      end
+    end
+  end
+end