dependabot-common 0.95.1 → 0.95.2

data/lib/dependabot/file_parsers.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module FileParsers
+    @file_parsers = {}
+
+    def self.for_package_manager(package_manager)
+      file_parser = @file_parsers[package_manager]
+      return file_parser if file_parser
+
+      raise "Unsupported package_manager #{package_manager}"
+    end
+
+    def self.register(package_manager, file_parser)
+      @file_parsers[package_manager] = file_parser
+    end
+  end
+end

data/lib/dependabot/file_parsers/README.md

@@ -0,0 +1,45 @@
+# File parsers
+
+File parsers take a set of dependency files and extract a list of dependencies
+for the project.
+
+There is a `Dependabot::FileParsers` class for each language Dependabot
+supports.
+
+## Public API
+
+Each `Dependabot::FileParsers` class implements the following methods:
+
+| Method              | Description                                                                                   |
+|---------------------|-----------------------------------------------------------------------------------------------|
+| `#parse`            | Returns an array of `Dependabot::Dependency` instances, representing the dependencies for the project. Each `Dependabot::Dependency` has a `name`, `version` and a `requirements` array |
+
+An integration might look as follows:
+
+```ruby
+require 'dependabot/file_parsers'
+
+files = fetcher.files
+
+parser_class = Dependabot::FileParsers::Ruby::Bundler
+source = Dependabot::Source.new(provider: 'github', repo: "gocardless/business")
+parser = parser_class.new(dependency_files: files, source: source)
+
+dependencies = parser.parse
+
+puts "Found the following dependencies: #{dependencies.map(&:name)}"
+```
+
+## Writing a file parser for a new language
+
+All new file parsers should inherit from `Dependabot::FileParsers::Base` and
+implement the following methods:
+
+| Method                  | Description                                                                                   |
+|-------------------------|-----------------------------------------------------------------------------------------------|
+| `#parse`                | See Public API section. |
+| `#check_required_files` | Raise a runtime error unless an appropriate set of files is provided. Private. |
+
+To ensure the above are implemented, you should include
+`it_behaves_like "a dependency file parser"` in your specs for the new file
+parser.

data/lib/dependabot/file_parsers/base.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module FileParsers
+    class Base
+      attr_reader :dependency_files, :credentials, :source
+
+      def initialize(dependency_files:, source:, credentials: [])
+        @dependency_files = dependency_files
+        @credentials = credentials
+        @source = source
+
+        check_required_files
+      end
+
+      def parse
+        raise NotImplementedError
+      end
+
+      private
+
+      def check_required_files
+        raise NotImplementedError
+      end
+
+      def get_original_file(filename)
+        dependency_files.find { |f| f.name == filename }
+      end
+    end
+  end
+end

data/lib/dependabot/file_parsers/base/dependency_set.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/file_parsers/base"
+require "dependabot/utils"
+
+module Dependabot
+  module FileParsers
+    class Base
+      class DependencySet
+        def initialize(dependencies = [])
+          unless dependencies.is_a?(Array) &&
+                 dependencies.all? { |dep| dep.is_a?(Dependency) }
+            raise ArgumentError, "must be an array of Dependency objects"
+          end
+
+          @dependencies = dependencies
+        end
+
+        attr_reader :dependencies
+
+        def <<(dep)
+          unless dep.is_a?(Dependency)
+            raise ArgumentError, "must be a Dependency object"
+          end
+
+          existing_dependency = dependencies.find { |d| d.name == dep.name }
+
+          return self if existing_dependency&.to_h == dep.to_h
+
+          if existing_dependency
+            dependencies[dependencies.index(existing_dependency)] =
+              combined_dependency(existing_dependency, dep)
+          else
+            dependencies << dep
+          end
+
+          self
+        end
+
+        def +(other)
+          unless other.is_a?(DependencySet)
+            raise ArgumentError, "must be a DependencySet"
+          end
+
+          other.dependencies.each { |dep| self << dep }
+          self
+        end
+
+        private
+
+        def combined_dependency(old_dep, new_dep)
+          package_manager = old_dep.package_manager
+          v_cls = Utils.version_class_for_package_manager(package_manager)
+
+          # If we already have a requirement use the existing version
+          # (if present). Otherwise, use whatever the lowest version is
+          new_version =
+            if old_dep.requirements.any? then old_dep.version || new_dep.version
+            elsif !v_cls.correct?(new_dep.version) then old_dep.version
+            elsif !v_cls.correct?(old_dep.version) then new_dep.version
+            elsif v_cls.new(new_dep.version) > v_cls.new(old_dep.version)
+              old_dep.version
+            else new_dep.version
+            end
+
+          Dependency.new(
+            name: old_dep.name,
+            version: new_version,
+            requirements: (old_dep.requirements + new_dep.requirements).uniq,
+            package_manager: package_manager
+          )
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/file_updaters.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module FileUpdaters
+    @file_updaters = {}
+
+    def self.for_package_manager(package_manager)
+      file_updater = @file_updaters[package_manager]
+      return file_updater if file_updater
+
+      raise "Unsupported package_manager #{package_manager}"
+    end
+
+    def self.register(package_manager, file_updater)
+      @file_updaters[package_manager] = file_updater
+    end
+  end
+end

data/lib/dependabot/file_updaters/README.md

@@ -0,0 +1,58 @@
+# File updaters
+
+File updaters update a dependency file to use the latest version of a given
+dependency. They rely on information provided to them by update checkers.
+
+There is a `Dependabot::FileUpdaters` class for each language Dependabot
+supports.
+
+## Public API
+
+Each `Dependabot::FileUpdaters` class implements the following methods:
+
+| Method                       | Description                                                                                   |
+|------------------------------|-----------------------------------------------------------------------------------------------|
+| `.updated_files_regex`       | An array of regular expressions matching the names of the files this class updates. Intended to be used by integrators when checking whether a commit may cause merge-conflicts with a dependency update pull request. |
+| `#updated_dependency_files`  | Returns an array of updated `Dependabot::DependencyFile` instances, with their content updated to include the updated dependency. |
+
+An integration might look as follows:
+
+```ruby
+require 'dependabot/file_updaters'
+
+unless update_checker.can_update?(requirements_to_update: :own)
+  raise "Dependency doesn't need update!"
+end
+dependencies = update_checker.updated_dependencies(requirements_to_update: :own)
+
+file_updater_class = Dependabot::FileUpdaters::Ruby::Bundler
+file_updater = file_updater_class.new(
+  dependencies: dependencies,
+  dependency_files: files,
+  credentials: [{
+    "type" => "git_source",
+    "host" => "github.com",
+    "username" => "x-access-token",
+    "password" => "token"
+  }]
+)
+
+file_updater.updated_dependency_files.each do |file|
+  puts "Updated #{file.name} with new content:\n\n#{file.content}"
+end
+```
+
+## Writing a file updater for a new language
+
+All new file updaters should inherit from `Dependabot::FileUpdaters::Base` and
+implement the following methods:
+
+| Method                      | Description             |
+|-----------------------------|-------------------------|
+| `.updated_files_regex`      | See Public API section. |
+| `#updated_dependency_files` | See Public API section. |
+
+To ensure the above are implemented, you should include
+`it_behaves_like "a dependency file updater"` in your specs for the new file
+updater.
+

data/lib/dependabot/file_updaters/base.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module FileUpdaters
+    class Base
+      attr_reader :dependencies, :dependency_files, :credentials
+
+      def self.updated_files_regex
+        raise NotImplementedError
+      end
+
+      def initialize(dependencies:, dependency_files:, credentials:)
+        @dependencies = dependencies
+        @dependency_files = dependency_files
+        @credentials = credentials
+
+        check_required_files
+      end
+
+      def updated_dependency_files
+        raise NotImplementedError
+      end
+
+      private
+
+      def check_required_files
+        raise NotImplementedError
+      end
+
+      def get_original_file(filename)
+        dependency_files.find { |f| f.name == filename }
+      end
+
+      def file_changed?(file)
+        dependencies.any? { |dep| requirement_changed?(file, dep) }
+      end
+
+      def requirement_changed?(file, dependency)
+        changed_requirements =
+          dependency.requirements - dependency.previous_requirements
+
+        changed_requirements.any? { |f| f[:file] == file.name }
+      end
+
+      def updated_file(file:, content:)
+        updated_file = file.dup
+        updated_file.content = content
+        updated_file
+      end
+    end
+  end
+end

data/lib/dependabot/git_commit_checker.rb

@@ -0,0 +1,412 @@
+# frozen_string_literal: true
+
+require "excon"
+require "gitlab"
+require "dependabot/clients/github_with_retries"
+require "dependabot/metadata_finders"
+require "dependabot/errors"
+require "dependabot/utils"
+require "dependabot/source"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  class GitCommitChecker
+    VERSION_REGEX = /(?<version>[0-9]+\.[0-9]+(?:\.[a-zA-Z0-9\-]+)*)$/.freeze
+    KNOWN_HOSTS = /github\.com|bitbucket\.org|gitlab.com/.freeze
+
+    def initialize(dependency:, credentials:, ignored_versions: [],
+                   requirement_class: nil, version_class: nil)
+      @dependency = dependency
+      @credentials = credentials
+      @ignored_versions = ignored_versions
+      @requirement_class = requirement_class
+      @version_class = version_class
+    end
+
+    def git_dependency?
+      return false if dependency_source_details.nil?
+
+      dependency_source_details.fetch(:type) == "git"
+    end
+
+    def pinned?
+      raise "Not a git dependency!" unless git_dependency?
+
+      ref = dependency_source_details.fetch(:ref)
+      branch = dependency_source_details.fetch(:branch)
+
+      return false if ref.nil?
+      return false if branch == ref
+      return true if branch
+      return true if dependency.version&.start_with?(ref)
+
+      # Check the specified `ref` isn't actually a branch
+      !local_upload_pack.match?("refs/heads/#{ref}")
+    end
+
+    def pinned_ref_looks_like_version?
+      return false unless pinned?
+
+      dependency_source_details.fetch(:ref).match?(VERSION_REGEX)
+    end
+
+    def branch_or_ref_in_release?(version)
+      pinned_ref_in_release?(version) || branch_behind_release?(version)
+    end
+
+    def head_commit_for_current_branch
+      return dependency.version if pinned?
+
+      branch_ref = ref_or_branch ? "refs/heads/#{ref_or_branch}" : "HEAD"
+
+      # Remove the opening clause of the upload pack as this isn't always
+      # followed by a line break. When it isn't (e.g., with Bitbucket) it causes
+      # problems for our `sha_for_update_pack_line` logic
+      line = local_upload_pack.
+             gsub(/.*git-upload-pack/, "").
+             lines.find { |l| l.include?(" #{branch_ref}") }
+
+      return sha_for_update_pack_line(line) if line
+
+      raise Dependabot::GitDependencyReferenceNotFound, dependency.name
+    end
+
+    def local_tag_for_latest_version
+      tag =
+        local_tags.
+        select { |t| t.name.match?(VERSION_REGEX) }.
+        reject { |t| tag_included_in_ignore_reqs?(t) }.
+        reject { |t| tag_is_prerelease?(t) && !wants_prerelease? }.
+        max_by do |t|
+          version = t.name.match(VERSION_REGEX).named_captures.fetch("version")
+          version_class.new(version)
+        end
+
+      return unless tag
+
+      {
+        tag: tag.name,
+        commit_sha: tag.commit_sha,
+        tag_sha: tag.tag_sha
+      }
+    end
+
+    private
+
+    attr_reader :dependency, :credentials, :ignored_versions
+
+    def pinned_ref_in_release?(version)
+      raise "Not a git dependency!" unless git_dependency?
+
+      return false unless pinned?
+      return false if listing_source_url.nil?
+
+      tag = listing_tag_for_version(version.to_s)
+      return false unless tag
+
+      commit_included_in_tag?(
+        commit: dependency_source_details.fetch(:ref),
+        tag: tag,
+        allow_identical: true
+      )
+    end
+
+    def branch_behind_release?(version)
+      raise "Not a git dependency!" unless git_dependency?
+
+      return false if ref_or_branch.nil?
+      return false if listing_source_url.nil?
+
+      tag = listing_tag_for_version(version.to_s)
+      return false unless tag
+
+      # Check if behind, excluding the case where it's identical, because
+      # we normally wouldn't switch you from tracking master to a release.
+      commit_included_in_tag?(
+        commit: ref_or_branch,
+        tag: tag,
+        allow_identical: false
+      )
+    end
+
+    def local_upload_pack
+      @local_upload_pack ||=
+        fetch_upload_pack_for(dependency_source_details.fetch(:url))
+    end
+
+    def local_tags
+      return [] unless local_upload_pack
+
+      tags_for_upload_pack(local_upload_pack)
+    end
+
+    def tags_for_upload_pack(upload_pack)
+      peeled_lines = []
+      unpeeled_lines = []
+
+      upload_pack.lines.each do |line|
+        next unless line.split(" ").last.start_with?("refs/tags")
+
+        if line.strip.end_with?("^{}") then peeled_lines << line
+        else unpeeled_lines << line
+        end
+      end
+
+      unpeeled_lines.map do |line|
+        tag_name    = line.split(" refs/tags/").last.strip
+        tag_sha     = sha_for_update_pack_line(line)
+        peeled_line = peeled_lines.find do |pl|
+          pl.split(" refs/tags/").last.strip == "#{tag_name}^{}"
+        end
+
+        commit_sha =
+          peeled_line ? sha_for_update_pack_line(peeled_line) : tag_sha
+
+        if dependency_source_details&.fetch(:ref, nil)&.start_with?("tags/")
+          tag_name = "tags/#{tag_name}"
+        end
+
+        OpenStruct.new(name: tag_name, tag_sha: tag_sha, commit_sha: commit_sha)
+      end
+    end
+
+    # rubocop:disable Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/PerceivedComplexity
+    def fetch_upload_pack_for(uri)
+      response = Excon.get(
+        service_pack_uri(uri),
+        idempotent: true,
+        **SharedHelpers.excon_defaults
+      )
+
+      return response.body if response.status == 200
+      if response.status >= 500 && uri.match?(KNOWN_HOSTS)
+        raise "Server error at #{uri}: #{response.body}"
+      end
+
+      raise Dependabot::GitDependenciesNotReachable, [uri]
+    rescue Excon::Error::Socket, Excon::Error::Timeout
+      retry_count ||= 0
+      retry_count += 1
+
+      sleep(rand(0.9)) && retry if retry_count < 2 && uri.match?(KNOWN_HOSTS)
+      raise if uri.match?(KNOWN_HOSTS)
+
+      raise Dependabot::GitDependenciesNotReachable, [uri]
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/PerceivedComplexity
+
+    def service_pack_uri(uri)
+      service_pack_uri = uri_with_auth(uri)
+      service_pack_uri = service_pack_uri.gsub(%r{/$}, "")
+      service_pack_uri += ".git" unless service_pack_uri.end_with?(".git")
+      service_pack_uri + "/info/refs?service=git-upload-pack"
+    end
+
+    def uri_with_auth(uri)
+      bare_uri =
+        if uri.include?("git@") then uri.split("git@").last.sub(":", "/")
+        else uri.sub(%r{.*?://}, "")
+        end
+      cred = credentials.select { |c| c["type"] == "git_source" }.
+             find { |c| bare_uri.start_with?(c["host"]) }
+
+      if bare_uri.match?(%r{[^/]+:[^/]+@})
+        # URI already has authentication details
+        "https://#{bare_uri}"
+      elsif cred
+        # URI doesn't have authentication details, but we have credentials
+        auth_string = "#{cred.fetch('username')}:#{cred.fetch('password')}"
+        "https://#{auth_string}@#{bare_uri}"
+      else
+        # No credentials, so just return the https URI
+        "https://#{bare_uri}"
+      end
+    end
+
+    def commit_included_in_tag?(tag:, commit:, allow_identical: false)
+      status =
+        case Source.from_url(listing_source_url)&.provider
+        when "github" then github_commit_comparison_status(tag, commit)
+        when "gitlab" then gitlab_commit_comparison_status(tag, commit)
+        when "bitbucket" then bitbucket_commit_comparison_status(tag, commit)
+        else raise "Unknown source"
+        end
+
+      return true if status == "behind"
+
+      allow_identical && status == "identical"
+    rescue Octokit::NotFound, Gitlab::Error::NotFound,
+           Octokit::InternalServerError
+      false
+    end
+
+    def github_commit_comparison_status(ref1, ref2)
+      client = Clients::GithubWithRetries.
+               for_github_dot_com(credentials: credentials)
+
+      client.compare(listing_source_repo, ref1, ref2).status
+    end
+
+    def gitlab_commit_comparison_status(ref1, ref2)
+      access_token = credentials.
+                     select { |cred| cred["type"] == "git_source" }.
+                     find { |cred| cred["host"] == "gitlab.com" }&.
+                     fetch("token")
+
+      client = Gitlab.client(endpoint: "https://gitlab.com/api/v4",
+                             private_token: access_token.to_s)
+
+      comparison = client.compare(listing_source_repo, ref1, ref2)
+
+      if comparison.commits.none? then "behind"
+      elsif comparison.compare_same_ref then "identical"
+      else "ahead"
+      end
+    end
+
+    def bitbucket_commit_comparison_status(ref1, ref2)
+      url = "https://api.bitbucket.org/2.0/repositories/"\
+            "#{listing_source_repo}/commits/?"\
+            "include=#{ref2}&exclude=#{ref1}"
+
+      response = Excon.get(url,
+                           headers: bitbucket_auth_header,
+                           idempotent: true,
+                           **SharedHelpers.excon_defaults)
+
+      # Conservatively assume that ref2 is ahead in the equality case, of
+      # if we get an unexpected format (e.g., due to a 404)
+      if JSON.parse(response.body).fetch("values", ["x"]).none? then "behind"
+      else "ahead"
+      end
+    end
+
+    def bitbucket_auth_header
+      token = credentials.
+              select { |cred| cred["type"] == "git_source" }.
+              find { |cred| cred["host"] == "bitbucket.org" }&.
+              fetch("token")
+
+      if token.nil? then {}
+      elsif token.include?(":")
+        encoded_token = Base64.encode64(token).delete("\n")
+        { "Authorization" => "Basic #{encoded_token}" }
+      elsif Base64.decode64(token).ascii_only? &&
+            Base64.decode64(token).include?(":")
+        { "Authorization" => "Basic #{token.delete("\n")}" }
+      else
+        { "Authorization" => "Bearer #{token}" }
+      end
+    end
+
+    def dependency_source_details
+      sources =
+        dependency.requirements.map { |r| r.fetch(:source) }.uniq.compact
+
+      return sources.first if sources.count <= 1
+
+      # If there are multiple source types, or multiple source URLs, then it's
+      # unclear how we should proceed
+      if sources.map { |s| [s.fetch(:type), s.fetch(:url, nil)] }.uniq.count > 1
+        raise "Multiple sources! #{sources.join(', ')}"
+      end
+
+      # Otherwise it's reasonable to take the first source and use that. This
+      # will happen if we have multiple git sources with difference references
+      # specified. In that case it's fine to update them all.
+      sources.first
+    end
+
+    def ref_or_branch
+      dependency_source_details.fetch(:ref) ||
+        dependency_source_details.fetch(:branch)
+    end
+
+    def listing_source_url
+      @listing_source_url ||=
+        begin
+          # Remove the git source, so the metadata finder looks on the
+          # registry
+          candidate_dep = Dependency.new(
+            name: dependency.name,
+            version: dependency.version,
+            requirements: [],
+            package_manager: dependency.package_manager
+          )
+
+          MetadataFinders.
+            for_package_manager(dependency.package_manager).
+            new(dependency: candidate_dep, credentials: credentials).
+            source_url
+        end
+    end
+
+    def listing_source_repo
+      return unless listing_source_url
+
+      Source.from_url(listing_source_url)&.repo
+    end
+
+    def listing_tag_for_version(version)
+      listing_tags.
+        find { |t| t.name =~ /(?:[^0-9\.]|\A)#{Regexp.escape(version)}\z/ }&.
+        name
+    end
+
+    def listing_tags
+      return [] unless listing_upload_pack
+
+      tags_for_upload_pack(listing_upload_pack)
+    rescue GitDependenciesNotReachable
+      []
+    end
+
+    def listing_upload_pack
+      return unless listing_source_url
+
+      @listing_upload_pack ||= fetch_upload_pack_for(listing_source_url)
+    end
+
+    def ignore_reqs
+      ignored_versions.map { |req| requirement_class.new(req.split(",")) }
+    end
+
+    def wants_prerelease?
+      return false unless dependency_source_details&.fetch(:ref, nil)
+      return false unless pinned_ref_looks_like_version?
+
+      version = dependency_source_details.fetch(:ref).match(VERSION_REGEX).
+                named_captures.fetch("version")
+      version_class.new(version).prerelease?
+    end
+
+    def tag_included_in_ignore_reqs?(tag)
+      version = tag.name.match(VERSION_REGEX).named_captures.fetch("version")
+      ignore_reqs.any? { |r| r.satisfied_by?(version_class.new(version)) }
+    end
+
+    def tag_is_prerelease?(tag)
+      version = tag.name.match(VERSION_REGEX).named_captures.fetch("version")
+      version_class.new(version).prerelease?
+    end
+
+    def version_class
+      return @version_class if @version_class
+
+      Utils.version_class_for_package_manager(dependency.package_manager)
+    end
+
+    def requirement_class
+      return @requirement_class if @requirement_class
+
+      Utils.requirement_class_for_package_manager(dependency.package_manager)
+    end
+
+    def sha_for_update_pack_line(line)
+      line.split(" ").first.chars.last(40).join
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength