dependabot-common 0.95.1 → 0.95.2

data/lib/dependabot/errors.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "dependabot/shared_helpers"
+
+module Dependabot
+  class DependabotError < StandardError
+    def initialize(msg = nil)
+      msg = sanitize_message(msg)
+      super(msg)
+    end
+
+    private
+
+    def sanitize_message(message)
+      return unless message
+
+      path_regex =
+        Regexp.escape(SharedHelpers::BUMP_TMP_DIR_PATH) + "\/" +
+        Regexp.escape(SharedHelpers::BUMP_TMP_FILE_PREFIX) + "[^/]*"
+
+      message.gsub(/#{path_regex}/, "dependabot_tmp_dir")
+    end
+  end
+
+  class OutOfMemory < DependabotError; end
+
+  #####################
+  # Repo leval errors #
+  #####################
+
+  class BranchNotFound < DependabotError
+    attr_reader :branch_name
+
+    def initialize(branch_name, msg = nil)
+      @branch_name = branch_name
+      msg = sanitize_message(msg)
+      super(msg)
+    end
+  end
+
+  class RepoNotFound < DependabotError
+    attr_reader :source
+
+    def initialize(source, msg = nil)
+      @source = source
+      super(msg)
+    end
+  end
+
+  #####################
+  # File level errors #
+  #####################
+
+  class DependencyFileNotFound < DependabotError
+    attr_reader :file_path
+
+    def initialize(file_path, msg = nil)
+      @file_path = file_path
+      super(msg)
+    end
+
+    def file_name
+      file_path.split("/").last
+    end
+
+    def directory
+      # Directory should always start with a `/`
+      file_path.split("/")[0..-2].join("/").sub(%r{^/*}, "/")
+    end
+  end
+
+  class DependencyFileNotParseable < DependabotError
+    attr_reader :file_path
+
+    def initialize(file_path, msg = nil)
+      @file_path = file_path
+      super(msg)
+    end
+
+    def file_name
+      file_path.split("/").last
+    end
+
+    def directory
+      # Directory should always start with a `/`
+      file_path.split("/")[0..-2].join("/").sub(%r{^/*}, "/")
+    end
+  end
+
+  class DependencyFileNotEvaluatable < DependabotError; end
+  class DependencyFileNotResolvable < DependabotError; end
+
+  #######################
+  # Source level errors #
+  #######################
+
+  class PrivateSourceAuthenticationFailure < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      msg = "The following source could not be reached as it requires "\
+            "authentication (and any provided details were invalid or lacked "\
+            "the required permissions): #{source}"
+      super(msg)
+    end
+  end
+
+  class PrivateSourceTimedOut < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      super("The following source timed out: #{source}")
+    end
+  end
+
+  class PrivateSourceCertificateFailure < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      super("Could not verify the SSL certificate for #{source}")
+    end
+  end
+
+  class MissingEnvironmentVariable < DependabotError
+    attr_reader :environment_variable
+
+    def initialize(environment_variable)
+      @environment_variable = environment_variable
+      super("Missing environment variable #{environment_variable}")
+    end
+  end
+
+  # Useful for JS file updaters, where the registry API sometimes returns
+  # different results to the actual update process
+  class InconsistentRegistryResponse < DependabotError; end
+
+  ###########################
+  # Dependency level errors #
+  ###########################
+
+  class GitDependenciesNotReachable < DependabotError
+    attr_reader :dependency_urls
+
+    def initialize(*dependency_urls)
+      @dependency_urls =
+        dependency_urls.flatten.map { |uri| uri.gsub(/x-access-token.*?@/, "") }
+
+      msg = "The following git URLs could not be retrieved: "\
+            "#{dependency_urls.join(', ')}"
+      super(msg)
+    end
+  end
+
+  class GitDependencyReferenceNotFound < DependabotError
+    attr_reader :dependency
+
+    def initialize(dependency)
+      @dependency = dependency
+
+      msg = "The branch or reference specified for #{dependency} could not "\
+            "be retrieved"
+      super(msg)
+    end
+  end
+
+  class PathDependenciesNotReachable < DependabotError
+    attr_reader :dependencies
+
+    def initialize(*dependencies)
+      @dependencies = dependencies.flatten
+      msg = "The following path based dependencies could not be retrieved: "\
+            "#{dependencies.join(', ')}"
+      super(msg)
+    end
+  end
+end

data/lib/dependabot/file_fetchers.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module FileFetchers
+    @file_fetchers = {}
+
+    def self.for_package_manager(package_manager)
+      file_fetcher = @file_fetchers[package_manager]
+      return file_fetcher if file_fetcher
+
+      raise "Unsupported package_manager #{package_manager}"
+    end
+
+    def self.register(package_manager, file_fetcher)
+      @file_fetchers[package_manager] = file_fetcher
+    end
+  end
+end

data/lib/dependabot/file_fetchers/README.md

@@ -0,0 +1,65 @@
+# File fetchers
+
+File fetchers are used to fetch the relevant dependency files for a project
+(e.g., the `Gemfile` and `Gemfile.lock`). They are also responsible for checking
+whether a repo has an admissable set of requirement files.
+
+There is a `Dependabot::FileFetchers` class for each language Dependabot
+supports.
+
+## Public API
+
+Each `Dependabot::FileFetchers` class implements the following methods:
+
+| Method                           | Description                                                                                   |
+|----------------------------------|-----------------------------------------------------------------------------------------------|
+| `.required_files_in?`            | Checks an array of filenames (string) and returns a boolean describing whether the language-specific dependency files required for an update run are present. |
+| `.required_files_message`        | Returns a static error message which can be displayed to a user if `required_files_in?` returns false. |
+| `#files`                         | Fetches the language-specific dependency files for the repo this instance was created with. Returns an array of `Dependabot::DependencyFile` instances. |
+| `#commit`                        | Returns the commit SHA-1 hash at the time the dependency files were fetched. If called before `files`, the returned value will be used in subsequent calls to `files`. |
+
+
+An integration might look as follows:
+
+```ruby
+require 'octokit'
+require 'dependabot/file_fetchers'
+require 'dependabot/source'
+
+target_repo_name = 'dependabot/dependabot-core'
+source = Dependabot::Source.new(provider: 'github', repo: target_repo_name)
+
+client = Octokit::Client.new
+fetcher_class = Dependabot::FileFetchers::Ruby::Bundler
+filenames = client.contents(target_repo_name).map(&:name)
+
+unless fetcher_class.required_files_in?(filenames)
+  raise fetcher_class.required_files_message
+end
+
+fetcher = fetcher_class.new(source: source, credentials: [])
+
+puts "Fetched #{fetcher.files.map(&:name)}, at commit SHA-1 '#{fetcher.commit}'"
+```
+
+## Writing a file fetcher for a new language
+
+All new file fetchers should inherit from `Dependabot::FileFetchers::Base` and
+implement the following methods:
+
+| Method                           | Description                                                                                   |
+|----------------------------------|-----------------------------------------------------------------------------------------------|
+| `.required_files_in?`            | See Public API section. |
+| `.required_files_message`        | See Public API section. |
+| `#fetch_files`                   | Private method to fetch the required files from GitHub. For each required file, you can use the `fetch_file_from_host(filename)` method from `Dependabot::FileFetchers::Base` to do the fetching. |
+
+To ensure the above are implemented, you should include
+`it_behaves_like "a dependency file fetcher"` in your specs for the new file
+fetcher.
+
+File fetchers tend to get complicated when the file requirements for an update
+to run are non-trivial - for example, for Ruby we could accept
+[`Gemfile`, `Gemfile.lock`] or [`Gemfile`, `example.gemspec`],
+but not just [`Gemfile.lock`]. When adding a new lanugage, it's normally easiest
+to pick a single case and implement it for all the update steps (parsing, update
+checking, etc.). You can then return and add other cases later.

data/lib/dependabot/file_fetchers/base.rb

@@ -0,0 +1,368 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency_file"
+require "dependabot/source"
+require "dependabot/errors"
+require "dependabot/clients/github_with_retries"
+require "dependabot/clients/bitbucket"
+require "dependabot/clients/gitlab"
+require "dependabot/shared_helpers"
+
+# rubocop:disable Metrics/ClassLength
+module Dependabot
+  module FileFetchers
+    class Base
+      attr_reader :source, :credentials
+
+      CLIENT_NOT_FOUND_ERRORS = [
+        Octokit::NotFound,
+        Gitlab::Error::NotFound,
+        Dependabot::Clients::Bitbucket::NotFound
+      ].freeze
+
+      def self.required_files_in?(_filename_array)
+        raise NotImplementedError
+      end
+
+      def self.required_files_message
+        raise NotImplementedError
+      end
+
+      def initialize(source:, credentials:)
+        @source = source
+        @credentials = credentials
+
+        @submodule_directories = {}
+      end
+
+      def repo
+        source.repo
+      end
+
+      def directory
+        source.directory || "/"
+      end
+
+      def target_branch
+        source.branch
+      end
+
+      def files
+        @files ||= fetch_files
+      end
+
+      def commit
+        branch = target_branch || default_branch_for_repo
+
+        @commit ||= client_for_provider.fetch_commit(repo, branch)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        raise Dependabot::BranchNotFound, branch
+      rescue Octokit::Conflict => error
+        raise unless error.message.include?("Repository is empty")
+      end
+
+      private
+
+      def fetch_file_if_present(filename, fetch_submodules: false)
+        dir = File.dirname(filename)
+        basename = File.basename(filename)
+
+        repo_includes_basename =
+          repo_contents(dir: dir, fetch_submodules: fetch_submodules).
+          reject { |f| f.type == "dir" }.
+          map(&:name).include?(basename)
+        return unless repo_includes_basename
+
+        fetch_file_from_host(filename, fetch_submodules: fetch_submodules)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
+        raise Dependabot::DependencyFileNotFound, path
+      end
+
+      def fetch_file_from_host(filename, type: "file", fetch_submodules: false)
+        path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
+
+        DependencyFile.new(
+          name: Pathname.new(filename).cleanpath.to_path,
+          directory: directory,
+          type: type,
+          content: _fetch_file_content(path, fetch_submodules: fetch_submodules)
+        )
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        raise Dependabot::DependencyFileNotFound, path
+      end
+
+      def repo_contents(dir: ".", ignore_base_directory: false,
+                        raise_errors: true, fetch_submodules: false)
+        dir = File.join(directory, dir) unless ignore_base_directory
+        path = Pathname.new(File.join(dir)).cleanpath.to_path.gsub(%r{^/*}, "")
+
+        @repo_contents ||= {}
+        @repo_contents[dir] ||= _fetch_repo_contents(
+          path,
+          raise_errors: raise_errors,
+          fetch_submodules: fetch_submodules
+        )
+      end
+
+      #################################################
+      # INTERNAL METHODS (not for use by sub-classes) #
+      #################################################
+
+      def _fetch_repo_contents(path, fetch_submodules: false,
+                               raise_errors: true)
+        path = path.gsub(" ", "%20")
+        provider, repo, tmp_path, commit =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          values_at(:provider, :repo, :path, :commit)
+
+        _fetch_repo_contents_fully_specified(provider, repo, tmp_path, commit)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        result = raise_errors ? -> { raise } : -> { [] }
+        retrying ||= false
+
+        # If the path changes after calling _fetch_repo_contents_fully_specified
+        # it's because we've found a sub-module (and are fetching them). Trigger
+        # a retry to get its contents.
+        updated_path =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          fetch(:path)
+        retry if updated_path != tmp_path
+
+        return result.call unless fetch_submodules && !retrying
+
+        _find_submodules(path)
+        return result.call unless _submodule_for(path)
+
+        retrying = true
+        retry
+      end
+
+      def _fetch_repo_contents_fully_specified(provider, repo, path, commit)
+        case provider
+        when "github"
+          _github_repo_contents(repo, path, commit)
+        when "gitlab"
+          _gitlab_repo_contents(repo, path, commit)
+        when "bitbucket"
+          _bitbucket_repo_contents(repo, path, commit)
+        else raise "Unsupported provider '#{provider}'."
+        end
+      end
+
+      def _github_repo_contents(repo, path, commit)
+        path = path.gsub(" ", "%20")
+        github_response = github_client.contents(repo, path: path, ref: commit)
+
+        if github_response.respond_to?(:type) &&
+           github_response.type == "submodule"
+          @submodule_directories[path] = github_response
+          raise Octokit::NotFound
+        elsif github_response.respond_to?(:type)
+          raise Octokit::NotFound
+        end
+
+        github_response.map { |f| _build_github_file_struct(f) }
+      end
+
+      def _build_github_file_struct(file)
+        OpenStruct.new(
+          name: file.name,
+          path: file.path,
+          type: file.type,
+          sha: file.sha,
+          size: file.size
+        )
+      end
+
+      def _gitlab_repo_contents(repo, path, commit)
+        gitlab_client.
+          repo_tree(repo, path: path, ref_name: commit, per_page: 100).
+          map do |file|
+            OpenStruct.new(
+              name: file.name,
+              path: file.path,
+              type: file.type == "blob" ? "file" : file.type,
+              size: 0 # GitLab doesn't return file size
+            )
+          end
+      end
+
+      def _bitbucket_repo_contents(repo, path, commit)
+        response = bitbucket_client.fetch_repo_contents(
+          repo,
+          commit,
+          path
+        )
+
+        response.map do |file|
+          type = case file.fetch("type")
+                 when "commit_file" then "file"
+                 when "commit_directory" then "dir"
+                 else file.fetch("type")
+                 end
+
+          OpenStruct.new(
+            name: File.basename(file.fetch("path")),
+            path: file.fetch("path"),
+            type: type,
+            size: file.fetch("size", 0)
+          )
+        end
+      end
+
+      def _full_specification_for(path, fetch_submodules:)
+        if fetch_submodules && _submodule_for(path) &&
+           Source.from_url(
+             @submodule_directories[_submodule_for(path)].submodule_git_url
+           )
+          submodule_details = @submodule_directories[_submodule_for(path)]
+          sub_source = Source.from_url(submodule_details.submodule_git_url)
+          {
+            repo: sub_source.repo,
+            commit: submodule_details.sha,
+            provider: sub_source.provider,
+            path: path.gsub(%r{^#{Regexp.quote(_submodule_for(path))}(/|$)}, "")
+          }
+        else
+          {
+            repo: source.repo,
+            path: path,
+            commit: commit,
+            provider: source.provider
+          }
+        end
+      end
+
+      def _fetch_file_content(path, fetch_submodules: false)
+        path = path.gsub(%r{^/*}, "")
+
+        provider, repo, path, commit =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          values_at(:provider, :repo, :path, :commit)
+
+        _fetch_file_content_fully_specified(provider, repo, path, commit)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        retrying ||= false
+
+        raise unless fetch_submodules && !retrying && !_submodule_for(path)
+
+        _find_submodules(path)
+        raise unless _submodule_for(path)
+
+        retrying = true
+        retry
+      end
+
+      def _fetch_file_content_fully_specified(provider, repo, path, commit)
+        case provider
+        when "github"
+          _fetch_file_content_from_github(path, repo, commit)
+        when "gitlab"
+          tmp = gitlab_client.get_file(repo, path, commit).content
+          Base64.decode64(tmp).force_encoding("UTF-8").encode
+        when "bitbucket"
+          bitbucket_client.fetch_file_contents(repo, commit, path)
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      def _fetch_file_content_from_github(path, repo, commit)
+        tmp = github_client.contents(repo, path: path, ref: commit)
+
+        raise Octokit::NotFound if tmp.is_a?(Array)
+
+        if tmp.type == "symlink"
+          tmp = github_client.contents(
+            repo,
+            path: tmp.target,
+            ref: commit
+          )
+        end
+
+        Base64.decode64(tmp.content).force_encoding("UTF-8").encode
+      rescue Octokit::Forbidden => error
+        raise unless error.message.include?("too_large")
+
+        # Fall back to Git Data API to fetch the file
+        prefix_dir = directory.gsub(%r{(^/|/$)}, "")
+        dir = File.dirname(path).gsub(%r{^/?#{Regexp.escape(prefix_dir)}/?}, "")
+        basename = File.basename(path)
+        file_details = repo_contents(dir: dir).find { |f| f.name == basename }
+        raise unless file_details
+
+        tmp = github_client.blob(repo, file_details.sha)
+        return tmp.content if tmp.encoding == "utf-8"
+
+        Base64.decode64(tmp.content).force_encoding("UTF-8").encode
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def default_branch_for_repo
+        @default_branch_for_repo ||= client_for_provider.
+                                     fetch_default_branch(repo)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        raise Dependabot::RepoNotFound, source
+      end
+
+      # Update the @submodule_directories hash by exploiting a side-effect of
+      # recursively calling `repo_contents` for each directory up the tree
+      # until a submodule is found
+      def _find_submodules(path)
+        path = Pathname.new(path).cleanpath.to_path.gsub(%r{^/*}, "")
+        dir = File.dirname(path)
+
+        return if [directory, "."].include?(dir)
+
+        repo_contents(
+          dir: dir,
+          ignore_base_directory: true,
+          fetch_submodules: true,
+          raise_errors: false
+        )
+      end
+
+      def _submodule_for(path)
+        submodules = @submodule_directories.keys
+        submodules.
+          select { |k| path.match?(%r{^#{Regexp.quote(k)}(/|$)}) }.
+          max_by(&:length)
+      end
+
+      def client_for_provider
+        case source.provider
+        when "github" then github_client
+        when "gitlab" then gitlab_client
+        when "bitbucket" then bitbucket_client
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      def github_client
+        @github_client ||=
+          Dependabot::Clients::GithubWithRetries.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def gitlab_client
+        @gitlab_client ||=
+          Dependabot::Clients::Gitlab.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def bitbucket_client
+        # TODO: When self-hosted Bitbucket is supported this should use
+        # `Bitbucket.for_source`
+        @bitbucket_client ||=
+          Dependabot::Clients::Bitbucket.
+          for_bitbucket_dot_org(credentials: credentials)
+      end
+    end
+  end
+end
+# rubocop:enable Metrics/ClassLength