dependabot-core 0.76.1

data/lib/dependabot/dependency_file.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Dependabot
+  class DependencyFile
+    attr_accessor :name, :content, :directory, :type, :support_file
+
+    def initialize(name:, content:, directory: "/", type: "file",
+                   support_file: false)
+      @name = name
+      @content = content
+      @directory = clean_directory(directory)
+      @type = type
+      @support_file = support_file
+    end
+
+    def to_h
+      {
+        "name" => name,
+        "content" => content,
+        "directory" => directory,
+        "type" => type
+      }
+    end
+
+    def path
+      Pathname.new(File.join(directory, name)).cleanpath.to_path
+    end
+
+    def ==(other)
+      other.instance_of?(self.class) && to_h == other.to_h
+    end
+
+    def hash
+      to_h.hash
+    end
+
+    def eql?(other)
+      self.==(other)
+    end
+
+    def support_file?
+      @support_file
+    end
+
+    private
+
+    def clean_directory(directory)
+      # Directory should always start with a `/`
+      directory.sub(%r{^/*}, "/")
+    end
+  end
+end

data/lib/dependabot/errors.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require "dependabot/shared_helpers"
+
+module Dependabot
+  class DependabotError < StandardError
+    def initialize(msg = nil)
+      msg = sanitize_message(msg)
+      super(msg)
+    end
+
+    private
+
+    def sanitize_message(message)
+      return unless message
+
+      path_regex =
+        Regexp.escape(SharedHelpers::BUMP_TMP_DIR_PATH) + "\/" +
+        Regexp.escape(SharedHelpers::BUMP_TMP_FILE_PREFIX) + "[^/]*"
+
+      message.gsub(/#{path_regex}/, "dependabot_tmp_dir")
+    end
+  end
+
+  class OutOfMemory < DependabotError; end
+
+  #####################
+  # Repo leval errors #
+  #####################
+
+  class BranchNotFound < DependabotError
+    attr_reader :branch_name
+
+    def initialize(branch_name, msg = nil)
+      @branch_name = branch_name
+      msg = sanitize_message(msg)
+      super(msg)
+    end
+  end
+
+  class RepoNotFound < DependabotError
+    attr_reader :source
+
+    def initialize(source, msg = nil)
+      @source = source
+      super(msg)
+    end
+  end
+
+  #####################
+  # File level errors #
+  #####################
+
+  class DependencyFileNotFound < DependabotError
+    attr_reader :file_path
+
+    def initialize(file_path, msg = nil)
+      @file_path = file_path
+      super(msg)
+    end
+
+    def file_name
+      file_path.split("/").last
+    end
+
+    def directory
+      # Directory should always start with a `/`
+      file_path.split("/")[0..-2].join("/").sub(%r{^/*}, "/")
+    end
+  end
+
+  class DependencyFileNotParseable < DependabotError
+    attr_reader :file_path
+
+    def initialize(file_path, msg = nil)
+      @file_path = file_path
+      super(msg)
+    end
+
+    def file_name
+      file_path.split("/").last
+    end
+
+    def directory
+      # Directory should always start with a `/`
+      file_path.split("/")[0..-2].join("/").sub(%r{^/*}, "/")
+    end
+  end
+
+  class DependencyFileNotEvaluatable < DependabotError; end
+  class DependencyFileNotResolvable < DependabotError; end
+
+  #######################
+  # Source level errors #
+  #######################
+
+  class PrivateSourceAuthenticationFailure < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      msg = "The following source could not be reached as it requires "\
+            "authentication (and any provided details were invalid or lacked "\
+            "the required permissions): #{source}"
+      super(msg)
+    end
+  end
+
+  class PrivateSourceTimedOut < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      super("The following source timed out: #{source}")
+    end
+  end
+
+  class PrivateSourceCertificateFailure < DependabotError
+    attr_reader :source
+
+    def initialize(source)
+      @source = source
+      super("Could not verify the SSL certificate for #{source}")
+    end
+  end
+
+  class MissingEnvironmentVariable < DependabotError
+    attr_reader :environment_variable
+
+    def initialize(environment_variable)
+      @environment_variable = environment_variable
+      super("Missing environment variable #{environment_variable}")
+    end
+  end
+
+  # Useful for JS file updaters, where the registry API sometimes returns
+  # different results to the actual update process
+  class InconsistentRegistryResponse < DependabotError; end
+
+  ###########################
+  # Dependency level errors #
+  ###########################
+
+  class GitDependenciesNotReachable < DependabotError
+    attr_reader :dependency_urls
+
+    def initialize(*dependency_urls)
+      @dependency_urls =
+        dependency_urls.flatten.map { |uri| uri.gsub(/x-access-token.*?@/, "") }
+
+      msg = "The following git URLs could not be retrieved: "\
+            "#{dependency_urls.join(', ')}"
+      super(msg)
+    end
+  end
+
+  class GitDependencyReferenceNotFound < DependabotError
+    attr_reader :dependency
+
+    def initialize(dependency)
+      @dependency = dependency
+
+      msg = "The branch or reference specified for #{dependency} could not "\
+            "be retrieved"
+      super(msg)
+    end
+  end
+
+  class PathDependenciesNotReachable < DependabotError
+    attr_reader :dependencies
+
+    def initialize(*dependencies)
+      @dependencies = dependencies.flatten
+      msg = "The following path based dependencies could not be retrieved: "\
+            "#{dependencies.join(', ')}"
+      super(msg)
+    end
+  end
+end

data/lib/dependabot/file_fetchers.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+require "dependabot/file_fetchers/ruby/bundler"
+require "dependabot/file_fetchers/python/pip"
+require "dependabot/file_fetchers/java_script/npm_and_yarn"
+require "dependabot/file_fetchers/java/maven"
+require "dependabot/file_fetchers/java/gradle"
+require "dependabot/file_fetchers/php/composer"
+require "dependabot/file_fetchers/git/submodules"
+require "dependabot/file_fetchers/docker/docker"
+require "dependabot/file_fetchers/elixir/hex"
+require "dependabot/file_fetchers/rust/cargo"
+require "dependabot/file_fetchers/dotnet/nuget"
+require "dependabot/file_fetchers/go/dep"
+require "dependabot/file_fetchers/go/modules"
+require "dependabot/file_fetchers/elm/elm_package"
+
+module Dependabot
+  module FileFetchers
+    @file_fetchers = {
+      "bundler" => FileFetchers::Ruby::Bundler,
+      "npm_and_yarn" => FileFetchers::JavaScript::NpmAndYarn,
+      "maven" => FileFetchers::Java::Maven,
+      "gradle" => FileFetchers::Java::Gradle,
+      "pip" => FileFetchers::Python::Pip,
+      "composer" => FileFetchers::Php::Composer,
+      "submodules" => FileFetchers::Git::Submodules,
+      "docker" => FileFetchers::Docker::Docker,
+      "hex" => FileFetchers::Elixir::Hex,
+      "cargo" => FileFetchers::Rust::Cargo,
+      "nuget" => FileFetchers::Dotnet::Nuget,
+      "dep" => FileFetchers::Go::Dep,
+      "go_modules" => FileFetchers::Go::Modules,
+      "elm-package" => FileFetchers::Elm::ElmPackage
+    }
+
+    def self.for_package_manager(package_manager)
+      file_fetcher = @file_fetchers[package_manager]
+      return file_fetcher if file_fetcher
+
+      raise "Unsupported package_manager #{package_manager}"
+    end
+
+    def self.register(package_manager, file_fetcher)
+      @file_fetchers[package_manager] = file_fetcher
+    end
+  end
+end

data/lib/dependabot/file_fetchers/README.md

@@ -0,0 +1,65 @@
+# File fetchers
+
+File fetchers are used to fetch the relevant dependency files for a project
+(e.g., the `Gemfile` and `Gemfile.lock`). They are also responsible for checking
+whether a repo has an admissable set of requirement files.
+
+There is a `Dependabot::FileFetchers` class for each language Dependabot
+supports.
+
+## Public API
+
+Each `Dependabot::FileFetchers` class implements the following methods:
+
+| Method                           | Description                                                                                   |
+|----------------------------------|-----------------------------------------------------------------------------------------------|
+| `.required_files_in?`            | Checks an array of filenames (string) and returns a boolean describing whether the language-specific dependency files required for an update run are present. |
+| `.required_files_message`        | Returns a static error message which can be displayed to a user if `required_files_in?` returns false. |
+| `#files`                         | Fetches the language-specific dependency files for the repo this instance was created with. Returns an array of `Dependabot::DependencyFile` instances. |
+| `#commit`                        | Returns the commit SHA-1 hash at the time the dependency files were fetched. If called before `files`, the returned value will be used in subsequent calls to `files`. |
+
+
+An integration might look as follows:
+
+```ruby
+require 'octokit'
+require 'dependabot/file_fetchers'
+require 'dependabot/source'
+
+target_repo_name = 'dependabot/dependabot-core'
+source = Dependabot::Source.new(provider: 'github', repo: target_repo_name)
+
+client = Octokit::Client.new
+fetcher_class = Dependabot::FileFetchers::Ruby::Bundler
+filenames = client.contents(target_repo_name).map(&:name)
+
+unless fetcher_class.required_files_in?(filenames)
+  raise fetcher_class.required_files_message
+end
+
+fetcher = fetcher_class.new(source: source, credentials: [])
+
+puts "Fetched #{fetcher.files.map(&:name)}, at commit SHA-1 '#{fetcher.commit}'"
+```
+
+## Writing a file fetcher for a new language
+
+All new file fetchers should inherit from `Dependabot::FileFetchers::Base` and
+implement the following methods:
+
+| Method                           | Description                                                                                   |
+|----------------------------------|-----------------------------------------------------------------------------------------------|
+| `.required_files_in?`            | See Public API section. |
+| `.required_files_message`        | See Public API section. |
+| `#fetch_files`                   | Private method to fetch the required files from GitHub. For each required file, you can use the `fetch_file_from_host(filename)` method from `Dependabot::FileFetchers::Base` to do the fetching. |
+
+To ensure the above are implemented, you should include
+`it_behaves_like "a dependency file fetcher"` in your specs for the new file
+fetcher.
+
+File fetchers tend to get complicated when the file requirements for an update
+to run are non-trivial - for example, for Ruby we could accept
+[`Gemfile`, `Gemfile.lock`] or [`Gemfile`, `example.gemspec`],
+but not just [`Gemfile.lock`]. When adding a new lanugage, it's normally easiest
+to pick a single case and implement it for all the update steps (parsing, update
+checking, etc.). You can then return and add other cases later.

data/lib/dependabot/file_fetchers/base.rb

@@ -0,0 +1,302 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency_file"
+require "dependabot/source"
+require "dependabot/errors"
+require "dependabot/clients/github_with_retries"
+require "dependabot/clients/bitbucket"
+require "dependabot/clients/gitlab"
+require "dependabot/shared_helpers"
+
+module Dependabot
+  module FileFetchers
+    class Base
+      attr_reader :source, :credentials
+
+      def self.required_files_in?(_filename_array)
+        raise NotImplementedError
+      end
+
+      def self.required_files_message
+        raise NotImplementedError
+      end
+
+      def initialize(source:, credentials:)
+        @source = source
+        @credentials = credentials
+
+        @submodule_directories = {}
+      end
+
+      def repo
+        source.repo
+      end
+
+      def directory
+        source.directory || "/"
+      end
+
+      def target_branch
+        source.branch
+      end
+
+      def files
+        @files ||= fetch_files
+      end
+
+      def commit
+        branch = target_branch || default_branch_for_repo
+
+        @commit ||= client_for_provider.fetch_commit(repo, branch)
+      rescue Octokit::NotFound, Gitlab::Error::NotFound,
+             Dependabot::Clients::Bitbucket::NotFound
+        raise Dependabot::BranchNotFound, branch
+      rescue Octokit::Conflict => error
+        raise unless error.message.include?("Repository is empty")
+      end
+
+      private
+
+      def client_for_provider
+        case source.provider
+        when "github"
+          github_client_for_source
+        when "gitlab"
+          gitlab_client
+        when "bitbucket"
+          bitbucket_client
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      def default_branch_for_repo
+        @default_branch_for_repo ||=
+          client_for_provider.fetch_default_branch(repo)
+      rescue Octokit::NotFound, Gitlab::Error::NotFound,
+             Dependabot::Clients::Bitbucket::NotFound
+        raise Dependabot::RepoNotFound, source
+      end
+
+      def fetch_file_if_present(filename)
+        dir = File.dirname(filename)
+        basename = File.basename(filename)
+        return unless repo_contents(dir: dir).map(&:name).include?(basename)
+
+        fetch_file_from_host(filename)
+      rescue Octokit::NotFound, Gitlab::Error::NotFound,
+             Dependabot::Clients::Bitbucket::NotFound
+        path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
+        raise Dependabot::DependencyFileNotFound, path
+      end
+
+      def fetch_file_from_host(filename, type: "file")
+        path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
+
+        DependencyFile.new(
+          name: Pathname.new(filename).cleanpath.to_path,
+          content: fetch_file_content(path),
+          directory: directory,
+          type: type
+        )
+      rescue Octokit::NotFound, Gitlab::Error::NotFound,
+             Dependabot::Clients::Bitbucket::NotFound
+        raise Dependabot::DependencyFileNotFound, path
+      end
+
+      def fetch_file_from_host_or_submodule(filename, type: "file")
+        fetch_file_from_host(filename, type: type)
+      rescue Dependabot::DependencyFileNotFound => error
+        begin
+          repo_contents(dir: File.dirname(filename))
+        rescue StandardError
+          raise error
+        end
+
+        fetch_file_from_host(filename, type: type)
+      end
+
+      def repo_contents(dir: ".", raise_errors: true)
+        path = Pathname.new(File.join(directory, dir)).
+               cleanpath.to_path.gsub(%r{^/*}, "")
+
+        # Don't fetch contents of repos nested in submodules
+        submods = @submodule_directories
+        if submods.keys.any? { |k| path.match?(%r{#{Regexp.quote(k)}(/|$)}) }
+          return []
+        end
+
+        @repo_contents ||= {}
+        @repo_contents[dir] ||=
+          case source.provider
+          when "github" then github_repo_contents(path)
+          when "gitlab" then gitlab_repo_contents(path)
+          when "bitbucket" then bitbucket_repo_contents(path)
+          else raise "Unsupported provider '#{source.provider}'."
+          end
+      rescue Octokit::NotFound, Gitlab::Error::NotFound,
+             Dependabot::Clients::Bitbucket::NotFound
+        raise if raise_errors
+
+        []
+      end
+
+      def github_repo_contents(path)
+        github_response = github_client_for_source.
+                          contents(repo, path: path, ref: commit)
+
+        if github_response.respond_to?(:type) &&
+           github_response.type == "submodule"
+          @submodule_directories[path] = github_response
+
+          sub_source = Source.from_url(github_response.submodule_git_url)
+          github_response = github_client_for_source.
+                            contents(sub_source.repo, ref: github_response.sha)
+        elsif github_response.respond_to?(:type)
+          raise Octokit::NotFound
+        end
+
+        github_response.map do |f|
+          OpenStruct.new(
+            name: f.name,
+            path: f.path,
+            type: f.type,
+            sha: f.sha,
+            size: f.size
+          )
+        end
+      end
+
+      def gitlab_repo_contents(path)
+        gitlab_client.
+          repo_tree(repo, path: path, ref_name: commit, per_page: 100).
+          map do |file|
+            OpenStruct.new(
+              name: file.name,
+              path: file.path,
+              type: file.type == "blob" ? "file" : file.type,
+              size: 0 # GitLab doesn't return file size
+            )
+          end
+      end
+
+      def bitbucket_repo_contents(path)
+        response = bitbucket_client.fetch_repo_contents(
+          repo,
+          commit,
+          path
+        )
+
+        response.map do |file|
+          type = case file.fetch("type")
+                 when "commit_file" then "file"
+                 when "commit_directory" then "dir"
+                 else file.fetch("type")
+                 end
+
+          OpenStruct.new(
+            name: File.basename(file.fetch("path")),
+            path: file.fetch("path"),
+            type: type,
+            size: file.fetch("size", 0)
+          )
+        end
+      end
+
+      def fetch_file_content(path)
+        path = path.gsub(%r{^/*}, "")
+        dir = Pathname.new(path).dirname.to_path.gsub(%r{^/*}, "")
+
+        if @submodule_directories.key?(dir)
+          return fetch_submodule_file_content(path)
+        end
+
+        case source.provider
+        when "github"
+          fetch_file_content_from_github(path, repo, commit)
+        when "gitlab"
+          tmp = gitlab_client.get_file(repo, path, commit).content
+          Base64.decode64(tmp).force_encoding("UTF-8").encode
+        when "bitbucket"
+          bitbucket_client.fetch_file_contents(repo, commit, path)
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      def fetch_submodule_file_content(path)
+        path = path.gsub(%r{^/*}, "")
+        dir = Pathname.new(path).dirname.to_path.gsub(%r{^/*}, "")
+        submodule = @submodule_directories[dir]
+
+        provider = Source.from_url(submodule.submodule_git_url).provider
+        repo = Source.from_url(submodule.submodule_git_url).repo
+        commit = submodule.sha
+        path = path.gsub("#{dir}/", "")
+
+        case provider
+        when "github"
+          fetch_file_content_from_github(path, repo, commit)
+        when "gitlab"
+          tmp = gitlab_client.get_file(repo, path, commit).content
+          Base64.decode64(tmp).force_encoding("UTF-8").encode
+        when "bitbucket"
+          bitbucket_client.fetch_file_contents(repo, commit, path)
+        else raise "Unsupported provider '#{provider}'."
+        end
+      end
+
+      # rubocop:disable Metrics/AbcSize
+      def fetch_file_content_from_github(path, repo, commit)
+        tmp = github_client_for_source.contents(repo, path: path, ref: commit)
+
+        if tmp.type == "symlink"
+          tmp = github_client_for_source.contents(
+            repo,
+            path: tmp.target,
+            ref: commit
+          )
+        end
+
+        Base64.decode64(tmp.content).force_encoding("UTF-8").encode
+      rescue Octokit::Forbidden => error
+        raise unless error.message.include?("too_large")
+
+        # Fall back to Git Data API to fetch the file
+        prefix_dir = directory.gsub(%r{(^/|/$)}, "")
+        dir = File.dirname(path).gsub(%r{^/?#{Regexp.escape(prefix_dir)}/?}, "")
+        basename = File.basename(path)
+        file_details = repo_contents(dir: dir).find { |f| f.name == basename }
+        raise unless file_details
+
+        tmp = github_client_for_source.blob(repo, file_details.sha)
+        return tmp.content if tmp.encoding == "utf-8"
+
+        Base64.decode64(tmp.content).force_encoding("UTF-8").encode
+      end
+      # rubocop:enable Metrics/AbcSize
+
+      def github_client_for_source
+        @github_client_for_source ||=
+          Dependabot::Clients::GithubWithRetries.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def gitlab_client
+        @gitlab_client ||=
+          Dependabot::Clients::Gitlab.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def bitbucket_client
+        # TODO: When self-hosted Bitbucket is supported this should use
+        # `Bitbucket.for_source`
+        @bitbucket_client ||=
+          Dependabot::Clients::Bitbucket.
+          for_bitbucket_dot_org(credentials: credentials)
+      end
+    end
+  end
+end