dependabot-core 0.94.3 → 0.94.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b9ca1b04d83b74e88958e5d5993404cfc65a0f1f732431c97ff2fc5c28c3528
-  data.tar.gz: 51925653e14e70c182fcd12bf232b4e65a500708d823dfe6d8a9791f5c4ca2cb
+  metadata.gz: 4b11e37fb0e013f8771d0a7d3d2d43eb70ca034a5b1f34750ba221dc869a43b6
+  data.tar.gz: 33eaade27acbe4aefe07238a9c760358e76c56541cfed364f0ea890dfe2effea
 SHA512:
-  metadata.gz: 000d8d39f5d6afe274272cbab57ae364e6ee571594820d5517b3a8b8583c660bb8527212ba34df392d702a73f204bdc37c92cac8aa299db6d6ebee4fbeba78b6
-  data.tar.gz: 91eae6edc6bfc30d26fb0c70b2bef65c707b524a2d298ba7734e1558caabeedd159c190e334a3022885a8e19a7a323efffda2afe9fb1bf4a121506f23eafdf97
+  metadata.gz: 2794a272a65b70afc8a1c70273994931320e5fc3ef186006d5dfdfc30651a4f802b91b44dcf54efa2eeb72afa72676f64c8abcd8a6f076af521e54ddb3b2ec50
+  data.tar.gz: 330ce862b3214c35c5ece45690422598e06cf5129bdf88d1d956a4cfc5f5db3e015d34aa5d6bdf9a3941fda2414f8a2bb5e795e0d94dad954332888577b511c6

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## v0.94.4, 3 February 2019
+
+- Fetch files that are nested in submodules if asked
+- Clean up file fetcher base class
+
 ## v0.94.3, 2 February 2019
 
 - Better name for language label details

data/lib/dependabot/file_fetchers/base.rb

@@ -8,11 +8,18 @@ require "dependabot/clients/bitbucket"
 require "dependabot/clients/gitlab"
 require "dependabot/shared_helpers"
 
+# rubocop:disable Metrics/ClassLength
 module Dependabot
   module FileFetchers
     class Base
       attr_reader :source, :credentials
 
+      CLIENT_NOT_FOUND_ERRORS = [
+        Octokit::NotFound,
+        Gitlab::Error::NotFound,
+        Dependabot::Clients::Bitbucket::NotFound
+      ].freeze
+
       def self.required_files_in?(_filename_array)
         raise NotImplementedError
       end
@@ -48,8 +55,7 @@ module Dependabot
         branch = target_branch || default_branch_for_repo
 
         @commit ||= client_for_provider.fetch_commit(repo, branch)
-      rescue Octokit::NotFound, Gitlab::Error::NotFound,
-             Dependabot::Clients::Bitbucket::NotFound
+      rescue *CLIENT_NOT_FOUND_ERRORS
         raise Dependabot::BranchNotFound, branch
       rescue Octokit::Conflict => error
         raise unless error.message.include?("Repository is empty")
@@ -57,117 +63,119 @@ module Dependabot
 
       private
 
-      def client_for_provider
-        case source.provider
-        when "github"
-          github_client_for_source
-        when "gitlab"
-          gitlab_client
-        when "bitbucket"
-          bitbucket_client
-        else raise "Unsupported provider '#{source.provider}'."
-        end
-      end
-
-      def default_branch_for_repo
-        @default_branch_for_repo ||=
-          client_for_provider.fetch_default_branch(repo)
-      rescue Octokit::NotFound, Gitlab::Error::NotFound,
-             Dependabot::Clients::Bitbucket::NotFound
-        raise Dependabot::RepoNotFound, source
-      end
-
-      def fetch_file_if_present(filename)
+      def fetch_file_if_present(filename, fetch_submodules: false)
         dir = File.dirname(filename)
         basename = File.basename(filename)
-        return unless repo_contents(dir: dir).map(&:name).include?(basename)
 
-        fetch_file_from_host(filename)
-      rescue Octokit::NotFound, Gitlab::Error::NotFound,
-             Dependabot::Clients::Bitbucket::NotFound
+        repo_includes_basename =
+          repo_contents(dir: dir, fetch_submodules: fetch_submodules).
+          reject { |f| f.type == "dir" }.
+          map(&:name).include?(basename)
+        return unless repo_includes_basename
+
+        fetch_file_from_host(filename, fetch_submodules: fetch_submodules)
+      rescue *CLIENT_NOT_FOUND_ERRORS
         path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
         raise Dependabot::DependencyFileNotFound, path
       end
 
-      def fetch_file_from_host(filename, type: "file")
+      def fetch_file_from_host(filename, type: "file", fetch_submodules: false)
         path = Pathname.new(File.join(directory, filename)).cleanpath.to_path
 
         DependencyFile.new(
           name: Pathname.new(filename).cleanpath.to_path,
-          content: fetch_file_content(path),
           directory: directory,
-          type: type
+          type: type,
+          content: _fetch_file_content(path, fetch_submodules: fetch_submodules)
         )
-      rescue Octokit::NotFound, Gitlab::Error::NotFound,
-             Dependabot::Clients::Bitbucket::NotFound
+      rescue *CLIENT_NOT_FOUND_ERRORS
         raise Dependabot::DependencyFileNotFound, path
       end
 
-      def fetch_file_from_host_or_submodule(filename, type: "file")
-        fetch_file_from_host(filename, type: type)
-      rescue Dependabot::DependencyFileNotFound => error
-        begin
-          repo_contents(dir: File.dirname(filename))
-        rescue StandardError
-          raise error
-        end
+      def repo_contents(dir: ".", ignore_base_directory: false,
+                        raise_errors: true, fetch_submodules: false)
+        dir = File.join(directory, dir) unless ignore_base_directory
+        path = Pathname.new(File.join(dir)).cleanpath.to_path.gsub(%r{^/*}, "")
 
-        fetch_file_from_host(filename, type: type)
+        @repo_contents ||= {}
+        @repo_contents[dir] ||= _fetch_repo_contents(
+          path,
+          raise_errors: raise_errors,
+          fetch_submodules: fetch_submodules
+        )
       end
 
-      def repo_contents(dir: ".", raise_errors: true)
-        path = Pathname.new(File.join(directory, dir)).
-               cleanpath.to_path.gsub(%r{^/*}, "")
-
-        # Don't fetch contents of repos nested in submodules
-        submods = @submodule_directories
-        if submods.keys.any? { |k| path.match?(%r{#{Regexp.quote(k)}(/|$)}) }
-          return []
-        end
+      #################################################
+      # INTERNAL METHODS (not for use by sub-classes) #
+      #################################################
 
-        @repo_contents ||= {}
-        @repo_contents[dir] ||=
-          case source.provider
-          when "github" then github_repo_contents(path)
-          when "gitlab" then gitlab_repo_contents(path)
-          when "bitbucket" then bitbucket_repo_contents(path)
-          else raise "Unsupported provider '#{source.provider}'."
-          end
-      rescue Octokit::NotFound, Gitlab::Error::NotFound,
-             Dependabot::Clients::Bitbucket::NotFound
-        raise if raise_errors
+      def _fetch_repo_contents(path, fetch_submodules: false,
+                               raise_errors: true)
+        path = path.gsub(" ", "%20")
+        provider, repo, tmp_path, commit =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          values_at(:provider, :repo, :path, :commit)
+
+        _fetch_repo_contents_fully_specified(provider, repo, tmp_path, commit)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        result = raise_errors ? -> { raise } : -> { [] }
+        retrying ||= false
+
+        # If the path changes after calling _fetch_repo_contents_fully_specified
+        # it's because we've found a sub-module (and are fetching them). Trigger
+        # a retry to get its contents.
+        updated_path =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          fetch(:path)
+        retry if updated_path != tmp_path
+
+        return result.call unless fetch_submodules && !retrying
+
+        _find_submodules(path)
+        return result.call unless _submodule_for(path)
+
+        retrying = true
+        retry
+      end
 
-        []
+      def _fetch_repo_contents_fully_specified(provider, repo, path, commit)
+        case provider
+        when "github"
+          _github_repo_contents(repo, path, commit)
+        when "gitlab"
+          _gitlab_repo_contents(repo, path, commit)
+        when "bitbucket"
+          _bitbucket_repo_contents(repo, path, commit)
+        else raise "Unsupported provider '#{provider}'."
+        end
       end
 
-      def github_repo_contents(path)
+      def _github_repo_contents(repo, path, commit)
         path = path.gsub(" ", "%20")
-        github_response = github_client_for_source.
-                          contents(repo, path: path, ref: commit)
+        github_response = github_client.contents(repo, path: path, ref: commit)
 
         if github_response.respond_to?(:type) &&
            github_response.type == "submodule"
           @submodule_directories[path] = github_response
-
-          sub_source = Source.from_url(github_response.submodule_git_url)
-          github_response = github_client_for_source.
-                            contents(sub_source.repo, ref: github_response.sha)
+          raise Octokit::NotFound
         elsif github_response.respond_to?(:type)
           raise Octokit::NotFound
         end
 
-        github_response.map do |f|
-          OpenStruct.new(
-            name: f.name,
-            path: f.path,
-            type: f.type,
-            sha: f.sha,
-            size: f.size
-          )
-        end
+        github_response.map { |f| _build_github_file_struct(f) }
+      end
+
+      def _build_github_file_struct(file)
+        OpenStruct.new(
+          name: file.name,
+          path: file.path,
+          type: file.type,
+          sha: file.sha,
+          size: file.size
+        )
       end
 
-      def gitlab_repo_contents(path)
+      def _gitlab_repo_contents(repo, path, commit)
         gitlab_client.
           repo_tree(repo, path: path, ref_name: commit, per_page: 100).
           map do |file|
@@ -180,7 +188,7 @@ module Dependabot
           end
       end
 
-      def bitbucket_repo_contents(path)
+      def _bitbucket_repo_contents(repo, path, commit)
         response = bitbucket_client.fetch_repo_contents(
           repo,
           commit,
@@ -203,54 +211,67 @@ module Dependabot
         end
       end
 
-      def fetch_file_content(path)
-        path = path.gsub(%r{^/*}, "")
-        dir = Pathname.new(path).dirname.to_path.gsub(%r{^/*}, "")
-
-        if @submodule_directories.key?(dir)
-          return fetch_submodule_file_content(path)
-        end
-
-        case source.provider
-        when "github"
-          fetch_file_content_from_github(path, repo, commit)
-        when "gitlab"
-          tmp = gitlab_client.get_file(repo, path, commit).content
-          Base64.decode64(tmp).force_encoding("UTF-8").encode
-        when "bitbucket"
-          bitbucket_client.fetch_file_contents(repo, commit, path)
-        else raise "Unsupported provider '#{source.provider}'."
+      def _full_specification_for(path, fetch_submodules:)
+        if fetch_submodules && _submodule_for(path)
+          submodule_details = @submodule_directories[_submodule_for(path)]
+          sub_source = Source.from_url(submodule_details.submodule_git_url)
+          {
+            repo: sub_source.repo,
+            commit: submodule_details.sha,
+            provider: sub_source.provider,
+            path: path.gsub(%r{^#{Regexp.quote(_submodule_for(path))}(/|$)}, "")
+          }
+        else
+          {
+            repo: source.repo,
+            path: path,
+            commit: commit,
+            provider: source.provider
+          }
         end
       end
 
-      def fetch_submodule_file_content(path)
+      def _fetch_file_content(path, fetch_submodules: false)
         path = path.gsub(%r{^/*}, "")
-        dir = Pathname.new(path).dirname.to_path.gsub(%r{^/*}, "")
-        submodule = @submodule_directories[dir]
 
-        provider = Source.from_url(submodule.submodule_git_url).provider
-        repo = Source.from_url(submodule.submodule_git_url).repo
-        commit = submodule.sha
-        path = path.gsub("#{dir}/", "")
+        provider, repo, path, commit =
+          _full_specification_for(path, fetch_submodules: fetch_submodules).
+          values_at(:provider, :repo, :path, :commit)
+
+        _fetch_file_content_fully_specified(provider, repo, path, commit)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        retrying ||= false
+
+        raise unless fetch_submodules && !retrying && !_submodule_for(path)
+
+        _find_submodules(path)
+        raise unless _submodule_for(path)
 
+        retrying = true
+        retry
+      end
+
+      def _fetch_file_content_fully_specified(provider, repo, path, commit)
         case provider
         when "github"
-          fetch_file_content_from_github(path, repo, commit)
+          _fetch_file_content_from_github(path, repo, commit)
         when "gitlab"
           tmp = gitlab_client.get_file(repo, path, commit).content
           Base64.decode64(tmp).force_encoding("UTF-8").encode
         when "bitbucket"
           bitbucket_client.fetch_file_contents(repo, commit, path)
-        else raise "Unsupported provider '#{provider}'."
+        else raise "Unsupported provider '#{source.provider}'."
         end
       end
 
       # rubocop:disable Metrics/AbcSize
-      def fetch_file_content_from_github(path, repo, commit)
-        tmp = github_client_for_source.contents(repo, path: path, ref: commit)
+      def _fetch_file_content_from_github(path, repo, commit)
+        tmp = github_client.contents(repo, path: path, ref: commit)
+
+        raise Octokit::NotFound if tmp.is_a?(Array)
 
         if tmp.type == "symlink"
-          tmp = github_client_for_source.contents(
+          tmp = github_client.contents(
             repo,
             path: tmp.target,
             ref: commit
@@ -268,15 +289,55 @@ module Dependabot
         file_details = repo_contents(dir: dir).find { |f| f.name == basename }
         raise unless file_details
 
-        tmp = github_client_for_source.blob(repo, file_details.sha)
+        tmp = github_client.blob(repo, file_details.sha)
         return tmp.content if tmp.encoding == "utf-8"
 
         Base64.decode64(tmp.content).force_encoding("UTF-8").encode
       end
       # rubocop:enable Metrics/AbcSize
 
-      def github_client_for_source
-        @github_client_for_source ||=
+      def default_branch_for_repo
+        @default_branch_for_repo ||= client_for_provider.
+                                     fetch_default_branch(repo)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        raise Dependabot::RepoNotFound, source
+      end
+
+      # Update the @submodule_directories hash by exploiting a side-effect of
+      # recursively calling `repo_contents` for each directory up the tree
+      # until a submodule is found
+      def _find_submodules(path)
+        path = Pathname.new(path).cleanpath.to_path.gsub(%r{^/*}, "")
+        dir = File.dirname(path)
+
+        return if [directory, "."].include?(dir)
+
+        repo_contents(
+          dir: dir,
+          ignore_base_directory: true,
+          fetch_submodules: true,
+          raise_errors: false
+        )
+      end
+
+      def _submodule_for(path)
+        submodules = @submodule_directories.keys
+        submodules.
+          select { |k| path.match?(%r{^#{Regexp.quote(k)}(/|$)}) }.
+          max_by(&:length)
+      end
+
+      def client_for_provider
+        case source.provider
+        when "github" then github_client
+        when "gitlab" then gitlab_client
+        when "bitbucket" then bitbucket_client
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      def github_client
+        @github_client ||=
           Dependabot::Clients::GithubWithRetries.for_source(
             source: source,
             credentials: credentials
@@ -301,3 +362,4 @@ module Dependabot
     end
   end
 end
+# rubocop:enable Metrics/ClassLength

data/lib/dependabot/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Dependabot
-  VERSION = "0.94.3"
+  VERSION = "0.94.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-core
 version: !ruby/object:Gem::Version
-  version: 0.94.3
+  version: 0.94.4
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-02 00:00:00.000000000 Z
+date: 2019-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-ecr