dependabot-common 0.212.0 → 0.214.0

data/lib/dependabot/file_fetchers/base.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "stringio"
 require "dependabot/config"
 require "dependabot/dependency_file"
 require "dependabot/source"
@@ -25,6 +26,12 @@ module Dependabot
         Dependabot::Clients::CodeCommit::NotFound
       ].freeze
 
+      GIT_SUBMODULE_INACCESSIBLE_ERROR =
+        /^fatal: unable to access '(?<url>.*)': The requested URL returned error: (?<code>\d+)$/
+      GIT_SUBMODULE_CLONE_ERROR =
+        /^fatal: clone of '(?<url>.*)' into submodule path '.*' failed$/
+      GIT_SUBMODULE_ERROR_REGEX = /(#{GIT_SUBMODULE_INACCESSIBLE_ERROR})|(#{GIT_SUBMODULE_CLONE_ERROR})/
+
       def self.required_files_in?(_filename_array)
         raise NotImplementedError
       end
@@ -69,6 +76,7 @@ module Dependabot
       end
 
       def commit
+        return cloned_commit if cloned_commit
         return source.commit if source.commit
 
         branch = target_branch || default_branch_for_repo
@@ -84,7 +92,11 @@ module Dependabot
       def clone_repo_contents
         @clone_repo_contents ||=
           _clone_repo_contents(target_directory: repo_contents_path)
-      rescue Dependabot::SharedHelpers::HelperSubprocessFailed
+      rescue Dependabot::SharedHelpers::HelperSubprocessFailed => e
+        if e.message.include?("fatal: Remote branch #{target_branch} not found in upstream origin")
+          raise Dependabot::BranchNotFound, target_branch
+        end
+
         raise Dependabot::RepoNotFound, source
       end
 
@@ -168,6 +180,97 @@ module Dependabot
                                 end
       end
 
+      def cloned_commit
+        return if repo_contents_path.nil? || !File.directory?(File.join(repo_contents_path, ".git"))
+
+        SharedHelpers.with_git_configured(credentials: credentials) do
+          Dir.chdir(repo_contents_path) do
+            return SharedHelpers.run_shell_command("git rev-parse HEAD")&.strip
+          end
+        end
+      end
+
+      def default_branch_for_repo
+        @default_branch_for_repo ||= client_for_provider.
+                                     fetch_default_branch(repo)
+      rescue *CLIENT_NOT_FOUND_ERRORS
+        raise Dependabot::RepoNotFound, source
+      end
+
+      def update_linked_paths(repo, path, commit, github_response)
+        case github_response.type
+        when "submodule"
+          sub_source = Source.from_url(github_response.submodule_git_url)
+          return unless sub_source
+
+          @linked_paths[path] = {
+            repo: sub_source.repo,
+            provider: sub_source.provider,
+            commit: github_response.sha,
+            path: "/"
+          }
+        when "symlink"
+          updated_path = File.join(File.dirname(path), github_response.target)
+          @linked_paths[path] = {
+            repo: repo,
+            provider: "github",
+            commit: commit,
+            path: Pathname.new(updated_path).cleanpath.to_path
+          }
+        end
+      end
+
+      def recurse_submodules_when_cloning?
+        false
+      end
+
+      def client_for_provider
+        case source.provider
+        when "github" then github_client
+        when "gitlab" then gitlab_client
+        when "azure" then azure_client
+        when "bitbucket" then bitbucket_client
+        when "codecommit" then codecommit_client
+        else raise "Unsupported provider '#{source.provider}'."
+        end
+      end
+
+      def github_client
+        @github_client ||=
+          Dependabot::Clients::GithubWithRetries.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def gitlab_client
+        @gitlab_client ||=
+          Dependabot::Clients::GitlabWithRetries.for_source(
+            source: source,
+            credentials: credentials
+          )
+      end
+
+      def azure_client
+        @azure_client ||=
+          Dependabot::Clients::Azure.
+          for_source(source: source, credentials: credentials)
+      end
+
+      def bitbucket_client
+        # TODO: When self-hosted Bitbucket is supported this should use
+        # `Bitbucket.for_source`
+        @bitbucket_client ||=
+          Dependabot::Clients::BitbucketWithRetries.
+          for_bitbucket_dot_org(credentials: credentials)
+      end
+
+      def codecommit_client
+        @codecommit_client ||=
+          Dependabot::Clients::CodeCommit.
+          for_source(source: source, credentials: credentials)
+      end
+
       #################################################
       # INTERNAL METHODS (not for use by sub-classes) #
       #################################################
@@ -254,29 +357,6 @@ module Dependabot
         end
       end
 
-      def update_linked_paths(repo, path, commit, github_response)
-        case github_response.type
-        when "submodule"
-          sub_source = Source.from_url(github_response.submodule_git_url)
-          return unless sub_source
-
-          @linked_paths[path] = {
-            repo: sub_source.repo,
-            provider: sub_source.provider,
-            commit: github_response.sha,
-            path: "/"
-          }
-        when "symlink"
-          updated_path = File.join(File.dirname(path), github_response.target)
-          @linked_paths[path] = {
-            repo: repo,
-            provider: "github",
-            commit: commit,
-            path: Pathname.new(updated_path).cleanpath.to_path
-          }
-        end
-      end
-
       def _build_github_file_struct(file)
         OpenStruct.new(
           name: file.name,
@@ -473,13 +553,6 @@ module Dependabot
       end
       # rubocop:enable Metrics/AbcSize
 
-      def default_branch_for_repo
-        @default_branch_for_repo ||= client_for_provider.
-                                     fetch_default_branch(repo)
-      rescue *CLIENT_NOT_FOUND_ERRORS
-        raise Dependabot::RepoNotFound, source
-      end
-
       # Update the @linked_paths hash by exploiting a side-effect of
       # recursively calling `repo_contents` for each directory up the tree
       # until a submodule or symlink is found
@@ -504,6 +577,10 @@ module Dependabot
           max_by(&:length)
       end
 
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      # rubocop:disable Metrics/PerceivedComplexity
+      # rubocop:disable Metrics/BlockLength
       def _clone_repo_contents(target_directory:)
         SharedHelpers.with_git_configured(credentials: credentials) do
           path = target_directory || File.join("tmp", source.repo)
@@ -512,62 +589,69 @@ module Dependabot
           return path if Dir.exist?(File.join(path, ".git"))
 
           FileUtils.mkdir_p(path)
-          br_opt = " --branch #{source.branch} --single-branch" if source.branch
-          SharedHelpers.run_shell_command(
-            <<~CMD
-              git clone --no-tags --no-recurse-submodules --depth 1#{br_opt} #{source.url} #{path}
-            CMD
-          )
-          path
-        end
-      end
 
-      def client_for_provider
-        case source.provider
-        when "github" then github_client
-        when "gitlab" then gitlab_client
-        when "azure" then azure_client
-        when "bitbucket" then bitbucket_client
-        when "codecommit" then codecommit_client
-        else raise "Unsupported provider '#{source.provider}'."
-        end
-      end
+          clone_options = StringIO.new
+          clone_options << "--no-tags --depth 1"
+          clone_options << if recurse_submodules_when_cloning?
+                             " --recurse-submodules --shallow-submodules"
+                           else
+                             " --no-recurse-submodules"
+                           end
+          clone_options << " --branch #{source.branch} --single-branch" if source.branch
 
-      def github_client
-        @github_client ||=
-          Dependabot::Clients::GithubWithRetries.for_source(
-            source: source,
-            credentials: credentials
-          )
-      end
+          submodule_cloning_failed = false
+          begin
+            SharedHelpers.run_shell_command(
+              <<~CMD
+                git clone #{clone_options.string} #{source.url} #{path}
+              CMD
+            )
+          rescue SharedHelpers::HelperSubprocessFailed => e
+            raise unless GIT_SUBMODULE_ERROR_REGEX && e.message.downcase.include?("submodule")
 
-      def gitlab_client
-        @gitlab_client ||=
-          Dependabot::Clients::GitlabWithRetries.for_source(
-            source: source,
-            credentials: credentials
-          )
-      end
+            submodule_cloning_failed = true
+            match = e.message.match(GIT_SUBMODULE_ERROR_REGEX)
+            url = match.named_captures["url"]
+            code = match.named_captures["code"]
 
-      def azure_client
-        @azure_client ||=
-          Dependabot::Clients::Azure.
-          for_source(source: source, credentials: credentials)
-      end
+            # Submodules might be in the repo but unrelated to dependencies,
+            # so ignoring this error to try the update anyway since the base repo exists.
+            Dependabot.logger.error("Cloning of submodule failed: #{url} error: #{code || 'unknown'}")
+          end
 
-      def bitbucket_client
-        # TODO: When self-hosted Bitbucket is supported this should use
-        # `Bitbucket.for_source`
-        @bitbucket_client ||=
-          Dependabot::Clients::BitbucketWithRetries.
-          for_bitbucket_dot_org(credentials: credentials)
-      end
+          if source.commit
+            # This code will only be called for testing. Production will never pass a commit
+            # since Dependabot always wants to use the latest commit on a branch.
+            Dir.chdir(path) do
+              fetch_options = StringIO.new
+              fetch_options << "--depth 1"
+              fetch_options << if recurse_submodules_when_cloning? && !submodule_cloning_failed
+                                 " --recurse-submodules=on-demand"
+                               else
+                                 " --no-recurse-submodules"
+                               end
+              # Need to fetch the commit due to the --depth 1 above.
+              SharedHelpers.run_shell_command("git fetch #{fetch_options.string} origin #{source.commit}")
+
+              reset_options = StringIO.new
+              reset_options << "--hard"
+              reset_options << if recurse_submodules_when_cloning? && !submodule_cloning_failed
+                                 " --recurse-submodules"
+                               else
+                                 " --no-recurse-submodules"
+                               end
+              # Set HEAD to this commit so later calls so git reset HEAD will work.
+              SharedHelpers.run_shell_command("git reset #{reset_options.string} #{source.commit}")
+            end
+          end
 
-      def codecommit_client
-        @codecommit_client ||=
-          Dependabot::Clients::CodeCommit.
-          for_source(source: source, credentials: credentials)
+          path
+        end
       end
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+      # rubocop:enable Metrics/PerceivedComplexity
+      # rubocop:enable Metrics/BlockLength
     end
   end
 end

data/lib/dependabot/file_parsers/base/dependency_set.rb

@@ -14,34 +14,42 @@ module Dependabot
             raise ArgumentError, "must be an array of Dependency objects"
           end
 
-          @dependencies = dependencies
           @case_sensitive = case_sensitive
+          @dependencies = Hash.new { |hsh, key| hsh[key] = DependencySlot.new }
+          dependencies.each { |dep| self << dep }
         end
 
-        attr_reader :dependencies
+        def dependencies
+          @dependencies.values.filter_map(&:combined)
+        end
 
         def <<(dep)
           raise ArgumentError, "must be a Dependency object" unless dep.is_a?(Dependency)
 
-          existing_dependency = dependency_for_name(dep.name)
+          @dependencies[key_for_dependency(dep)] << dep
+          self
+        end
 
-          return self if existing_dependency&.to_h == dep.to_h
+        def +(other)
+          raise ArgumentError, "must be a DependencySet" unless other.is_a?(DependencySet)
 
-          if existing_dependency
-            dependencies[dependencies.index(existing_dependency)] =
-              combined_dependency(existing_dependency, dep)
-          else
-            dependencies << dep
+          other_names = other.dependencies.map(&:name)
+          other_names.each do |name|
+            all_versions = other.all_versions_for_name(name)
+            all_versions.each { |dep| self << dep }
           end
 
           self
         end
 
-        def +(other)
-          raise ArgumentError, "must be a DependencySet" unless other.is_a?(DependencySet)
+        def all_versions_for_name(name)
+          key = key_for_name(name)
+          @dependencies.key?(key) ? @dependencies[key].all_versions : []
+        end
 
-          other.dependencies.each { |dep| self << dep }
-          self
+        def dependency_for_name(name)
+          key = key_for_name(name)
+          @dependencies.key?(key) ? @dependencies[key].combined : nil
         end
 
         private
@@ -50,41 +58,98 @@ module Dependabot
           @case_sensitive
         end
 
-        def dependency_for_name(name)
-          return dependencies.find { |d| d.name == name } if case_sensitive?
+        def key_for_name(name)
+          case_sensitive? ? name : name.downcase
+        end
 
-          dependencies.find { |d| d.name&.downcase == name&.downcase }
+        def key_for_dependency(dep)
+          key_for_name(dep.name)
         end
 
-        def combined_dependency(old_dep, new_dep)
-          package_manager = old_dep.package_manager
-          v_cls = Utils.version_class_for_package_manager(package_manager)
-
-          # If we already have a requirement use the existing version
-          # (if present). Otherwise, use whatever the lowest version is
-          new_version =
-            if old_dep.requirements.any? then old_dep.version || new_dep.version
-            elsif !v_cls.correct?(new_dep.version) then old_dep.version
-            elsif !v_cls.correct?(old_dep.version) then new_dep.version
-            elsif v_cls.new(new_dep.version) > v_cls.new(old_dep.version)
-              old_dep.version
+        # There can only be one entry per dependency name in a `DependencySet`. Each entry
+        # is assigned a `DependencySlot`.
+        #
+        # In some ecosystems (like `npm_and_yarn`), however, multiple versions of a
+        # dependency may be encountered and added to the set. The `DependencySlot` retains
+        # all added versions and presents a single unified dependency for the entry
+        # that combines the attributes of these versions.
+        #
+        # The combined dependency is accessible via `DependencySet#dependencies` or
+        # `DependencySet#dependency_for_name`. The list of individual versions of the
+        # dependency is accessible via `DependencySet#all_versions_for_name`.
+        class DependencySlot
+          attr_reader :all_versions, :combined
+
+          def initialize
+            @all_versions = []
+            @combined = nil
+          end
+
+          def <<(dep)
+            return self if @all_versions.include?(dep)
+
+            @combined = if @combined
+                          combined_dependency(@combined, dep)
+                        else
+                          Dependency.new(
+                            name: dep.name,
+                            version: dep.version,
+                            requirements: dep.requirements,
+                            package_manager: dep.package_manager,
+                            subdependency_metadata: dep.subdependency_metadata
+                          )
+                        end
+
+            index_of_same_version =
+              @all_versions.find_index { |other| other.version == dep.version }
+
+            if index_of_same_version.nil?
+              @all_versions << dep
             else
-              new_dep.version
+              same_version = @all_versions[index_of_same_version]
+              @all_versions[index_of_same_version] = combined_dependency(same_version, dep)
             end
 
-          subdependency_metadata = (
-            (old_dep.subdependency_metadata || []) +
-            (new_dep.subdependency_metadata || [])
-          ).uniq
-
-          Dependency.new(
-            name: old_dep.name,
-            version: new_version,
-            requirements: (old_dep.requirements + new_dep.requirements).uniq,
-            package_manager: package_manager,
-            subdependency_metadata: subdependency_metadata
-          )
+            self
+          end
+
+          private
+
+          # Produces a new dependency by merging the attributes of `old_dep` with those of
+          # `new_dep`. Requirements and subdependency metadata will be combined and deduped.
+          # The version of the combined dependency is determined by the logic below.
+          def combined_dependency(old_dep, new_dep)
+            version = if old_dep.top_level? # Prefer a direct dependency over a transitive one
+                        old_dep.version || new_dep.version
+                      elsif !version_class.correct?(new_dep.version)
+                        old_dep.version
+                      elsif !version_class.correct?(old_dep.version)
+                        new_dep.version
+                      elsif version_class.new(new_dep.version) > version_class.new(old_dep.version)
+                        old_dep.version
+                      else
+                        new_dep.version
+                      end
+            requirements = (old_dep.requirements + new_dep.requirements).uniq
+            subdependency_metadata = (
+              (old_dep.subdependency_metadata || []) +
+              (new_dep.subdependency_metadata || [])
+            ).uniq
+
+            Dependency.new(
+              name: old_dep.name,
+              version: version,
+              requirements: requirements,
+              package_manager: old_dep.package_manager,
+              subdependency_metadata: subdependency_metadata
+            )
+          end
+
+          def version_class
+            @version_class ||= Utils.version_class_for_package_manager(@combined.package_manager)
+          end
         end
+        private_constant :DependencySlot
       end
     end
   end