dependabot-go_modules 0.128.2 → 0.129.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 829d0ac68d17ff0e98eb1cea23df08e6d0dda6f28b9f9f7315b99b7ed2a6a355
-  data.tar.gz: f90a05b616834eea6573db46d5e327ffc2fc233fc9018562746205f88972a2c0
+  metadata.gz: d8f7a2c4bb77e9289dd8673f71692a0ad30600fa56a924057b13c0539c2abbf1
+  data.tar.gz: 0c532fbe591362a72d8a063f91b0a416cbbd788c806a248d4d94ddcb60371edf
 SHA512:
-  metadata.gz: 1fe6f621f9af602d6c4f8245e17376f6fb3ea010d97b0ccd31d832f1a91d119133ca9cc2a6c386595320770214a08891bf9c3260035476985417242a59d495a6
-  data.tar.gz: 73f66e0d0165db078aee5ea0ddbe088306955eb7be220e43118bc5d46ae06c302836472d6565d19a9ec04e4285695ed0b8e4f53576b972c7f1d2dd6e0a1e1ef3
+  metadata.gz: 4fa94982678e5c4a78481f43b90ecbf882da31bacddf3fbe34aa51f0d47e50a09a26d72d626d9fa6d8f832a6e21d7205568a4a9a8e1049f10df4d6a1ade7f25a
+  data.tar.gz: c0e66a28d79c1e82747e8844887332ff3da98c1c3914bf6a4d911c112e61be3df70bd68daa60069e045329b831daf1faa50b4586fa176717acb58dcd4c40a7e5

data/lib/dependabot/go_modules/file_parser.rb

@@ -16,17 +16,8 @@ module Dependabot
       def parse
         dependency_set = Dependabot::FileParsers::Base::DependencySet.new
 
-        i = 0
-        chunks = module_info.lines.
-                 group_by { |line| line == "{\n" ? i += 1 : i }
-        deps = chunks.values.map { |chunk| JSON.parse(chunk.join) }
-
-        deps.each do |dep|
-          # The project itself appears in this list as "Main"
-          next if dep["Main"]
-
-          dependency = dependency_from_details(dep)
-          dependency_set << dependency if dependency
+        required_packages.each do |dep|
+          dependency_set << dependency_from_details(dep) unless dep["Indirect"]
         end
 
         dependency_set.dependencies
@@ -65,39 +56,36 @@ module Dependabot
         )
       end
 
-      def module_info
-        @module_info ||=
+      def required_packages
+        @required_packages ||=
           SharedHelpers.in_a_temporary_directory do |path|
-            SharedHelpers.with_git_configured(credentials: credentials) do
-              # Create a fake empty module for each local module so that
-              # `go list` works, even if some modules have been `replace`d with
-              # a local module that we don't have access to.
-              local_replacements.each do |_, stub_path|
-                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                FileUtils.touch(File.join(stub_path, "go.mod"))
-              end
-
-              File.write("go.mod", go_mod_content)
-
-              command = "go mod edit -print > /dev/null"
-              command += " && go list -m -json all"
-
-              # Turn off the module proxy for now, as it's causing issues with
-              # private git dependencies
-              env = { "GOPRIVATE" => "*" }
-
-              stdout, stderr, status = Open3.capture3(env, command)
-              handle_parser_error(path, stderr) unless status.success?
-              stdout
-            rescue Dependabot::DependencyFileNotResolvable
-              # We sometimes see this error if a host times out.
-              # In such cases, retrying (a maximum of 3 times) may fix it.
-              retry_count ||= 0
-              raise if retry_count >= 3
-
-              retry_count += 1
-              retry
+            # Create a fake empty module for each local module so that
+            # `go mod edit` works, even if some modules have been `replace`d with
+            # a local module that we don't have access to.
+            local_replacements.each do |_, stub_path|
+              Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+              FileUtils.touch(File.join(stub_path, "go.mod"))
             end
+
+            File.write("go.mod", go_mod_content)
+
+            command = "go mod edit -json"
+
+            # Turn off the module proxy for now, as it's causing issues with
+            # private git dependencies
+            env = { "GOPRIVATE" => "*" }
+
+            stdout, stderr, status = Open3.capture3(env, command)
+            handle_parser_error(path, stderr) unless status.success?
+            JSON.parse(stdout)["Require"] || []
+          rescue Dependabot::DependencyFileNotResolvable
+            # We sometimes see this error if a host times out.
+            # In such cases, retrying (a maximum of 3 times) may fix it.
+            retry_count ||= 0
+            raise if retry_count >= 3
+
+            retry_count += 1
+            retry
           end
       end
 
@@ -135,46 +123,9 @@ module Dependabot
         end
       end
 
-      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/m.freeze
       def handle_parser_error(path, stderr)
-        case stderr
-        when /go: .*: unknown revision/m
-          line = stderr.lines.grep(/unknown revision/).first.strip
-          handle_github_unknown_revision(line) if line.start_with?("go: github.com/")
-          raise Dependabot::DependencyFileNotResolvable, line
-        when /go: .*: unrecognized import path/m
-          line = stderr.lines.grep(/unrecognized import/).first
-          raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: errors parsing go.mod/m
-          msg = stderr.gsub(path.to_s, "").strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
-        when GIT_ERROR_REGEX
-          lines = stderr.lines.drop_while { |l| GIT_ERROR_REGEX !~ l }
-          raise Dependabot::DependencyFileNotResolvable.new, lines.join
-        else
-          msg = stderr.gsub(path.to_s, "").strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
-        end
-      end
-
-      GITHUB_REPO_REGEX = %r{github.com/[^@]*}.freeze
-      def handle_github_unknown_revision(line)
-        repo_path = line.scan(GITHUB_REPO_REGEX).first
-        return unless repo_path
-
-        # Query for _any_ version of this module, to know if it doesn't exist (or is private)
-        # or we were just given a bad revision by this manifest
-        SharedHelpers.in_a_temporary_directory do
-          SharedHelpers.with_git_configured(credentials: credentials) do
-            File.write("go.mod", "module dummy\n")
-
-            env = { "GOPRIVATE" => "*" }
-            _, _, status = Open3.capture3(env, SharedHelpers.escape_command("go get #{repo_path}"))
-            raise Dependabot::DependencyFileNotResolvable, line if status.success?
-
-            raise Dependabot::GitDependenciesNotReachable, [repo_path]
-          end
-        end
+        msg = stderr.gsub(path.to_s, "").strip
+        raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
       end
 
       def rev_identifier?(dep)

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -4,6 +4,7 @@ require "dependabot/shared_helpers"
 require "dependabot/errors"
 require "dependabot/go_modules/file_updater"
 require "dependabot/go_modules/native_helpers"
+require "dependabot/go_modules/resolvability_errors"
 
 module Dependabot
   module GoModules
@@ -14,19 +15,21 @@ module Dependabot
         ENVIRONMENT = { "GOPRIVATE" => "*" }.freeze
 
         RESOLVABILITY_ERROR_REGEXES = [
-          # (Private) module could not be fetched
-          /go: .*: git fetch .*: exit status 128/.freeze,
           # The checksum in go.sum does not match the dowloaded content
           /verifying .*: checksum mismatch/.freeze,
+          /go: .*: go.mod has post-v\d+ module path/
+        ].freeze
+
+        REPO_RESOLVABILITY_ERROR_REGEXES = [
+          # (Private) module could not be fetched
+          /go: .*: git fetch .*: exit status 128/.freeze,
           # (Private) module could not be found
           /cannot find module providing package/.freeze,
           # Package in module was likely renamed or removed
           /module .* found \(.*\), but does not contain package/m.freeze,
           # Package does not exist, has been pulled or cannot be reached due to
           # auth problems with either git or the go proxy
-          /go: .*: unknown revision/m.freeze,
-          # Package version doesn't match the module major version
-          /go: .*: go.mod has post-v\d+ module path/m.freeze
+          /go: .*: unknown revision/m.freeze
         ].freeze
 
         MODULE_PATH_MISMATCH_REGEXES = [
@@ -40,6 +43,8 @@ module Dependabot
           /no space left on device/.freeze
         ].freeze
 
+        GO_MOD_VERSION = /^go 1\.[\d]+$/.freeze
+
         def initialize(dependencies:, credentials:, repo_contents_path:,
                        directory:, options:)
           @dependencies = dependencies
@@ -67,10 +72,9 @@ module Dependabot
           @updated_files ||= update_files
         end
 
-        def update_files # rubocop:disable Metrics/AbcSize
+        def update_files # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity
           in_repo_path do
             # Map paths in local replace directives to path hashes
-
             original_go_mod = File.read("go.mod")
             original_manifest = parse_manifest
             original_go_sum = File.read("go.sum") if File.exist?("go.sum")
@@ -87,35 +91,36 @@ module Dependabot
             # Then run `go get` to pick up other changes to the file caused by
             # the upgrade
             run_go_get
-            run_go_vendor
-            run_go_mod_tidy
-
-            # At this point, the go.mod returned from run_go_get contains the
-            # correct set of modules, but running `go get` can change the file
-            # in undesirable ways (such as injecting the current Go version),
-            # so we need to update the original go.mod with the updated set of
-            # requirements rather than using the regenerated file directly
-            original_reqs = original_manifest["Require"] || []
-            updated_reqs = parse_manifest["Require"] || []
-
-            original_paths = original_reqs.map { |r| r["Path"] }
-            updated_paths = updated_reqs.map { |r| r["Path"] }
-            req_paths_to_remove = original_paths - updated_paths
 
-            # Put back the original content before we replace just the updated
-            # dependencies.
-            write_go_mod(original_go_mod)
-
-            remove_requirements(req_paths_to_remove)
-            deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
-            update_go_mod(deps)
-
-            # put the old replace directives back again
-            substitute_all(substitutions.invert)
+            # If we stubbed modules, don't run `go mod {tidy,vendor}` as
+            # dependencies are incomplete
+            if substitutions.empty?
+              run_go_mod_tidy
+              run_go_vendor
+            else
+              substitute_all(substitutions.invert)
+            end
 
             updated_go_sum = original_go_sum ? File.read("go.sum") : nil
             updated_go_mod = File.read("go.mod")
 
+            # running "go get" may inject the current go version, remove it
+            original_go_version = original_go_mod.match(GO_MOD_VERSION)&.to_a&.first
+            updated_go_version = updated_go_mod.match(GO_MOD_VERSION)&.to_a&.first
+            if original_go_version != updated_go_version
+              go_mod_lines = updated_go_mod.lines
+              go_mod_lines.each_with_index do |line, i|
+                next unless line&.match?(GO_MOD_VERSION)
+
+                # replace with the original version
+                go_mod_lines[i] = original_go_version
+                # avoid a stranded newline if there was no version originally
+                go_mod_lines[i + 1] = nil if original_go_version.nil?
+              end
+
+              updated_go_mod = go_mod_lines.compact.join
+            end
+
             { go_mod: updated_go_mod, go_sum: updated_go_sum }
           end
         end
@@ -184,24 +189,6 @@ module Dependabot
           JSON.parse(stdout) || {}
         end
 
-        def remove_requirements(requirement_paths)
-          requirement_paths.each do |path|
-            escaped_path = Shellwords.escape(path)
-            command = "go mod edit -droprequire #{escaped_path}"
-            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
-            handle_subprocess_error(stderr) unless status.success?
-          end
-        end
-
-        def add_requirements(requirements)
-          requirements.each do |r|
-            escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
-            command = "go mod edit -require #{escaped_req}"
-            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
-            handle_subprocess_error(stderr) unless status.success?
-          end
-        end
-
         def in_repo_path(&block)
           SharedHelpers.
             in_a_temporary_repo_directory(directory, repo_contents_path) do
@@ -268,7 +255,7 @@ module Dependabot
         end
 
         def module_pathname
-          @module_pathname ||= Pathname.new(repo_contents_path).join(directory)
+          @module_pathname ||= Pathname.new(repo_contents_path).join(directory.sub(%r{^/}, ""))
         end
 
         def substitute_all(substitutions)
@@ -279,13 +266,22 @@ module Dependabot
           write_go_mod(body)
         end
 
+        # rubocop:disable Metrics/AbcSize
+        # rubocop:disable Metrics/PerceivedComplexity
         def handle_subprocess_error(stderr)
           stderr = stderr.gsub(Dir.getwd, "")
 
+          # Package version doesn't match the module major version
           error_regex = RESOLVABILITY_ERROR_REGEXES.find { |r| stderr =~ r }
           if error_regex
             lines = stderr.lines.drop_while { |l| error_regex !~ l }
-            raise Dependabot::DependencyFileNotResolvable.new, lines.join
+            raise Dependabot::DependencyFileNotResolvable, lines.join
+          end
+
+          repo_error_regex = REPO_RESOLVABILITY_ERROR_REGEXES.find { |r| stderr =~ r }
+          if repo_error_regex
+            lines = stderr.lines.drop_while { |l| repo_error_regex !~ l }
+            ResolvabilityErrors.handle(lines.join, credentials: credentials)
           end
 
           path_regex = MODULE_PATH_MISMATCH_REGEXES.find { |r| stderr =~ r }
@@ -305,6 +301,8 @@ module Dependabot
           msg = stderr.lines.last(10).join.strip
           raise Dependabot::DependabotError, msg
         end
+        # rubocop:enable Metrics/PerceivedComplexity
+        # rubocop:enable Metrics/AbcSize
 
         def go_mod_path
           return "go.mod" if directory == "/"
@@ -312,24 +310,6 @@ module Dependabot
           File.join(directory, "go.mod")
         end
 
-        def requirement_to_dependency_obj(req)
-          # This is an approximation - we're not correctly populating `source`
-          # for instance, but it's only to plug the requirement into the
-          # `update_go_mod` method so this mapping doesn't need to be perfect
-          dep_req = {
-            file: "go.mod",
-            requirement: req["Version"],
-            groups: [],
-            source: nil
-          }
-          Dependency.new(
-            name: req["Path"],
-            version: req["Version"],
-            requirements: req["Indirect"] ? [] : [dep_req],
-            package_manager: "go_modules"
-          )
-        end
-
         def write_go_mod(body)
           File.write("go.mod", body)
         end

data/lib/dependabot/go_modules/resolvability_errors.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module GoModules
+    module ResolvabilityErrors
+      GITHUB_REPO_REGEX = %r{github.com/[^:@]*}.freeze
+
+      def self.handle(message, credentials:)
+        mod_path = message.scan(GITHUB_REPO_REGEX).first
+        raise Dependabot::DependencyFileNotResolvable, message unless mod_path
+
+        # Module not found on github.com - query for _any_ version to know if it
+        # doesn't exist (or is private) or we were just given a bad revision by this manifest
+        SharedHelpers.in_a_temporary_directory do
+          SharedHelpers.with_git_configured(credentials: credentials) do
+            File.write("go.mod", "module dummy\n")
+
+            env = { "GOPRIVATE" => "*" }
+            _, _, status = Open3.capture3(env, SharedHelpers.escape_command("go get #{mod_path}"))
+            raise Dependabot::DependencyFileNotResolvable, message if status.success?
+
+            mod_split = mod_path.split("/")
+            repo_path = if mod_split.size > 3
+                          mod_split[0..2].join("/")
+                        else
+                          mod_path
+                        end
+            raise Dependabot::GitDependenciesNotReachable, [repo_path]
+          end
+        end
+      end
+    end
+  end
+end

data/lib/dependabot/go_modules/update_checker.rb

@@ -5,6 +5,7 @@ require "dependabot/update_checkers/base"
 require "dependabot/shared_helpers"
 require "dependabot/errors"
 require "dependabot/go_modules/native_helpers"
+require "dependabot/go_modules/resolvability_errors"
 require "dependabot/go_modules/version"
 
 module Dependabot
@@ -14,7 +15,8 @@ module Dependabot
         # Package url/proxy doesn't include any redirect meta tags
         /no go-import meta tags/,
         # Package url 404s
-        /404 Not Found/
+        /404 Not Found/,
+        /Repository not found/
       ].freeze
 
       def latest_resolvable_version
@@ -86,7 +88,7 @@ module Dependabot
 
       def handle_subprocess_error(error)
         if RESOLVABILITY_ERROR_REGEXES.any? { |rgx| error.message =~ rgx }
-          raise Dependabot::DependencyFileNotResolvable, error.message
+          ResolvabilityErrors.handle(error.message, credentials: credentials)
         end
 
         raise

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.128.2
+  version: 0.129.4
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-12-14 00:00:00.000000000 Z
+date: 2021-01-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.128.2
+        version: 0.129.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.128.2
+        version: 0.129.4
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -100,28 +100,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.7.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.6.0
+        version: 1.7.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: 0.21.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.20.0
+        version: 0.21.0
 - !ruby/object:Gem::Dependency
   name: simplecov-console
   requirement: !ruby/object:Gem::Requirement
@@ -191,6 +191,7 @@ files:
 - lib/dependabot/go_modules/native_helpers.rb
 - lib/dependabot/go_modules/path_converter.rb
 - lib/dependabot/go_modules/requirement.rb
+- lib/dependabot/go_modules/resolvability_errors.rb
 - lib/dependabot/go_modules/update_checker.rb
 - lib/dependabot/go_modules/version.rb
 homepage: https://github.com/dependabot/dependabot-core