dependabot-go_modules 0.117.5 → 0.117.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fceadfcc210c9dc64acc8d92d504afbd0e21e8dbfea0a7109a7dcae163251ed8
-  data.tar.gz: 04c67017af0b4d2443c8e5dd4800d3b8631eb9cd587afdda7a18f4dfaaefa520
+  metadata.gz: 71e2a6e2a03792cf25a5d76d5933b8511805bd589022c553d97d3264c79ced66
+  data.tar.gz: 2369ac59c46fbb38b3663ce10c6db8f5a6d7ab250688455eac39196907dd3b34
 SHA512:
-  metadata.gz: a5c8da9265bd35b9e3f07af7be78c6187d43cb4da8bea56a8cc0b24bb4f4f9f711f4e0f1c453543314962c5f4901a1b8288fadb5c235e9bef7d6bdfde25fce65
-  data.tar.gz: 22b5cece236b8040309ee3e9399aea64e87c8cdfc69ba6cda6428a3f3b5beb40d33e20f78cf4415a779dcc0ec134cbf6fc60d70a6234346dbd05075cf32bcdd0
+  metadata.gz: 825ea487e046564cdd545df45475000dd86c8cfe2e3aef65feec2f8343b59f9d432dd493db0acb94d951ed0e58d25792d2d373296524c5fb4800053de630e9cf
+  data.tar.gz: d01b64ed69bc6494d09c9f967f65c91e7f4dad14bc830bb018e1cb02fd429c99930584991b4c5d3758262e6249c7d5dc3e1b7461c45304b3519b8686a3916079

data/helpers/updatechecker/main.go

@@ -36,6 +36,12 @@ func GetUpdatedVersion(args *Args) (interface{}, error) {
 		return nil, errors.New("Expected args.dependency to not be nil")
 	}
 
+	currentVersion := args.Dependency.Version
+	currentPrerelease := semver.Prerelease(currentVersion)
+	if pseudoVersionRegexp.MatchString(currentPrerelease) {
+		return currentVersion, nil
+	}
+
 	modload.InitMod()
 
 	repo, err := modfetch.Lookup(args.Dependency.Name)
@@ -53,15 +59,9 @@ func GetUpdatedVersion(args *Args) (interface{}, error) {
 		return nil, err
 	}
 
-	currentVersion := args.Dependency.Version
 	currentMajor := semver.Major(currentVersion)
-	currentPrerelease := semver.Prerelease(currentVersion)
 	latestVersion := args.Dependency.Version
 
-	if pseudoVersionRegexp.MatchString(currentPrerelease) {
-		return latestVersion, nil
-	}
-
 Outer:
 	for _, v := range versions {
 		if semver.Major(v) != currentMajor {

data/lib/dependabot/go_modules/file_parser.rb

@@ -135,16 +135,16 @@ module Dependabot
         end
       end
 
-      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/.freeze
+      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/m.freeze
       def handle_parser_error(path, stderr)
         case stderr
-        when /go: .*: unknown revision/
+        when /go: .*: unknown revision/m
           line = stderr.lines.grep(/unknown revision/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: .*: unrecognized import path/
+        when /go: .*: unrecognized import path/m
           line = stderr.lines.grep(/unrecognized import/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: errors parsing go.mod/
+        when /go: errors parsing go.mod/m
           msg = stderr.gsub(path.to_s, "").strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         when GIT_ERROR_REGEX

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -9,6 +9,22 @@ module Dependabot
   module GoModules
     class FileUpdater
       class GoModUpdater
+        # Turn off the module proxy for now, as it's causing issues with
+        # private git dependencies
+        ENVIRONMENT = { "GOPRIVATE" => "*" }.freeze
+
+        RESOLVABILITY_ERROR_REGEXES = [
+          /go: .*: git fetch .*: exit status 128/.freeze,
+          /verifying .*: checksum mismatch/.freeze,
+          /build .*: cannot find module providing package/.freeze
+        ].freeze
+
+        MODULE_PATH_MISMATCH_REGEXES = [
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
+        ].freeze
+
         def initialize(dependencies:, go_mod:, go_sum:, credentials:)
           @dependencies = dependencies
           @go_mod = go_mod
@@ -17,96 +33,171 @@ module Dependabot
         end
 
         def updated_go_mod_content
-          # Turn off the module proxy for now, as it's causing issues with
-          # private git dependencies
-          env = { "GOPRIVATE" => "*" }
-
-          @updated_go_mod_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                File.write("go.mod", go_mod.content)
-
-                deps = dependencies.map do |dep|
-                  {
-                    name: dep.name,
-                    version: "v" + dep.version.sub(/^v/i, ""),
-                    indirect: dep.requirements.empty?
-                  }
-                end
-
-                SharedHelpers.run_helper_subprocess(
-                  command: NativeHelpers.helper_path,
-                  env: env,
-                  function: "updateDependencyFile",
-                  args: { dependencies: deps }
-                )
-              end
-            end
+          updated_files[:go_mod]
         end
 
         def updated_go_sum_content
-          return nil unless go_sum
-
-          # This needs to be run separately so we don't nest subprocess calls
-          prepared_go_mod_content
-
-          @updated_go_sum_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                # Create a fake empty module for each local module so that
-                # `go get -d` works, even if some modules have been `replace`d
-                # with a local module that we don't have access to.
-                local_replacements.each do |_, stub_path|
-                  Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                  FileUtils.touch(File.join(stub_path, "go.mod"))
-                end
-
-                File.write("go.mod", prepared_go_mod_content)
-                File.write("go.sum", go_sum.content)
-                File.write("main.go", dummy_main_go)
-
-                # Turn off the module proxy for now, as it's causing issues
-                # with private git dependencies
-                env = { "GOPRIVATE" => "*" }
-
-                _, stderr, status = Open3.capture3(env, "go get -d")
-                unless status.success?
-                  handle_subprocess_error(go_sum.path, stderr)
-                end
-
-                File.read("go.sum")
-              end
-            end
+          updated_files[:go_sum]
         end
 
         private
 
-        RESOLVABILITY_ERROR_REGEXES = [
-          /go: .*: git fetch .*: exit status 128/.freeze,
-          /verifying .*: checksum mismatch/.freeze,
-          /build .*: cannot find module providing package/.freeze
-        ].freeze
-        MODULE_PATH_MISMATCH_REGEXES = [
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
-        ].freeze
+        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+
+        def updated_files
+          @updated_files ||= update_files
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        def update_files
+          # Map paths in local replace directives to path hashes
+          substitutions = replace_directive_substitutions(go_mod.content)
+          stub_dirs = substitutions.values
+
+          # Replace full paths with path hashes in the go.mod
+          clean_go_mod = substitute_all(go_mod.content, substitutions)
+
+          # Set the new dependency versions in the go.mod
+          updated_go_mod = in_temp_dir(stub_dirs) do
+            update_go_mod(clean_go_mod, dependencies)
+          end
+
+          # Then run `go get` to pick up other changes to the file caused by
+          # the upgrade
+          regenerated_files = in_temp_dir(stub_dirs) do
+            run_go_get(updated_go_mod, go_sum)
+          end
+
+          # At this point, the go.mod returned from run_go_get contains the
+          # correct set of modules, but running `go get` can change the file in
+          # undesirable ways (such as injecting the current Go version), so we
+          # need to update the original go.mod with the updated set of
+          # requirements rather than using the regenerated file directly
+          original_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(go_mod.content)
+          end
+          updated_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(regenerated_files[:go_mod])
+          end
+
+          original_paths = original_reqs.map { |r| r["Path"] }
+          updated_paths = updated_reqs.map { |r| r["Path"] }
+          req_paths_to_remove = original_paths - updated_paths
 
-        def local_replacements
-          @local_replacements ||=
+          output_go_mod = in_temp_dir(stub_dirs) do
+            remove_requirements(go_mod.content, req_paths_to_remove)
+          end
+
+          output_go_mod = in_temp_dir(stub_dirs) do
+            deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
+            update_go_mod(output_go_mod, deps)
+          end
+
+          { go_mod: output_go_mod, go_sum: regenerated_files[:go_sum] }
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def update_go_mod(go_mod_content, dependencies)
+          File.write("go.mod", go_mod_content)
+
+          deps = dependencies.map do |dep|
+            {
+              name: dep.name,
+              version: "v" + dep.version.sub(/^v/i, ""),
+              indirect: dep.requirements.empty?
+            }
+          end
+
+          SharedHelpers.run_helper_subprocess(
+            command: NativeHelpers.helper_path,
+            env: ENVIRONMENT,
+            function: "updateDependencyFile",
+            args: { dependencies: deps }
+          )
+        end
+
+        def run_go_get(go_mod_content, go_sum)
+          File.write("go.mod", go_mod_content)
+          File.write("go.sum", go_sum.content) if go_sum
+          File.write("main.go", dummy_main_go)
+
+          _, stderr, status = Open3.capture3(ENVIRONMENT, "go get -d")
+          handle_subprocess_error(stderr) unless status.success?
+
+          updated_go_sum = go_sum ? File.read("go.sum") : nil
+          { go_mod: File.read("go.mod"), go_sum: updated_go_sum }
+        end
+
+        def parse_manifest_requirements(go_mod_content)
+          File.write("go.mod", go_mod_content)
+
+          command = "go mod edit -json"
+          stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+
+          JSON.parse(stdout)["Require"] || []
+        end
+
+        def remove_requirements(go_mod_content, requirement_paths)
+          File.write("go.mod", go_mod_content)
+
+          requirement_paths.each do |path|
+            escaped_path = Shellwords.escape(path)
+            command = "go mod edit -droprequire #{escaped_path}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def add_requirements(go_mod_content, requirements)
+          File.write("go.mod", go_mod_content)
+
+          requirements.each do |r|
+            escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
+            command = "go mod edit -require #{escaped_req}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def in_temp_dir(stub_paths, &block)
+          SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              # Create a fake empty module for each local module so that
+              # `go get -d` works, even if some modules have been `replace`d
+              # with a local module that we don't have access to.
+              stub_paths.each do |stub_path|
+                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+                FileUtils.touch(File.join(stub_path, "go.mod"))
+              end
+
+              block.call
+            end
+          end
+        end
+
+        # Given a go.mod file, find all `replace` directives pointing to a path
+        # on the local filesystem, and return an array of pairs mapping the
+        # original path to a hash of the path.
+        #
+        # This lets us substitute all parts of the go.mod that are dependent on
+        # the layout of the filesystem with a structure we can reproduce (i.e.
+        # no paths such as ../../../foo), run the Go tooling, then reverse the
+        # process afterwards.
+        def replace_directive_substitutions(go_mod_content)
+          @replace_directive_substitutions ||=
             SharedHelpers.in_a_temporary_directory do |path|
-              File.write("go.mod", go_mod.content)
+              File.write("go.mod", go_mod_content)
 
               # Parse the go.mod to get a JSON representation of the replace
               # directives
               command = "go mod edit -json"
-
-              # Turn off the module proxy for now, as it's causing issues with
-              # private git dependencies
-              env = { "GOPRIVATE" => "*" }
-
-              stdout, stderr, status = Open3.capture3(env, command)
-              handle_parser_error(path, stderr) unless status.success?
+              stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+              handle_subprocess_error(path, stderr) unless status.success?
 
               # Find all the local replacements, and return them with a stub
               # path we can use in their place. Using generated paths is safer
@@ -116,18 +207,20 @@ module Dependabot
                 map { |r| r["New"]["Path"] }.
                 compact.
                 select { |p| p.start_with?(".") || p.start_with?("/") }.
-                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }
+                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }.
+                to_h
             end
         end
 
-        def prepared_go_mod_content
-          content = updated_go_mod_content
-          local_replacements.reduce(content) do |body, (path, stub_path)|
-            body.sub(path, stub_path)
+        def substitute_all(file, substitutions)
+          substitutions.reduce(file) do |text, (a, b)|
+            text.sub(a, b)
           end
         end
 
-        def handle_subprocess_error(path, stderr)
+        def handle_subprocess_error(stderr)
+          stderr = stderr.gsub(Dir.getwd, "")
+
           error_regex = RESOLVABILITY_ERROR_REGEXES.find { |r| stderr =~ r }
           if error_regex
             lines = stderr.lines.drop_while { |l| error_regex !~ l }
@@ -141,7 +234,7 @@ module Dependabot
               new(go_mod.path, match[1], match[2])
           end
 
-          msg = stderr.gsub(path.to_s, "").lines.last(10).join.strip
+          msg = stderr.lines.last(10).join.strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         end
 
@@ -156,13 +249,29 @@ module Dependabot
           # good to switch back to `main` so we can surface more errors.
           lines = ["package dummypkg", "import ("]
           dependencies.each do |dep|
-            lines << "_ \"#{dep.name}\""
+            lines << "_ \"#{dep.name}\"" unless dep.requirements.empty?
           end
           lines << ")"
           lines.join("\n")
         end
 
-        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+        def requirement_to_dependency_obj(req)
+          # This is an approximation - we're not correctly populating `source`
+          # for instance, but it's only to plug the requirement into the
+          # `update_go_mod` method so this mapping doesn't need to be perfect
+          dep_req = {
+            file: "go.mod",
+            requirement: req["Version"],
+            groups: [],
+            source: nil
+          }
+          Dependency.new(
+            name: req["Path"],
+            version: req["Version"],
+            requirements: req["Indirect"] ? [] : [dep_req],
+            package_manager: "go_modules"
+          )
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.117.5
+  version: 0.117.10
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-31 00:00:00.000000000 Z
+date: 2020-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.5
+        version: 0.117.10
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.5
+        version: 0.117.10
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.1
+        version: 0.83.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.1
+        version: 0.83.0
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement