dependabot-go_modules 0.117.6 → 0.117.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f7372dd4c4672d221065b7fa54cc53cb5f0430f9987942d06defb0dbe429140a
-  data.tar.gz: cae04c71ebcf2588dc2c5c919a8c1a4561df683c25a3d52a4c007b34ea1e80dc
+  metadata.gz: dd8c69512adb380e04e618dd8aee5babf1cb2c9ecce5db883d75cbe4f6e5da8d
+  data.tar.gz: d8f0ac8addff60780b4a2c0f8c0511a22927b707a56ed36e57621db867065a7a
 SHA512:
-  metadata.gz: 3b121dd8bb6e9863ac9270535540c91f2c52b4469f39acb2c8f3d5a8d243eac41598356c86b628a0ffdc8db6fbcfa6897714609c3860cfc3ab598bea5cf2c55f
-  data.tar.gz: '0881fa05874b7f704207d12ba344fe4f5c032b339795d28133aa0482427b6ad3e64650ea563308bcbb7db3ad3bb258a110a1a2fee2a35bfa237fce6c4687c322'
+  metadata.gz: 451e6577853ff04a4533020294c3dbd9bca7a1ad2bcc61900d68af7b76fcc4af3b3ac5930c5b01b7d9da6dce9227b1f7ef5a7c80c814bd06c3366f9eebca16f3
+  data.tar.gz: 851ef0e19e28d029fdcda001b25b51b4255e34f4ebe8ccc480bd9285ab0f54d618fbc411ffdc9d144858425119a1e4ca508e294f41cc3a4f7ba2ff4f53f2cec7

data/lib/dependabot/go_modules/file_parser.rb

@@ -135,16 +135,16 @@ module Dependabot
         end
       end
 
-      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/.freeze
+      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/m.freeze
       def handle_parser_error(path, stderr)
         case stderr
-        when /go: .*: unknown revision/
+        when /go: .*: unknown revision/m
           line = stderr.lines.grep(/unknown revision/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: .*: unrecognized import path/
+        when /go: .*: unrecognized import path/m
           line = stderr.lines.grep(/unrecognized import/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: errors parsing go.mod/
+        when /go: errors parsing go.mod/m
           msg = stderr.gsub(path.to_s, "").strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         when GIT_ERROR_REGEX

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -9,6 +9,22 @@ module Dependabot
   module GoModules
     class FileUpdater
       class GoModUpdater
+        # Turn off the module proxy for now, as it's causing issues with
+        # private git dependencies
+        ENVIRONMENT = { "GOPRIVATE" => "*" }.freeze
+
+        RESOLVABILITY_ERROR_REGEXES = [
+          /go: .*: git fetch .*: exit status 128/.freeze,
+          /verifying .*: checksum mismatch/.freeze,
+          /build .*: cannot find module providing package/.freeze
+        ].freeze
+
+        MODULE_PATH_MISMATCH_REGEXES = [
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
+        ].freeze
+
         def initialize(dependencies:, go_mod:, go_sum:, credentials:)
           @dependencies = dependencies
           @go_mod = go_mod
@@ -17,96 +33,171 @@ module Dependabot
         end
 
         def updated_go_mod_content
-          # Turn off the module proxy for now, as it's causing issues with
-          # private git dependencies
-          env = { "GOPRIVATE" => "*" }
-
-          @updated_go_mod_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                File.write("go.mod", go_mod.content)
-
-                deps = dependencies.map do |dep|
-                  {
-                    name: dep.name,
-                    version: "v" + dep.version.sub(/^v/i, ""),
-                    indirect: dep.requirements.empty?
-                  }
-                end
-
-                SharedHelpers.run_helper_subprocess(
-                  command: NativeHelpers.helper_path,
-                  env: env,
-                  function: "updateDependencyFile",
-                  args: { dependencies: deps }
-                )
-              end
-            end
+          updated_files[:go_mod]
         end
 
         def updated_go_sum_content
-          return nil unless go_sum
-
-          # This needs to be run separately so we don't nest subprocess calls
-          prepared_go_mod_content
-
-          @updated_go_sum_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                # Create a fake empty module for each local module so that
-                # `go get -d` works, even if some modules have been `replace`d
-                # with a local module that we don't have access to.
-                local_replacements.each do |_, stub_path|
-                  Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                  FileUtils.touch(File.join(stub_path, "go.mod"))
-                end
-
-                File.write("go.mod", prepared_go_mod_content)
-                File.write("go.sum", go_sum.content)
-                File.write("main.go", dummy_main_go)
-
-                # Turn off the module proxy for now, as it's causing issues
-                # with private git dependencies
-                env = { "GOPRIVATE" => "*" }
-
-                _, stderr, status = Open3.capture3(env, "go get -d")
-                unless status.success?
-                  handle_subprocess_error(go_sum.path, stderr)
-                end
-
-                File.read("go.sum")
-              end
-            end
+          updated_files[:go_sum]
         end
 
         private
 
-        RESOLVABILITY_ERROR_REGEXES = [
-          /go: .*: git fetch .*: exit status 128/.freeze,
-          /verifying .*: checksum mismatch/.freeze,
-          /build .*: cannot find module providing package/.freeze
-        ].freeze
-        MODULE_PATH_MISMATCH_REGEXES = [
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
-        ].freeze
+        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+
+        def updated_files
+          @updated_files ||= update_files
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        def update_files
+          # Map paths in local replace directives to path hashes
+          substitutions = replace_directive_substitutions(go_mod.content)
+          stub_dirs = substitutions.values
+
+          # Replace full paths with path hashes in the go.mod
+          clean_go_mod = substitute_all(go_mod.content, substitutions)
+
+          # Set the new dependency versions in the go.mod
+          updated_go_mod = in_temp_dir(stub_dirs) do
+            update_go_mod(clean_go_mod, dependencies)
+          end
+
+          # Then run `go get` to pick up other changes to the file caused by
+          # the upgrade
+          regenerated_files = in_temp_dir(stub_dirs) do
+            run_go_get(updated_go_mod, go_sum)
+          end
+
+          # At this point, the go.mod returned from run_go_get contains the
+          # correct set of modules, but running `go get` can change the file in
+          # undesirable ways (such as injecting the current Go version), so we
+          # need to update the original go.mod with the updated set of
+          # requirements rather than using the regenerated file directly
+          original_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(go_mod.content)
+          end
+          updated_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(regenerated_files[:go_mod])
+          end
+
+          original_paths = original_reqs.map { |r| r["Path"] }
+          updated_paths = updated_reqs.map { |r| r["Path"] }
+          req_paths_to_remove = original_paths - updated_paths
 
-        def local_replacements
-          @local_replacements ||=
+          output_go_mod = in_temp_dir(stub_dirs) do
+            remove_requirements(go_mod.content, req_paths_to_remove)
+          end
+
+          output_go_mod = in_temp_dir(stub_dirs) do
+            deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
+            update_go_mod(output_go_mod, deps)
+          end
+
+          { go_mod: output_go_mod, go_sum: regenerated_files[:go_sum] }
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def update_go_mod(go_mod_content, dependencies)
+          File.write("go.mod", go_mod_content)
+
+          deps = dependencies.map do |dep|
+            {
+              name: dep.name,
+              version: "v" + dep.version.sub(/^v/i, ""),
+              indirect: dep.requirements.empty?
+            }
+          end
+
+          SharedHelpers.run_helper_subprocess(
+            command: NativeHelpers.helper_path,
+            env: ENVIRONMENT,
+            function: "updateDependencyFile",
+            args: { dependencies: deps }
+          )
+        end
+
+        def run_go_get(go_mod_content, go_sum)
+          File.write("go.mod", go_mod_content)
+          File.write("go.sum", go_sum.content) if go_sum
+          File.write("main.go", dummy_main_go)
+
+          _, stderr, status = Open3.capture3(ENVIRONMENT, "go get -d")
+          handle_subprocess_error(stderr) unless status.success?
+
+          updated_go_sum = go_sum ? File.read("go.sum") : nil
+          { go_mod: File.read("go.mod"), go_sum: updated_go_sum }
+        end
+
+        def parse_manifest_requirements(go_mod_content)
+          File.write("go.mod", go_mod_content)
+
+          command = "go mod edit -json"
+          stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+
+          JSON.parse(stdout)["Require"] || []
+        end
+
+        def remove_requirements(go_mod_content, requirement_paths)
+          File.write("go.mod", go_mod_content)
+
+          requirement_paths.each do |path|
+            escaped_path = Shellwords.escape(path)
+            command = "go mod edit -droprequire #{escaped_path}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def add_requirements(go_mod_content, requirements)
+          File.write("go.mod", go_mod_content)
+
+          requirements.each do |r|
+            escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
+            command = "go mod edit -require #{escaped_req}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def in_temp_dir(stub_paths, &block)
+          SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              # Create a fake empty module for each local module so that
+              # `go get -d` works, even if some modules have been `replace`d
+              # with a local module that we don't have access to.
+              stub_paths.each do |stub_path|
+                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+                FileUtils.touch(File.join(stub_path, "go.mod"))
+              end
+
+              block.call
+            end
+          end
+        end
+
+        # Given a go.mod file, find all `replace` directives pointing to a path
+        # on the local filesystem, and return an array of pairs mapping the
+        # original path to a hash of the path.
+        #
+        # This lets us substitute all parts of the go.mod that are dependent on
+        # the layout of the filesystem with a structure we can reproduce (i.e.
+        # no paths such as ../../../foo), run the Go tooling, then reverse the
+        # process afterwards.
+        def replace_directive_substitutions(go_mod_content)
+          @replace_directive_substitutions ||=
             SharedHelpers.in_a_temporary_directory do |path|
-              File.write("go.mod", go_mod.content)
+              File.write("go.mod", go_mod_content)
 
               # Parse the go.mod to get a JSON representation of the replace
               # directives
               command = "go mod edit -json"
-
-              # Turn off the module proxy for now, as it's causing issues with
-              # private git dependencies
-              env = { "GOPRIVATE" => "*" }
-
-              stdout, stderr, status = Open3.capture3(env, command)
-              handle_parser_error(path, stderr) unless status.success?
+              stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+              handle_subprocess_error(path, stderr) unless status.success?
 
               # Find all the local replacements, and return them with a stub
               # path we can use in their place. Using generated paths is safer
@@ -116,18 +207,20 @@ module Dependabot
                 map { |r| r["New"]["Path"] }.
                 compact.
                 select { |p| p.start_with?(".") || p.start_with?("/") }.
-                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }
+                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }.
+                to_h
             end
         end
 
-        def prepared_go_mod_content
-          content = updated_go_mod_content
-          local_replacements.reduce(content) do |body, (path, stub_path)|
-            body.sub(path, stub_path)
+        def substitute_all(file, substitutions)
+          substitutions.reduce(file) do |text, (a, b)|
+            text.sub(a, b)
           end
         end
 
-        def handle_subprocess_error(path, stderr)
+        def handle_subprocess_error(stderr)
+          stderr = stderr.gsub(Dir.getwd, "")
+
           error_regex = RESOLVABILITY_ERROR_REGEXES.find { |r| stderr =~ r }
           if error_regex
             lines = stderr.lines.drop_while { |l| error_regex !~ l }
@@ -141,7 +234,7 @@ module Dependabot
               new(go_mod.path, match[1], match[2])
           end
 
-          msg = stderr.gsub(path.to_s, "").lines.last(10).join.strip
+          msg = stderr.lines.last(10).join.strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         end
 
@@ -156,13 +249,29 @@ module Dependabot
           # good to switch back to `main` so we can surface more errors.
           lines = ["package dummypkg", "import ("]
           dependencies.each do |dep|
-            lines << "_ \"#{dep.name}\""
+            lines << "_ \"#{dep.name}\"" unless dep.requirements.empty?
           end
           lines << ")"
           lines.join("\n")
         end
 
-        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+        def requirement_to_dependency_obj(req)
+          # This is an approximation - we're not correctly populating `source`
+          # for instance, but it's only to plug the requirement into the
+          # `update_go_mod` method so this mapping doesn't need to be perfect
+          dep_req = {
+            file: "go.mod",
+            requirement: req["Version"],
+            groups: [],
+            source: nil
+          }
+          Dependency.new(
+            name: req["Path"],
+            version: req["Version"],
+            requirements: req["Indirect"] ? [] : [dep_req],
+            package_manager: "go_modules"
+          )
+        end
       end
     end
   end

data/lib/dependabot/go_modules/update_checker.rb

@@ -125,7 +125,8 @@ module Dependabot
           GitCommitChecker.new(
             dependency: dependency,
             credentials: credentials,
-            ignored_versions: ignored_versions
+            ignored_versions: ignored_versions,
+            raise_on_ignored: raise_on_ignored
           )
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.117.6
+  version: 0.117.11
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-09 00:00:00.000000000 Z
+date: 2020-05-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.6
+        version: 0.117.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.6
+        version: 0.117.11
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.1
+        version: 0.83.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.80.1
+        version: 0.83.0
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement