dependabot-go_modules 0.117.7 → 0.118.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 39ce7bcb4b1a8d66343e6a186c0deab95ce6f7c0a3e7f577e55dfcb127d73e99
-  data.tar.gz: 4351710af1e7c33b4e597464d43725a976064a413c7d7f7acf5a55d0ec74af39
+  metadata.gz: 7799098c9d1777821ce42059f6a47702bb887af1179ab82c9266d65ad2ce73fc
+  data.tar.gz: 06c2fdfb2b91cd8e1d5ac93d027c6d25a6f744b17b333cd9fca392daa6176673
 SHA512:
-  metadata.gz: 10761c29d848916a0447d5ead96ee3050c671667193fc05b6f142f221034fa368b19c6c28ccb2c57efb7d889709bf1532d1b7602cec6725746c07f416b469e32
-  data.tar.gz: 0b201a3a611e0730a1b01db6b6a21af6f3fb10fd3de0d21586fb03f07ef0150194e3f6a9e063562acd0ebdd9266b20f8658cd753908f0076fc8b5b1b8ef87e8b
+  metadata.gz: aee189a098fc42fcdb31a2449f941c318e1a7a14bc1475eeec0dc6d0da4e97c05b764e9ff3ae6a3d1ab5e7084f75db7139243bacbe54386de5970067283a7ceb
+  data.tar.gz: a73d5d3884e68bd67ec2138c8c6a35009355da02d8a7a73c1993a0ef5a148ed6baf83711f32d2a3bc5405a19a5672b96b73b70d844bb97709b3036d79578581b

data/lib/dependabot/go_modules/file_parser.rb

@@ -135,16 +135,16 @@ module Dependabot
         end
       end
 
-      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/.freeze
+      GIT_ERROR_REGEX = /go: .*: git fetch .*: exit status 128/m.freeze
       def handle_parser_error(path, stderr)
         case stderr
-        when /go: .*: unknown revision/
+        when /go: .*: unknown revision/m
           line = stderr.lines.grep(/unknown revision/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: .*: unrecognized import path/
+        when /go: .*: unrecognized import path/m
           line = stderr.lines.grep(/unrecognized import/).first
           raise Dependabot::DependencyFileNotResolvable, line.strip
-        when /go: errors parsing go.mod/
+        when /go: errors parsing go.mod/m
           msg = stderr.gsub(path.to_s, "").strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         when GIT_ERROR_REGEX

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -9,6 +9,22 @@ module Dependabot
   module GoModules
     class FileUpdater
       class GoModUpdater
+        # Turn off the module proxy for now, as it's causing issues with
+        # private git dependencies
+        ENVIRONMENT = { "GOPRIVATE" => "*" }.freeze
+
+        RESOLVABILITY_ERROR_REGEXES = [
+          /go: .*: git fetch .*: exit status 128/.freeze,
+          /verifying .*: checksum mismatch/.freeze,
+          /build .*: cannot find module providing package/.freeze
+        ].freeze
+
+        MODULE_PATH_MISMATCH_REGEXES = [
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
+          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
+        ].freeze
+
         def initialize(dependencies:, go_mod:, go_sum:, credentials:)
           @dependencies = dependencies
           @go_mod = go_mod
@@ -17,96 +33,171 @@ module Dependabot
         end
 
         def updated_go_mod_content
-          # Turn off the module proxy for now, as it's causing issues with
-          # private git dependencies
-          env = { "GOPRIVATE" => "*" }
-
-          @updated_go_mod_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                File.write("go.mod", go_mod.content)
-
-                deps = dependencies.map do |dep|
-                  {
-                    name: dep.name,
-                    version: "v" + dep.version.sub(/^v/i, ""),
-                    indirect: dep.requirements.empty?
-                  }
-                end
-
-                SharedHelpers.run_helper_subprocess(
-                  command: NativeHelpers.helper_path,
-                  env: env,
-                  function: "updateDependencyFile",
-                  args: { dependencies: deps }
-                )
-              end
-            end
+          updated_files[:go_mod]
         end
 
         def updated_go_sum_content
-          return nil unless go_sum
-
-          # This needs to be run separately so we don't nest subprocess calls
-          prepared_go_mod_content
-
-          @updated_go_sum_content ||=
-            SharedHelpers.in_a_temporary_directory do
-              SharedHelpers.with_git_configured(credentials: credentials) do
-                # Create a fake empty module for each local module so that
-                # `go get -d` works, even if some modules have been `replace`d
-                # with a local module that we don't have access to.
-                local_replacements.each do |_, stub_path|
-                  Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                  FileUtils.touch(File.join(stub_path, "go.mod"))
-                end
-
-                File.write("go.mod", prepared_go_mod_content)
-                File.write("go.sum", go_sum.content)
-                File.write("main.go", dummy_main_go)
-
-                # Turn off the module proxy for now, as it's causing issues
-                # with private git dependencies
-                env = { "GOPRIVATE" => "*" }
-
-                _, stderr, status = Open3.capture3(env, "go get -d")
-                unless status.success?
-                  handle_subprocess_error(go_sum.path, stderr)
-                end
-
-                File.read("go.sum")
-              end
-            end
+          updated_files[:go_sum]
         end
 
         private
 
-        RESOLVABILITY_ERROR_REGEXES = [
-          /go: .*: git fetch .*: exit status 128/.freeze,
-          /verifying .*: checksum mismatch/.freeze,
-          /build .*: cannot find module providing package/.freeze
-        ].freeze
-        MODULE_PATH_MISMATCH_REGEXES = [
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* has non-.* module path "(.*)" at/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* unexpected module path "(.*)"/,
-          /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
-        ].freeze
+        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+
+        def updated_files
+          @updated_files ||= update_files
+        end
+
+        # rubocop:disable Metrics/AbcSize
+        def update_files
+          # Map paths in local replace directives to path hashes
+          substitutions = replace_directive_substitutions(go_mod.content)
+          stub_dirs = substitutions.values
+
+          # Replace full paths with path hashes in the go.mod
+          clean_go_mod = substitute_all(go_mod.content, substitutions)
+
+          # Set the new dependency versions in the go.mod
+          updated_go_mod = in_temp_dir(stub_dirs) do
+            update_go_mod(clean_go_mod, dependencies)
+          end
+
+          # Then run `go get` to pick up other changes to the file caused by
+          # the upgrade
+          regenerated_files = in_temp_dir(stub_dirs) do
+            run_go_get(updated_go_mod, go_sum)
+          end
+
+          # At this point, the go.mod returned from run_go_get contains the
+          # correct set of modules, but running `go get` can change the file in
+          # undesirable ways (such as injecting the current Go version), so we
+          # need to update the original go.mod with the updated set of
+          # requirements rather than using the regenerated file directly
+          original_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(go_mod.content)
+          end
+          updated_reqs = in_temp_dir(stub_dirs) do
+            parse_manifest_requirements(regenerated_files[:go_mod])
+          end
+
+          original_paths = original_reqs.map { |r| r["Path"] }
+          updated_paths = updated_reqs.map { |r| r["Path"] }
+          req_paths_to_remove = original_paths - updated_paths
 
-        def local_replacements
-          @local_replacements ||=
+          output_go_mod = in_temp_dir(stub_dirs) do
+            remove_requirements(go_mod.content, req_paths_to_remove)
+          end
+
+          output_go_mod = in_temp_dir(stub_dirs) do
+            deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
+            update_go_mod(output_go_mod, deps)
+          end
+
+          { go_mod: output_go_mod, go_sum: regenerated_files[:go_sum] }
+        end
+        # rubocop:enable Metrics/AbcSize
+
+        def update_go_mod(go_mod_content, dependencies)
+          File.write("go.mod", go_mod_content)
+
+          deps = dependencies.map do |dep|
+            {
+              name: dep.name,
+              version: "v" + dep.version.sub(/^v/i, ""),
+              indirect: dep.requirements.empty?
+            }
+          end
+
+          SharedHelpers.run_helper_subprocess(
+            command: NativeHelpers.helper_path,
+            env: ENVIRONMENT,
+            function: "updateDependencyFile",
+            args: { dependencies: deps }
+          )
+        end
+
+        def run_go_get(go_mod_content, go_sum)
+          File.write("go.mod", go_mod_content)
+          File.write("go.sum", go_sum.content) if go_sum
+          File.write("main.go", dummy_main_go)
+
+          _, stderr, status = Open3.capture3(ENVIRONMENT, "go get -d")
+          handle_subprocess_error(stderr) unless status.success?
+
+          updated_go_sum = go_sum ? File.read("go.sum") : nil
+          { go_mod: File.read("go.mod"), go_sum: updated_go_sum }
+        end
+
+        def parse_manifest_requirements(go_mod_content)
+          File.write("go.mod", go_mod_content)
+
+          command = "go mod edit -json"
+          stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+
+          JSON.parse(stdout)["Require"] || []
+        end
+
+        def remove_requirements(go_mod_content, requirement_paths)
+          File.write("go.mod", go_mod_content)
+
+          requirement_paths.each do |path|
+            escaped_path = Shellwords.escape(path)
+            command = "go mod edit -droprequire #{escaped_path}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def add_requirements(go_mod_content, requirements)
+          File.write("go.mod", go_mod_content)
+
+          requirements.each do |r|
+            escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
+            command = "go mod edit -require #{escaped_req}"
+            _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+            handle_subprocess_error(stderr) unless status.success?
+          end
+
+          File.read("go.mod")
+        end
+
+        def in_temp_dir(stub_paths, &block)
+          SharedHelpers.in_a_temporary_directory do
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              # Create a fake empty module for each local module so that
+              # `go get -d` works, even if some modules have been `replace`d
+              # with a local module that we don't have access to.
+              stub_paths.each do |stub_path|
+                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+                FileUtils.touch(File.join(stub_path, "go.mod"))
+              end
+
+              block.call
+            end
+          end
+        end
+
+        # Given a go.mod file, find all `replace` directives pointing to a path
+        # on the local filesystem, and return an array of pairs mapping the
+        # original path to a hash of the path.
+        #
+        # This lets us substitute all parts of the go.mod that are dependent on
+        # the layout of the filesystem with a structure we can reproduce (i.e.
+        # no paths such as ../../../foo), run the Go tooling, then reverse the
+        # process afterwards.
+        def replace_directive_substitutions(go_mod_content)
+          @replace_directive_substitutions ||=
             SharedHelpers.in_a_temporary_directory do |path|
-              File.write("go.mod", go_mod.content)
+              File.write("go.mod", go_mod_content)
 
               # Parse the go.mod to get a JSON representation of the replace
               # directives
               command = "go mod edit -json"
-
-              # Turn off the module proxy for now, as it's causing issues with
-              # private git dependencies
-              env = { "GOPRIVATE" => "*" }
-
-              stdout, stderr, status = Open3.capture3(env, command)
-              handle_parser_error(path, stderr) unless status.success?
+              stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
+              handle_subprocess_error(path, stderr) unless status.success?
 
               # Find all the local replacements, and return them with a stub
               # path we can use in their place. Using generated paths is safer
@@ -116,18 +207,20 @@ module Dependabot
                 map { |r| r["New"]["Path"] }.
                 compact.
                 select { |p| p.start_with?(".") || p.start_with?("/") }.
-                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }
+                map { |p| [p, "./" + Digest::SHA2.hexdigest(p)] }.
+                to_h
             end
         end
 
-        def prepared_go_mod_content
-          content = updated_go_mod_content
-          local_replacements.reduce(content) do |body, (path, stub_path)|
-            body.sub(path, stub_path)
+        def substitute_all(file, substitutions)
+          substitutions.reduce(file) do |text, (a, b)|
+            text.sub(a, b)
           end
         end
 
-        def handle_subprocess_error(path, stderr)
+        def handle_subprocess_error(stderr)
+          stderr = stderr.gsub(Dir.getwd, "")
+
           error_regex = RESOLVABILITY_ERROR_REGEXES.find { |r| stderr =~ r }
           if error_regex
             lines = stderr.lines.drop_while { |l| error_regex !~ l }
@@ -141,7 +234,7 @@ module Dependabot
               new(go_mod.path, match[1], match[2])
           end
 
-          msg = stderr.gsub(path.to_s, "").lines.last(10).join.strip
+          msg = stderr.lines.last(10).join.strip
           raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
         end
 
@@ -156,13 +249,29 @@ module Dependabot
           # good to switch back to `main` so we can surface more errors.
           lines = ["package dummypkg", "import ("]
           dependencies.each do |dep|
-            lines << "_ \"#{dep.name}\""
+            lines << "_ \"#{dep.name}\"" unless dep.requirements.empty?
           end
           lines << ")"
           lines.join("\n")
         end
 
-        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+        def requirement_to_dependency_obj(req)
+          # This is an approximation - we're not correctly populating `source`
+          # for instance, but it's only to plug the requirement into the
+          # `update_go_mod` method so this mapping doesn't need to be perfect
+          dep_req = {
+            file: "go.mod",
+            requirement: req["Version"],
+            groups: [],
+            source: nil
+          }
+          Dependency.new(
+            name: req["Path"],
+            version: req["Version"],
+            requirements: req["Indirect"] ? [] : [dep_req],
+            package_manager: "go_modules"
+          )
+        end
       end
     end
   end

data/lib/dependabot/go_modules/update_checker.rb

@@ -125,7 +125,8 @@ module Dependabot
           GitCommitChecker.new(
             dependency: dependency,
             credentials: credentials,
-            ignored_versions: ignored_versions
+            ignored_versions: ignored_versions,
+            raise_on_ignored: raise_on_ignored
           )
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.117.7
+  version: 0.118.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-04-20 00:00:00.000000000 Z
+date: 2020-05-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.7
+        version: 0.118.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.117.7
+        version: 0.118.0
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -114,14 +114,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.82.0
+        version: 0.83.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.82.0
+        version: 0.83.0
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement