dependabot-go_modules 0.120.4 → 0.122.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fd5bd86e2db4a63bf585f2811548d8bedcff1a1b9c4fcf7ef46c12e9e174894a
-  data.tar.gz: 734b450561f99006c351af51ca91a07a5cef07312faa3cf5adf44b6d0fd1daa4
+  metadata.gz: da8437c4b2e42e473b12e787510f09a045cf46d79c6efd807d276cea8048ea52
+  data.tar.gz: 21b14468c658ed748c3adfa41e90fa1f86dbf9dc90510dfc963b70f9d49f8839
 SHA512:
-  metadata.gz: de4e7c309a4c299a9ab32e1372d1d06949ef3cdf15dc405e03e4eff0731af8f5c1dae4a377c8321d10eba0c2c64d332ba3e382fa08bbe586396903c58d9ee373
-  data.tar.gz: 54461612c6078e1ba42257d0b7519d76da0ac53339d68f6792eedc35480803c03a6f95576b77f352dab5668744af4f80d2df57668e30638d5c9dd0c18c57f8a5
+  metadata.gz: 38d199a506af5f10c71d48e381f33dcb480f80baf39905ac507bea51d57dae65ec7c200fe77a3e5b0ff7a0fa00b487372d45705b97a293064b0eaf9f6c2ff179
+  data.tar.gz: 132db9b235fbe89ec4f39c759e95d6b0d60ad539762aada40794245020cbe23c68bee171c6115fd33766688e30e9ad4a92e719f6ffbe53de53d1f53f1b145a8b

data/lib/dependabot/go_modules.rb

@@ -17,3 +17,6 @@ Dependabot::PullRequestCreator::Labeler.
 require "dependabot/dependency"
 Dependabot::Dependency.
   register_production_check("go_modules", ->(_) { true })
+
+require "dependabot/utils"
+Dependabot::Utils.register_always_clone("go_modules")

data/lib/dependabot/go_modules/file_fetcher.rb

@@ -17,23 +17,31 @@ module Dependabot
       private
 
       def fetch_files
-        unless go_mod
-          raise(
-            Dependabot::DependencyFileNotFound,
-            File.join(directory, "go.mod")
-          )
+        # Ensure we always check out the full repo contents for go_module
+        # updates.
+        SharedHelpers.in_a_temporary_repo_directory(
+          directory,
+          clone_repo_contents
+        ) do
+          unless go_mod
+            raise(
+              Dependabot::DependencyFileNotFound,
+              Pathname.new(File.join(directory, "go.mod")).
+              cleanpath.to_path
+            )
+          end
+
+          fetched_files = [go_mod]
+
+          # Fetch the (optional) go.sum
+          fetched_files << go_sum if go_sum
+
+          # Fetch the main.go file if present, as this will later identify
+          # this repo as an app.
+          fetched_files << main if main
+
+          fetched_files
         end
-
-        fetched_files = [go_mod]
-
-        # Fetch the (optional) go.sum
-        fetched_files << go_sum if go_sum
-
-        # Fetch the main.go file if present, as this will later identify
-        # this repo as an app.
-        fetched_files << main if main
-
-        fetched_files
       end
 
       def go_mod
@@ -45,15 +53,21 @@ module Dependabot
       end
 
       def main
-        return @main if @main
+        return @main if defined?(@main)
 
-        go_files = repo_contents.select { |f| f.name.end_with?(".go") }
+        go_files = Dir.glob("*.go")
 
-        go_files.each do |go_file|
-          file = fetch_file_from_host(go_file.name, type: "package_main")
-          next unless file.content.match?(/\s*package\s+main/)
+        go_files.each do |filename|
+          file_content = File.read(filename)
+          next unless file_content.match?(/\s*package\s+main/)
 
-          return @main = file.tap { |f| f.support_file = true }
+          return @main = DependencyFile.new(
+            name: Pathname.new(filename).cleanpath.to_path,
+            directory: "/",
+            type: "package_main",
+            support_file: true,
+            content: file_content
+          )
         end
 
         nil

data/lib/dependabot/go_modules/file_updater.rb

@@ -3,12 +3,34 @@
 require "dependabot/shared_helpers"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 
 module Dependabot
   module GoModules
     class FileUpdater < Dependabot::FileUpdaters::Base
       require_relative "file_updater/go_mod_updater"
 
+      def initialize(dependencies:, dependency_files:, repo_contents_path: nil,
+                     credentials:, options: {})
+        super
+        return unless repo_contents_path.nil?
+
+        # masquerade repo_contents_path for GoModUpdater during transition
+        tmp = Dir.mktmpdir
+        Dir.chdir(tmp) do
+          dependency_files.each do |file|
+            File.write(file.name, file.content)
+          end
+          `git config --global user.email "no-reply@github.com"`
+          `git config --global user.name "Dependabot"`
+          `git init .`
+          `git add .`
+          `git commit -m'fake repo_contents_path'`
+        end
+        @repo_contents_path = tmp
+        @repo_contents_stub = true
+      end
+
       def self.updated_files_regex
         [
           /^go\.mod$/,
@@ -33,6 +55,12 @@ module Dependabot
                 content: file_updater.updated_go_sum_content
               )
           end
+
+          vendor_updater.
+            updated_vendor_cache_files(base_directory: directory).
+            each do |file|
+            updated_files << file
+          end
         end
 
         raise "No files changed!" if updated_files.none?
@@ -56,15 +84,40 @@ module Dependabot
         @go_sum ||= get_original_file("go.sum")
       end
 
+      def directory
+        dependency_files.first.directory
+      end
+
+      def vendor_dir
+        File.join(repo_contents_path, directory, "vendor")
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_dir
+        )
+      end
+
       def file_updater
         @file_updater ||=
           GoModUpdater.new(
             dependencies: dependencies,
-            go_mod: go_mod,
-            go_sum: go_sum,
-            credentials: credentials
+            credentials: credentials,
+            repo_contents_path: repo_contents_path,
+            directory: directory,
+            options: { tidy: tidy?, vendor: vendor? }
           )
       end
+
+      def tidy?
+        !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+      end
+
+      def vendor?
+        File.exist?(File.join(vendor_dir, "modules.txt")) &&
+          options.fetch(:go_mod_vendor, false)
+      end
     end
   end
 end

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -16,7 +16,8 @@ module Dependabot
         RESOLVABILITY_ERROR_REGEXES = [
           /go: .*: git fetch .*: exit status 128/.freeze,
           /verifying .*: checksum mismatch/.freeze,
-          /build .*: cannot find module providing package/.freeze
+          /build .*: cannot find module providing package/.freeze,
+          /module .* found \(.*\), but does not contain package/m.freeze
         ].freeze
 
         MODULE_PATH_MISMATCH_REGEXES = [
@@ -25,11 +26,14 @@ module Dependabot
           /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
         ].freeze
 
-        def initialize(dependencies:, go_mod:, go_sum:, credentials:)
+        def initialize(dependencies:, credentials:, repo_contents_path:,
+                       directory:, options:)
           @dependencies = dependencies
-          @go_mod = go_mod
-          @go_sum = go_sum
           @credentials = credentials
+          @repo_contents_path = repo_contents_path
+          @directory = directory
+          @tidy = options.fetch(:tidy, false)
+          @vendor = options.fetch(:vendor, false)
         end
 
         def updated_go_mod_content
@@ -42,64 +46,83 @@ module Dependabot
 
         private
 
-        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+        attr_reader :dependencies, :credentials, :repo_contents_path,
+                    :directory
 
         def updated_files
           @updated_files ||= update_files
         end
 
-        # rubocop:disable Metrics/AbcSize
-        def update_files
-          # Map paths in local replace directives to path hashes
-          substitutions = replace_directive_substitutions(go_mod.content)
-          stub_dirs = substitutions.values
+        def update_files # rubocop:disable Metrics/AbcSize
+          in_repo_path do
+            # Map paths in local replace directives to path hashes
 
-          # Replace full paths with path hashes in the go.mod
-          clean_go_mod = substitute_all(go_mod.content, substitutions)
+            original_go_mod = File.read("go.mod")
+            original_manifest = parse_manifest
+            original_go_sum = File.read("go.sum") if File.exist?("go.sum")
 
-          # Set the new dependency versions in the go.mod
-          updated_go_mod = in_temp_dir(stub_dirs) do
-            update_go_mod(clean_go_mod, dependencies)
-          end
+            substitutions = replace_directive_substitutions(original_manifest)
+            build_module_stubs(substitutions.values)
 
-          # Then run `go get` to pick up other changes to the file caused by
-          # the upgrade
-          regenerated_files = in_temp_dir(stub_dirs) do
-            run_go_get(updated_go_mod, go_sum)
-          end
+            # Replace full paths with path hashes in the go.mod
+            substitute_all(substitutions)
 
-          # At this point, the go.mod returned from run_go_get contains the
-          # correct set of modules, but running `go get` can change the file in
-          # undesirable ways (such as injecting the current Go version), so we
-          # need to update the original go.mod with the updated set of
-          # requirements rather than using the regenerated file directly
-          original_reqs = in_temp_dir(stub_dirs) do
-            parse_manifest_requirements(go_mod.content)
-          end
-          updated_reqs = in_temp_dir(stub_dirs) do
-            parse_manifest_requirements(regenerated_files[:go_mod])
-          end
+            # Set the stubbed replace directives
+            update_go_mod(dependencies)
 
-          original_paths = original_reqs.map { |r| r["Path"] }
-          updated_paths = updated_reqs.map { |r| r["Path"] }
-          req_paths_to_remove = original_paths - updated_paths
+            # Then run `go get` to pick up other changes to the file caused by
+            # the upgrade
+            run_go_get
+            run_go_vendor
+            run_go_mod_tidy
 
-          output_go_mod = in_temp_dir(stub_dirs) do
-            remove_requirements(go_mod.content, req_paths_to_remove)
-          end
+            # At this point, the go.mod returned from run_go_get contains the
+            # correct set of modules, but running `go get` can change the file
+            # in undesirable ways (such as injecting the current Go version),
+            # so we need to update the original go.mod with the updated set of
+            # requirements rather than using the regenerated file directly
+            original_reqs = original_manifest["Require"] || []
+            updated_reqs = parse_manifest["Require"] || []
 
-          output_go_mod = in_temp_dir(stub_dirs) do
+            original_paths = original_reqs.map { |r| r["Path"] }
+            updated_paths = updated_reqs.map { |r| r["Path"] }
+            req_paths_to_remove = original_paths - updated_paths
+
+            # Put back the original content before we replace just the updated
+            # dependencies.
+            write_go_mod(original_go_mod)
+
+            remove_requirements(req_paths_to_remove)
             deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
-            update_go_mod(output_go_mod, deps)
+            update_go_mod(deps)
+
+            # put the old replace directives back again
+            substitute_all(substitutions.invert)
+
+            updated_go_sum = original_go_sum ? File.read("go.sum") : nil
+            updated_go_mod = File.read("go.mod")
+
+            { go_mod: updated_go_mod, go_sum: updated_go_sum }
           end
+        end
+
+        def run_go_mod_tidy
+          return unless tidy?
 
-          { go_mod: output_go_mod, go_sum: regenerated_files[:go_sum] }
+          command = "go mod tidy"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
         end
-        # rubocop:enable Metrics/AbcSize
 
-        def update_go_mod(go_mod_content, dependencies)
-          File.write("go.mod", go_mod_content)
+        def run_go_vendor
+          return unless vendor?
+
+          command = "go mod vendor"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+        end
 
+        def update_go_mod(dependencies)
           deps = dependencies.map do |dep|
             {
               name: dep.name,
@@ -108,78 +131,75 @@ module Dependabot
             }
           end
 
-          SharedHelpers.run_helper_subprocess(
+          body = SharedHelpers.run_helper_subprocess(
             command: NativeHelpers.helper_path,
             env: ENVIRONMENT,
             function: "updateDependencyFile",
             args: { dependencies: deps }
           )
+
+          write_go_mod(body)
         end
 
-        def run_go_get(go_mod_content, go_sum)
-          File.write("go.mod", go_mod_content)
-          File.write("go.sum", go_sum.content) if go_sum
-          File.write("main.go", dummy_main_go)
+        def run_go_get
+          tmp_go_file = "#{SecureRandom.hex}.go"
+
+          unless Dir.glob("*.go").any?
+            File.write(tmp_go_file, "package dummypkg\n")
+          end
 
           _, stderr, status = Open3.capture3(ENVIRONMENT, "go get -d")
           handle_subprocess_error(stderr) unless status.success?
-
-          updated_go_sum = go_sum ? File.read("go.sum") : nil
-          { go_mod: File.read("go.mod"), go_sum: updated_go_sum }
+        ensure
+          File.delete(tmp_go_file) if File.exist?(tmp_go_file)
         end
 
-        def parse_manifest_requirements(go_mod_content)
-          File.write("go.mod", go_mod_content)
-
+        def parse_manifest
           command = "go mod edit -json"
           stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
           handle_subprocess_error(stderr) unless status.success?
 
-          JSON.parse(stdout)["Require"] || []
+          JSON.parse(stdout) || {}
         end
 
-        def remove_requirements(go_mod_content, requirement_paths)
-          File.write("go.mod", go_mod_content)
-
+        def remove_requirements(requirement_paths)
           requirement_paths.each do |path|
             escaped_path = Shellwords.escape(path)
             command = "go mod edit -droprequire #{escaped_path}"
             _, stderr, status = Open3.capture3(ENVIRONMENT, command)
             handle_subprocess_error(stderr) unless status.success?
           end
-
-          File.read("go.mod")
         end
 
-        def add_requirements(go_mod_content, requirements)
-          File.write("go.mod", go_mod_content)
-
+        def add_requirements(requirements)
           requirements.each do |r|
             escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
             command = "go mod edit -require #{escaped_req}"
             _, stderr, status = Open3.capture3(ENVIRONMENT, command)
             handle_subprocess_error(stderr) unless status.success?
           end
-
-          File.read("go.mod")
         end
 
-        def in_temp_dir(stub_paths, &block)
-          SharedHelpers.in_a_temporary_directory do
+        def in_repo_path(&block)
+          SharedHelpers.
+            in_a_temporary_repo_directory(directory, repo_contents_path) do
             SharedHelpers.with_git_configured(credentials: credentials) do
-              # Create a fake empty module for each local module so that
-              # `go get -d` works, even if some modules have been `replace`d
-              # with a local module that we don't have access to.
-              stub_paths.each do |stub_path|
-                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                FileUtils.touch(File.join(stub_path, "go.mod"))
-              end
-
               block.call
             end
           end
         end
 
+        def build_module_stubs(stub_paths)
+          # Create a fake empty module for each local module so that
+          # `go get -d` works, even if some modules have been `replace`d
+          # with a local module that we don't have access to.
+          stub_paths.each do |stub_path|
+            Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+            FileUtils.touch(File.join(stub_path, "go.mod"))
+            FileUtils.touch(File.join(stub_path, "main.go"))
+          end
+        end
+
         # Given a go.mod file, find all `replace` directives pointing to a path
         # on the local filesystem, and return an array of pairs mapping the
         # original path to a hash of the path.
@@ -188,22 +208,14 @@ module Dependabot
         # the layout of the filesystem with a structure we can reproduce (i.e.
         # no paths such as ../../../foo), run the Go tooling, then reverse the
         # process afterwards.
-        def replace_directive_substitutions(go_mod_content)
+        def replace_directive_substitutions(manifest)
           @replace_directive_substitutions ||=
-            SharedHelpers.in_a_temporary_directory do |path|
-              File.write("go.mod", go_mod_content)
-
-              # Parse the go.mod to get a JSON representation of the replace
-              # directives
-              command = "go mod edit -json"
-              stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
-              handle_subprocess_error(path, stderr) unless status.success?
-
+            begin
               # Find all the local replacements, and return them with a stub
               # path we can use in their place. Using generated paths is safer
               # as it means we don't need to worry about references to parent
               # directories, etc.
-              (JSON.parse(stdout)["Replace"] || []).
+              (manifest["Replace"] || []).
                 map { |r| r["New"]["Path"] }.
                 compact.
                 select { |p| p.start_with?(".") || p.start_with?("/") }.
@@ -212,10 +224,12 @@ module Dependabot
             end
         end
 
-        def substitute_all(file, substitutions)
-          substitutions.reduce(file) do |text, (a, b)|
+        def substitute_all(substitutions)
+          body = substitutions.reduce(File.read("go.mod")) do |text, (a, b)|
             text.sub(a, b)
           end
+
+          write_go_mod(body)
         end
 
         def handle_subprocess_error(stderr)
@@ -231,28 +245,18 @@ module Dependabot
           if path_regex
             match = path_regex.match(stderr)
             raise Dependabot::GoModulePathMismatch.
-              new(go_mod.path, match[1], match[2])
+              new(go_mod_path, match[1], match[2])
           end
 
           msg = stderr.lines.last(10).join.strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
+          raise Dependabot::DependencyFileNotParseable.
+            new(go_mod_path, msg)
         end
 
-        def dummy_main_go
-          # If we use `main` as the package name, running `go get -d` seems to
-          # invoke the build systems, which can cause problems. For instance,
-          # if the go.mod includes a module that doesn't have a top-level
-          # package, we have no way of working out the import path, so the
-          # build step fails.
-          #
-          # In due course, if we end up fetching the full repo, it might be
-          # good to switch back to `main` so we can surface more errors.
-          lines = ["package dummypkg", "import ("]
-          dependencies.each do |dep|
-            lines << "_ \"#{dep.name}\"" unless dep.requirements.empty?
-          end
-          lines << ")"
-          lines.join("\n")
+        def go_mod_path
+          return "go.mod" if directory == "/"
+
+          File.join(directory, "go.mod")
         end
 
         def requirement_to_dependency_obj(req)
@@ -272,6 +276,18 @@ module Dependabot
             package_manager: "go_modules"
           )
         end
+
+        def write_go_mod(body)
+          File.write("go.mod", body)
+        end
+
+        def tidy?
+          !!@tidy
+        end
+
+        def vendor?
+          !!@vendor
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.120.4
+  version: 0.122.1
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-01 00:00:00.000000000 Z
+date: 2020-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.120.4
+        version: 0.122.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.120.4
+        version: 0.122.1
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.91.0
+        version: 0.93.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.91.0
+        version: 0.93.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement