dependabot-go_modules 0.120.3 → 0.122.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3ba69f6aa10d39e7c438ea6daca7e59083639eb56ab219a750b347100408e055
-  data.tar.gz: 5e61dbc6788267f0742040df46585e1f99ed85f2d39a5bfb35e253bcdb89392a
+  metadata.gz: 4668f28343a248c1d0e2ec3d61ffb59b79be51eb0df6ece487db719379935e3f
+  data.tar.gz: a5957a544166a39647cee725a66c85503887a8e141589adbfed2987a47d987b3
 SHA512:
-  metadata.gz: 8956cf5e519f92b08051f41cd2cbec50568fbb8c108fecab2483807e0d34e93c77b36d9f7891ba5cc0a9612721b3a8f56f26312a06412c0a231c99b3ec0c22db
-  data.tar.gz: '019c64586f858ec3ed1ff6bad2cbb52656bc290b8cdec09557f15c4ee3098a4976d01c5dc9d0868dac85b212639aca415811a02123bf792d3df45a11462beac5'
+  metadata.gz: ffe602aaff9f86aa15d94ebe13d5fd533e614eeb293819a6521403fc65cbff4158a0d4b1aa00aa1373e757ec47039f95d69b39e2d9ff386e9a42055e1e8283ef
+  data.tar.gz: 0b30b162084a264df70c145b1fecc99f9895e9c5594a08b2a90730dd8b1aa56c9bd535eeb6bc78430e140812dacca17a9e61017c1dfa76eebaf4ffef739b22d8

data/lib/dependabot/go_modules.rb

@@ -17,3 +17,6 @@ Dependabot::PullRequestCreator::Labeler.
 require "dependabot/dependency"
 Dependabot::Dependency.
   register_production_check("go_modules", ->(_) { true })
+
+require "dependabot/utils"
+Dependabot::Utils.register_always_clone("go_modules")

data/lib/dependabot/go_modules/file_fetcher.rb

@@ -17,23 +17,31 @@ module Dependabot
       private
 
       def fetch_files
-        unless go_mod
-          raise(
-            Dependabot::DependencyFileNotFound,
-            File.join(directory, "go.mod")
-          )
+        # Ensure we always check out the full repo contents for go_module
+        # updates.
+        SharedHelpers.in_a_temporary_repo_directory(
+          directory,
+          clone_repo_contents
+        ) do
+          unless go_mod
+            raise(
+              Dependabot::DependencyFileNotFound,
+              Pathname.new(File.join(directory, "go.mod")).
+              cleanpath.to_path
+            )
+          end
+
+          fetched_files = [go_mod]
+
+          # Fetch the (optional) go.sum
+          fetched_files << go_sum if go_sum
+
+          # Fetch the main.go file if present, as this will later identify
+          # this repo as an app.
+          fetched_files << main if main
+
+          fetched_files
         end
-
-        fetched_files = [go_mod]
-
-        # Fetch the (optional) go.sum
-        fetched_files << go_sum if go_sum
-
-        # Fetch the main.go file if present, as this will later identify
-        # this repo as an app.
-        fetched_files << main if main
-
-        fetched_files
       end
 
       def go_mod
@@ -45,15 +53,21 @@ module Dependabot
       end
 
       def main
-        return @main if @main
+        return @main if defined?(@main)
 
-        go_files = repo_contents.select { |f| f.name.end_with?(".go") }
+        go_files = Dir.glob("*.go")
 
-        go_files.each do |go_file|
-          file = fetch_file_from_host(go_file.name, type: "package_main")
-          next unless file.content.match?(/\s*package\s+main/)
+        go_files.each do |filename|
+          file_content = File.read(filename)
+          next unless file_content.match?(/\s*package\s+main/)
 
-          return @main = file.tap { |f| f.support_file = true }
+          return @main = DependencyFile.new(
+            name: Pathname.new(filename).cleanpath.to_path,
+            directory: "/",
+            type: "package_main",
+            support_file: true,
+            content: file_content
+          )
         end
 
         nil

data/lib/dependabot/go_modules/file_updater.rb

@@ -3,12 +3,34 @@
 require "dependabot/shared_helpers"
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
+require "dependabot/file_updaters/vendor_updater"
 
 module Dependabot
   module GoModules
     class FileUpdater < Dependabot::FileUpdaters::Base
       require_relative "file_updater/go_mod_updater"
 
+      def initialize(dependencies:, dependency_files:, repo_contents_path: nil,
+                     credentials:, options: {})
+        super
+        return unless repo_contents_path.nil?
+
+        # masquerade repo_contents_path for GoModUpdater during transition
+        tmp = Dir.mktmpdir
+        Dir.chdir(tmp) do
+          dependency_files.each do |file|
+            File.write(file.name, file.content)
+          end
+          `git config --global user.email "no-reply@github.com"`
+          `git config --global user.name "Dependabot"`
+          `git init .`
+          `git add .`
+          `git commit -m'fake repo_contents_path'`
+        end
+        @repo_contents_path = tmp
+        @repo_contents_stub = true
+      end
+
       def self.updated_files_regex
         [
           /^go\.mod$/,
@@ -33,6 +55,12 @@ module Dependabot
                 content: file_updater.updated_go_sum_content
               )
           end
+
+          vendor_updater.
+            updated_vendor_cache_files(base_directory: directory).
+            each do |file|
+            updated_files << file
+          end
         end
 
         raise "No files changed!" if updated_files.none?
@@ -56,15 +84,40 @@ module Dependabot
         @go_sum ||= get_original_file("go.sum")
       end
 
+      def directory
+        dependency_files.first.directory
+      end
+
+      def vendor_dir
+        File.join(repo_contents_path, directory, "vendor")
+      end
+
+      def vendor_updater
+        Dependabot::FileUpdaters::VendorUpdater.new(
+          repo_contents_path: repo_contents_path,
+          vendor_dir: vendor_dir
+        )
+      end
+
       def file_updater
         @file_updater ||=
           GoModUpdater.new(
             dependencies: dependencies,
-            go_mod: go_mod,
-            go_sum: go_sum,
-            credentials: credentials
+            credentials: credentials,
+            repo_contents_path: repo_contents_path,
+            directory: directory,
+            options: { tidy: tidy?, vendor: vendor? }
           )
       end
+
+      def tidy?
+        !@repo_contents_stub && options.fetch(:go_mod_tidy, false)
+      end
+
+      def vendor?
+        File.exist?(File.join(vendor_dir, "modules.txt")) &&
+          options.fetch(:go_mod_vendor, false)
+      end
     end
   end
 end

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -25,11 +25,14 @@ module Dependabot
           /go: ([^@\s]+)(?:@[^\s]+)?: .* declares its path as: ([\S]*)/m
         ].freeze
 
-        def initialize(dependencies:, go_mod:, go_sum:, credentials:)
+        def initialize(dependencies:, credentials:, repo_contents_path:,
+                       directory:, options:)
           @dependencies = dependencies
-          @go_mod = go_mod
-          @go_sum = go_sum
           @credentials = credentials
+          @repo_contents_path = repo_contents_path
+          @directory = directory
+          @tidy = options.fetch(:tidy, false)
+          @vendor = options.fetch(:vendor, false)
         end
 
         def updated_go_mod_content
@@ -42,64 +45,83 @@ module Dependabot
 
         private
 
-        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+        attr_reader :dependencies, :credentials, :repo_contents_path,
+                    :directory
 
         def updated_files
           @updated_files ||= update_files
         end
 
-        # rubocop:disable Metrics/AbcSize
-        def update_files
-          # Map paths in local replace directives to path hashes
-          substitutions = replace_directive_substitutions(go_mod.content)
-          stub_dirs = substitutions.values
+        def update_files # rubocop:disable Metrics/AbcSize
+          in_repo_path do
+            # Map paths in local replace directives to path hashes
 
-          # Replace full paths with path hashes in the go.mod
-          clean_go_mod = substitute_all(go_mod.content, substitutions)
+            original_go_mod = File.read("go.mod")
+            original_manifest = parse_manifest
+            original_go_sum = File.read("go.sum") if File.exist?("go.sum")
 
-          # Set the new dependency versions in the go.mod
-          updated_go_mod = in_temp_dir(stub_dirs) do
-            update_go_mod(clean_go_mod, dependencies)
-          end
+            substitutions = replace_directive_substitutions(original_manifest)
+            build_module_stubs(substitutions.values)
 
-          # Then run `go get` to pick up other changes to the file caused by
-          # the upgrade
-          regenerated_files = in_temp_dir(stub_dirs) do
-            run_go_get(updated_go_mod, go_sum)
-          end
+            # Replace full paths with path hashes in the go.mod
+            substitute_all(substitutions)
 
-          # At this point, the go.mod returned from run_go_get contains the
-          # correct set of modules, but running `go get` can change the file in
-          # undesirable ways (such as injecting the current Go version), so we
-          # need to update the original go.mod with the updated set of
-          # requirements rather than using the regenerated file directly
-          original_reqs = in_temp_dir(stub_dirs) do
-            parse_manifest_requirements(go_mod.content)
-          end
-          updated_reqs = in_temp_dir(stub_dirs) do
-            parse_manifest_requirements(regenerated_files[:go_mod])
-          end
+            # Set the stubbed replace directives
+            update_go_mod(dependencies)
 
-          original_paths = original_reqs.map { |r| r["Path"] }
-          updated_paths = updated_reqs.map { |r| r["Path"] }
-          req_paths_to_remove = original_paths - updated_paths
+            # Then run `go get` to pick up other changes to the file caused by
+            # the upgrade
+            run_go_get
+            run_go_vendor
+            run_go_mod_tidy
 
-          output_go_mod = in_temp_dir(stub_dirs) do
-            remove_requirements(go_mod.content, req_paths_to_remove)
-          end
+            # At this point, the go.mod returned from run_go_get contains the
+            # correct set of modules, but running `go get` can change the file
+            # in undesirable ways (such as injecting the current Go version),
+            # so we need to update the original go.mod with the updated set of
+            # requirements rather than using the regenerated file directly
+            original_reqs = original_manifest["Require"] || []
+            updated_reqs = parse_manifest["Require"] || []
 
-          output_go_mod = in_temp_dir(stub_dirs) do
+            original_paths = original_reqs.map { |r| r["Path"] }
+            updated_paths = updated_reqs.map { |r| r["Path"] }
+            req_paths_to_remove = original_paths - updated_paths
+
+            # Put back the original content before we replace just the updated
+            # dependencies.
+            write_go_mod(original_go_mod)
+
+            remove_requirements(req_paths_to_remove)
             deps = updated_reqs.map { |r| requirement_to_dependency_obj(r) }
-            update_go_mod(output_go_mod, deps)
+            update_go_mod(deps)
+
+            # put the old replace directives back again
+            substitute_all(substitutions.invert)
+
+            updated_go_sum = original_go_sum ? File.read("go.sum") : nil
+            updated_go_mod = File.read("go.mod")
+
+            { go_mod: updated_go_mod, go_sum: updated_go_sum }
           end
+        end
+
+        def run_go_mod_tidy
+          return unless tidy?
 
-          { go_mod: output_go_mod, go_sum: regenerated_files[:go_sum] }
+          command = "go mod tidy"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
         end
-        # rubocop:enable Metrics/AbcSize
 
-        def update_go_mod(go_mod_content, dependencies)
-          File.write("go.mod", go_mod_content)
+        def run_go_vendor
+          return unless vendor?
+
+          command = "go mod vendor"
+          _, stderr, status = Open3.capture3(ENVIRONMENT, command)
+          handle_subprocess_error(stderr) unless status.success?
+        end
 
+        def update_go_mod(dependencies)
           deps = dependencies.map do |dep|
             {
               name: dep.name,
@@ -108,78 +130,75 @@ module Dependabot
             }
           end
 
-          SharedHelpers.run_helper_subprocess(
+          body = SharedHelpers.run_helper_subprocess(
             command: NativeHelpers.helper_path,
             env: ENVIRONMENT,
             function: "updateDependencyFile",
             args: { dependencies: deps }
           )
+
+          write_go_mod(body)
         end
 
-        def run_go_get(go_mod_content, go_sum)
-          File.write("go.mod", go_mod_content)
-          File.write("go.sum", go_sum.content) if go_sum
-          File.write("main.go", dummy_main_go)
+        def run_go_get
+          tmp_go_file = "#{SecureRandom.hex}.go"
+
+          unless Dir.glob("*.go").any?
+            File.write(tmp_go_file, "package dummypkg\n")
+          end
 
           _, stderr, status = Open3.capture3(ENVIRONMENT, "go get -d")
           handle_subprocess_error(stderr) unless status.success?
-
-          updated_go_sum = go_sum ? File.read("go.sum") : nil
-          { go_mod: File.read("go.mod"), go_sum: updated_go_sum }
+        ensure
+          File.delete(tmp_go_file) if File.exist?(tmp_go_file)
         end
 
-        def parse_manifest_requirements(go_mod_content)
-          File.write("go.mod", go_mod_content)
-
+        def parse_manifest
           command = "go mod edit -json"
           stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
           handle_subprocess_error(stderr) unless status.success?
 
-          JSON.parse(stdout)["Require"] || []
+          JSON.parse(stdout) || {}
         end
 
-        def remove_requirements(go_mod_content, requirement_paths)
-          File.write("go.mod", go_mod_content)
-
+        def remove_requirements(requirement_paths)
           requirement_paths.each do |path|
             escaped_path = Shellwords.escape(path)
             command = "go mod edit -droprequire #{escaped_path}"
             _, stderr, status = Open3.capture3(ENVIRONMENT, command)
             handle_subprocess_error(stderr) unless status.success?
           end
-
-          File.read("go.mod")
         end
 
-        def add_requirements(go_mod_content, requirements)
-          File.write("go.mod", go_mod_content)
-
+        def add_requirements(requirements)
           requirements.each do |r|
             escaped_req = Shellwords.escape("#{r['Path']}@#{r['Version']}")
             command = "go mod edit -require #{escaped_req}"
             _, stderr, status = Open3.capture3(ENVIRONMENT, command)
             handle_subprocess_error(stderr) unless status.success?
           end
-
-          File.read("go.mod")
         end
 
-        def in_temp_dir(stub_paths, &block)
-          SharedHelpers.in_a_temporary_directory do
+        def in_repo_path(&block)
+          SharedHelpers.
+            in_a_temporary_repo_directory(directory, repo_contents_path) do
             SharedHelpers.with_git_configured(credentials: credentials) do
-              # Create a fake empty module for each local module so that
-              # `go get -d` works, even if some modules have been `replace`d
-              # with a local module that we don't have access to.
-              stub_paths.each do |stub_path|
-                Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
-                FileUtils.touch(File.join(stub_path, "go.mod"))
-              end
-
               block.call
             end
           end
         end
 
+        def build_module_stubs(stub_paths)
+          # Create a fake empty module for each local module so that
+          # `go get -d` works, even if some modules have been `replace`d
+          # with a local module that we don't have access to.
+          stub_paths.each do |stub_path|
+            Dir.mkdir(stub_path) unless Dir.exist?(stub_path)
+            FileUtils.touch(File.join(stub_path, "go.mod"))
+            FileUtils.touch(File.join(stub_path, "main.go"))
+          end
+        end
+
         # Given a go.mod file, find all `replace` directives pointing to a path
         # on the local filesystem, and return an array of pairs mapping the
         # original path to a hash of the path.
@@ -188,22 +207,14 @@ module Dependabot
         # the layout of the filesystem with a structure we can reproduce (i.e.
         # no paths such as ../../../foo), run the Go tooling, then reverse the
         # process afterwards.
-        def replace_directive_substitutions(go_mod_content)
+        def replace_directive_substitutions(manifest)
           @replace_directive_substitutions ||=
-            SharedHelpers.in_a_temporary_directory do |path|
-              File.write("go.mod", go_mod_content)
-
-              # Parse the go.mod to get a JSON representation of the replace
-              # directives
-              command = "go mod edit -json"
-              stdout, stderr, status = Open3.capture3(ENVIRONMENT, command)
-              handle_subprocess_error(path, stderr) unless status.success?
-
+            begin
               # Find all the local replacements, and return them with a stub
               # path we can use in their place. Using generated paths is safer
               # as it means we don't need to worry about references to parent
               # directories, etc.
-              (JSON.parse(stdout)["Replace"] || []).
+              (manifest["Replace"] || []).
                 map { |r| r["New"]["Path"] }.
                 compact.
                 select { |p| p.start_with?(".") || p.start_with?("/") }.
@@ -212,10 +223,12 @@ module Dependabot
             end
         end
 
-        def substitute_all(file, substitutions)
-          substitutions.reduce(file) do |text, (a, b)|
+        def substitute_all(substitutions)
+          body = substitutions.reduce(File.read("go.mod")) do |text, (a, b)|
             text.sub(a, b)
           end
+
+          write_go_mod(body)
         end
 
         def handle_subprocess_error(stderr)
@@ -231,28 +244,18 @@ module Dependabot
           if path_regex
             match = path_regex.match(stderr)
             raise Dependabot::GoModulePathMismatch.
-              new(go_mod.path, match[1], match[2])
+              new(go_mod_path, match[1], match[2])
           end
 
           msg = stderr.lines.last(10).join.strip
-          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
+          raise Dependabot::DependencyFileNotParseable.
+            new(go_mod_path, msg)
         end
 
-        def dummy_main_go
-          # If we use `main` as the package name, running `go get -d` seems to
-          # invoke the build systems, which can cause problems. For instance,
-          # if the go.mod includes a module that doesn't have a top-level
-          # package, we have no way of working out the import path, so the
-          # build step fails.
-          #
-          # In due course, if we end up fetching the full repo, it might be
-          # good to switch back to `main` so we can surface more errors.
-          lines = ["package dummypkg", "import ("]
-          dependencies.each do |dep|
-            lines << "_ \"#{dep.name}\"" unless dep.requirements.empty?
-          end
-          lines << ")"
-          lines.join("\n")
+        def go_mod_path
+          return "go.mod" if directory == "/"
+
+          File.join(directory, "go.mod")
         end
 
         def requirement_to_dependency_obj(req)
@@ -272,6 +275,18 @@ module Dependabot
             package_manager: "go_modules"
           )
         end
+
+        def write_go_mod(body)
+          File.write("go.mod", body)
+        end
+
+        def tidy?
+          !!@tidy
+        end
+
+        def vendor?
+          !!@vendor
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-go_modules
 version: !ruby/object:Gem::Version
-  version: 0.120.3
+  version: 0.122.0
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-28 00:00:00.000000000 Z
+date: 2020-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.120.3
+        version: 0.122.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.120.3
+        version: 0.122.0
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,42 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.91.0
+        version: 0.92.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.91.0
+        version: 0.92.0
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: simplecov-console
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.7.2
 - !ruby/object:Gem::Dependency
   name: vcr
   requirement: !ruby/object:Gem::Requirement