dependabot-go_modules 0.87.0

data/lib/dependabot/go_modules/file_parser.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "open3"
+require "dependabot/dependency"
+require "dependabot/file_parsers/base/dependency_set"
+require "dependabot/go_modules/path_converter"
+require "dependabot/errors"
+require "dependabot/file_parsers"
+require "dependabot/file_parsers/base"
+
+module Dependabot
+  module GoModules
+    class FileParser < Dependabot::FileParsers::Base
+      GIT_VERSION_REGEX = /^v\d+\.\d+\.\d+-.*-(?<sha>[0-9a-f]{12})$/.freeze
+
+      def parse
+        dependency_set = Dependabot::FileParsers::Base::DependencySet.new
+
+        i = 0
+        chunks = module_info(go_mod).lines.
+                 group_by { |line| line == "{\n" ? i += 1 : i }
+        deps = chunks.values.map { |chunk| JSON.parse(chunk.join) }
+
+        deps.each do |dep|
+          # The project itself appears in this list as "Main"
+          next if dep["Main"]
+
+          dependency = dependency_from_details(dep)
+          dependency_set << dependency if dependency
+        end
+
+        dependency_set.dependencies
+      end
+
+      private
+
+      def go_mod
+        @go_mod ||= get_original_file("go.mod")
+      end
+
+      def check_required_files
+        raise "No go.mod!" unless go_mod
+      end
+
+      def dependency_from_details(details)
+        source =
+          if rev_identifier?(details) then git_source(details)
+          else { type: "default", source: details["Path"] }
+          end
+
+        version = details["Version"]&.sub(/^v?/, "")
+
+        reqs = [{
+          requirement: rev_identifier?(details) ? nil : details["Version"],
+          file: go_mod.name,
+          source: source,
+          groups: []
+        }]
+
+        Dependency.new(
+          name: details["Path"],
+          version: version,
+          requirements: details["Indirect"] ? [] : reqs,
+          package_manager: "dep"
+        )
+      end
+
+      def module_info(go_mod)
+        @module_info ||=
+          SharedHelpers.in_a_temporary_directory do |path|
+            SharedHelpers.with_git_configured(credentials: credentials) do
+              File.write("go.mod", go_mod.content)
+
+              command = "GO111MODULE=on go mod edit -print > /dev/null"
+              command += " && GO111MODULE=on go list -m -json all"
+              stdout, stderr, status = Open3.capture3(command)
+              handle_parser_error(path, stderr) unless status.success?
+              stdout
+            end
+          end
+      end
+
+      def handle_parser_error(path, stderr)
+        case stderr
+        when /go: .*: unknown revision/
+          line = stderr.lines.grep(/unknown revision/).first
+          raise Dependabot::DependencyFileNotResolvable, line.strip
+        when /go: .*: unrecognized import path/
+          line = stderr.lines.grep(/unrecognized import/).first
+          raise Dependabot::DependencyFileNotResolvable, line.strip
+        when /go: errors parsing go.mod/
+          msg = stderr.gsub(path.to_s, "").strip
+          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
+        else
+          msg = stderr.gsub(path.to_s, "").strip
+          raise Dependabot::DependencyFileNotParseable.new(go_mod.path, msg)
+        end
+      end
+
+      def rev_identifier?(dep)
+        dep["Version"]&.match?(GIT_VERSION_REGEX)
+      end
+
+      def git_source(dep)
+        url = PathConverter.git_url_for_path(dep["Path"])
+
+        # Currently, we have no way of knowing whether the commit tagged
+        # is being used because a branch is being followed or because a
+        # particular ref is in use. We *assume* that a particular ref is in
+        # use (which means we'll only propose updates when its included in
+        # a release)
+        {
+          type: "git",
+          url: url || dep["Path"],
+          ref: git_revision(dep),
+          branch: nil
+        }
+      end
+
+      def git_revision(dep)
+        raw_version = dep.fetch("Version")
+        return raw_version unless raw_version.match?(GIT_VERSION_REGEX)
+
+        raw_version.match(GIT_VERSION_REGEX).named_captures.fetch("sha")
+      end
+    end
+  end
+end
+
+Dependabot::FileParsers.
+  register("go_modules", Dependabot::GoModules::FileParser)

data/lib/dependabot/go_modules/file_updater.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "dependabot/shared_helpers"
+require "dependabot/file_updaters"
+require "dependabot/file_updaters/base"
+
+module Dependabot
+  module GoModules
+    class FileUpdater < Dependabot::FileUpdaters::Base
+      require_relative "file_updater/go_mod_updater"
+
+      def self.updated_files_regex
+        [
+          /^go\.mod$/,
+          /^go\.sum$/
+        ]
+      end
+
+      def updated_dependency_files
+        updated_files = []
+
+        if go_mod && file_changed?(go_mod)
+          updated_files <<
+            updated_file(
+              file: go_mod,
+              content: file_updater.updated_go_mod_content
+            )
+
+          if go_sum && go_sum.content != file_updater.updated_go_sum_content
+            updated_files <<
+              updated_file(
+                file: go_sum,
+                content: file_updater.updated_go_sum_content
+              )
+          end
+        end
+
+        raise "No files changed!" if updated_files.none?
+
+        updated_files
+      end
+
+      private
+
+      def check_required_files
+        return if go_mod
+
+        raise "No go.mod!"
+      end
+
+      def go_mod
+        @go_mod ||= get_original_file("go.mod")
+      end
+
+      def go_sum
+        @go_sum ||= get_original_file("go.sum")
+      end
+
+      def file_updater
+        @file_updater ||=
+          GoModUpdater.new(
+            dependencies: dependencies,
+            go_mod: go_mod,
+            go_sum: go_sum,
+            credentials: credentials
+          )
+      end
+    end
+  end
+end
+
+Dependabot::FileUpdaters.
+  register("go_modules", Dependabot::GoModules::FileUpdater)

data/lib/dependabot/go_modules/file_updater/go_mod_updater.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+require "dependabot/utils/go/shared_helper"
+require "dependabot/go_modules/file_updater"
+require "dependabot/go_modules/native_helpers"
+
+module Dependabot
+  module GoModules
+    class FileUpdater
+      class GoModUpdater
+        def initialize(dependencies:, go_mod:, go_sum:, credentials:)
+          @dependencies = dependencies
+          @go_mod = go_mod
+          @go_sum = go_sum
+          @credentials = credentials
+        end
+
+        def updated_go_mod_content
+          @updated_go_mod_content ||=
+            SharedHelpers.in_a_temporary_directory do
+              SharedHelpers.with_git_configured(credentials: credentials) do
+                File.write("go.mod", go_mod.content)
+
+                deps = dependencies.map do |dep|
+                  {
+                    name: dep.name,
+                    version: "v" + dep.version.sub(/^v/i, ""),
+                    indirect: dep.requirements.empty?
+                  }
+                end
+
+                SharedHelpers.run_helper_subprocess(
+                  command: "GO111MODULE=on #{NativeHelpers.helper_path}",
+                  function: "updateDependencyFile",
+                  args: { dependencies: deps }
+                )
+              end
+            end
+        end
+
+        def updated_go_sum_content
+          return nil unless go_sum
+
+          # This needs to be run separately so we don't nest subprocess calls
+          updated_go_mod_content
+
+          @updated_go_sum_content ||=
+            SharedHelpers.in_a_temporary_directory do
+              SharedHelpers.with_git_configured(credentials: credentials) do
+                File.write("go.mod", updated_go_mod_content)
+                File.write("go.sum", go_sum.content)
+                File.write("main.go", dummy_main_go)
+
+                `GO111MODULE=on go get -d`
+                unless $CHILD_STATUS.success?
+                  raise Dependabot::DependencyFileNotParseable, go_sum.path
+                end
+
+                File.read("go.sum")
+              end
+            end
+        end
+
+        private
+
+        def dummy_main_go
+          lines = ["package main", "import ("]
+          dependencies.each do |dep|
+            lines << "_ \"#{dep.name}\""
+          end
+          lines << ")"
+          lines << "func main() {}"
+          lines.join("\n")
+        end
+
+        attr_reader :dependencies, :go_mod, :go_sum, :credentials
+      end
+    end
+  end
+end

data/lib/dependabot/go_modules/metadata_finder.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require "dependabot/metadata_finders"
+require "dependabot/metadata_finders/base"
+require "dependabot/go_modules/path_converter"
+
+module Dependabot
+  module GoModules
+    class MetadataFinder < Dependabot::MetadataFinders::Base
+      private
+
+      def look_up_source
+        return look_up_git_dependency_source if git_dependency?
+
+        path_str = (specified_source_string || dependency.name)
+        url = Dependabot::GoModules::PathConverter.
+              git_url_for_path_without_go_helper(path_str)
+        Source.from_url(url) if url
+      end
+
+      def git_dependency?
+        return false unless declared_source_details
+
+        dependency_type =
+          declared_source_details.fetch(:type, nil) ||
+          declared_source_details.fetch("type")
+
+        dependency_type == "git"
+      end
+
+      def look_up_git_dependency_source
+        specified_url =
+          declared_source_details.fetch(:url, nil) ||
+          declared_source_details.fetch("url")
+
+        Source.from_url(specified_url)
+      end
+
+      def specified_source_string
+        declared_source_details&.fetch(:source, nil) ||
+          declared_source_details&.fetch("source", nil)
+      end
+
+      def declared_source_details
+        sources = dependency.requirements.
+                  map { |r| r.fetch(:source) }.
+                  uniq.compact
+
+        raise "Multiple sources! #{sources.join(', ')}" if sources.count > 1
+
+        sources.first
+      end
+    end
+  end
+end
+
+Dependabot::MetadataFinders.
+  register("go_modules", Dependabot::GoModules::MetadataFinder)

data/lib/dependabot/go_modules/native_helpers.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Dependabot
+  module GoModules
+    module NativeHelpers
+      def self.helper_path
+        clean_path(File.join(native_helpers_root, "go_modules/bin/helper"))
+      end
+
+      def self.native_helpers_root
+        default_path = File.join(__dir__, "../../../helpers/install-dir")
+        ENV.fetch("DEPENDABOT_NATIVE_HELPERS_PATH", default_path)
+      end
+
+      def self.clean_path(path)
+        Pathname.new(path).cleanpath.to_path
+      end
+    end
+  end
+end

data/lib/dependabot/go_modules/path_converter.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+require "excon"
+require "nokogiri"
+
+require "dependabot/shared_helpers"
+require "dependabot/source"
+require "dependabot/go_modules/native_helpers"
+
+module Dependabot
+  module GoModules
+    module PathConverter
+      def self.git_url_for_path(path)
+        # Save a query by manually converting golang.org/x names
+        import_path = path.gsub(%r{^golang\.org/x}, "github.com/golang")
+
+        SharedHelpers.run_helper_subprocess(
+          command: NativeHelpers.helper_path,
+          function: "getVcsRemoteForImport",
+          args: { import: import_path }
+        )
+      end
+
+      # Used in dependabot-backend, which doesn't have access to any Go
+      # helpers.
+      # TODO: remove the need for this.
+      def self.git_url_for_path_without_go_helper(path)
+        # Save a query by manually converting golang.org/x names
+        tmp_path = path.gsub(%r{^golang\.org/x}, "github.com/golang")
+
+        # Currently, Dependabot::Source.new will return `nil` if it can't
+        # find a git SCH associated with a path. If it is ever extended to
+        # handle non-git sources we'll need to add an additional check here.
+        return Source.from_url(tmp_path).url if Source.from_url(tmp_path)
+        return "https://#{tmp_path}" if tmp_path.end_with?(".git")
+        return unless (metadata_response = fetch_path_metadata(path))
+
+        # Look for a GitHub, Bitbucket or GitLab URL in the response
+        metadata_response.scan(Dependabot::Source::SOURCE_REGEX) do
+          source_url = Regexp.last_match.to_s
+          return Source.from_url(source_url).url
+        end
+
+        # If none are found, parse the response and return the go-import path
+        doc = Nokogiri::XML(metadata_response)
+        doc.remove_namespaces!
+        import_details =
+          doc.xpath("//meta").
+          find { |n| n.attributes["name"]&.value == "go-import" }&.
+          attributes&.fetch("content")&.value&.split(/\s+/)
+        return unless import_details && import_details[1] == "git"
+
+        import_details[2]
+      end
+
+      def self.fetch_path_metadata(path)
+        # TODO: This is not robust! Instead, we should shell out to Go and
+        # use https://github.com/Masterminds/vcs.
+        response = Excon.get(
+          "https://#{path}?go-get=1",
+          idempotent: true,
+          **SharedHelpers.excon_defaults
+        )
+
+        return unless response.status == 200
+
+        response.body
+      end
+      private_class_method :fetch_path_metadata
+    end
+  end
+end

data/lib/dependabot/go_modules/requirement.rb

@@ -0,0 +1,148 @@
+# frozen_string_literal: true
+
+################################################################################
+# For more details on Go version constraints, see:                             #
+# - https://github.com/Masterminds/semver                                      #
+# - https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md               #
+################################################################################
+
+require "dependabot/go_modules/version"
+
+module Dependabot
+  module GoModules
+    class Requirement < Gem::Requirement
+      WILDCARD_REGEX = /(?:\.|^)[xX*]/.freeze
+      OR_SEPARATOR = /(?<=[a-zA-Z0-9*])\s*\|{2}/.freeze
+
+      # Override the version pattern to allow a 'v' prefix
+      quoted = OPS.keys.map { |k| Regexp.quote(k) }.join("|")
+      version_pattern = "v?#{Version::VERSION_PATTERN}"
+
+      PATTERN_RAW = "\\s*(#{quoted})?\\s*(#{version_pattern})\\s*"
+      PATTERN = /\A#{PATTERN_RAW}\z/.freeze
+
+      # Use GoModules::Version rather than Gem::Version to ensure that
+      # pre-release versions aren't transformed.
+      def self.parse(obj)
+        return ["=", Version.new(obj.to_s)] if obj.is_a?(Gem::Version)
+
+        unless (matches = PATTERN.match(obj.to_s))
+          msg = "Illformed requirement [#{obj.inspect}]"
+          raise BadRequirementError, msg
+        end
+
+        return DefaultRequirement if matches[1] == ">=" && matches[2] == "0"
+
+        [matches[1] || "=", Version.new(matches[2])]
+      end
+
+      # Returns an array of requirements. At least one requirement from the
+      # returned array must be satisfied for a version to be valid.
+      def self.requirements_array(requirement_string)
+        return [new(nil)] if requirement_string.nil?
+
+        requirement_string.strip.split(OR_SEPARATOR).map do |req_string|
+          new(req_string)
+        end
+      end
+
+      def initialize(*requirements)
+        requirements = requirements.flatten.flat_map do |req_string|
+          req_string.split(",").map do |r|
+            convert_go_constraint_to_ruby_constraint(r.strip)
+          end
+        end
+
+        super(requirements)
+      end
+
+      private
+
+      def convert_go_constraint_to_ruby_constraint(req_string)
+        req_string = req_string
+        req_string = convert_wildcard_characters(req_string)
+
+        if req_string.match?(WILDCARD_REGEX)
+          ruby_range(req_string.gsub(WILDCARD_REGEX, "").gsub(/^[^\d]/, ""))
+        elsif req_string.match?(/^~[^>]/) then convert_tilde_req(req_string)
+        elsif req_string.include?(" - ") then convert_hyphen_req(req_string)
+        elsif req_string.match?(/^[\dv^]/) then convert_caret_req(req_string)
+        elsif req_string.match?(/[<=>]/) then req_string
+        else ruby_range(req_string)
+        end
+      end
+
+      def convert_wildcard_characters(req_string)
+        if req_string.match?(/^[\dv^>~]/)
+          replace_wildcard_in_lower_bound(req_string)
+        elsif req_string.start_with?("<")
+          parts = req_string.split(".")
+          parts.map.with_index do |part, index|
+            next "0" if part.match?(WILDCARD_REGEX)
+            next part.to_i + 1 if parts[index + 1]&.match?(WILDCARD_REGEX)
+
+            part
+          end.join(".")
+        else
+          req_string
+        end
+      end
+
+      def replace_wildcard_in_lower_bound(req_string)
+        after_wildcard = false
+
+        if req_string.start_with?("~")
+          req_string = req_string.gsub(/(?:(?:\.|^)[xX*])(\.[xX*])+/, "")
+        end
+
+        req_string.split(".").
+          map do |part|
+            part.split("-").map.with_index do |p, i|
+              # Before we hit a wildcard we just return the existing part
+              next p unless p.match?(WILDCARD_REGEX) || after_wildcard
+
+              # On or after a wildcard we replace the version part with zero
+              after_wildcard = true
+              i.zero? ? "0" : "a"
+            end.join("-")
+          end.join(".")
+      end
+
+      def convert_tilde_req(req_string)
+        version = req_string.gsub(/^~/, "")
+        parts = version.split(".")
+        parts << "0" if parts.count < 3
+        "~> #{parts.join('.')}"
+      end
+
+      def convert_hyphen_req(req_string)
+        lower_bound, upper_bound = req_string.split(/\s+-\s+/)
+        [">= #{lower_bound}", "<= #{upper_bound}"]
+      end
+
+      def ruby_range(req_string)
+        parts = req_string.split(".")
+
+        # If we have three or more parts then this is an exact match
+        return req_string if parts.count >= 3
+
+        # If we have no parts then the version is completely unlocked
+        return ">= 0" if parts.count.zero?
+
+        # If we have fewer than three parts we do a partial match
+        parts << "0"
+        "~> #{parts.join('.')}"
+      end
+
+      # Note: Dep's caret notation implementation doesn't distinguish between
+      # pre and post-1.0.0 requirements (unlike in JS)
+      def convert_caret_req(req_string)
+        version = req_string.gsub(/^\^?v?/, "")
+        parts = version.split(".")
+        upper_bound = [parts.first.to_i + 1, 0, 0, "a"].map(&:to_s).join(".")
+
+        [">= #{version}", "< #{upper_bound}"]
+      end
+    end
+  end
+end