dependabot-terraform 0.146.0 → 0.148.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e420475dee8741febcbdbb02fbffc509cf328bc767ba0f3126dc384403cf8d3
-  data.tar.gz: 728d0bcf47d8891d39ea7de8c6e4794f2866492aae8328cce43f1d72156abd7e
+  metadata.gz: 7deaa50339570bf2aa6a092465692a047db22f62342284d484c9bd447130e8d7
+  data.tar.gz: bc1e4776ea8c45cb9e4967456c23372a5ba4c02fb467e46171b2d87a3d77b026
 SHA512:
-  metadata.gz: b9ae9da9838932d029c73c2d9ede6cf13f0405440f590bb74ad2e7bcaf37394e2ce2c4ba8d8d8746699f8fe238f0c06c4756f5d3a919817b91132ee3e816efcd
-  data.tar.gz: a530a3b1fba4dae111238e23d67e89f3794fb812ebf658f3a357cf7b588bcb9637c582f55619aee83f87f75fca4b7302efc172363008cf6833154d848be575f4
+  metadata.gz: 47b0dee55cc993cf107e74577df8ae26c75050defe0d564fef43718e5c335ac63819848455db69994ae1a0becd9061a02cb5f40a4c38555e4d9a4a887161589d
+  data.tar.gz: f5e38dccd33ba13e417cb6a3dc6dd1ae4c3bb316d4ab95fa72c86eb5b9873729d433ea8f08550ceb38ddd4d1e240d8f1df0be85b6ed22366f59ff9f3c92b7f7b

data/helpers/build

@@ -13,7 +13,10 @@ if [ ! -d "$install_dir/bin" ]; then
 fi
 
 os="$(uname -s | tr '[:upper:]' '[:lower:]')"
-github_url="https://github.com/kvz/json2hcl"
-url="${github_url}/releases/download/v0.0.6/json2hcl_v0.0.6_${os}_amd64"
-wget -O "$install_dir/bin/json2hcl" "$url"
-chmod +x "$install_dir/bin/json2hcl"
+
+hcl2json_checksum="24068f1e25a34d8f8ca763f34fce11527472891bfa834d1504f665855021d5d4"
+hcl2json_url="https://github.com/tmccombs/hcl2json/releases/download/v0.3.3/hcl2json_${os}_amd64"
+hcl2json_path="$install_dir/bin/hcl2json"
+wget -O "$hcl2json_path" "$hcl2json_url"
+echo "$hcl2json_checksum  $hcl2json_path" | sha256sum -c
+chmod +x "$install_dir/bin/hcl2json"

data/lib/dependabot/terraform/file_fetcher.rb

@@ -2,12 +2,15 @@
 
 require "dependabot/file_fetchers"
 require "dependabot/file_fetchers/base"
+require "dependabot/terraform/file_selector"
 
 module Dependabot
   module Terraform
     class FileFetcher < Dependabot::FileFetchers::Base
+      include FileSelector
+
       def self.required_files_in?(filenames)
-        filenames.any? { |f| f.end_with?(".tf", ".tfvars") }
+        filenames.any? { |f| f.end_with?(".tf", ".hcl") }
       end
 
       def self.required_files_message
@@ -39,7 +42,7 @@ module Dependabot
       def terragrunt_files
         @terragrunt_files ||=
           repo_contents(raise_errors: false).
-          select { |f| f.type == "file" && f.name.end_with?(".tfvars") }.
+          select { |f| f.type == "file" && terragrunt_file?(f.name) }.
           map { |f| fetch_file_from_host(f.name) }
       end
     end

data/lib/dependabot/terraform/file_parser.rb

@@ -10,27 +10,38 @@ require "dependabot/file_parsers/base"
 require "dependabot/git_commit_checker"
 require "dependabot/shared_helpers"
 require "dependabot/errors"
+require "dependabot/terraform/file_selector"
 
 module Dependabot
   module Terraform
     class FileParser < Dependabot::FileParsers::Base
       require "dependabot/file_parsers/base/dependency_set"
 
+      include FileSelector
+
       ARCHIVE_EXTENSIONS = %w(.zip .tbz2 .tgz .txz).freeze
 
       def parse
         dependency_set = DependencySet.new
 
         terraform_files.each do |file|
-          modules = parsed_file(file).fetch("module", []).map(&:first)
+          modules = parsed_file(file).fetch("module", {})
           modules.each do |name, details|
-            dependency_set << build_terraform_dependency(file, name, details)
+            dependency_set << build_terraform_dependency(file, name, details, false)
+          end
+
+          parsed_file(file).fetch("terraform", []).each do |terraform|
+            required_providers = terraform.fetch("required_providers", {})
+            required_providers.each do |provider|
+              provider.each do |name, details|
+                dependency_set << build_terraform_dependency(file, name, details, true)
+              end
+            end
           end
         end
 
         terragrunt_files.each do |file|
-          modules = parsed_file(file).fetch("terragrunt", []).first || {}
-          modules = modules.fetch("terraform", [])
+          modules = parsed_file(file).fetch("terraform", [])
           modules.each do |details|
             next unless details["source"]
 
@@ -38,17 +49,20 @@ module Dependabot
           end
         end
 
-        dependency_set.dependencies
+        dependency_set.dependencies.sort_by(&:name)
       end
 
       private
 
-      def build_terraform_dependency(file, name, details)
-        details = details.first
+      def build_terraform_dependency(file, name, details, provider)
+        details = details.is_a?(Array) ? details.first : details
 
-        source = source_from(details)
-        dep_name =
-          source[:type] == "registry" ? source[:module_identifier] : name
+        source = source_from(details, provider)
+        dep_name = case source[:type]
+                   when "registry" then source[:module_identifier]
+                   when "provider" then details["source"]
+                   else name
+                   end
         version_req = details["version"]&.strip
         version =
           if source[:type] == "git" then version_from_ref(source[:ref])
@@ -69,7 +83,7 @@ module Dependabot
       end
 
       def build_terragrunt_dependency(file, details)
-        source = source_from(details)
+        source = source_from(details, false)
         dep_name =
           if Source.from_url(source[:url])
             Source.from_url(source[:url]).repo
@@ -93,7 +107,7 @@ module Dependabot
       end
 
       # Full docs at https://www.terraform.io/docs/modules/sources.html
-      def source_from(details_hash)
+      def source_from(details_hash, provider)
         raw_source = details_hash.fetch("source")
         bare_source = get_proxied_source(raw_source)
 
@@ -104,17 +118,23 @@ module Dependabot
           when :github, :bitbucket, :git
             git_source_details_from(bare_source)
           when :registry
-            registry_source_details_from(bare_source)
+            registry_source_details_from(bare_source, provider)
           end
 
         source_details[:proxy_url] = raw_source if raw_source != bare_source
         source_details
       end
 
-      def registry_source_details_from(source_string)
+      def registry_source_details_from(source_string, provider)
         parts = source_string.split("//").first.split("/")
 
-        if parts.count == 3
+        if provider && parts.count == 2
+          {
+            "type": "provider",
+            "registry_hostname": "registry.terraform.io",
+            "module_identifier": source_string
+          }
+        elsif parts.count == 3
           {
             type: "registry",
             registry_hostname: "registry.terraform.io",
@@ -210,30 +230,51 @@ module Dependabot
       end
       # rubocop:enable Metrics/PerceivedComplexity
 
+      # == Returns:
+      # A Hash representing each module found in the specified file
+      #
+      # E.g.
+      # {
+      #   "module" => {
+      #     {
+      #       "consul" => [
+      #         {
+      #           "source"=>"consul/aws",
+      #           "version"=>"0.1.0"
+      #         }
+      #       ]
+      #     }
+      #   },
+      #   "terragrunt"=>[
+      #     {
+      #       "include"=>[{ "path"=>"${find_in_parent_folders()}" }],
+      #       "terraform"=>[{ "source" => "git::git@github.com:gruntwork-io/modules-example.git//consul?ref=v0.0.2" }]
+      #     }
+      #   ],
+      # }
       def parsed_file(file)
         @parsed_buildfile ||= {}
-        @parsed_buildfile[file.name] ||=
-          SharedHelpers.in_a_temporary_directory do
-            File.write("tmp.tf", file.content)
-
-            command = "#{terraform_parser_path} -reverse < tmp.tf"
-            start = Time.now
-            stdout, stderr, process = Open3.capture3(command)
-            time_taken = Time.now - start
-
-            unless process.success?
-              raise SharedHelpers::HelperSubprocessFailed.new(
-                message: stderr,
-                error_context: {
-                  command: command,
-                  time_taken: time_taken,
-                  process_exit_value: process.to_s
-                }
-              )
-            end
-
-            JSON.parse(stdout)
+        @parsed_buildfile[file.name] ||= SharedHelpers.in_a_temporary_directory do
+          File.write("tmp.tf", file.content)
+
+          command = "#{terraform_hcl2_parser_path} < tmp.tf"
+          start = Time.now
+          stdout, stderr, process = Open3.capture3(command)
+          time_taken = Time.now - start
+
+          unless process.success?
+            raise SharedHelpers::HelperSubprocessFailed.new(
+              message: stderr,
+              error_context: {
+                command: command,
+                time_taken: time_taken,
+                process_exit_value: process.to_s
+              }
+            )
           end
+
+          JSON.parse(stdout)
+        end
       rescue SharedHelpers::HelperSubprocessFailed => e
         msg = e.message.strip
         raise Dependabot::DependencyFileNotParseable.new(file.path, msg)
@@ -244,19 +285,16 @@ module Dependabot
         Pathname.new(File.join(helper_bin_dir, "json2hcl")).cleanpath.to_path
       end
 
+      def terraform_hcl2_parser_path
+        helper_bin_dir = File.join(native_helpers_root, "terraform/bin")
+        Pathname.new(File.join(helper_bin_dir, "hcl2json")).cleanpath.to_path
+      end
+
       def native_helpers_root
         default_path = File.join(__dir__, "../../../helpers/install-dir")
         ENV.fetch("DEPENDABOT_NATIVE_HELPERS_PATH", default_path)
       end
 
-      def terraform_files
-        dependency_files.select { |f| f.name.end_with?(".tf") }
-      end
-
-      def terragrunt_files
-        dependency_files.select { |f| f.name.end_with?(".tfvars") }
-      end
-
       def check_required_files
         return if [*terraform_files, *terragrunt_files].any?
 

data/lib/dependabot/terraform/file_selector.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module FileSelector
+  private
+
+  def terraform_files
+    dependency_files.select { |f| f.name.end_with?(".tf") }
+  end
+
+  def terragrunt_files
+    dependency_files.select { |f| terragrunt_file?(f.name) }
+  end
+
+  def terragrunt_file?(file_name)
+    file_name != ".terraform.lock.hcl" && file_name.end_with?(".hcl")
+  end
+end

data/lib/dependabot/terraform/file_updater.rb

@@ -3,12 +3,15 @@
 require "dependabot/file_updaters"
 require "dependabot/file_updaters/base"
 require "dependabot/errors"
+require "dependabot/terraform/file_selector"
 
 module Dependabot
   module Terraform
     class FileUpdater < Dependabot::FileUpdaters::Base
+      include FileSelector
+
       def self.updated_files_regex
-        [/\.tf$/, /\.tfvars$/]
+        [/\.tf$/, /\.hcl$/]
       end
 
       def updated_dependency_files
@@ -44,7 +47,7 @@ module Dependabot
           case new_req[:source][:type]
           when "git"
             update_git_declaration(new_req, old_req, content, file.name)
-          when "registry"
+          when "registry", "provider"
             update_registry_declaration(new_req, old_req, content)
           else
             raise "Don't know how to update a #{new_req[:source][:type]} "\
@@ -87,14 +90,6 @@ module Dependabot
         dependency_files.select { |file| filenames.include?(file.name) }
       end
 
-      def terraform_files
-        dependency_files.select { |f| f.name.end_with?(".tf") }
-      end
-
-      def terragrunt_files
-        dependency_files.select { |f| f.name.end_with?(".tfvars") }
-      end
-
       def check_required_files
         return if [*terraform_files, *terragrunt_files].any?
 
@@ -113,7 +108,7 @@ module Dependabot
       def git_declaration_regex(filename)
         # For terragrunt dependencies there's not a lot we can base the
         # regex on. Just look for declarations within a `terraform` block
-        return /terraform\s*\{(?:(?!^\}).)*/m if filename.end_with?(".tfvars")
+        return /terraform\s*\{(?:(?!^\}).)*/m if terragrunt_file?(filename)
 
         # For modules we can do better - filter for module blocks that use the
         # name of the dependency

data/lib/dependabot/terraform/metadata_finder.rb

@@ -15,6 +15,7 @@ module Dependabot
         case new_source_type
         when "git" then find_source_from_git_url
         when "registry" then find_source_from_registry_details
+        when "provider" then find_source_from_provider_details
         else raise "Unexpected source type: #{new_source_type}"
         end
       end
@@ -36,29 +37,11 @@ module Dependabot
         Source.from_url(url)
       end
 
-      # Registry API docs:
-      # https://www.terraform.io/docs/registry/api.html
       def find_source_from_registry_details
         info = dependency.requirements.map { |r| r[:source] }.compact.first
-
         hostname = info[:registry_hostname] || info["registry_hostname"]
 
-        # TODO: Implement service discovery for custom registries
-        return unless hostname == "registry.terraform.io"
-
-        url = "https://registry.terraform.io/v1/modules/"\
-              "#{dependency.name}/#{dependency.version}"
-
-        response = Excon.get(
-          url,
-          idempotent: true,
-          **SharedHelpers.excon_defaults
-        )
-
-        raise "Response from registry was #{response.status}" unless response.status == 200
-
-        source_url = JSON.parse(response.body).fetch("source")
-        Source.from_url(source_url) if source_url
+        RegistryClient.new(hostname: hostname).source(dependency: dependency)
       end
     end
   end

data/lib/dependabot/terraform/registry_client.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require "dependabot/dependency"
+require "dependabot/source"
+require "dependabot/terraform/version"
+
+module Dependabot
+  module Terraform
+    # Terraform::RegistryClient is a basic API client to interact with a
+    # terraform registry: https://www.terraform.io/docs/registry/api.html
+    class RegistryClient
+      PUBLIC_HOSTNAME = "registry.terraform.io"
+
+      def initialize(hostname:)
+        @hostname = hostname
+      end
+
+      # Fetch all the versions of a provider, and return a Version
+      # representation of them.
+      #
+      # @param identifier [String] the identifier for the dependency, i.e:
+      # "hashicorp/aws"
+      # @return [Array<Dependabot::Terraform::Version>]
+      # @raise [RuntimeError] when the versions cannot be retrieved
+      def all_provider_versions(identifier:)
+        # TODO: Implement service discovery for custom registries
+        return [] unless hostname == PUBLIC_HOSTNAME
+
+        response = get(endpoint: "providers/#{identifier}/versions")
+
+        JSON.parse(response).
+          fetch("versions").
+          map { |release| version_class.new(release.fetch("version")) }
+      end
+
+      # Fetch all the versions of a module, and return a Version
+      # representation of them.
+      #
+      # @param identifier [String] the identifier for the dependency, i.e:
+      # "hashicorp/consul/aws"
+      # @return [Array<Dependabot::Terraform::Version>]
+      # @raise [RuntimeError] when the versions cannot be retrieved
+      def all_module_versions(identifier:)
+        # TODO: Implement service discovery for custom registries
+        return [] unless hostname == PUBLIC_HOSTNAME
+
+        response = get(endpoint: "modules/#{identifier}/versions")
+
+        JSON.parse(response).
+          fetch("modules").first.fetch("versions").
+          map { |release| version_class.new(release.fetch("version")) }
+      end
+
+      # Fetch the "source" for a module or provider. We use the API to fetch
+      # the source for a dependency, this typically points to a source code
+      # repository, and then instantiate a Dependabot::Source object that we
+      # can use to fetch Metadata about a specific version of the dependency.
+      #
+      # @param dependency [Dependabot::Dependency] the dependency who's source
+      # we're attempting to find
+      # @return Dependabot::Source
+      # @raise [RuntimeError] when the source cannot be retrieved
+      def source(dependency:)
+        # TODO: Implement service discovery for custom registries
+        return unless hostname == PUBLIC_HOSTNAME
+
+        endpoint = if dependency.requirements.first[:source][:type] == "registry"
+                     "modules/#{dependency.name}/#{dependency.version}"
+                   elsif dependency.source[:type] == "provider"
+                     "providers/#{dependency.name}/#{dependency.version}"
+                   else
+                     raise "Invalid source type"
+                   end
+        response = get(endpoint: endpoint)
+
+        source_url = JSON.parse(response).fetch("source")
+        Source.from_url(source_url) if source_url
+      end
+
+      private
+
+      attr_reader :hostname
+
+      def get(endpoint:)
+        url = "https://#{hostname}/v1/#{endpoint}"
+
+        response = Excon.get(
+          url,
+          idempotent: true,
+          **SharedHelpers.excon_defaults
+        )
+
+        raise "Response from registry was #{response.status}" unless response.status == 200
+
+        response.body
+      end
+
+      def version_class
+        Version
+      end
+    end
+  end
+end

data/lib/dependabot/terraform/requirements_updater.rb

@@ -72,7 +72,7 @@ module Dependabot
         requirements.map do |req|
           case req.dig(:source, :type)
           when "git" then update_git_requirement(req)
-          when "registry" then update_registry_requirement(req)
+          when "registry", "provider" then update_registry_requirement(req)
           else req
           end
         end

data/lib/dependabot/terraform/update_checker.rb

@@ -6,6 +6,7 @@ require "dependabot/git_commit_checker"
 require "dependabot/terraform/requirements_updater"
 require "dependabot/terraform/requirement"
 require "dependabot/terraform/version"
+require "dependabot/terraform/registry_client"
 
 module Dependabot
   module Terraform
@@ -13,6 +14,7 @@ module Dependabot
       def latest_version
         return latest_version_for_git_dependency if git_dependency?
         return latest_version_for_registry_dependency if registry_dependency?
+        return latest_version_for_provider_dependency if provider_dependency?
         # Other sources (mercurial, path dependencies) just return `nil`
       end
 
@@ -65,34 +67,40 @@ module Dependabot
 
         return @latest_version_for_registry_dependency if @latest_version_for_registry_dependency
 
-        versions = all_registry_versions
+        versions = all_module_versions
         versions.reject!(&:prerelease?) unless wants_prerelease?
         versions.reject! { |v| ignore_requirements.any? { |r| r.satisfied_by?(v) } }
 
         @latest_version_for_registry_dependency = versions.max
       end
 
-      def all_registry_versions
-        hostname = dependency_source_details.fetch(:registry_hostname)
+      def all_module_versions
         identifier = dependency_source_details.fetch(:module_identifier)
+        registry_client.all_module_versions(identifier: identifier)
+      end
+
+      def all_provider_versions
+        identifier = dependency_source_details.fetch(:module_identifier)
+        registry_client.all_provider_versions(identifier: identifier)
+      end
 
-        # TODO: Implement service discovery for custom registries
-        return [] unless hostname == "registry.terraform.io"
+      def registry_client
+        @registry_client ||= begin
+          hostname = dependency_source_details.fetch(:registry_hostname)
+          RegistryClient.new(hostname: hostname)
+        end
+      end
 
-        url = "https://registry.terraform.io/v1/modules/"\
-              "#{identifier}/versions"
+      def latest_version_for_provider_dependency
+        return unless provider_dependency?
 
-        response = Excon.get(
-          url,
-          idempotent: true,
-          **SharedHelpers.excon_defaults
-        )
+        return @latest_version_for_provider_dependency if @latest_version_for_provider_dependency
 
-        raise "Response from registry was #{response.status}" unless response.status == 200
+        versions = all_provider_versions
+        versions.reject!(&:prerelease?) unless wants_prerelease?
+        versions.reject! { |v| ignore_requirements.any? { |r| r.satisfied_by?(v) } }
 
-        JSON.parse(response.body).
-          fetch("modules").first.fetch("versions").
-          map { |release| version_class.new(release.fetch("version")) }
+        @latest_version_for_provider_dependency = versions.max
       end
 
       def wants_prerelease?
@@ -160,6 +168,12 @@ module Dependabot
         dependency_source_details.fetch(:type) == "registry"
       end
 
+      def provider_dependency?
+        return false if dependency_source_details.nil?
+
+        dependency_source_details.fetch(:type) == "provider"
+      end
+
       def dependency_source_details
         sources =
           dependency.requirements.map { |r| r.fetch(:source) }.uniq.compact

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dependabot-terraform
 version: !ruby/object:Gem::Version
-  version: 0.146.0
+  version: 0.148.1
 platform: ruby
 authors:
 - Dependabot
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-11 00:00:00.000000000 Z
+date: 2021-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dependabot-common
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.146.0
+        version: 0.148.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.146.0
+        version: 0.148.1
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
@@ -100,14 +100,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 1.15.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.14.0
+        version: 1.15.0
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -189,8 +189,10 @@ files:
 - lib/dependabot/terraform.rb
 - lib/dependabot/terraform/file_fetcher.rb
 - lib/dependabot/terraform/file_parser.rb
+- lib/dependabot/terraform/file_selector.rb
 - lib/dependabot/terraform/file_updater.rb
 - lib/dependabot/terraform/metadata_finder.rb
+- lib/dependabot/terraform/registry_client.rb
 - lib/dependabot/terraform/requirement.rb
 - lib/dependabot/terraform/requirements_updater.rb
 - lib/dependabot/terraform/update_checker.rb