cfndsl 1.0.6 → 1.3.1

data/README.md

@@ -6,25 +6,23 @@ cfndsl
 [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/cfndsl/cfndsl?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) [![Join the chat at https://gitter.im/cfndsl/cfndsl](https://badges.gitter.im/cfndsl/cfndsl.svg)](https://gitter.im/cfndsl/cfndsl?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 [AWS Cloudformation](http://docs.amazonwebservices.com/AWSCloudFormation/latest/UserGuide/GettingStarted.html) templates are an incredibly powerful way to build
-sets of resources in Amazon's AWS environment. Unfortunately, because
-they are specified in JSON, they are also difficult to write and
-maintain:
+sets of resources in Amazon's AWS environment.
+Unfortunately, because they are specified in JSON or YAML, they are also difficult to write and maintain:
 
-* JSON does not allow comments
+* JSON does not allow comments, although CloudFormation now supports YAML
 
-* All structures are JSON, so it is sometimes easy for a person
-  reading a template to get lost.
+* JSON/YAML templates do not allow re-usable logic
 
-* References and internal functions have a particularly unpleasant syntax.
+* It is easy for a person reading a template to miss the association and dependencies between entries
 
+* References and internal functions have a particularly unpleasant syntax.
 
-The cnfdsl gem provides a simple DSL that allows you to write equivalent
-templates in a more friendly language and generate the correct json
-templates by running ruby.
+The cnfdsl gem provides a DSL that allows you to write templates using friendly Ruby logic to generate and validate
+  cloudformation templates.
 
 ## Getting Started
 
-ruby version > 2.4 is required to run cfndsl, you should look at using rbenv example for installing with rbenv
+ruby version > 2.7 is required to run cfndsl, you should look at using rbenv example for installing with rbenv
 
     rbenv exec gem install cfndsl
 

data/Rakefile

@@ -73,6 +73,8 @@ task :bump, :type do |_, args|
 
   GitHubChangelogGenerator::RakeTask.new :changelog do |config|
     config.future_release = version
+    config.user = 'cfndsl'
+    config.project = 'cfndsl'
   end
 
   puts "Bumping gem from version #{CfnDsl::VERSION} to #{version} as a '#{type.capitalize}' release"
@@ -81,7 +83,7 @@ task :bump, :type do |_, args|
   Rake::Task[:changelog].execute
 
   contents         = File.read version_path
-  updated_contents = contents.gsub(/'[0-9\.]+'/, "'#{version}'")
+  updated_contents = contents.gsub(/'[0-9.]+'/, "'#{version}'")
   File.write(version_path, updated_contents)
 
   puts 'Commiting version update'

data/cfndsl.gemspec

@@ -17,10 +17,10 @@ Gem::Specification.new do |s|
   s.test_files            = s.files.grep(%r{^(test|spec|features)/})
   s.require_paths         = ['lib']
   s.bindir                = 'exe'
-  s.required_ruby_version = '~> 2.4'
+  s.required_ruby_version = '>= 2.7'
 
   s.executables << 'cfndsl'
 
-  s.add_development_dependency 'bundler', '~> 2.1'
+  s.add_development_dependency 'bundler', '~> 2.2'
   s.add_runtime_dependency 'hana', '~> 1.3'
 end

data/lib/cfndsl/aws/patches/500_BadTagsv16.2.0_patch.json

@@ -0,0 +1,23 @@
+{
+  "broken": "16.2.0",
+  "ResourceTypes": {
+    "AWS::IoT::ProvisioningTemplate": {
+      "patch": {
+        "operations": [
+          {
+            "op": "replace",
+            "path": "/Properties/Tags",
+            "value": {
+              "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iot-provisioningtemplate.html#cfn-iot-provisioningtemplate-tags",
+              "UpdateType": "Mutable",
+              "Required": false,
+              "Type": "List",
+              "ItemType": "Tag"
+            }
+          }
+        ]
+      }
+    }
+  }
+}
+

data/lib/cfndsl/aws/patches/900_SageMakerTags_patch.json

@@ -0,0 +1,41 @@
+{
+  "broken": "22.0.0",
+  "ResourceTypes": {
+    "AWS::SageMaker::Device": {
+      "patch": {
+        "operations": [
+          {
+            "op": "replace",
+            "path": "/Properties/Tags",
+            "value": {
+              "Type": "List",
+              "ItemType": "Tag",
+              "Required": false,
+              "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sagemaker-device.html#cfn-sagemaker-device-tags",
+              "UpdateType": "Mutable"
+            }
+          }
+        ],
+        "description": "Fix SageMaker Device to be List of Tag"
+      }
+    },
+    "AWS::SageMaker::DeviceFleet": {
+      "patch": {
+        "operations": [
+          {
+            "op": "replace",
+            "path": "/Properties/Tags",
+            "value": {
+              "Type": "List",
+              "ItemType": "Tag",
+              "Required": false,
+              "Documentation": "http://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sagemaker-device.html#cfn-sagemaker-device-tags",
+              "UpdateType": "Mutable"
+            }
+          }
+        ],
+        "description": "Fix SageMaker Device to be List of Tag"
+      }
+    }
+  }
+}

data/lib/cfndsl/cloudformation.rb

@@ -43,9 +43,10 @@ module CfnDsl
         params.load_file file
       when :raw
         file_parts = file.split('=')
-        if file_parts[1].downcase == 'true'
+        case file_parts[1].downcase
+        when 'true'
           params.set_param(file_parts[0], true)
-        elsif file_parts[1].downcase == 'false'
+        when 'false'
           params.set_param(file_parts[0], false)
         else
           params.set_param(*file.split('='))

data/lib/cfndsl/globals.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'version'
+require 'fileutils'
 
 # Global variables to adjust CfnDsl behavior
 module CfnDsl

data/lib/cfndsl/jsonable.rb

@@ -87,7 +87,7 @@ module CfnDsl
     def FnSub(string, substitutions = nil)
       raise ArgumentError, 'The first argument passed to Fn::Sub must be a string' unless string.is_a? String
 
-      refs = string.scan(FN_SUB_SCANNER).map(&:first)
+      refs = string.scan(FN_SUB_SCANNER).map(&:first).map { |r| r.split('.', 2).first }
 
       if substitutions
         raise ArgumentError, 'The second argument passed to Fn::Sub must be a Hash' unless substitutions.is_a? Hash
@@ -136,12 +136,13 @@ module CfnDsl
       check_names
       hash = {}
       instance_variables.each do |var|
-        name = var[1..-1]
+        name = var[1..]
 
-        if name =~ /^__/
+        case name
+        when /^__/
           # if a variable starts with double underscore, strip one off
-          name = name[1..-1]
-        elsif name =~ /^_/
+          name = name[1..]
+        when /^_/
           # Hide variables that start with single underscore
           name = nil
         end

data/lib/cfndsl/orchestration_template.rb

@@ -22,7 +22,7 @@ module CfnDsl
   # Handles the overall template object
   # rubocop:disable Metrics/ClassLength
   class OrchestrationTemplate < JSONable
-    dsl_attr_setter :AWSTemplateFormatVersion, :Description, :Metadata, :Transform
+    dsl_attr_setter :AWSTemplateFormatVersion, :Description, :Metadata, :Transform, :Hooks
     dsl_content_object :Condition, :Parameter, :Output, :Resource, :Mapping, :Rule
 
     GLOBAL_REFS = {
@@ -83,7 +83,6 @@ module CfnDsl
         resource_name
       end
 
-      # rubocop:disable Metrics/PerceivedComplexity
       def create_array_property_def(resource, pname, pclass, info)
         singular_name = CfnDsl::Plurals.singularize pname
         plural_name = singular_name == pname ? CfnDsl::Plurals.pluralize(pname) : pname
@@ -111,7 +110,6 @@ module CfnDsl
         # Singular form understands concatenation and Fn::If property
         create_singular_property_def(resource, pname, pclass, singular_name) if singular_name
       end
-      # rubocop:enable Metrics/PerceivedComplexity
 
       def create_resource_accessor(accessor, resource, type)
         class_eval do
@@ -245,7 +243,7 @@ module CfnDsl
       end
     end
 
-    # rubocop:disable Metrics/PerceivedComplexity
+    # rubocop:disable Metrics/CyclomaticComplexity
     def _check_refs(container_name, method, source_containers)
       container = instance_variable_get("@#{container_name}s")
       return [] unless container
@@ -279,7 +277,7 @@ module CfnDsl
 
       invalids
     end
-    # rubocop:enable Metrics/PerceivedComplexity
+    # rubocop:enable Metrics/CyclomaticComplexity
 
     def validate
       errors = check_refs || []

data/lib/cfndsl/plurals.rb

@@ -20,7 +20,7 @@ module CfnDsl
     @singles = @plurals.invert
 
     def pluralize(name)
-      @plurals.fetch(name.to_s) { |key| key + 's' }
+      @plurals.fetch(name.to_s) { |key| "#{key}s" }
     end
 
     def singularize(name)
@@ -29,7 +29,7 @@ module CfnDsl
         when /List$/
           key[0..-5]
         when /ies$/
-          key[0..-4] + 'y'
+          "#{key[0..-4]}y"
         when /s$/
           key[0..-2]
         else

data/lib/cfndsl/rake_task.rb

@@ -79,7 +79,7 @@ module CfnDsl
     #    then any existing file is considered sufficient, and 'latest' is the version used for downloading
     #
     # @todo Add capability to provide a user spec/patches dir
-    def specification(name: nil, file:, version: nil)
+    def specification(file:, name: nil, version: nil)
       if name
         desc 'Update Resource Specification' unless ::Rake.application.last_description
         task name, [:cfn_spec_version] => file
@@ -218,7 +218,7 @@ module CfnDsl
     end
 
     def verbose
-      (Rake.verbose? || cfndsl_opts&.fetch(:verbose, false)) && STDERR
+      (Rake.verbose? || cfndsl_opts&.fetch(:verbose, false)) && $stderr
     end
 
     def generate(opts)
@@ -238,8 +238,8 @@ module CfnDsl
       verbose&.puts("Writing to #{type}")
     end
 
-    def outputter(opts)
-      opts[:output].nil? ? yield(STDOUT) : file_output(opts[:output]) { |f| yield f }
+    def outputter(opts, &block)
+      opts[:output].nil? ? yield($stdout) : file_output(opts[:output], &block)
     end
 
     def model(filename)
@@ -253,8 +253,8 @@ module CfnDsl
       cfndsl_opts.fetch(:extras, [])
     end
 
-    def file_output(path)
-      File.open(File.expand_path(path), 'w') { |f| yield f }
+    def file_output(path, &block)
+      File.open(File.expand_path(path), 'w', &block)
     end
   end
   # rubocop:enable Metrics/ClassLength

data/lib/cfndsl/ref_check.rb

@@ -10,7 +10,7 @@ module RefCheck
   end
 
   # Build up a set of references.
-  # rubocop:disable Metrics/PerceivedComplexity
+  # rubocop:disable Metrics/CyclomaticComplexity
   def build_references(refs = [], origin = nil, method = :all_refs)
     if respond_to?(method)
       send(method).each do |ref|
@@ -30,7 +30,7 @@ module RefCheck
 
     refs
   end
-  # rubocop:enable Metrics/PerceivedComplexity
+  # rubocop:enable Metrics/CyclomaticComplexity
 
   def ref_children
     []

data/lib/cfndsl/resources.rb

@@ -5,7 +5,7 @@ require_relative 'jsonable'
 module CfnDsl
   # Handles Resource objects
   class ResourceDefinition < JSONable
-    dsl_attr_setter :Type, :DependsOn, :DeletionPolicy, :Condition, :Metadata
+    dsl_attr_setter :Type, :UpdateReplacePolicy, :DeletionPolicy, :Condition, :Metadata
     dsl_content_object :Property, :UpdatePolicy, :CreationPolicy
 
     def add_tag(name, value, propagate = nil)
@@ -16,6 +16,23 @@ module CfnDsl
       end
     end
 
+    # DependsOn can be a single value or a list
+    def DependsOn(value)
+      case @DependsOn
+      when nil
+        @DependsOn = value
+      when Array
+        @DependsOn << value
+      else
+        @DependsOn = [@DependsOn, value]
+      end
+      if @DependsOn.is_a?(Array)
+        @DependsOn.flatten!
+        @DependsOn.uniq!
+      end
+      @DependsOn
+    end
+
     def condition_refs
       [@Condition].flatten.compact.map(&:to_s)
     end

data/lib/cfndsl/runner.rb

@@ -7,7 +7,6 @@ require_relative 'globals'
 
 module CfnDsl
   # Runner class to handle commandline invocation
-  # rubocop:disable Metrics/ClassLength
   class Runner
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
     def self.invoke!
@@ -118,14 +117,14 @@ module CfnDsl
       end
 
       filename = File.expand_path(ARGV[0])
-      verbose = options[:verbose] && STDERR
+      verbose = options[:verbose] && $stderr
 
       verbose.puts "Using specification file #{CfnDsl.specification_file}" if verbose
 
       require_relative 'cloudformation'
       model = CfnDsl.eval_file_with_extras(filename, options[:extras], verbose)
 
-      output = STDOUT
+      output = $stdout
       if options[:output] != '-'
         verbose.puts("Writing to #{options[:output]}") if verbose
         output = File.open(File.expand_path(options[:output]), 'w')
@@ -143,4 +142,3 @@ module CfnDsl
   end
   # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
 end
-# rubocop:enable Metrics/ClassLength

data/lib/cfndsl/specification.rb

@@ -70,20 +70,18 @@ module CfnDsl
         next unless %w[ResourceTypes PropertyTypes].include?(top_level_type)
 
         patches.each_pair do |property_type_name, patch_details|
-          begin
-            applies_to = spec[top_level_type]
-            unless property_type_name == 'patch'
-              # Patch applies within a specific property type
-              applies_to = applies_to[property_type_name]
-              patch_details = patch_details['patch']
-            end
-
-            Hana::Patch.new(patch_details['operations']).apply(applies_to) if patch_required?(patch_details)
-          rescue Hana::Patch::MissingTargetException => e
-            raise "Failed specification patch #{top_level_type} #{property_type_name} from #{from_file}" if fail_patches
-
-            warn "Ignoring failed specification patch #{top_level_type} #{property_type_name} from #{from_file} - #{e.class.name}:#{e.message}"
+          applies_to = spec[top_level_type]
+          unless property_type_name == 'patch'
+            # Patch applies within a specific property type
+            applies_to = applies_to[property_type_name]
+            patch_details = patch_details['patch']
           end
+
+          Hana::Patch.new(patch_details['operations']).apply(applies_to) if patch_required?(patch_details)
+        rescue Hana::Patch::MissingTargetException => e
+          raise "Failed specification patch #{top_level_type} #{property_type_name} from #{from_file}" if fail_patches
+
+          warn "Ignoring failed specification patch #{top_level_type} #{property_type_name} from #{from_file} - #{e.class.name}:#{e.message}"
         end
       end
     end

data/lib/cfndsl/types.rb

@@ -16,7 +16,7 @@ module CfnDsl
       { 'Resources' => resources, 'Types' => types, 'Version' => spec.version, 'File' => spec.file }
     end
 
-    # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/MethodLength
+    # rubocop:disable Metrics/AbcSize, Metrics/MethodLength, Metrics/CyclomaticComplexity
     def self.extract_resources(spec)
       spec.each_with_object({}) do |(resource_name, resource_info), resources|
         properties = resource_info['Properties'].each_with_object({}) do |(property_name, property_info), extracted|
@@ -43,7 +43,8 @@ module CfnDsl
             # resource name for uniqueness and connection
             property_type = resource_name.split('::').join + property_info['Type']
           else
-            warn "could not extract resource type from #{resource_name}"
+            warn "could not extract resource type for property #{property_name} from #{resource_name}, assuming Json"
+            property_type = 'Json'
           end
           extracted[property_name] = property_type
           extracted
@@ -52,9 +53,9 @@ module CfnDsl
         resources
       end
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/MethodLength
+    # rubocop:enable Metrics/AbcSize, Metrics/MethodLength
 
-    # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    # rubocop:disable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/MethodLength
     def self.extract_types(spec)
       primitive_types = {
         'String' => 'String',
@@ -99,6 +100,8 @@ module CfnDsl
               nested_prop_type =
                 if nested_prop_info['ItemType'] == 'Tag'
                   ['Tag']
+                elsif (nested_prop_info['ItemType'] == 'Json') && (nested_prop_info['Type'] == 'List')
+                  ['Json']
                 else
                   Array(root_resource_name + nested_prop_info['ItemType'])
                 end
@@ -106,7 +109,8 @@ module CfnDsl
             elsif nested_prop_info['Type']
               nested_prop_type = root_resource_name + nested_prop_info['Type']
             else
-              warn "could not extract property type from #{property_name}"
+              warn "could not extract property type for #{nested_prop_name} from #{property_name}, assuming Json"
+              nested_prop_type = 'Json'
               p nested_prop_info
             end
             extracted[nested_prop_name] = nested_prop_type
@@ -117,9 +121,9 @@ module CfnDsl
         types
       end
     end
-    # rubocop:enable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity, Metrics/MethodLength
+    # rubocop:enable Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/MethodLength
 
-    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity
     def self.included(type_def)
       types_list = extract_from_resource_spec
       type_def.const_set('Types_Internal', types_list)