cfndsl 0.12.11 → 0.13.0

data/lib/cfndsl/globals.rb

@@ -1,10 +1,22 @@
+# Global variables to adjust CfnDsl behavior
 module CfnDsl
-  # Set global variables
-  class Globals
-    class << self
-      def reserved_items
-        %w[Resource Parameter Output].freeze
-      end
-    end
+  module_function
+
+  def disable_binding
+    @disable_binding = true
+  end
+
+  def disable_binding?
+    @disable_binding
+  end
+
+  def specification_file(file = nil)
+    @specification_file = file if file
+    @specification_file ||= File.join(ENV['HOME'], '.cfndsl/resource_specification.json')
+    @specification_file
+  end
+
+  def reserved_items
+    %w[Resource Parameter Output].freeze
   end
 end

data/lib/cfndsl/orchestration_template.rb

@@ -28,7 +28,7 @@ module CfnDsl
           resource_name = create_resource_def(resource, info)
           parts = resource.split('::')
           until parts.empty?
-            break if Globals.reserved_items.include? parts.first
+            break if CfnDsl.reserved_items.include? parts.first
             abreve_name = parts.join('_')
             if accessors.key? abreve_name
               accessors.delete abreve_name # Delete potentially ambiguous names

data/lib/cfndsl/patches.rb

@@ -0,0 +1,98 @@
+module CfnDsl
+  # Module for handling inconsistencies in the published resource specification from AWS
+  module Patches
+    # Missing/malformed resources from the resource specification
+    # rubocop:disable Metrics/MethodLength
+    def self.resources
+      {
+        'AWS::Serverless::Function' => {
+          'Properties' => {
+            'Handler'     => { 'PrimitiveType' => 'String' },
+            'Runtime'     => { 'PrimitiveType' => 'String' },
+            'CodeUri'     => { 'PrimitiveType' => 'String' },
+            'Description' => { 'PrimitiveType' => 'String' },
+            'MemorySize'  => { 'PrimitiveType' => 'Integer' },
+            'Timeout'     => { 'PrimitiveType' => 'Integer' },
+            'Environment' => { 'PrimitiveType' => 'Json' },
+            'Events'      => { 'PrimitiveType' => 'Json' },
+            'Policies'    => { 'Type' => 'List', 'ItemType' => 'Policy' }
+          }
+        },
+        'AWS::Serverless::Api' => {
+          'Properties' => {
+            'StageName'           => { 'PrimitiveType' => 'String' },
+            'DefinitionUri'       => { 'PrimitiveType' => 'String' },
+            'CacheClusterEnabled' => { 'PrimitiveType' => 'Boolean' },
+            'CacheClusterSize'    => { 'PrimitiveType' => 'String' },
+            'Variables'           => { 'PrimitiveType' => 'Json' }
+          }
+        },
+        'AWS::Serverless::SimpleTable' => {
+          'Properties' => {
+            'PrimaryKey' => { 'Type' => 'PrimaryKey' },
+            'ProvisionedThroughput' => { 'Type' => 'ProvisionedThroughput' }
+          }
+        },
+        'AWS::SSM::Parameter' => {
+          'Properties' => {
+            'Name'        => { 'PrimitiveType' => 'String' },
+            'Description' => { 'PrimitiveType' => 'String' },
+            'Type'        => { 'PrimitiveType' => 'String' },
+            'Value'       => { 'PrimitiveType' => 'String' }
+          }
+        },
+        'AWS::EC2::VPNGatewayConnection' => {
+          'Properties' => {
+            'Type' => { 'PrimitiveType' => 'String' },
+            'Tags' => { 'Type' => 'List', 'ItemType' => 'Tag' }
+          }
+        },
+        'AWS::EC2::EIPAssociation' => {
+          'Properties' => {
+            'AllocationId'       => { 'PrimitiveType' => 'String' },
+            'EIP'                => { 'PrimitiveType' => 'String' },
+            'InstanceId'         => { 'PrimitiveType' => 'String' },
+            'NetworkInterfaceId' => { 'PrimitiveType' => 'String' },
+            'PrivateIpAddress'   => { 'PrimitiveType' => 'String' }
+          }
+        },
+        'AWS::Config::ConfigurationRecorder' => {
+          'Properties' => {
+            'Name'           => { 'PrimitiveType' => 'String' },
+            'RecordingGroup' => { 'Type' => 'RecordingGroup' },
+            'RoleARN'        => { 'PrimitiveType' => 'String' }
+          }
+        }
+      }
+    end
+
+    # Missing/malformed types from the resource specification
+    def self.types
+      {
+        'AWS::Serverless::SimpleTable.PrimaryKey' => {
+          'Properties' => {
+            'Name' => { 'PrimitiveType' => 'String' },
+            'Type' => { 'PrimitiveType' => 'String' }
+          }
+        },
+        'AWS::Serverless::SimpleTable.ProvisionedThroughput' => {
+          'Properties' => {
+            'ReadCapacityUnits'  => { 'PrimitiveType' => 'Integer' },
+            'WriteCapacityUnits' => { 'PrimitiveType' => 'Integer' }
+          }
+        },
+        'AWS::Serverless::Function.Policy' => {
+          'Properties' => {
+            'PolicyDocument' => { 'PrimitiveType' => 'Json' },
+            'PolicyName'     => { 'PrimitiveType' => 'String' }
+          }
+        },
+        'AWS::Cognito::IdentityPoolRoleAttachment.RulesConfigurationType' => {
+          'Properties' => {
+            'Rules' => { 'Type' => 'List', 'ItemType' => 'MappingRule' }
+          }
+        }
+      }
+    end
+  end
+end

data/lib/cfndsl/specification.rb

@@ -0,0 +1,89 @@
+module CfnDsl
+  # Helper module for bridging the gap between a static types file included in the repo
+  # and dynamically generating the types directly from the AWS specification
+  module Specification
+    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity
+    def self.extract_resources(spec)
+      spec.each_with_object({}) do |(resource_name, resource_info), resources|
+        properties = resource_info['Properties'].each_with_object({}) do |(property_name, property_info), extracted|
+          # some json incorrectly labelled as Type -> Json instead of PrimitiveType
+          # also, AWS now has the concept of Map which cfndsl had never defined
+          if property_info['Type'] == 'Map' || property_info['Type'] == 'Json'
+            property_type = 'Json'
+          elsif property_info['PrimitiveType']
+            property_type = property_info['PrimitiveType']
+          elsif property_info['PrimitiveItemType']
+            property_type = Array(property_info['PrimitiveItemType'])
+          elsif property_info['ItemType']
+            # Tag is a reused type, but not quite primitive
+            # and not all resources use the general form
+            property_type = if property_info['ItemType'] == 'Tag'
+                              ['Tag']
+                            else
+                              Array(resource_name.split('::').join + property_info['ItemType'])
+                            end
+          elsif property_info['Type']
+            # Special types (defined below) are joined with their parent
+            # resource name for uniqueness and connection
+            property_type = resource_name.split('::').join + property_info['Type']
+          else
+            warn "could not extract type from #{resource_name}"
+          end
+          extracted[property_name] = property_type
+          extracted
+        end
+        resources[resource_name] = { 'Properties' => properties }
+        resources
+      end
+    end
+
+    def self.extract_types(spec)
+      primitive_types = {
+        'String'    => 'String',
+        'Boolean'   => 'Boolean',
+        'Json'      => 'Json',
+        'Integer'   => 'Integer',
+        'Number'    => 'Number',
+        'Double'    => 'Double',
+        'Timestamp' => 'Timestamp',
+        'Long'      => 'Long'
+      }
+      spec.each_with_object(primitive_types) do |(property_name, property_info), types|
+        # In order to name things uniquely and allow for connections
+        # we extract the resource name from the property
+        # AWS::IAM::User.Policy becomes AWSIAMUserPolicy
+        root_resource = property_name.match(/(.*)\./)
+        root_resource_name = root_resource ? root_resource[1].gsub(/::/, '') : property_name
+        property_name = property_name.gsub(/::|\./, '')
+        next unless property_info['Properties']
+        properties = property_info['Properties'].each_with_object({}) do |(nested_prop_name, nested_prop_info), extracted|
+          if nested_prop_info['Type'] == 'Map' || nested_prop_info['Type'] == 'Json'
+            # The Map type and the incorrectly labelled Json type
+            nested_prop_type = 'Json'
+          elsif nested_prop_info['PrimitiveType']
+            nested_prop_type = nested_prop_info['PrimitiveType']
+          elsif nested_prop_info['PrimitiveItemType']
+            nested_prop_type = Array(nested_prop_info['PrimitiveItemType'])
+          elsif nested_prop_info['ItemType']
+            nested_prop_type = root_resource_name + nested_prop_info['ItemType']
+          elsif nested_prop_info['Type']
+            nested_prop_type = root_resource_name + nested_prop_info['Type']
+          else
+            warn "could not extract type from #{property_name}"
+          end
+          extracted[nested_prop_name] = nested_prop_type
+          extracted
+        end
+        types[property_name] = properties
+        types
+      end
+    end
+
+    def self.extract_from_resource_spec!
+      spec_file = JSON.parse File.read(CfnDsl.specification_file)
+      resources = extract_resources spec_file['ResourceTypes'].merge(Patches.resources)
+      types = extract_types spec_file['PropertyTypes'].merge(Patches.types)
+      { 'Resources' => resources, 'Types' => types }
+    end
+  end
+end

data/lib/cfndsl/types.rb

@@ -10,9 +10,12 @@ module CfnDsl
   module Types
     # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
     def self.included(type_def)
-      types_list = YAML.safe_load(File.open("#{File.dirname(__FILE__)}/#{type_def::TYPE_PREFIX}/types.yaml"))
+      types_list = if type_def::TYPE_PREFIX == 'aws'
+                     Specification.extract_from_resource_spec!
+                   else
+                     YAML.safe_load(File.open("#{File.dirname(__FILE__)}/#{type_def::TYPE_PREFIX}/types.yaml"))
+                   end
       type_def.const_set('Types_Internal', types_list)
-
       # Do a little sanity checking - all of the types referenced in Resources
       # should be represented in Types
       types_list['Resources'].keys.each do |resource_name|

data/lib/cfndsl/version.rb

@@ -1,3 +1,3 @@
 module CfnDsl
-  VERSION = '0.12.11'.freeze
+  VERSION = '0.13.0'.freeze
 end

data/spec/aws/serverless_spec.rb

@@ -33,7 +33,7 @@ describe CfnDsl::CloudFormationTemplate do
   end
 
   it 'Serverless_Api' do
-    template.Serverless_API(:Test) do
+    template.Serverless_Api(:Test) do
       StageName 'prod'
       DefinitionUri 'swagger.yml'
       CacheClusterEnabled false

data/spec/cfndsl_spec.rb

@@ -89,7 +89,7 @@ describe CfnDsl::CloudFormationTemplate do
       PolicyDocument(a: 7)
     end
 
-    expect(result2).to be_a(CfnDsl::AWS::Types::IAMEmbeddedPolicy)
+    expect(result2).to be_a(CfnDsl::AWS::Types::AWSIAMUserPolicy)
     expect(user.instance_variable_get('@Properties')['Policies'].value.length).to eq(2)
   end
 

data/spec/cli_spec.rb

@@ -13,6 +13,8 @@ describe 'cfndsl', type: :aruba do
           -D, --define "VARIABLE=VALUE"    Directly set local VARIABLE as VALUE
           -v, --verbose                    Turn on verbose ouptut
           -b, --disable-binding            Disable binding configuration
+          -s, --specification-file FILE    Location of Cloudformation Resource Specification file
+          -u, --update-specification       Update the Cloudformation Resource Specification file
           -h, --help                       Display this screen
     USAGE
   end
@@ -27,6 +29,17 @@ describe 'cfndsl', type: :aruba do
 
   before(:each) { write_file('template.rb', template_content) }
 
+  context 'cfndsl -u' do
+    it 'updates the specification file' do
+      run 'cfndsl -u'
+      expect(last_command_started).to have_output_on_stderr(<<-OUTPUT.gsub(/^ {8}/, '').chomp)
+        Updating specification file
+        Specification successfully written to #{ENV['HOME']}/.cfndsl/resource_specification.json
+      OUTPUT
+      expect(last_command_started).to have_exit_status(0)
+    end
+  end
+
   context 'cfndsl' do
     it 'displays the usage' do
       run 'cfndsl'
@@ -134,6 +147,7 @@ describe 'cfndsl', type: :aruba do
     it 'displays the variables as they are interpolated in the CloudFormation template' do
       run_simple 'cfndsl template.rb --yaml params.yaml --verbose'
       verbose = /
+        Using \s specification \s file .* \.json \n
         Loading \s YAML \s file \s .* params\.yaml \n
         Setting \s local \s variable \s DESC \s to \s yaml \n
         Loading \s template \s file \s .* template.rb \n

data/spec/fixtures/serverless-api.json

@@ -1 +1 @@
-{"AWSTemplateFormatVersion":"2010-09-09","Resources":{"Test":{"Properties":{"StageName":"prod","DefinitionUri":"swagger.yml","CacheClusterEnabled":false,"CacheClusterSize":"512M","Variables":{"Var1":"value1"}},"Type":"AWS::Serverless::API"}}}
+{"AWSTemplateFormatVersion":"2010-09-09","Resources":{"Test":{"Properties":{"StageName":"prod","DefinitionUri":"swagger.yml","CacheClusterEnabled":false,"CacheClusterSize":"512M","Variables":{"Var1":"value1"}},"Type":"AWS::Serverless::Api"}}}

data/spec/spec_helper.rb

@@ -9,6 +9,9 @@ if ENV['CFNDSL_COV']
   end
 end
 
+require 'cfndsl/globals'
+CfnDsl.specification_file File.expand_path('../../lib/cfndsl/aws/resource_specification.json', __FILE__)
+# use local fixture for tests
 require 'cfndsl'
 
 bindir = File.expand_path('../../bin', __FILE__)

data/spec/types_definition_spec.rb

@@ -0,0 +1,43 @@
+require 'spec_helper'
+
+# This is a somewhat temporary test class to compare functionality
+# between the AWS, OS and new ways of defining types
+RSpec.describe 'Type Definitions' do
+  aws_spec = YAML.load_file File.expand_path('../../lib/cfndsl/aws/types.yaml', __FILE__)
+  os_spec = YAML.load_file File.expand_path('../../lib/cfndsl/os/types.yaml', __FILE__)
+  new_spec = CfnDsl::Specification.extract_from_resource_spec!
+
+  { 'AWS' => aws_spec, 'OS' => os_spec, 'New' => new_spec }.each_pair do |cloud, specdef|
+    context cloud do
+      resources = specdef['Resources']
+      types = specdef['Types']
+
+      context 'Resources' do
+        resources.each do |name, info|
+          it "#{name} has all property types defined" do
+            properties = info['Properties']
+            properties.each do |_, type|
+              type = type.first if type.is_a?(Array)
+              expect(types).to have_key(type)
+            end
+          end
+        end
+      end
+
+      context 'Types' do
+        types.each do |name, type|
+          it "#{name} has all property types defined" do
+            type = type.first if type.is_a?(Array)
+            if type.is_a?(String)
+              expect(types).to have_key(type)
+            elsif type.is_a?(Hash)
+              type.values.flatten.each { |t| expect(types).to have_key(t) }
+            else
+              raise "A defined type should only be of the form String, Array or Hash, got #{type.class}"
+            end
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cfndsl
 version: !ruby/object:Gem::Version
-  version: 0.12.11
+  version: 0.13.0
 platform: ruby
 authors:
 - Steven Jack
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-10 00:00:00.000000000 Z
+date: 2017-05-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -46,13 +46,13 @@ files:
 - cfndsl.gemspec
 - lib/cfndsl.rb
 - lib/cfndsl/aws/cloud_formation_template.rb
+- lib/cfndsl/aws/resource_specification.json
 - lib/cfndsl/aws/types.rb
 - lib/cfndsl/aws/types.yaml
 - lib/cfndsl/conditions.rb
 - lib/cfndsl/creation_policy.rb
 - lib/cfndsl/errors.rb
 - lib/cfndsl/external_parameters.rb
-- lib/cfndsl/generate_types.rb
 - lib/cfndsl/globals.rb
 - lib/cfndsl/json_serialisable_object.rb
 - lib/cfndsl/jsonable.rb
@@ -65,11 +65,13 @@ files:
 - lib/cfndsl/os/types.yaml
 - lib/cfndsl/outputs.rb
 - lib/cfndsl/parameters.rb
+- lib/cfndsl/patches.rb
 - lib/cfndsl/plurals.rb
 - lib/cfndsl/properties.rb
 - lib/cfndsl/rake_task.rb
 - lib/cfndsl/ref_check.rb
 - lib/cfndsl/resources.rb
+- lib/cfndsl/specification.rb
 - lib/cfndsl/types.rb
 - lib/cfndsl/update_policy.rb
 - lib/cfndsl/version.rb
@@ -114,6 +116,7 @@ files:
 - spec/spec_helper.rb
 - spec/support/shared_examples/orchestration_template.rb
 - spec/transform_spec.rb
+- spec/types_definition_spec.rb
 homepage: https://github.com/stevenjack/cfndsl
 licenses:
 - MIT
@@ -165,3 +168,4 @@ test_files:
 - spec/spec_helper.rb
 - spec/support/shared_examples/orchestration_template.rb
 - spec/transform_spec.rb
+- spec/types_definition_spec.rb

data/lib/cfndsl/generate_types.rb

@@ -1,154 +0,0 @@
-require 'yaml'
-require 'cfndsl/jsonable'
-require 'cfndsl/plurals'
-require 'cfndsl/names'
-
-module CfnDsl
-  # Type generation helper
-  module GenerateTypes
-    # declare classes for all of the types with named methods for setting the values
-    class Type < JSONable
-    end
-
-    # rubocop:disable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
-    def generate_types(filename)
-      types = YAML.safe_load(File.open(filename))
-      const_set('Types_Internal', types)
-
-      validate_types(types)
-
-      classes = {}
-
-      # Go through and declare all of the types first
-      types['Types'].each_key do |typename|
-        if !const_defined?(typename)
-          klass = const_set(typename, Class.new(self))
-          classes[typename] = klass
-        else
-          classes[typename] = const_get(typename)
-        end
-      end
-
-      # Now go through them again and define attribute setter methods
-      classes.each_pair do |typename, type|
-        typeval = types['Types'][typename]
-        next unless typeval.respond_to?(:each_pair)
-        typeval.each_pair do |attr_name, attr_type|
-          if attr_type.is_a?(Array)
-            klass = const_get(attr_type[0])
-            variable = "@#{attr_name}".to_sym
-
-            method = CfnDsl::Plurals.singularize(attr_name)
-            methods = attr_name
-            all_methods = CfnDsl.method_names(method) + CfnDsl.method_names(methods)
-            type.class_eval do
-              all_methods.each do |method_name|
-                define_method(method_name) do |value = nil, *rest, &block|
-                  existing = instance_variable_get(variable)
-                  # For no-op invocations, get out now
-                  return existing if value.nil? && rest.empty? && !block
-
-                  # We are going to modify the value in some
-                  # way, make sure that we have an array to mess
-                  # with if we start with nothing
-                  existing = instance_variable_set(variable, []) unless existing
-
-                  # special case for just a block, no args
-                  if value.nil? && rest.empty? && block
-                    val = klass.new
-                    existing.push val
-                    value.instance_eval(&block(val))
-                    return existing
-                  end
-
-                  # Glue all of our parameters together into
-                  # a giant array - flattening one level deep, if needed
-                  array_params = []
-                  if value.is_a?(Array)
-                    value.each { |x| array_params.push x }
-                  else
-                    array_params.push value
-                  end
-                  unless rest.empty?
-                    rest.each do |v|
-                      if v.is_a?(Array)
-                        array_params += rest
-                      else
-                        array_params.push v
-                      end
-                    end
-                  end
-
-                  # Here, if we were given multiple arguments either
-                  # as method [a,b,c], method(a,b,c), or even
-                  # method( a, [b], c) we end up with
-                  # array_params = [a,b,c]
-                  #
-                  # array_params will have at least one item
-                  # unless the user did something like pass in
-                  # a bunch of empty arrays.
-                  if block
-                    # TODO: Is this a bug? We don't do anything with the array conetns
-                    array_params.each do |_array_params_value|
-                      value = klass.new
-                      existing.push value
-                      value.instance_eval(&block(val)) if block
-                    end
-                  else
-                    # List of parameters with no block -
-                    # hope that the user knows what he is
-                    # doing and stuff them into our existing
-                    # array
-                    # TODO: Is this a bug? We don't do anything with the array conetns
-                    array_params.each do |_array_params_value|
-                      existing.push value
-                    end
-                  end
-                  return existing
-                end
-              end
-            end
-          else
-            klass = const_get(attr_type)
-            variable = "@#{attr_name}".to_sym
-
-            type.class_eval do
-              CfnDsl.method_names(attr_name) do |inner_method|
-                define_method(inner_method) do |value = nil, *_rest, &block|
-                  value ||= klass.new
-                  instance_variable_set(variable, value)
-                  value.instance_eval(&block) if block
-                  value
-                end
-              end
-            end
-          end
-        end
-      end
-    end
-    # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity
-
-    private
-
-    # Do a little sanity checking - all of the types referenced in Resources
-    # should be represented in Types
-    def validate_types(types)
-      types['Resources'].values.each do |resource|
-        resource.values.each do |thing|
-          thing.values.flatten.each do |type|
-            puts "unknown type #{type}" unless types['Types'].key?(type)
-          end
-        end
-      end
-
-      # All of the type values should also be references
-      types['Types'].values do |type|
-        next unless type.respond_to?(:values)
-
-        type.values.each do |tv|
-          puts "unknown type #{tv}" unless types['Types'].key?(tv)
-        end
-      end
-    end
-  end
-end