cfndsl 0.14.0 → 0.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 17794b4c98827341ca1be479e9b41394e351bf0a
-  data.tar.gz: 96c8c7fb7868190f9e69268c9b0e331671c9ff92
+  metadata.gz: c35481efeaa53234104932119ca1f2d30618dce3
+  data.tar.gz: a80db4b44cce2a4e0fc4591fc1df3496b5adda2d
 SHA512:
-  metadata.gz: 140566d5b12c819b586e000deb5c1dca773bc2aecce051935fec314b9c72f3103e976afea2daf058fe61d9a8400e95e2b7ccd4ae7a6356c0239fac23ab287124
-  data.tar.gz: 354452707c917d0db1e41dc98fdddff6aa20f9ba35301a27dbfdd5af643c92e555542353a02142b5f7ea5c51bb3c76e10e1667cb7819ecd0e1501b5a5e743f10
+  metadata.gz: 0bddda0ad62f9d9d1c89378c41add30867e0cf9ef37a4fe505f0b00cf76eac09231406c8f28f4e09b0ed7382376577f7209b1979927179b320411f11b52eba5d
+  data.tar.gz: a46c41614907b56f121ea34f8eaa42216df064c349f85076352fd80d770e60d2da1ea4d34793fcd27d0adc1f9b57765be06efb929018eeb408ec7a3441f6c56a

data/CHANGELOG.md

@@ -1,7 +1,20 @@
 # Change Log
 
-## [0.14.0](https://github.com/stevenjack/cfndsl/tree/0.14.0) (2017-06-15)
-[Full Changelog](https://github.com/stevenjack/cfndsl/compare/v0.13.1...0.14.0)
+## [0.15.0](https://github.com/stevenjack/cfndsl/tree/0.15.0) (2017-06-19)
+[Full Changelog](https://github.com/stevenjack/cfndsl/compare/v0.14.0...0.15.0)
+
+**Closed issues:**
+
+- Please add support for multiple input files with deep merge [\#327](https://github.com/stevenjack/cfndsl/issues/327)
+
+**Merged pull requests:**
+
+- Clean up of README [\#330](https://github.com/stevenjack/cfndsl/pull/330) ([elmobp](https://github.com/elmobp))
+- remove 'disable\_binding', merge 0.x changes [\#329](https://github.com/stevenjack/cfndsl/pull/329) ([gergnz](https://github.com/gergnz))
+- enable deep merge as the default for yaml [\#328](https://github.com/stevenjack/cfndsl/pull/328) ([gergnz](https://github.com/gergnz))
+
+## [v0.14.0](https://github.com/stevenjack/cfndsl/tree/v0.14.0) (2017-06-15)
+[Full Changelog](https://github.com/stevenjack/cfndsl/compare/v0.13.1...v0.14.0)
 
 **Implemented enhancements:**
 

data/README.md

@@ -431,14 +431,16 @@ Usage: cfndsl [options] FILE
     -r, --ruby FILE                  Evaluate ruby file before template
     -j, --json FILE                  Import json file as local variables
     -p, --pretty                     Pretty-format output JSON
+    -f, --format FORMAT              Specify the output format (JSON default)
     -D, --define "VARIABLE=VALUE"    Directly set local VARIABLE as VALUE
     -v, --verbose                    Turn on verbose ouptut
     -b, --disable-binding            Disable binding configuration
+    -m, --disable-deep-merge         Disable deep merging of yaml
+    -s, --specification-file FILE    Location of Cloudformation Resource Specification file
+    -u, --update-specification       Update the Cloudformation Resource Specification file
     -g RESOURCE_TYPE,RESOURCE_LOGICAL_NAME,
-         --generate                   Add resource type and logical name    
-    -i, --indent TOKEN               Use TOKEN for indent character (default space)
+        --generate                   Add resource type and logical name
     -l, --list                       List supported resources
-    -c, --indent-count [COUNT]       Count of characters to use for indenting. (default: 2)
     -h, --help                       Display this screen
 ```
 
@@ -549,6 +551,9 @@ $ cfndsl sample/t1.rb -y sample/t1.yaml
 
 would generate a template with 5 instances declared.
 
+Specifying multiple -y options will default deep_merge all the yaml in the order specified.
+You can disable this with -m.
+
 Finally, the -r option gives you the opportunity to execute some
 arbitrary ruby code in the evaluation context before the cloudformation
 template is evaluated (this is not available if `--disable-binding` is used).
@@ -580,11 +585,13 @@ $ bin/rake generate
 ```
 
 ### Generating CloudFormation resources from cfndsl
-By supplying the -g paramater you are now able to generate cloudformation resources for support objects, for a list of supported resources run cfndsl -l
+By supplying the -g paramater you are now able to generate cloudformation resources for supported objects, for a list of supported resources run cfndsl -l
 
 Example
+```bash
+cfndsl -g AWS::EC2::EIP,EIP
 ```
- cfndsl -g AWS::EC2::EIP,EIP
+```ruby
 require 'cfndsl'
 CloudFormation do
   Description 'auto generated cloudformation cfndsl template'

data/bin/cfndsl

@@ -38,8 +38,7 @@ optparse = OptionParser.new do |opts|
   end
 
   opts.on('-f', '--format FORMAT', 'Specify the output format (JSON default)') do |format|
-    types = %w[json yaml]
-    raise "Format #{format} not supported" unless types.include? format
+    raise "Format #{format} not supported" unless %w[json yaml].include?(format)
     options[:outformat] = format
   end
 
@@ -56,6 +55,10 @@ optparse = OptionParser.new do |opts|
     CfnDsl.disable_binding
   end
 
+  opts.on('-m', '--disable-deep-merge', 'Disable deep merging of yaml') do
+    CfnDsl.disable_deep_merge
+  end
+
   opts.on('-s', '--specification-file FILE', 'Location of Cloudformation Resource Specification file') do |file|
     CfnDsl.specification_file File.expand_path(file)
   end

data/cfndsl.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
 
   s.executables << 'cfndsl'
 
-  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'bundler', '~> 1.13'
 
   s.post_install_message = "'addTag' is now deprecated in favour of 'add_tag'. 'addTag' will be removed in the next major version."
 end

data/lib/cfndsl.rb

@@ -1,6 +1,7 @@
 require 'forwardable'
 require 'json'
 
+require 'deep_merge/deep_merge'
 require 'cfndsl/globals'
 require 'cfndsl/module'
 require 'cfndsl/errors'

data/lib/cfndsl/external_parameters.rb

@@ -31,6 +31,10 @@ module CfnDsl
       parameters[k.to_sym] = v
     end
 
+    def merge_param(x)
+      parameters.deep_merge!(x)
+    end
+
     def get_param(k)
       parameters[k.to_sym]
     end
@@ -57,7 +61,13 @@ module CfnDsl
       else
         raise "Unrecognized extension #{format}"
       end
-      params.each { |key, val| set_param(key, val) }
+      if CfnDsl.disable_deep_merge?
+        params.each { |key, val| set_param(key, val) }
+      else
+        x = {}
+        params.map { |k, v| x[k.to_sym] = v }
+        merge_param(x)
+      end
     end
   end
 end

data/lib/cfndsl/globals.rb

@@ -10,6 +10,14 @@ module CfnDsl
     @disable_binding
   end
 
+  def disable_deep_merge
+    @disable_deep_merge = true
+  end
+
+  def disable_deep_merge?
+    @disable_deep_merge
+  end
+
   def specification_file(file = nil)
     @specification_file = file if file
     @specification_file ||= File.join(ENV['HOME'], '.cfndsl/resource_specification.json')

data/lib/cfndsl/version.rb

@@ -1,3 +1,3 @@
 module CfnDsl
-  VERSION = '0.14.0'.freeze
+  VERSION = '0.15.0'.freeze
 end

data/lib/deep_merge/core.rb

@@ -0,0 +1,244 @@
+# rubocop:disable Metrics/AbcSize, Metrics/BlockNesting, Metrics/CyclomaticComplexity, Metrics/MethodLength
+# rubocop:disable Metrics/ModuleLength, Metrics/PerceivedComplexity, Style/IfInsideElse, Style/Semicolon
+#
+# Totally borrowed from https://github.com/danielsdeleo/deep_merge
+module DeepMerge
+  class InvalidParameter < StandardError; end
+
+  DEFAULT_FIELD_KNOCKOUT_PREFIX = '--'.freeze
+
+  # Deep Merge core documentation.
+  # deep_merge! method permits merging of arbitrary child elements. The two top level
+  # elements must be hashes. These hashes can contain unlimited (to stack limit) levels
+  # of child elements. These child elements to not have to be of the same types.
+  # Where child elements are of the same type, deep_merge will attempt to merge them together.
+  # Where child elements are not of the same type, deep_merge will skip or optionally overwrite
+  # the destination element with the contents of the source element at that level.
+  # So if you have two hashes like this:
+  #   source = {:x => [1,2,3], :y => 2}
+  #   dest =   {:x => [4,5,'6'], :y => [7,8,9]}
+  #   dest.deep_merge!(source)
+  #   Results: {:x => [1,2,3,4,5,'6'], :y => 2}
+  # By default, "deep_merge!" will overwrite any unmergeables and merge everything else.
+  # To avoid this, use "deep_merge" (no bang/exclamation mark)
+  #
+  # Options:
+  #   Options are specified in the last parameter passed, which should be in hash format:
+  #   hash.deep_merge!({:x => [1,2]}, {:knockout_prefix => '--'})
+  #   :preserve_unmergeables  DEFAULT: false
+  #      Set to true to skip any unmergeable elements from source
+  #   :knockout_prefix        DEFAULT: nil
+  #      Set to string value to signify prefix which deletes elements from existing element
+  #   :overwrite_arrays       DEFAULT: false
+  #      Set to true if you want to avoid merging arrays
+  #   :sort_merged_arrays     DEFAULT: false
+  #      Set to true to sort all arrays that are merged together
+  #   :unpack_arrays          DEFAULT: nil
+  #      Set to string value to run "Array::join" then "String::split" against all arrays
+  #   :merge_hash_arrays      DEFAULT: false
+  #      Set to true to merge hashes within arrays
+  #   :keep_array_duplicates  DEFAULT: false
+  #      Set to true to preserve duplicate array entries
+  #   :merge_debug            DEFAULT: false
+  #      Set to true to get console output of merge process for debugging
+  #
+  # Selected Options Details:
+  # :knockout_prefix => The purpose of this is to provide a way to remove elements
+  #   from existing Hash by specifying them in a special way in incoming hash
+  #    source = {:x => ['--1', '2']}
+  #    dest   = {:x => ['1', '3']}
+  #    dest.ko_deep_merge!(source)
+  #    Results: {:x => ['2','3']}
+  #   Additionally, if the knockout_prefix is passed alone as a string, it will cause
+  #   the entire element to be removed:
+  #    source = {:x => '--'}
+  #    dest   = {:x => [1,2,3]}
+  #    dest.ko_deep_merge!(source)
+  #    Results: {:x => ""}
+  # :unpack_arrays => The purpose of this is to permit compound elements to be passed
+  #   in as strings and to be converted into discrete array elements
+  #   irsource = {:x => ['1,2,3', '4']}
+  #   dest   = {:x => ['5','6','7,8']}
+  #   dest.deep_merge!(source, {:unpack_arrays => ','})
+  #   Results: {:x => ['1','2','3','4','5','6','7','8'}
+  #   Why: If receiving data from an HTML form, this makes it easy for a checkbox
+  #    to pass multiple values from within a single HTML element
+  #
+  # :merge_hash_arrays => merge hashes within arrays
+  #   source = {:x => [{:y => 1}]}
+  #   dest   = {:x => [{:z => 2}]}
+  #   dest.deep_merge!(source, {:merge_hash_arrays => true})
+  #   Results: {:x => [{:y => 1, :z => 2}]}
+  #
+  # :keep_array_duplicates => merges arrays within hashes but keeps duplicate elements
+  #   source = {:x => {:y => [1,2,2,2,3]}}
+  #   dest   = {:x => {:y => [4,5,6]}}
+  #   dest.deep_merge!(source, {:keep_array_duplicates => true})
+  #   Results: {:x => {:y => [1,2,2,2,3,4,5,6]}}
+  #
+  # There are many tests for this library - and you can learn more about the features
+  # and usages of deep_merge! by just browsing the test examples
+  def self.deep_merge!(source, dest, options = {})
+    # turn on this line for stdout debugging text
+    merge_debug = options[:merge_debug] || false
+    overwrite_unmergeable = !options[:preserve_unmergeables]
+    knockout_prefix = options[:knockout_prefix] || nil
+    raise InvalidParameter, 'knockout_prefix cannot be an empty string in deep_merge!' if knockout_prefix == ''
+    raise InvalidParameter, 'overwrite_unmergeable must be true if knockout_prefix is specified in deep_merge!' if knockout_prefix && !overwrite_unmergeable
+    # if present: we will split and join arrays on this char before merging
+    array_split_char = options[:unpack_arrays] || false
+    # request that we avoid merging arrays
+    overwrite_arrays = options[:overwrite_arrays] || false
+    # request that we sort together any arrays when they are merged
+    sort_merged_arrays = options[:sort_merged_arrays] || false
+    # request that arrays of hashes are merged together
+    merge_hash_arrays = options[:merge_hash_arrays] || false
+    # request to extend existing arrays, instead of overwriting them
+    extend_existing_arrays = options[:extend_existing_arrays] || false
+    # request that arrays keep duplicate elements
+    keep_array_duplicates = options[:keep_array_duplicates] || false
+    # request that nil values are merged or skipped (Skipped/false by default)
+    merge_nil_values = options[:merge_nil_values] || false
+
+    di = options[:debug_indent] || ''
+    # do nothing if source is nil
+    return dest if !merge_nil_values && source.nil?
+    # if dest doesn't exist, then simply copy source to it
+    if !dest && overwrite_unmergeable
+      dest = source; return dest
+    end
+
+    puts "#{di}Source class: #{source.class.inspect} :: Dest class: #{dest.class.inspect}" if merge_debug
+    if source.is_a?(Hash)
+      puts "#{di}Hashes: #{source.inspect} :: #{dest.inspect}" if merge_debug
+      source.each do |src_key, src_value|
+        if dest.is_a?(Hash)
+          puts "#{di} looping: #{src_key.inspect} => #{src_value.inspect} :: #{dest.inspect}" if merge_debug
+          if dest[src_key]
+            puts "#{di} ==>merging: #{src_key.inspect} => #{src_value.inspect} :: #{dest[src_key].inspect}" if merge_debug
+            dest[src_key] = deep_merge!(src_value, dest[src_key], options.merge(debug_indent: di + '  '))
+          else # dest[src_key] doesn't exist so we want to create and overwrite it (but we do this via deep_merge!)
+            puts "#{di} ==>merging over: #{src_key.inspect} => #{src_value.inspect}" if merge_debug
+            # note: we rescue here b/c some classes respond to "dup" but don't implement it (Numeric, TrueClass, FalseClass, NilClass among maybe others)
+            begin
+              src_dup = src_value.dup # we dup src_value if possible because we're going to merge into it (since dest is empty)
+            rescue TypeError
+              src_dup = src_value
+            end
+            dest[src_key] = deep_merge!(src_value, src_dup, options.merge(debug_indent: di + '  '))
+          end
+        elsif dest.is_a?(Array) && extend_existing_arrays
+          dest.push(source)
+        else # dest isn't a hash, so we overwrite it completely (if permitted)
+          if overwrite_unmergeable
+            puts "#{di}  overwriting dest: #{src_key.inspect} => #{src_value.inspect} -over->  #{dest.inspect}" if merge_debug
+            dest = overwrite_unmergeables(source, dest, options)
+          end
+        end
+      end
+    elsif source.is_a?(Array)
+      puts "#{di}Arrays: #{source.inspect} :: #{dest.inspect}" if merge_debug
+      if overwrite_arrays
+        puts "#{di} overwrite arrays" if merge_debug
+        dest = source
+      else
+        # if we are instructed, join/split any source arrays before processing
+        if array_split_char
+          puts "#{di} split/join on source: #{source.inspect}" if merge_debug
+          source = source.join(array_split_char).split(array_split_char)
+          if dest.is_a?(Array)
+            dest = dest.join(array_split_char).split(array_split_char)
+          end
+        end
+        # if there's a naked knockout_prefix in source, that means we are to truncate dest
+        if knockout_prefix && source.index(knockout_prefix)
+          dest = clear_or_nil(dest); source.delete(knockout_prefix)
+        end
+        if dest.is_a?(Array)
+          if knockout_prefix
+            print "#{di} knocking out: " if merge_debug
+            # remove knockout prefix items from both source and dest
+            source.delete_if do |ko_item|
+              retval = false
+              item = ko_item.respond_to?(:gsub) ? ko_item.gsub(/^#{knockout_prefix}/, '') : ko_item
+              if item != ko_item
+                print "#{ko_item} - " if merge_debug
+                dest.delete(item)
+                dest.delete(ko_item)
+                retval = true
+              end
+              retval
+            end
+            puts if merge_debug
+          end
+          puts "#{di} merging arrays: #{source.inspect} :: #{dest.inspect}" if merge_debug
+          source_all_hashes = source.all? { |i| i.is_a?(Hash) }
+          dest_all_hashes = dest.all? { |i| i.is_a?(Hash) }
+          if merge_hash_arrays && source_all_hashes && dest_all_hashes
+            # merge hashes in lists
+            list = []
+            dest.each_index do |i|
+              list[i] = deep_merge!(source[i] || {}, dest[i],
+                                    options.merge(debug_indent: di + '  '))
+            end
+            list += source[dest.count..-1] if source.count > dest.count
+            dest = list
+          elsif keep_array_duplicates
+            dest = dest.concat(source)
+          else
+            dest |= source
+          end
+          dest.sort! if sort_merged_arrays
+        elsif overwrite_unmergeable
+          puts "#{di} overwriting dest: #{source.inspect} -over-> #{dest.inspect}" if merge_debug
+          dest = overwrite_unmergeables(source, dest, options)
+        end
+      end
+    else # src_hash is not an array or hash, so we'll have to overwrite dest
+      if dest.is_a?(Array) && extend_existing_arrays
+        dest.push(source)
+      else
+        puts "#{di}Others: #{source.inspect} :: #{dest.inspect}" if merge_debug
+        dest = overwrite_unmergeables(source, dest, options)
+      end
+    end
+    puts "#{di}Returning #{dest.inspect}" if merge_debug
+    dest
+  end # deep_merge!
+
+  # allows deep_merge! to uniformly handle overwriting of unmergeable entities
+  def self.overwrite_unmergeables(source, dest, options)
+    merge_debug = options[:merge_debug] || false
+    overwrite_unmergeable = !options[:preserve_unmergeables]
+    knockout_prefix = options[:knockout_prefix] || false
+    di = options[:debug_indent] || ''
+    if knockout_prefix && overwrite_unmergeable
+      src_tmp = if source.is_a?(String) # remove knockout string from source before overwriting dest
+                  source.gsub(/^#{knockout_prefix}/, '')
+                elsif source.is_a?(Array) # remove all knockout elements before overwriting dest
+                  source.delete_if { |ko_item| ko_item.is_a?(String) && ko_item.match(/^#{knockout_prefix}/) }
+                else
+                  source
+                end
+      if src_tmp == source # if we didn't find a knockout_prefix then we just overwrite dest
+        puts "#{di}#{src_tmp.inspect} -over-> #{dest.inspect}" if merge_debug
+        dest = src_tmp
+      else # if we do find a knockout_prefix, then we just delete dest
+        puts "#{di}\"\" -over-> #{dest.inspect}" if merge_debug
+        dest = ''
+      end
+    elsif overwrite_unmergeable
+      dest = source
+    end
+    dest
+  end
+
+  def self.clear_or_nil(obj)
+    if obj.respond_to?(:clear)
+      obj.clear
+    else
+      obj = nil
+    end
+    obj
+  end
+end # module DeepMerge

data/lib/deep_merge/deep_merge.rb

@@ -0,0 +1,18 @@
+# Totally borrowed from https://github.com/danielsdeleo/deep_merge
+require 'deep_merge/core'
+
+module DeepMerge
+  # Implements a deep merge with default options
+  module DeepMergeHash
+    # deep_merge! will merge and overwrite any unmergeables in destination hash
+    def deep_merge!(source, options = {})
+      default_opts = { preserve_unmergeables: false }
+      DeepMerge.deep_merge!(source, self, default_opts.merge(options))
+    end
+  end # DeepMergeHashExt
+end
+
+# Extends hash with deep merge
+class Hash
+  include DeepMerge::DeepMergeHash
+end

data/spec/cli_spec.rb

@@ -13,6 +13,7 @@ describe 'cfndsl', type: :aruba do
           -D, --define "VARIABLE=VALUE"    Directly set local VARIABLE as VALUE
           -v, --verbose                    Turn on verbose ouptut
           -b, --disable-binding            Disable binding configuration
+          -m, --disable-deep-merge         Disable deep merging of yaml
           -s, --specification-file FILE    Location of Cloudformation Resource Specification file
           -u, --update-specification       Update the Cloudformation Resource Specification file
           -g RESOURCE_TYPE,RESOURCE_LOGICAL_NAME,

data/spec/deep_merge_spec.rb

@@ -0,0 +1,12 @@
+require 'spec_helper'
+describe DeepMerge do
+  source = { key1: { keya1: 1, keya2: 2 }, key2: [1, 2] }
+  let(:merged_hash) { source.deep_merge!(key1: { keya1: '1a', keya3: 3 }, key2: [2, 3]) }
+
+  context 'deep_merge' do
+    it 'merges correctly' do
+      test_hash = { key1: { keya1: '1a', keya2: 2, keya3: 3 }, key2: [1, 2, 3] }
+      expect(merged_hash).to eq test_hash
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cfndsl
 version: !ruby/object:Gem::Version
-  version: 0.14.0
+  version: 0.15.0
 platform: ruby
 authors:
 - Steven Jack
@@ -9,22 +9,22 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-15 00:00:00.000000000 Z
+date: 2017-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.13'
 description: DSL for creating AWS Cloudformation templates
 email:
 - stevenmajack@gmail.com
@@ -79,6 +79,8 @@ files:
 - lib/cfnlego/cloudformation.erb
 - lib/cfnlego/cloudformation.rb
 - lib/cfnlego/resource.rb
+- lib/deep_merge/core.rb
+- lib/deep_merge/deep_merge.rb
 - sample/autoscale.rb
 - sample/autoscale.template
 - sample/autoscale2.rb
@@ -105,6 +107,7 @@ files:
 - spec/cfndsl_spec.rb
 - spec/cli_spec.rb
 - spec/cloud_formation_template_spec.rb
+- spec/deep_merge_spec.rb
 - spec/external_parameters_spec.rb
 - spec/fixtures/heattest.rb
 - spec/fixtures/serverless-api.json
@@ -158,6 +161,7 @@ test_files:
 - spec/cfndsl_spec.rb
 - spec/cli_spec.rb
 - spec/cloud_formation_template_spec.rb
+- spec/deep_merge_spec.rb
 - spec/external_parameters_spec.rb
 - spec/fixtures/heattest.rb
 - spec/fixtures/serverless-api.json