cfhighlander 0.6.0.alpha.1538010880 → 0.6.0.alpha.1539120146

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a1ca60058a558ba23ed08273fa7ec940794476f1fc9228d6077da3521c6d7b19
-  data.tar.gz: 83b85f5799b9d48263174b5adc2ec5db6a6fde698a21e1b0698c36c1a4091ff7
+  metadata.gz: 4a73647965281a89f8bb2bc78e8282e81604f13b5aa06a48ab2ad0f7c21ec78b
+  data.tar.gz: 60a0df940c9a7675ec62de5f636144496bd88dad4ddfcaeae1e5c6f6669dca94
 SHA512:
-  metadata.gz: 682f0fb7fe00e2ecb5315ec1d5e599591d8a9a8721d55bc40d96fdafbf0fbfadc227b2272aaf20c19802d27fde22c1215a2ef10987fdefb306d42c8ec65ccfe3
-  data.tar.gz: 553b3e35fe13a125bf633da11486167f4946a5f7ea5cc51be92b9189ffb95ef4d614446e191fbbd5b7648952739337419435858b467cc54efd68c7e72939288d
+  metadata.gz: b743ab97b1cc2051d98f048630ee114a2ebd976b3acb14f14d9b989373bcbd0964b10e726330d12b569aa443504f052111bc0ad78febd61a54c8f26b5eb7ea10
+  data.tar.gz: 9a5de349f486e6c8cea90a1c7f7672ca178179815f489c16795714684da4dd8d0862a54bee672c083fc0a3d106d3bb5161443e23fb9bc3f06cb0cf8053f61b3c

data/README.md

@@ -699,6 +699,46 @@ CfhighlanderTemplate do
 end
 
 ```
+## Render mode for components
+
+Rendering component resources in resulting cloudformation stack is available in 2 modes. These modes
+are controlled using `render` keyword of `Component` DSL statement
+
+`Substack` - creates additional substack for cfhighlander component and points to it using [CloudFormation
+Stack](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-stack.html) resource type
+This is also default render mode - if no render mode is specified `Substack` will be used
+
+`Inline` - places all defined resources from inner component in outer component cloudformation template. Resources,
+Outputs, Conditions, Parameters and Mappings are all inlined - please note that some of the template elements may be renamed in this
+process in order to assure unique names. 
+
+There are some limitations when using inline components - Inlined component parameters, having values as outputs from another component (inlined or not)
+can't be referenced in component conditions. However, conditions referencing mapping values or parameters passed as mapping values,
+are allowed.
+
+**`SIDE EFFECTS`** Side effect of moving from substack based to fully inlined stack may be revealing some of the implicit dependencies within an environment
+
+*Example:* Component A defines Hosted Zone, while component B defines Record Set for given hosted zone. Record set is defined
+by referencing Zone Name (rather than ZoneId), meaning there is no explicit dependency between the resources. When both components
+are rendered as substack, implicit dependency is created if there is at least one output from component A passed as parameter
+to component B. Rendering components inlined removes this implicitly defined dependency, as a consequence stack deletion or creation
+may be halted, as record set is being created/deleted before prior the record set. 
+
+**`WARNING`** Be aware of [resource, condition, parameter, output and mapping limits](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cloudformation-limits.html) on a single template
+when rendering inner components inlined.
+
+**`EXAMPLE`** All of the VPC resources will be rendered in outer component template, while bastion
+will be referenced as substack in example below. 
+
+```ruby
+CfhighlanderTemplate do
+
+    Component template:'vpc@1.5.0', name: 'vpc', render: Inline
+    Component template:'bastion@1.2.0', name: 'bastion', render: Substack
+
+end
+```
+
 
 
 ## Rendering CloudFormation templates

data/bin/cfhighlander.rb

@@ -125,6 +125,7 @@ class HighlanderCli < Thor
     publisher.publishFiles(compiler.cfn_template_paths + compiler.lambda_src_paths)
 
     puts "\n\nUse following url to launch CloudFormation stack\n\n#{publisher.getLaunchStackUrl}\n\n"
+    puts "\n\nUse following template url to update the stack\n\n#{publisher.getTemplateUrl}\n\n"
 
   end
 

data/hl_ext/mapping_helper.rb

@@ -5,11 +5,11 @@ def mappings_provider(provider_name, is_legacy = false)
   return nil if provider_name.nil?
   provider = nil
   module_name = is_legacy ? 'Highlander': 'Cfhighlander'
-  providers = Object.const_get(module_name).const_get('MapProviders')
   begin
+    providers = Object.const_get(module_name).const_get('MapProviders')
     providers.const_get(provider_name)
   rescue NameError => e
-    if e.to_s.include? "uninitialized constant #{module_name}::MapProviders::"
+    if e.to_s.include? "uninitialized constant"
       return mappings_provider(provider_name, true) unless is_legacy
       return nil
     end

data/lib/cfhighlander.compiler.rb

@@ -11,6 +11,7 @@ require 'net/https'
 require 'highline/import'
 require 'zip'
 require_relative './util/zip.util'
+require_relative './util/cloudformation.util'
 
 module Cfhighlander
 
@@ -140,7 +141,22 @@ module Cfhighlander
 
 
         # grab cfndsl model
+
+        # 1st pass of cloudformation compiling - does the substacks normally
         model = evaluateCloudFormation
+        @component.set_cfndsl_model model
+
+        # compile sub-component templates
+        @sub_components.each do |sub_component|
+          sub_component.compileCloudFormation format
+          @cfn_template_paths += sub_component.cfn_template_paths
+          @lambda_src_paths += sub_component.lambda_src_paths
+        end
+
+        # 2nd pass will flatten any inlined components
+        model = Cfhighlander::Util::CloudFormation.flattenCloudformation(
+            component: @component
+        )
 
         # write resulting cloud formation template
         if format == 'json'
@@ -155,12 +171,8 @@ module Cfhighlander
         # `cfndsl #{@cfndsl_compiled_path} -p -f #{format} -o #{output_path} --disable-binding`
         puts "CloudFormation #{format.upcase} template for #{dsl.name} written to #{output_path}"
 
-        # compile sub-component templates
-        @sub_components.each do |sub_component|
-          sub_component.compileCloudFormation format
-          @cfn_template_paths += sub_component.cfn_template_paths
-          @lambda_src_paths += sub_component.lambda_src_paths
-        end
+
+
 
       end
 

data/lib/cfhighlander.dsl.base.rb

@@ -35,6 +35,10 @@ module Cfhighlander
         return { 'Fn::Split' => [delimiter, source] }
       end
 
+      def FnJoin(glue, pieces)
+        return { 'Fn::Join' => [glue, pieces]}
+      end
+
       def FnSelect(index, list)
         return { 'Fn::Select' => [index, list] }
       end

data/lib/cfhighlander.dsl.subcomponent.rb

@@ -28,13 +28,13 @@ module Cfhighlander
           :component_config_override,
           :export_config
 
-
       attr_reader :cfn_name,
           :conditional,
           :parent,
           :name,
           :template,
-          :template_version
+          :template_version,
+          :inlined
 
       def initialize(parent,
           name,
@@ -45,6 +45,7 @@ module Cfhighlander
           export_config = {},
           conditional = false,
           enabled = true,
+          inline = false,
           distribution_format = 'yaml')
 
         @parent = parent
@@ -52,6 +53,7 @@ module Cfhighlander
         @export_config = export_config
         @component_sources = component_sources
         @conditional = conditional
+        @inlined = inline
 
         template_name = template
         template_version = 'latest'
@@ -227,7 +229,12 @@ module Cfhighlander
         # TODO wire mapping parameters outside of component
         if param.class == Cfhighlander::Dsl::MappingParam
           puts " mapping parameter"
-          return self.resolveMappingParamValue(component, sub_component, param)
+          mapping_param_value = self.resolveMappingParamValue(component, sub_component, param)
+
+          # if mapping param is not resolved, e.g. mapping not provided
+          # parameters will bubble to parent component if not matched by outputs from
+          # other components
+          return mapping_param_value unless mapping_param_value.nil?
         end
 
         # rule #2: match output values from other components
@@ -287,11 +294,6 @@ module Cfhighlander
             key_name: key_name
         )
 
-        if value.nil?
-          return "'#{param.default_value}'" unless param.default_value.empty?
-          return "''"
-        end
-
         return value
       end
 

data/lib/cfhighlander.dsl.template.rb

@@ -117,6 +117,7 @@ module Cfhighlander
           export_config: {},
           conditional: false,
           enabled: true,
+          render: Cfhighlander::Model::Component::Substack,
           &block)
         puts "INFO: Requested subcomponent #{name} with template #{template}"
         if ((not template_arg.nil?) and template.nil?)
@@ -137,7 +138,8 @@ module Cfhighlander
             config,
             export_config,
             conditional,
-            enabled
+            enabled,
+            render == Cfhighlander::Model::Component::Inline
         )
         # component.instance_eval(&block) unless block.nil?
         @subcomponents_exec[name] = block unless block.nil?
@@ -482,9 +484,6 @@ def CfhighlanderTemplate(&block)
   unless @distribution_prefix.nil?
     instance.DistributionPrefix(@distribution_prefix)
   end
-  unless @distribution_format.nil?
-    instance.Forma
-  end
 
   # process convention over configuration componentname.config.yaml files
   @potential_subcomponent_overrides.each do |name, config|

data/lib/cfhighlander.factory.templatefinder.rb

@@ -15,8 +15,9 @@ module Cfhighlander
         default_locations = [
             LOCAL_CFHIGHLANDER_CACHE_LOCATION,
             File.expand_path('components'),
-            File.expand_path('.')
+            File.expand_path('.'),
         ]
+        default_locations << ENV['CFHIGHLANDER_WORKDIR'] if ENV.key? 'CFHIGHLANDER_WORKDIR'
         default_locations.each do |predefined_path|
           component_sources.unshift(predefined_path)
         end

data/lib/cfhighlander.model.component.rb

@@ -6,6 +6,9 @@ module Cfhighlander
 
     class Component
 
+      Inline = 'inline'
+      Substack = 'substack'
+
       attr_accessor :component_dir,
           :config,
           :highlander_dsl_path,
@@ -26,12 +29,12 @@ module Cfhighlander
           :factory,
           :extended_component,
           :is_parent_component
-
-      attr_reader :cfn_model,
-          :outputs,
+      attr_reader :outputs,
           :factory,
           :extended_component,
-          :potential_subcomponent_overrides
+          :potential_subcomponent_overrides,
+          :cfn_model,
+          :cfn_model_raw
 
 
       def initialize(template_meta, component_name, factory)
@@ -90,7 +93,7 @@ module Cfhighlander
       def loadDepandantExt()
         @highlander_dsl.dependson_components.each do |requirement|
           requirement.component_loaded.cfndsl_ext_files.each do |file|
-            cfndsl_ext_files << file
+            @cfndsl_ext_files << file
           end
         end
       end
@@ -230,12 +233,17 @@ module Cfhighlander
       end
 
       # evaluates cfndsl with current config
+      def set_cfndsl_model(value)
+        @cfn_model = value.as_json
+        @cfn_model_raw = JSON.parse(@cfn_model.to_json)
+      end
       def eval_cfndsl
         compiler = Cfhighlander::Compiler::ComponentCompiler.new self
         # there is no need for processing lambda source code during cloudformation evaluation,
         # this version never gets published
         compiler.process_lambdas = false
         @cfn_model = compiler.evaluateCloudFormation().as_json
+        @cfn_model_raw = JSON.parse(@cfn_model.to_json)
         @outputs = (
         if @cfn_model.key? 'Outputs'
         then

data/lib/cfhighlander.publisher.rb

@@ -40,12 +40,17 @@ module Cfhighlander
 
       end
 
-      def getLaunchStackUrl
+      def getTemplateUrl
         template_url = "https://#{@component.highlander_dsl.distribution_bucket}.s3.amazonaws.com/"
         template_url += @component.highlander_dsl.distribution_prefix + "/"
         template_url += @component.highlander_dsl.version
-        region = s3_bucket_region(@component.highlander_dsl.distribution_bucket)
         template_url += "/#{@component.name}.compiled.yaml"
+        return template_url
+      end
+
+      def getLaunchStackUrl
+        template_url = getTemplateUrl
+        region = s3_bucket_region(@component.highlander_dsl.distribution_bucket)
         return "https://console.aws.amazon.com/cloudformation/home?region=#{region}#/stacks/create/review?filter=active&templateURL=" +
             "#{URI::encode(template_url)}&stackName=#{@component.name}"
       end

data/lib/cfhighlander.validator.rb

@@ -7,7 +7,6 @@ module Cfhighlander
 
   module Cloudformation
 
-
     class Validator
 
       def initialize(component)

data/lib/util/cloudformation.util.rb

@@ -0,0 +1,450 @@
+require_relative '../cfhighlander.model.component'
+require_relative './debug.util'
+require 'duplicate'
+
+module Cfhighlander
+
+  module Util
+
+    class CloudFormation
+
+      def self.flattenCloudformation(args = {})
+
+        component = args.fetch(:component)
+        template = component.highlander_dsl
+
+        # make sure all mappings, resources and conditions
+        # are named uniquely in all of the templates
+        flatten_key_names(component, template)
+        Debug.debug_dump_cfn(template, 'namespace_flat')
+
+        # collect output values
+        output_values = collect_output_values(template)
+        Debug.debug_dump(output_values, 'outputs')
+
+        # collect referenced parameters and convert to replacements
+        component_replacements = collect_replacements(component, template, output_values)
+        Debug.debug_dump(component_replacements, 'replacements')
+
+        # apply replacements in referenced templates
+        process_replacements(component, template, component_replacements)
+        Debug.debug_dump_cfn(template, 'transformed')
+
+        # inline all of the resources
+        inline_resources(component, template)
+
+        # remove substacks
+        remove_inlined_component_stacks(component, template)
+
+        # return inlined model
+        return component.cfn_model_raw
+      end
+
+      def self.remove_inlined_component_stacks(component, template)
+        model = component.cfn_model_raw
+        template.subcomponents.each do |sub_component|
+          next unless sub_component.inlined
+          model['Resources'].delete(sub_component.component_loaded.name)
+        end
+      end
+
+      def self.flatten_namespace(element_type, component, template)
+        if component.cfn_model_raw.key? element_type
+          keys_taken = component.cfn_model_raw[element_type].keys
+        else
+          keys_taken = []
+        end
+        template.subcomponents.each do |sub_component|
+          next unless sub_component.inlined
+          model = sub_component.component_loaded.cfn_model_raw
+
+          model[element_type].keys.each do |key|
+            if keys_taken.include? key
+              candidate = "#{sub_component.component_loaded.name}#{key}"
+              counter = 1
+              while keys_taken.include? candidate
+                candidate = "#{sub_component.component_loaded.name}#{key}#{counter}"
+                counter = counter + 1
+              end
+              actual_key = candidate
+              # we need to replace all as
+              # resources can reference conditions
+              # outputs can and will reference resources
+              model[element_type][actual_key] = model[element_type][key]
+              model[element_type].delete(key)
+              case element_type
+              when 'Resources'
+                rename_resource(model, key, actual_key)
+              when 'Mappings'
+                rename_mapping(model, key, actual_key)
+              when 'Conditions'
+                rename_condition(model, key, actual_key)
+              when 'Outputs'
+                # outputs are not effecting anything within the same template
+              end
+              keys_taken << actual_key
+            else
+              keys_taken << key
+            end
+          end if model.key? element_type
+        end
+      end
+
+      def self.inline_resources(component, template)
+        inline_elements('Conditions', component, template)
+        inline_elements('Mappings', component, template)
+        inline_elements('Resources', component, template)
+        inline_elements('Outputs', component, template)
+      end
+
+      def self.inline_elements(element_name, component, template)
+        parent_model = component.cfn_model_raw
+        template.subcomponents.each do |sub_component|
+          next unless sub_component.inlined
+          model = sub_component.component_loaded.cfn_model_raw
+          model[element_name].each do |resource, value|
+            # effective extraction of child resource into parent
+            parent_model[element_name] = {} unless parent_model.key? element_name
+            parent_model[element_name][resource] = value
+          end if model.key? element_name
+        end
+      end
+
+      def self.process_replacements(component, template, component_replacements)
+
+        # replacement processing is done from least to most dependant component
+        dependency_sorted_subcomponents = template.subcomponents.sort {|sc1, sc2|
+          sc1_params = component.cfn_model_raw['Resources'][sc1.cfn_name]['Properties']['Parameters']
+          sc2_params = component.cfn_model_raw['Resources'][sc2.cfn_name]['Properties']['Parameters']
+          outval_refs_sc1 = find_outval_refs(sc1_params)
+          outval_refs_sc2 = find_outval_refs(sc2_params)
+
+          # if sc1 is dependant on sc2,
+          # sc2 param outval refs should have sc1 output
+          # and vice versa
+          sc1_depends_sc2 = if outval_refs_sc1.find{|oref| oref[:component] == sc2.cfn_name}.nil? then false else true end
+          sc2_depends_sc1 = if outval_refs_sc2.find{|oref| oref[:component] == sc1.cfn_name}.nil? then false else true end
+
+          if (sc1_depends_sc2 and sc2_depends_sc1)
+            raise StandardError, "Components #{sc1.cfn_name} and #{sc2.cfn_name} have circular dependency!!"
+          end
+          if sc1_depends_sc2 then
+            +1
+          elsif sc2_depends_sc1 then
+            -1
+          else
+            0
+          end
+        }
+
+        # process replacements in order from least dependant to
+        # most dependant
+        dependency_sorted_subcomponents.each_with_index do |sub_component, index|
+          next unless sub_component.inlined
+          component_name = sub_component.component_loaded.name
+          if component_replacements.key? component_name
+            if sub_component.component_loaded.cfn_model_raw.key? 'Outputs'
+              outputs_apriori = duplicate(sub_component.component_loaded.cfn_model_raw['Outputs'])
+            else
+              outputs_apriori = {}
+            end
+            component_replacements[component_name].each do |replacement|
+              node_replace(
+                  sub_component.component_loaded.cfn_model_raw,
+                  replacement[:search],
+                  replacement[:replace]
+              ) # some of the component outputs may be changed and thus replacements need be updated
+            end
+            iteration_index = 2
+            outputs_apriori.each do |out_name, out_value|
+              value_after_transform = sub_component.component_loaded.cfn_model_raw['Outputs'][out_name]
+              # value of the output was changed by replacement
+              unless out_value == value_after_transform
+                # for all downstream dependant components
+                propagated_update_index = index + 1
+                while propagated_update_index < dependency_sorted_subcomponents.size
+                  pc_name = dependency_sorted_subcomponents[propagated_update_index].component_loaded.name
+                  component_replacements[pc_name].each do |replacement|
+                    # replacements for dependant component needs to be updated as well
+                    replace = replacement[:replace]
+
+                    if out_value['Value'] == replace
+                      replacement[:replace] = value_after_transform['Value']
+                    else
+                      node_replace(replacement[:replace], out_value['Value'], value_after_transform['Value'])
+                    end
+
+                  end if component_replacements.include? pc_name
+                  propagated_update_index += 1
+                end
+                Debug.debug_dump(component_replacements, "replacements.#{iteration_index}")
+                iteration_index += 1
+              end
+            end
+          end
+        end
+
+        # process replacements on component itself
+        component_replacements[component.name].each do |replacement|
+          node_replace(component.cfn_model_raw, replacement[:search], replacement[:replace])
+        end
+      end
+
+      def self.find_outval_refs(tree, outval_refs = [])
+        tree.each do |key, val|
+
+          # if we have located get att, it may be output value
+          if key == 'Fn::GetAtt'
+            if val.is_a? Array and val.size == 2
+              if val[1].start_with? 'Outputs.'
+                component = val[0]
+                output = val[1].split('.')[1]
+                outval_refs << { component: component, outputName: output }
+              end
+            end
+          elsif val.is_a? Hash or val.is_a? Array
+            # however we may also find output deeper in the tree
+            # example being FnIf(condition, out1, out2)
+            find_outval_refs(val, outval_refs)
+          end
+
+        end if tree.is_a? Hash
+
+        tree.each do |element|
+          find_outval_refs(element, outval_refs)
+        end if tree.is_a? Array
+        return outval_refs
+      end
+
+      def self.collect_replacements(component, template, output_values)
+        replacements = {}
+
+        # collect replacements for inlined components
+        template.subcomponents.each do |sub_component|
+          next unless sub_component.inlined
+          component_loaded = sub_component.component_loaded
+          replacements[component_loaded.name] = []
+          sub_stack_def = component.cfn_model_raw['Resources'][sub_component.cfn_name]
+          next unless sub_stack_def['Properties'].key? 'Parameters'
+          params = sub_stack_def['Properties']['Parameters']
+          params.each do |param_name, param_value|
+            # if param value is hash, we may find output values
+            # these should be replaced with inlined values
+            if param_value.is_a? Hash
+              outval_refs = find_outval_refs(param_value)
+              outval_refs.each do |out_ref|
+                # replacement only takes place if
+                # source component is inlined as well
+                # if source component is not inlined
+                # it's output won't be collected
+                source_sub_component = template.subcomponents.find {|sc| sc.component_loaded.name == out_ref[:component]}
+
+                # if source component is not inlined we can replacement as-is
+                next unless source_sub_component.inlined
+                search = { 'Fn::GetAtt' => [
+                    out_ref[:component],
+                    "Outputs.#{out_ref[:outputName]}"
+                ] }
+                replacement = output_values[out_ref[:component]][out_ref[:outputName]]
+                if param_value == search
+                  param_value = replacement
+                else
+                  # parameter value may be deeper in the structure, e.g.
+                  # member of Fn::If intrinsic function
+                  node_replace(
+                      param_value,
+                      search,
+                      replacement
+                  )
+                end if output_values.key? out_ref[:component]
+
+              end
+            end
+            replacements[component_loaded.name] << {
+                search: { 'Ref' => param_name },
+                replace: param_value
+            }
+          end
+
+        end
+
+        # collect replacements to be performed on parameters of non-inlined components
+        # that are referencing inlined components
+        replacements[component.name] = []
+        template.subcomponents.each do |sub_component|
+          next if sub_component.inlined
+          sub_stack_def = component.cfn_model_raw['Resources'][sub_component.cfn_name]
+          next unless sub_stack_def['Properties'].key? 'Parameters'
+          params = sub_stack_def['Properties']['Parameters']
+          params.each do |param_name, param_value|
+            if param_value.is_a? Hash
+              outval_refs = find_outval_refs(param_value)
+
+              # component is NOT inlined and has out references to components that MAY be inlined
+              outval_refs.each do |out_ref|
+                component_name = out_ref[:component]
+                ref_sub_component = template.subcomponents.find {|sc| sc.name == component_name}
+                if ref_sub_component.inlined
+                  # out refs here need to be replaced with actual values
+                  replacement = output_values[out_ref[:component]][out_ref[:outputName]]
+                  replacements[component.name] << {
+                      search: { 'Fn::GetAtt' => [component_name, "Outputs.#{out_ref[:outputName]}"] },
+                      replace: replacement
+                  }
+                end
+              end
+            end
+          end
+        end
+        return replacements
+      end
+
+      def self.collect_output_values(template)
+        output_vals = {}
+        template.subcomponents.each do |sub_component|
+          # we collect outputs only from inlined components
+          model = sub_component.component_loaded.cfn_model_raw
+          model['Outputs'].each do |name, value|
+            output_vals[sub_component.component_loaded.name] = {} unless output_vals.key? sub_component.component_loaded.name
+            output_vals[sub_component.component_loaded.name][name] = value['Value']
+          end if model.key? 'Outputs'
+        end
+        return output_vals
+      end
+
+      ## if hash is treated as
+      ## collection of tree structures
+      ## where each key in Hash is root of the tree
+      ## and value is subtree
+      ## replace hash subtree with another subtree
+      def self.node_replace(tree, search, replacement)
+        if tree.is_a? Hash
+          tree.each do |root, subtree|
+            if subtree == search
+              tree[root] = replacement
+            elsif subtree.is_a? Hash or subtree.is_a? Array
+              node_replace(subtree, search, replacement)
+            end
+          end
+        elsif tree.is_a? Array
+          tree.each do |element|
+            if element == search
+              tree[tree.index element] = replacement
+            elsif element.is_a? Hash or element.is_a? Array
+              node_replace(element, search, replacement)
+            end
+          end
+        end
+      end
+
+      # rename cloudformation resource in model
+      def self.rename_resource(tree, search, replacement)
+        tree.keys.each do |k|
+          v = tree[k]
+          if k == 'Ref' and v == search
+            tree[k] = replacement
+          end
+
+          if k == 'Fn::GetAtt' and v[0] == search
+            tree[k] = [replacement, v[1]]
+          end
+
+          if v.is_a? Array or v.is_a? Hash
+            rename_resource(v, search, replacement)
+          end
+        end if tree.is_a? Hash
+
+        tree.each do |element|
+          rename_resource(element, search, replacement)
+        end if tree.is_a? Array
+
+      end
+
+      # rename cloudformation mapping in cfn model
+      def self.rename_mapping(tree, search, replacement)
+        tree.keys.each do |k|
+          v = tree[k]
+          if k == 'Fn::FindInMap' and v[0] = search
+            tree[k] = [replacement, v[1], v[2]]
+          end
+
+          if v.is_a? Array or v.is_a? Hash
+            rename_mapping(v, search, replacement)
+          end
+        end if tree.is_a? Hash
+
+        tree.each do |element|
+          rename_mapping(element, search, replacement)
+        end if tree.is_a? Array
+
+      end
+
+      # rename cloudformation condition in cfn model
+      def self.rename_condition(tree, search, replacement)
+        # conditions can be referenced by Fn::If and Condition => cond
+        tree.keys.each do |k|
+          v = tree[k]
+          if k == 'Fn::If' and v[0] == search
+            tree[k] = [replacement, v[1], v[2]]
+          end
+
+          if k == 'Condition' and v == search
+            tree[k] = replacement
+          end
+
+          if v.is_a? Array or v.is_a? Hash
+            rename_condition(v, search, replacement)
+          end
+        end if tree.is_a? Hash
+
+        tree.each do |element|
+          rename_condition(element, search, replacement)
+        end if tree.is_a? Array
+
+      end
+
+      ## Replace single value in tree structure (represented)
+      # either as Hash or Array with another value
+      def self.value_replace(tree, search, replacement)
+        if tree.is_a? Hash
+          tree.keys.each do |root|
+            subtree = tree[root]
+            if root == search
+              tree[replacement] = subtree
+              tree.delete(root)
+            end
+            if subtree == search
+              tree[root] = replacement
+            end
+            if subtree.is_a? Hash or subtree.is_a? Array
+              value_replace(subtree, search, replacement)
+            end
+          end
+        elsif tree.is_a? Array
+          tree.each do |element|
+            if element == search
+              tree[tree.index element] = replacement
+            elsif element.is_a? Hash or element.is_a? Array
+              value_replace(element, search, replacement)
+            end
+          end
+        end
+      end
+
+      def self.flatten_key_names(component, template)
+        flatten_namespace('Conditions', component, template)
+        Debug.debug_dump_cfn(template, 'flat.conditions')
+
+        flatten_namespace('Mappings', component, template)
+        Debug.debug_dump_cfn(template, 'flat.mappings')
+
+        flatten_namespace('Resources', component, template)
+        Debug.debug_dump_cfn(template, 'flat.resources')
+
+        flatten_namespace('Outputs', component, template)
+        Debug.debug_dump_cfn(template, 'flat.outputs')
+      end
+
+    end
+  end
+end

data/lib/util/debug.util.rb

@@ -0,0 +1,37 @@
+module Cfhighlander
+
+  module Util
+
+    class Debug
+
+      @@debug_folder_exists = false
+
+      def self.create_debug_folder
+        if ((ENV.key? 'CFHIGHLANDER_DEBUG') and (ENV['CFHIGHLANDER_DEBUG'] == '1'))
+          FileUtils.mkdir_p "#{ENV['CFHIGHLANDER_WORKDIR']}/out/debug/"
+          @@debug_folder_exists = true
+        end unless @@debug_folder_exists
+      end
+
+      def self.debug_dump_cfn(template, step_name)
+        create_debug_folder
+        if ((ENV.key? 'CFHIGHLANDER_DEBUG') and (ENV['CFHIGHLANDER_DEBUG'] == '1'))
+          template.subcomponents.each do |sub_component|
+            path = "#{ENV['CFHIGHLANDER_WORKDIR']}/out/debug/#{sub_component.cfn_name}_#{step_name}.yaml"
+            File.write(path, sub_component.component_loaded.cfn_model_raw.to_yaml)
+          end
+        end
+      end
+
+      def self.debug_dump(model, dump_name)
+        create_debug_folder
+        if ((ENV.key? 'CFHIGHLANDER_DEBUG') and (ENV['CFHIGHLANDER_DEBUG'] == '1'))
+          path = "#{ENV['CFHIGHLANDER_WORKDIR']}/out/debug/#{dump_name}.yaml"
+          File.write(path, model.to_yaml)
+        end
+      end
+
+
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cfhighlander
 version: !ruby/object:Gem::Version
-  version: 0.6.0.alpha.1538010880
+  version: 0.6.0.alpha.1539120146
 platform: ruby
 authors:
 - Nikola Tosic
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-27 00:00:00.000000000 Z
+date: 2018-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: highline
@@ -212,6 +212,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 1.5.1
+- !ruby/object:Gem::Dependency
+  name: duplicate
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -243,7 +257,6 @@ files:
 - cfndsl_ext/config/managed_policies.yaml
 - cfndsl_ext/iam_helper.rb
 - cfndsl_ext/lambda_helper.rb
-- cfndsl_ext/sg.rb
 - hl_ext/aws_helper.rb
 - hl_ext/common_helper.rb
 - hl_ext/mapping_helper.rb
@@ -261,6 +274,8 @@ files:
 - lib/cfhighlander.model.templatemeta.rb
 - lib/cfhighlander.publisher.rb
 - lib/cfhighlander.validator.rb
+- lib/util/cloudformation.util.rb
+- lib/util/debug.util.rb
 - lib/util/zip.util.rb
 - templates/cfndsl.component.template.erb
 homepage: https://github.com/theonestack/cfhighlander/blob/master/README.md

data/cfndsl_ext/sg.rb

@@ -1,53 +0,0 @@
-require 'netaddr'
-
-def sg_create_rules (x, ip_blocks={})
-  rules = []
-  x.each do | group |
-    group['ips'].each do |ip|
-      group['rules'].each do |rule|
-        lookup_ips_for_sg(ip_blocks, ip).each do |cidr|
-          rules << { IpProtocol: "#{rule['IpProtocol']}", FromPort: "#{rule['FromPort']}", ToPort: "#{rule['ToPort']}", CidrIp: cidr }
-        end
-      end
-    end
-  end
-  return rules
-end
-
-
-def lookup_ips_for_sg (ips, ip_block_name={})
-  cidr = []
-  if ip_block_name == 'stack'
-    cidr = [FnJoin( "", [ "10.", Ref('StackOctet'), ".", "0.0/16" ] )]
-  elsif ips.has_key? ip_block_name
-    ips[ip_block_name].each do |ip|
-      if (ips.include?(ip) || ip_block_name == 'stack')
-        cidr += lookup_ips_for_sg(ips, ip) unless ip == ip_block_name
-      else
-        if ip == 'stack'
-          cidr << [FnJoin( "", [ "10.", Ref('StackOctet'), ".", "0.0/16" ] )]
-        elsif(isCidr(ip))
-          cidr << ip
-        else
-          STDERR.puts("WARN: ip #{ip} is not a valid CIDR. Ignoring IP")
-        end
-      end
-    end
-  else
-    if isCidr(ip_block_name)
-      cidr = [ip_block_name]
-    else
-      STDERR.puts("WARN: ip #{ip_block_name} is not a valid CIDR. Ignoring IP")
-    end
-  end
-  cidr
-end
-
-def isCidr(block)
-  begin
-    NetAddr::CIDR.create(block)
-    return block.include?('/')
-  rescue NetAddr::ValidationError
-    return false
-  end
-end