aws-cfn-compiler 0.6.0 → 0.7.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MWFiNWI1NTIzY2VhNjQ4NjM3ZDgzOWIwNTg4MzczZDI3OWFlNzFlMw==
+    NTljZDM3YzBkMWZhYzVhM2RhMDBkMjA4MjQ3MTljZjdlNjczZDUyYg==
   data.tar.gz: !binary |-
-    YjIwMjMwNzlkY2Q2YzliM2U3ZWM0YjE2ODgyZDcyMjUzMDM2MzJmMw==
+    YWIzN2E1YTE3YTk3OWVlNDVlYTYyZGQzNWZiYjc5N2QzNDY1NjU5YQ==
 SHA512:
   metadata.gz: !binary |-
-    NDhjYjgyZjZlNzM4ZWExMTA0Y2QxNGIzMWM3ZTNmZWQ2ZWUxNDdlMDlhNzA3
-    ZjI4Y2QxNDgwNDYwZjNiNTRjODY5ZmE3ZjBlYmJjYmYxMzhkYmJjN2YzMjNh
-    YTMwYjQ2Y2U4NDE2N2Q0NTJlZWZiNDVlZjg5MjFhMDRhMWRjNzk=
+    Y2RhZTI5ODI1Y2MzODU2OTQzYjI3ODFlOGEzMDU4MjM2ZTJiMDU5MWIyODg3
+    YWJmOGQwYTFlNDNiMWQ5Y2FjMmJhN2JiMzBlMTg1YTE3ZGQ5NDk3N2FiZTU5
+    YTYyMjU0ZjU2YWY5MWUxYTMxOThmYzc5NjU3MDlhNjI0ZTQ2MjQ=
   data.tar.gz: !binary |-
-    NGYyNzU2MWE2NTZhOGFkZTRjYTgxZTMzNzVjOGVmZDJjZjQzOTQxNzg3M2Vh
-    ODY1YzQ1YWI5OWMxOGU5OGY5MzEyMTg0YjY2NzdiODcyYzUzNTk0ODVjNjQw
-    MjBhMmQzMmZmYzM3MTIxNTc1MzFlZTkyZWUxNGUzYWYxNTFkZDg=
+    NDdkYjUyOTA4Y2U1MDM0YjZlZmM5YmZhN2E2NzIyMWZiZjQ1ODQwNWRmOGM2
+    MTZjMzExOWVhNWE0MjM5YjQyZTc3ZGFmZDk1OTY5MTFmN2RlYmVmZjljMThj
+    M2Q4YzIyNzJkNGFlZDFiMDFmYWRkOTdlZjYwNzcxNDhiN2ZhOWI=

data/Gemfile

@@ -9,9 +9,9 @@ gem 'dldinternet-mixlib-logging', :path => '../dldinternet-mixlib-logging', :gro
 gem 'dldinternet-mixlib-cli', :path => '../dldinternet-mixlib-cli', :group => :development
 gem 'aws-cfn-dsl', :path => '../aws-cfn-dsl', :group => :development
 gem 'aws-cfn-decompiler', :path => '../aws-cfn-decompiler', :group => :development
-gem 'aws-cfn-compiler', :path => '../aws-cfn-compiler', :group => :development
 gem 'aws-cfn-yats', :path => '../aws-cfn-yats', :group => :development
 gem 'aws-cfn-stacker', :path => '../aws-cfn-stacker', :group => :development
 
+gem 'json_pure', :group => :development
 gem 'bundler', "~> 1.6", :group => :development
 gem 'rake', :group => :development

data/aws-cfn-compiler.gemspec

@@ -18,13 +18,11 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "awesome_print", '~> 1.2', '>= 1.2.0'
-  spec.add_dependency "psych"
-  spec.add_dependency "json"
-  spec.add_dependency "slop"
-  spec.add_dependency 'colorize'
-  spec.add_dependency 'inifile'
-  spec.add_dependency 'aws-cfn-dsl', '>= 0.2.0', '~> 0.2'
+  spec.add_dependency 'awesome_print', '~> 1.2', '>= 1.2.0'
+  spec.add_dependency 'psych'
+  spec.add_dependency 'json'
+  spec.add_dependency 'slop'
+  spec.add_dependency 'aws-cfn-dsl', '>= 0.7.0', '~> 0.7'
   spec.add_dependency 'dldinternet-mixlib-cli', ">= 0.1.0", '~> 0.1'
   spec.add_dependency 'dldinternet-mixlib-logging', '>= 0.1.6', '~> 0.1'
 

data/aws-cfn-compiler.iml

@@ -52,6 +52,7 @@
     <orderEntry type="library" scope="PROVIDED" name="detabulator (v0.1.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="inifile (v2.0.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="json (v1.8.1, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="json_pure (v1.8.1, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="little-plugger (v1.1.3, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="logging (v1.8.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="mixlib-cli (v1.5.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />

data/lib/aws/cfn/compiler/base.rb

@@ -21,7 +21,7 @@ module Aws
 
         def validate(compiled)
           abort! 'No Resources!?' unless compiled['Resources']
-          logStep 'Validating compiled file...'
+          logStep 'Validating template'
 
           # Mappings => Resources
           maps  = find_maps(compiled) #.select { |a| !(a =~ /^AWS::/) }
@@ -55,8 +55,12 @@ module Aws
         end
 
         def save_template(output_file,compiled)
-          output_file = File.realpath(File.expand_path(output_file)) if @config[:expandedpaths]
-          logStep "Writing compiled file to #{output_file}..."
+          file = output_file
+          file = File.expand_path(output_file) if @config[:expandedpaths]
+          if File.exists?(file)
+            file = File.realpath(file)
+          end
+          logStep "Writing compiled file to #{file}..."
           begin
             hash = {}
             compiled.each do |item,value|
@@ -67,13 +71,15 @@ module Aws
               end
             end
 
-            File.open output_file, 'w' do |f|
-              f.write JSON.pretty_generate(hash, { indent: "\t", space: ' '})
-            end
+            dir  = File.dirname(output_file)
+            file = File.basename(output_file)
+            sect = dir == '.' ? Dir.pwd : dir
+            sect = File.basename(sect) unless @config[:expandedpaths]
+            save_section(dir, file, @config[:format], sect, hash, '', 'template')
+
             @logger.info '  Compiled file written.'
           rescue
-            @logger.error "!!! Could not write compiled file: #{$!}"
-            abort!
+            abort! "!!! Could not write compiled file: #{$!}"
           end
         end
 
@@ -107,295 +113,29 @@ module Aws
                   abort! "Unsupported file type for specification: #{spec}"
               end
             else
-              abort! "Unable to open specification"+ (abs.nil? ? " or {,#{@config[:directory]}/}#{spec} not found" : ": #{abs}")
-            end
-            @dsl ||= Aws::Cfn::Dsl::Template.new(@config[:directory])
-            %w( Mappings Parameters Resources Outputs ).each do |dir|
-              load_dir(dir,@spec)
+              abort! 'Unable to open specification'+ (abs.nil? ? " or {,#{@config[:directory]}/}#{spec} not found" : ": #{abs}")
             end
           else
-            raise "No specification provided"
+            abort! 'No specification provided'
           end
         end
 
         protected
 
-        # noinspection RubyGlobalVariableNamingConvention
-        def load_dir(dir,spec=nil)
-          logStep "Loading #{dir}..."
-
-          if spec and spec[dir]
-            raise "No such directory: #{@config[:directory]}" unless File.directory?(@config[:directory])
-            path = vet_path(dir)
-            @items      ||= {}
-            @items[dir] ||= {}
-            set = {}
-            get = {}
-            get[path] = get_file_set([".*"], path, @config[:precedence])
-
-            item = {}
-            spec[dir].each do |rsrc|
-              @logger.info "\tUsing #{dir}::#{rsrc}"
-              set = get[path]
-              refp,sub,base,rel = map_resource_reference(rsrc)
-              unless refp.nil?
-                path = vet_path(sub ? sub : dir,refp, rel)
-                unless get[path]
-                  get[path] = get_file_set([".*"], path, @config[:precedence])
-                  set = get[path]
-                end
-              end
-              if set[base]
-                if item.has_key?(base)
-                  @logger.error "  !! error: Duplicate item: #{dir}/#{base}"
-                  abort!
-                end
-
-                filename = set[base]
-                unless filename =~ /\.(ru?by?|ya?ml|js(|on))\z/i
-                  @logger.info "Brick not supported/ relevant: #{filename}"
-                  next
-                end
-
-                begin
-                  @logger.debug "  reading #{filename}"
-                  content = File.read(filename)
-                  next if content.size==0
-
-                  if filename =~ /\.(rb|ruby)\z/i
-                    compile_rb_file(base, dir, filename)
-                    item.merge! @dsl.dict[dir.to_sym]
-                  elsif filename =~ /\.js(|on)\z/i
-                    item.merge! JSON.parse(content)
-                  elsif filename =~ /\.ya?ml\z/i
-                    item.merge! YAML.load(content)
-                  else
-                    next
-                  end
-
-                rescue
-                  @logger.error "  !! error: #{$!}"
-                  abort!
-                end
-              else
-                @logger.error "  !! error: #{dir}/#{base} not found!"
-                abort!
-              end
-            end
-            item.keys.each { |key|
-              if @items[dir].has_key?(key)
-                @logger.error "  !! error: Duplicate item: #{dir}/#{key}"
-                abort!
-              end
-            }
-            @items[dir].merge! item
-
-            unless @items[dir].keys.count == spec[dir].count
-              @logger.error "  !! error: Suspect that a #{dir} item was missed! \nRequested: #{spec[dir]}\n    Found: #{@items[dir].keys}"
-              abort!
-            end
-          end
-
-        end
-
-        def compile_rb_file(base, dir, filename)
-          Aws::Cfn::Compiler.binding ||= {}
-          Aws::Cfn::Compiler.binding[dir] ||= {}
-          Aws::Cfn::Compiler.binding[dir][base] ||= {
-              brick_path: @config[:directory],
-              template: @dsl,
-              logger: @logger
-          }
-          source_file = File.expand_path(filename)
-          # source      = IO.read(source_file)
-          eval "require source_file", binding
-          unless @dsl.dict[dir.to_sym]
-            abort! "Unable to compile/expand #{filename} for #{dir}/#{base}"
-          end
-        end
-
-        def find_refs(hash, type='Reference', parent='')
-          h = {}
-          newparent = parent
-          if hash.is_a? Hash
-            hash.keys.collect do |key|
-              if %w{Mappings Parameters Resources Outputs}.include? key
-                type = key#.gsub(/s$/, '')
-                newparent = key
-              elsif %w{Mappings Parameters Resources Outputs}.include? parent
-                newparent = key
-              end
-              if %w{Ref}.include? key
-                h = { hash[key] => [type,newparent] }
-              elsif 'Fn::GetAtt' == key
-                h = { hash[key].first => [type,newparent] }
-                # elsif %w{SourceSecurityGroupName CacheSecurityGroupNames SecurityGroupNames}.include? key
-                #   a = find_refs(hash[key],type,newparent)
-                #   h = merge(h, a, *[type,newparent])
-              else
-                a = find_refs(hash[key],type,newparent)
-                h = merge(h, a, *[type,newparent])
-              end
-            end.flatten.compact.uniq
-          elsif hash.is_a? Array
-            a = hash.map{|i| find_refs(i,type,newparent) }
-            h = merge(h, a, type, *[type,newparent])
-          end
-          h
-        end
-
-        def merge(h, a, *type)
-          if a.is_a? Hash
-            if a.size > 0
-              h.merge! a
-            end
-          else
-            a.flatten.compact.uniq.map { |i|
-              if i.is_a? Hash
-                if i.size > 0
-                  h.merge! i
-                  h
-                end
-              else
-                h[i] = type
-              end
-            }
-          end
-          h
-        end
+        require 'aws/cfn/compiler/mixins/options'
+        include Aws::Cfn::Compiler::Options
 
-        def find_maps(hash)
-          if hash.is_a? Hash
-            tr = []
-            hash.keys.collect do |key|
-              if 'Fn::FindInMap' == key
-                hash[key].first
-              else
-                find_maps(hash[key])
-              end
-            end.flatten.compact.uniq
-          elsif hash.is_a? Array
-            hash.collect{|a| find_maps(a)}.flatten.compact.uniq
-          end
-        end
+        require 'aws/cfn/compiler/mixins/load'
+        include Aws::Cfn::Compiler::Load
 
-        # --------------------------------------------------------------------------------
-        def get_file_set(want, path, exts=[])
-          raise "Bad call to #{self.class.name}.get_file_set: want == nil" unless want
-          @logger.debug "Look for #{want.ai} in #{[path]} with #{exts} extensions"
-          if exts.nil?
-            exts = @config[:precedence]
-          end
-          file_regex=%r/^(\S+)\.(#{exts.join('|')})$/
-          if exts.empty?
-            file_regex=%r/^(\S+)()$/
-            exts=['']
-          end
-          regex = "^(#{want.join('|')})$"
-          set = {}
-          abs = File.expand_path(path)
-          abs = path unless @config[:expandedpaths]
-          raise "Oops! Does '#{path}' directory exist?" unless File.directory?(abs)
-          #abs = File.realpath(abs)
-          begin
-            Dir.glob("#{abs}/*").each{ |f|
-              match = File.basename(f).match(file_regex)
-              if match
-                name = match[1]
-                ext  = match[2]
-                set[ext] = {} unless set[ext]
-                @logger.trace "#{name} =~ #{regex}"
-                set[ext][name] = f if name.match(regex)
-              end
-            }
-          rescue RegexpError => e
-            raise "The regular expression attempting to match resources in '#{path}' is incorrect! #{e.message}"
-          end
-          @logger.debug "getPathSet set=#{set.ai}"
-          res = {}
-          # Iterate extension sets in increasing precedence order ...
-          # Survivor will be the most desireable version of the item
-          # i.e. the .rb environment, role, data bag, etc. will be preferred over the .json version
-          exts.each{ |e|
-            h = set[e]
-            if h
-              h.each{ |n,f|
-                @logger.info "Ignoring #{File.basename(res[n])}" if res[n]
-                res[n] = f
-              }
-            else
-              @logger.debug "'#{e}' set is empty! (No #{path}/*.#{e} files found using precedence #{exts})"
-            end
-          }
-          res
-        end
+        require 'aws/cfn/compiler/mixins/parse'
+        include Aws::Cfn::Compiler::Parse
 
-        def map_resource_reference(rsrc)
-          path = nil
-          sub  = nil
-          ref  = nil
-          rel  = false
-          # noinspection RubyParenthesesAroundConditionInspection
-          if rsrc.match %r'^(\.\./.*?)::(.*)$'
-            # Relative path stack reference
-            path,sub,ref,rel  = map_resource_reference(File.basename(rsrc))
-          elsif rsrc.match %r'^(~/.*?)$'
-            # Relative to HOME
-            path,sub,ref,rel  = map_resource_reference(File.expand_path(rsrc))
-          elsif rsrc.match %r'^(\.\./[^:]*?)$'
-            # Relative path
-            path = File.dirname(rsrc)
-            sub  = File.basename(path)
-            path = File.dirname(path)
-            ref  = File.basename(rsrc)
-            rel  = true
-          elsif rsrc.match %r'(^/.*?)::(.*)$'
-            # Absolute path
-            _,sub,ref,rel  = map_resource_reference(File.basename(rsrc))
-            path = File.realpath(File.join(File.dirname(rsrc),_))
-          elsif rsrc.match %r'(^/.*?[^:]*?)$'
-            # Absolute path
-            path = File.dirname(rsrc)
-            sub  = File.basename(path)
-            path = File.dirname(path)
-            ref  = File.basename(rsrc)
-          elsif (match = rsrc.match %r'^(.*?)::(.*)$')
-            # Inherited stack reference
-            ref = match[2]
-            # noinspection RubyParenthesesAroundConditionInspection
-            if (subm = match[1].match(%r'^(.+?)/(.+)$'))
-              path = File.join(File.dirname(@config[:directory]),subm[1])
-              sub = subm[2]
-            else
-              # sub = nil
-              path = File.join(File.dirname(@config[:directory]),match[1])
-            end
-          else
-            # Otherwise it is what it seems ;)
-            ref  = rsrc
-          end
-          [path,sub,ref,rel]
-        end
+        require 'aws/cfn/compiler/mixins/compile'
+        include Aws::Cfn::Compiler::Compile
 
-        def vet_path(dir,base=nil,rel=false)
-          if rel
-            base = File.realpath(File.expand_path(File.join(@config[:directory], base)))
-          else
-            base = @config[:directory] unless base
-          end
-          path = nil
-          [dir, dir.downcase].each do |d|
-            path = File.join(base, dir)
-            if File.directory?(path)
-              break
-            end
-          end
-          unless File.directory?(path)
-            @logger.error "  !! error: Cannot load bricks from #{path} (started with #{File.join(base, dir)}')"
-            abort!
-          end
-          path
-        end
+        require 'aws/cfn/compiler/mixins/save'
+        include Aws::Cfn::Compiler::Save
 
       end
     end

data/lib/aws/cfn/compiler/main.rb

@@ -1,8 +1,4 @@
 
-require 'json'
-require 'ap'
-require 'yaml'
-require 'slop'
 
 module Aws
   module Cfn
@@ -11,60 +7,19 @@ module Aws
 
         def run
 
-          @opts = Slop.parse(help: true) do
-            # command File.basename(__FILE__,'.rb')
-            on :d, :directory=,     'The directory containing template sections', as: String
-            on :o, :output=,        'The JSON file to output', as: String
-            on :s, :specification=, 'The specification to use when selecting components. A JSON or YAML file or JSON object', as: String
-            on :f, :formatversion=, 'The AWS Template format version. ', {  as: String,
-                                                                            optional_argument: true,
-                                                                            default: '2010-09-09' }
-            on :p, :precedence=,    'The precedence of template component types. Default: rb,ruby,yaml,yml,json,js', { as: String,
-                                                                                                                  default: 'rb,ruby,yaml,yml,json,js' }
-            on :t, :description=,   "The AWS Template description. Default: output basename or #{File.basename(__FILE__,'.rb')}", { as: String,
-                                                                                                                                    default: File.basename(__FILE__,'.rb') }
-            on :x, :expandedpaths,  'Show expanded paths in output', { as: String,
-                                                                       optional_argument: true,
-                                                                       default: 'off',
-                                                                       match: %r/0|1|yes|no|on|off|enable|disable|set|unset|true|false|raw/i }
-            on :O, :overwrite,      'Overwrite existing generated source files. (HINT: Think twice ...)', { as: String,
-                                                                                                            optional_argument: true,
-                                                                                                            default: 'off',
-                                                                                                            match: %r/0|1|yes|no|on|off|enable|disable|set|unset|true|false|raw/i }
-          end
+          parse_options
 
-          unless @opts[:directory]
-            puts @opts
-            exit
-          end
+          set_config_options
 
-          @config[:precedence]    = @opts[:precedence].split(%r',+\s*').reverse
-          @config[:expandedpaths] = @opts[:expandedpaths].downcase.match %r'^(1|true|on|yes|enable|set)$'
-          @config[:directory]     = @opts[:directory]
+          load_spec @config[:specification]
 
-          load_spec @opts[:specification]
+          parse
 
-          desc = @opts[:output] ? File.basename(@opts[:output]).gsub(%r/\.(json|yaml)/, '') : File.basename(__FILE__,'.rb')
-          if @spec and @spec['Description']
-            desc = @spec['Description']
-          end
-          vers = '2010-09-09'
-          if @spec and @spec['AWSTemplateFormatVersion']
-            vers = @spec['AWSTemplateFormatVersion']
-          end
-          # noinspection RubyStringKeysInHashInspection
-          compiled = {
-              'AWSTemplateFormatVersion' => (@opts[:formatversion].nil? ? vers : @opts[:formatversion]),
-              'Description'              => (@opts[:description].nil? ? desc : @opts[:description]),
-              'Mappings'                 => @items['Mappings'],
-              'Parameters'               => @items['Parameters'],
-              'Resources'                => @items['Resources'],
-              'Outputs'                  => @items['Outputs'],
-          }
+          compiled = compile
 
           validate(compiled)
 
-          output_file = @opts[:output] || 'compiled.json'
+          output_file = @config[:template] || 'compiled.json'
           save_template(output_file,compiled)
 
           @logger.step '*** Compiled Successfully ***'

data/lib/aws/cfn/compiler/mixins/compile.rb

@@ -0,0 +1,154 @@
+module Aws
+  module Cfn
+    module Compiler
+      module Compile
+
+        def compile
+          desc =  if @config[:description]
+                    @config[:description]
+                  elsif @spec and @spec['Description']
+                    @spec['Description']
+                  elsif @config[:template]
+                    File.basename(@config[:template]).gsub(%r/\..*$/, '')
+                  else
+                    'compiled template'
+                  end
+          vers =  if @config[:formatversion]
+                    @config[:formatversion]
+                  else
+                    if @spec and @spec['AWSTemplateFormatVersion']
+                      @spec['AWSTemplateFormatVersion']
+                    else
+                      '2010-09-09'
+                    end
+                  end
+          # [2014-06-29 Christo] IIRC it is important that the section names be strings instead of symbols ...
+          # noinspection RubyStringKeysInHashInspection
+          compiled =
+          {
+              'AWSTemplateFormatVersion' => vers,
+              'Description'              => desc,
+              'Mappings'                 => @items['Mappings'],
+              'Parameters'               => @items['Parameters'],
+              'Resources'                => @items['Resources'],
+              'Outputs'                  => @items['Outputs'],
+          }
+        end
+
+        def find_refs(hash, type='Reference', parent='')
+          h = {}
+          newparent = parent
+          if hash.is_a? Hash
+            hash.keys.collect do |key|
+              if %w{Mappings Parameters Resources Outputs}.include? key
+                type = key#.gsub(/s$/, '')
+                newparent = key
+              elsif %w{Mappings Parameters Resources Outputs}.include? parent
+                newparent = key
+              end
+              if %w{Ref}.include? key
+                h = { hash[key] => [type,newparent] }
+              elsif 'Fn::GetAtt' == key
+                h = { hash[key].first => [type,newparent] }
+                # elsif %w{SourceSecurityGroupName CacheSecurityGroupNames SecurityGroupNames}.include? key
+                #   a = find_refs(hash[key],type,newparent)
+                #   h = merge(h, a, *[type,newparent])
+              else
+                a = find_refs(hash[key],type,newparent)
+                h = merge(h, a, *[type,newparent])
+              end
+            end.flatten.compact.uniq
+          elsif hash.is_a? Array
+            a = hash.map{|i| find_refs(i,type,newparent) }
+            h = merge(h, a, type, *[type,newparent])
+          end
+          h
+        end
+
+        def merge(h, a, *type)
+          if a.is_a? Hash
+            if a.size > 0
+              h.merge! a
+            end
+          else
+            a.flatten.compact.uniq.map { |i|
+              if i.is_a? Hash
+                if i.size > 0
+                  h.merge! i
+                  h
+                end
+              else
+                h[i] = type
+              end
+            }
+          end
+          h
+        end
+
+        def find_maps(hash)
+          if hash.is_a? Hash
+            tr = []
+            hash.keys.collect do |key|
+              if 'Fn::FindInMap' == key
+                hash[key].first
+              else
+                find_maps(hash[key])
+              end
+            end.flatten.compact.uniq
+          elsif hash.is_a? Array
+            hash.collect{|a| find_maps(a)}.flatten.compact.uniq
+          end
+        end
+
+        def map_resource_reference(rsrc)
+          path = nil
+          sub  = nil
+          ref  = nil
+          rel  = false
+          # noinspection RubyParenthesesAroundConditionInspection
+          if rsrc.match %r'^(\.\./.*?)::(.*)$'
+            # Relative path stack reference
+            path,sub,ref,rel  = map_resource_reference(File.basename(rsrc))
+          elsif rsrc.match %r'^(~/.*?)$'
+            # Relative to HOME
+            path,sub,ref,rel  = map_resource_reference(File.expand_path(rsrc))
+          elsif rsrc.match %r'^(\.\./[^:]*?)$'
+            # Relative path
+            path = File.dirname(rsrc)
+            sub  = File.basename(path)
+            path = File.dirname(path)
+            ref  = File.basename(rsrc)
+            rel  = true
+          elsif rsrc.match %r'(^/.*?)::(.*)$'
+            # Absolute path
+            _,sub,ref,rel  = map_resource_reference(File.basename(rsrc))
+            path = File.realpath(File.join(File.dirname(rsrc),_))
+          elsif rsrc.match %r'(^/.*?[^:]*?)$'
+            # Absolute path
+            path = File.dirname(rsrc)
+            sub  = File.basename(path)
+            path = File.dirname(path)
+            ref  = File.basename(rsrc)
+          elsif (match = rsrc.match %r'^(.*?)::(.*)$')
+            # Inherited stack reference
+            ref = match[2]
+            # noinspection RubyParenthesesAroundConditionInspection
+            if (subm = match[1].match(%r'^(.+?)/(.+)$'))
+              path = File.join(File.dirname(@config[:directory]),subm[1])
+              sub = subm[2]
+            else
+              # sub = nil
+              path = File.join(File.dirname(@config[:directory]),match[1])
+            end
+          else
+            # Otherwise it is what it seems ;)
+            ref  = rsrc
+          end
+          [path,sub,ref,rel]
+        end
+
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/mixins/load.rb

@@ -0,0 +1,81 @@
+module Aws
+  module Cfn
+    module Compiler
+      module Load
+
+        # --------------------------------------------------------------------------------
+        def get_file_set(want, path, exts=[])
+          raise "Bad call to #{self.class.name}.get_file_set: want == nil" unless want
+          @logger.debug "Look for #{want.ai} in #{[path]} with #{exts} extensions"
+          if exts.nil?
+            exts = @config[:precedence]
+          end
+          file_regex=%r/^(\S+)\.(#{exts.join('|')})$/
+          if exts.empty?
+            file_regex=%r/^(\S+)()$/
+            exts=['']
+          end
+          regex = "^(#{want.join('|')})$"
+          set = {}
+          abs = File.expand_path(path)
+          abs = path unless @config[:expandedpaths]
+          raise "Oops! Does '#{path}' directory exist?" unless File.directory?(abs)
+          #abs = File.realpath(abs)
+          begin
+            Dir.glob("#{abs}/*").each{ |f|
+              match = File.basename(f).match(file_regex)
+              if match
+                name = match[1]
+                ext  = match[2]
+                set[ext] = {} unless set[ext]
+                @logger.trace "#{name} =~ #{regex}"
+                set[ext][name] = f if name.match(regex)
+              end
+            }
+          rescue RegexpError => e
+            raise "The regular expression attempting to match resources in '#{path}' is incorrect! #{e.message}"
+          end
+          @logger.debug "getPathSet set=#{set.ai}"
+          res = {}
+          # Iterate extension sets in increasing precedence order ...
+          # Survivor will be the most desireable version of the item
+          # i.e. the .rb environment, role, data bag, etc. will be preferred over the .json version
+          exts.each{ |e|
+            h = set[e]
+            if h
+              h.each{ |n,f|
+                @logger.info "Ignoring #{File.basename(res[n])}" if res[n]
+                res[n] = f
+              }
+            else
+              @logger.debug "'#{e}' set is empty! (No #{path}/*.#{e} files found using precedence #{exts})"
+            end
+          }
+          res
+        end
+
+        def vet_path(dir,base=nil,rel=false)
+          if rel
+            base = File.realpath(File.expand_path(File.join(@config[:directory], base)))
+          else
+            base = @config[:directory] unless base
+          end
+          path = nil
+          [dir, dir.downcase].each do |d|
+            path = File.join(base, dir)
+            if File.directory?(path)
+              break
+            end
+          end
+          unless File.directory?(path)
+            @logger.error "  !! error: Cannot load bricks from #{path} (started with #{File.join(base, dir)}')"
+            abort!
+          end
+          path
+        end
+
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/mixins/options.rb

@@ -0,0 +1,40 @@
+module Aws
+  module Cfn
+    module Compiler
+      module Options
+
+        # TODO: [2014-06-29 Christo] Hook into super class and add on options instead of starting from scratch every time
+        def parse_options
+
+          setup_options
+
+          @opts.on :F, :format=,        'The output format of the components. [JSON|YAML|Ruby]', { as: String, match: @format_regex, default: 'yaml' }
+          @opts.on :s, :specification=, 'The specification to use when selecting components. A JSON or YAML file', { as: String
+          }
+          @opts.on :f, :formatversion=, 'The AWS Template format version. ', {as: String,
+                                                                              default: '2010-09-09'
+          }
+          @opts.on :D, :description=, "The AWS Template description. Default: template name", { as: String }
+          @opts.on :p, :precedence=, 'The precedence of template component types. Default: rb,ruby,yaml,yml,json,js', { as: String,
+                                                                                                                        default: 'rb,ruby,yaml,yml,json,js', }
+
+          @opts.parse!
+
+          if ARGV.size > 0
+            puts @opts
+            abort! "Extra arguments! #{ARGV}"
+          end
+        end
+
+        def set_config_options
+
+          @config[:precedence]    = @opts[:precedence].split(%r',+\s*').reverse
+
+          setup_config
+
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/mixins/parse.rb

@@ -0,0 +1,119 @@
+module Aws
+  module Cfn
+    module Compiler
+      module Parse
+
+        def parse
+          @dsl ||= Aws::Cfn::Dsl::Template.new(@config[:directory])
+          %w( Mappings Parameters Resources Outputs ).each do |section|
+            parse_section(section,@spec)
+          end
+        end
+
+        # noinspection RubyGlobalVariableNamingConvention
+        def parse_section(section,spec=nil)
+          logStep "Parsing #{section}..."
+
+          if spec and spec[section]
+            abort! "No such directory: #{@config[:directory]} (I am here: #{Dir.pwd})" unless File.directory?(@config[:directory])
+            @items          ||= {}
+            @items[section] ||= {}
+            get  = {}
+            item = {}
+            spec[section].each do |rsrc|
+              @logger.info "\tUsing #{section}::#{rsrc}"
+              refp,sub,base,rel = map_resource_reference(rsrc)
+              if refp.nil?
+                path = vet_path(section)
+              else
+                path = vet_path(sub ? sub : section, refp, rel)
+              end
+              unless get[path]
+                get[path] = get_file_set([".*"], path, @config[:precedence])
+              end
+              set = get[path]
+              if set[base]
+                if item.has_key?(base)
+                  @logger.error "  !! error: Duplicate item: #{section}/#{base}"
+                  abort!
+                end
+
+                filename = set[base]
+                unless filename =~ /\.(ru?by?|ya?ml|js(|on))\z/i
+                  @logger.info "Brick not supported/ relevant: #{filename}"
+                  next
+                end
+
+                begin
+                  @logger.debug "  reading #{filename}"
+                  content = File.read(filename)
+                  next if content.size==0
+
+                  if filename =~ /\.(rb|ruby)\z/i
+                    item.merge! parse_rb_file(base, section, filename)
+                  elsif filename =~ /\.js(|on)\z/i
+                    item.merge! JSON.parse(content)
+                  elsif filename =~ /\.ya?ml\z/i
+                    item.merge! YAML.load(content)
+                  else
+                    next
+                  end
+
+                rescue
+                  abort! "  !! error: #{$!}"
+                end
+              else
+                abort! "  !! error: #{section}/#{base} not found!"
+              end
+            end
+            item.keys.each { |key|
+              if @items[section].has_key?(key)
+                abort! "  !! error: Duplicate item: #{section}/#{key}"
+              end
+            }
+            @items[section].merge! item
+
+            unless @items[section].keys.count == spec[section].count
+              abort! "  !! error: Suspect that a #{section} item was missed! \nRequested: #{spec[section]}\n    Found: #{@items[section].keys}"
+            end
+          end
+
+        end
+
+        def parse_rb_file(base, section, filename)
+          Aws::Cfn::Compiler.binding ||= {}
+          Aws::Cfn::Compiler.binding[section] ||= {}
+          Aws::Cfn::Compiler.binding[section][base] ||= {
+              brick_path: @config[:directory],
+              template: @dsl,
+              logger: @logger
+          }
+          source_file = File.expand_path(filename)
+          # source      = IO.read(source_file)
+          eval "require source_file", binding
+          unless @dsl.dict[section.to_sym]
+            abort! "Unable to compile/expand #{filename} for #{section}/#{base}"
+          end
+          sym_to_s(@dsl.dict[section.to_sym])
+        end
+
+        def sym_to_s(hash)
+          if hash.is_a?(Hash)
+            item = {}
+            hash.each { |k,v|
+              item[k.to_s] = sym_to_s(v)
+            }
+            item
+          elsif hash.is_a?(Array)
+            hash.map{|e| sym_to_s(e) }
+          elsif hash.is_a?(String)
+            hash
+          else
+            abort! "#{hash} is a #{hash.class.name}"
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/mixins/save.rb

@@ -0,0 +1,54 @@
+module Aws
+  module Cfn
+    module Compiler
+      module Save
+
+        def save_section(dir, file, format, section, hash, join='/', brick=nil)
+          brick ||= "brick #{hash.keys[0]}"
+          path = File.join(File.expand_path(dir), file)
+          filn = if @config[:expandedpaths]
+                   path
+                 else
+                   File.join(dir, file)
+                 end
+          logStep "Saving #{brick} to #{filn} "
+          # noinspection RubyAssignmentExpressionInConditionalInspection
+          if match = file.match(%r'\.(.*)$')
+            if (section != '') and (not formats_compatible?(format, match[1]))
+              msg = "The file extension (#{match[1]}) does not match the chosen output format (#{format})!"
+              if @config[:force]
+                logger.warn msg
+              else
+                abort! "#{msg}\n\tIf you are SURE this is what you want then use the --force option!"
+              end
+            end
+          end
+
+          if i_am_maintainer(path) or @config[:force]
+            begin
+              File.open path, File::CREAT|File::TRUNC|File::RDWR, 0644 do |f|
+                case format
+                  when /ruby|rb|yaml|yml/
+                    f.write maintainer_comment('')
+                    f.write hash.to_yaml line_width: 1024, indentation: 4, canonical: false
+                  when /json|js/
+                    # You wish ... f.write maintainer_comment('')
+                    f.write JSON.pretty_generate(hash, { indent: "\t", space: ' '})
+                  else
+                    abort! "Internal: Unsupported format #{format}. Should have noticed this earlier!"
+                end
+                f.close
+              end
+              @logger.info "  decompiled #{filn}."
+            rescue
+              abort! "!!! Could not write compiled file #{path}: #{$!}"
+            end
+          else
+            @logger.warn "  Did not overwrite #{filn}."
+          end
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/version.rb

@@ -1,7 +1,7 @@
 module Aws
   module Cfn
     module Compiler
-      VERSION = "0.6.0"
+      VERSION = '0.7.0'
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aws-cfn-compiler
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - PKinney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-28 00:00:00.000000000 Z
+date: 2014-06-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
@@ -72,54 +72,26 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: colorize
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: inifile
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: aws-cfn-dsl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: 0.7.0
     - - ~>
       - !ruby/object:Gem::Version
-        version: '0.2'
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: dldinternet-mixlib-cli
   requirement: !ruby/object:Gem::Requirement
@@ -197,6 +169,11 @@ files:
 - lib/aws/cfn/compiler.rb
 - lib/aws/cfn/compiler/base.rb
 - lib/aws/cfn/compiler/main.rb
+- lib/aws/cfn/compiler/mixins/compile.rb
+- lib/aws/cfn/compiler/mixins/load.rb
+- lib/aws/cfn/compiler/mixins/options.rb
+- lib/aws/cfn/compiler/mixins/parse.rb
+- lib/aws/cfn/compiler/mixins/save.rb
 - lib/aws/cfn/compiler/version.rb
 homepage: https://github.com/dldinternet/aws-cfn-compiler
 licenses: