aws-cfn-compiler 0.3.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 98579909e2c94b50d6d0f5905160c7bb2e033bb9
-  data.tar.gz: 4c5f6152afa7a9853c0a80ade457367124f65598
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MzY5MDJkOWY3MWRlNGZhOTEzNDMxN2IzMjVlMmZhNmI3ZGE4YWYyMg==
+  data.tar.gz: !binary |-
+    MmI4Zjc4OGI3NmI0NzQzMTJmZTEyNTUxMjMzMDcxMmZjMDJiOTk5OQ==
 SHA512:
-  metadata.gz: 0d37543235669ff4bfbdf7b6f17aaaaba20eb7375734f633902c49bd5df38aa5308f359f2e4ad82fbb9d8d2d8d4f8e56770a8347fc01c27dfdede867969daffc
-  data.tar.gz: d342f24025ce2b7b2880d1870318b83cae302b9027e8ed6357cb4f35542146db853d195c0687716209b0f55efa958d812db05235db8917f82ab6039e8ca02d2f
+  metadata.gz: !binary |-
+    NWU0YTAyZTU1MDUxMWYxMGI1NDhhNTBlMzhmMmFiNTViMjhjZjBkYzM5MTMz
+    YzQwNzVlMjRmN2JkYmFlOTJhN2M4OWJkYzlmNjkzM2JjMGEzNTEyYzYxMGJk
+    ZWU4Y2NhNDQ0NGQ4NzBhMmRiN2Y5ZDc4ZGY2ZTE2MjViNDMxOGY=
+  data.tar.gz: !binary |-
+    M2QzYTdlMmEyZWI3NDUwODA1YmEzNDc4Nzc2Y2UyODU0MmFjZmQ5MmQzODUx
+    N2RlYjFkZDlkYWNjZjM3NGExZThkZmIwNDYyNWY0NjIwNmZhYmM4YWQxMDEx
+    NGQwNjkyYzhjMzYzMDZhMzlmZjQzNjM0ZjFlYTBlNTBiMTViNjU=

data/Gemfile

@@ -2,3 +2,9 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in aws-cfn-compiler.gemspec
 gemspec
+
+gem 'cloudformation-ruby-dsl', :path => '../cloudformation-ruby-dsl', :group => :development
+gem 'aws-cfn-dsl', :path => '../aws-cfn-dsl', :group => :development
+gem 'aws-cfn-decompiler', :path => '../aws-cfn-decompiler', :group => :development
+gem 'bundler', "~> 1.6", :group => :development
+gem 'rake', :group => :development

data/aws-cfn-compiler.gemspec

@@ -18,12 +18,14 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "awesome_print"
+  spec.add_dependency "awesome_print", '~> 1.2', '>= 1.2.0'
   spec.add_dependency "psych"
   spec.add_dependency "json"
-  spec.add_dependency 'aws-cfn-dsl', '>= 0.0.1', '~> 0.0'
   spec.add_dependency "slop"
+  spec.add_dependency 'colorize'
+  spec.add_dependency 'inifile'
+  spec.add_dependency 'aws-cfn-dsl', '>= 0.2.0', '~> 0.2'
+  spec.add_dependency 'dldinternet-mixlib-cli', ">= 0.1.0", '~> 0.1'
+  spec.add_dependency 'dldinternet-mixlib-logging', '>= 0.1.6', '~> 0.1'
 
-  spec.add_development_dependency 'bundler', "~> 1.6"
-  spec.add_development_dependency 'rake'
 end

data/aws-cfn-compiler.iml

@@ -3,26 +3,25 @@
   <component name="CompassSettings">
     <option name="compassSupportEnabled" value="true" />
   </component>
-  <component name="FacetManager">
-    <facet type="gem" name="Ruby Gem">
-      <configuration>
-        <option name="GEM_APP_ROOT_PATH" value="$MODULE_DIR$" />
-        <option name="GEM_APP_TEST_PATH" value="" />
-        <option name="GEM_APP_LIB_PATH" value="$MODULE_DIR$/lib" />
-      </configuration>
-    </facet>
-  </component>
   <component name="NewModuleRootManager" inherit-compiler-output="true">
     <exclude-output />
     <content url="file://$MODULE_DIR$" />
-    <orderEntry type="jdk" jdkName="RVM: ruby-1.9.3-p547 [aws-cfn-compile]" jdkType="RUBY_SDK" />
+    <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="module" module-name="aws-cfn-decompiler" />
+    <orderEntry type="module" module-name="aws-cfn-dsl" />
+    <orderEntry type="module" module-name="cloudformation-ruby-dsl" />
     <orderEntry type="library" scope="PROVIDED" name="awesome_print (v1.2.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="aws-cfn-dsl (v0.0.4, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="bundler (v1.6.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
-    <orderEntry type="library" scope="PROVIDED" name="cloudformation-ruby-dsl (v0.4.3, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="colorize (v0.7.3, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="detabulator (v0.1.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="dldinternet-mixlib-cli (v0.1.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="dldinternet-mixlib-logging (v0.1.6, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="inifile (v2.0.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="json (v1.8.1, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="little-plugger (v1.1.3, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="logging (v1.8.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="mixlib-cli (v1.5.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="multi_json (v1.10.1, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="psych (v2.0.5, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="rake (v10.3.2, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />
     <orderEntry type="library" scope="PROVIDED" name="slop (v3.5.0, RVM: ruby-1.9.3-p547 [aws-cfn-compile]) [gem]" level="application" />

data/bin/cfn-compiler

@@ -2,27 +2,30 @@
 
 require 'rubygems'
 
-# require 'safe_yaml'
-# SafeYAML::OPTIONS[:default_mode] = :safe
-
+path = File.dirname(__FILE__)
+if File.exists?("#{path}/../.gitignore")
+  %w(aws-cfn-decompiler aws-cfn-compiler cloudformation-ruby-dsl).each do |mod|
+    add_path = File.expand_path(File.join(path, "../../#{mod}", "lib"))
+    $:.unshift(add_path)
+  end
+else
 # Borrowing from "whiches" gem ...
-cmd  = File.basename(__FILE__, '.rb')
-exes = []
-exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
-ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
-  exts.each { |ext|
-    exe = File.join(path, "#{cmd}#{ext}")
-    exes << exe if File.executable? exe
-  }
-end
-path = if exes.size > 0
-         File.dirname(exes[0])
-       else
-         File.dirname(__FILE__)
-       end
+  cmd  = File.basename(__FILE__, '.rb')
+  exes = []
+  exts = ENV['PATHEXT'] ? ENV['PATHEXT'].split(';') : ['']
+  ENV['PATH'].split(File::PATH_SEPARATOR).each do |path|
+    exts.each { |ext|
+      exe = File.join(path, "#{cmd}#{ext}")
+      exes << exe if File.executable? exe
+    }
+  end
+  if exes.size > 0
+    path = File.dirname(exes[0])
+  end
 
+end
 add_path = File.expand_path(File.join(path, "..", "lib"))
 $:.unshift(add_path)
 
 require 'aws/cfn/compiler'
-Aws::Cfn::Compiler::Main.new().run
+Aws::Cfn::Compiler::Main.new.run

data/lib/aws/cfn/compiler/base.rb

@@ -0,0 +1,328 @@
+
+require 'json'
+require 'ap'
+require 'yaml'
+require 'slop'
+require 'aws/cfn/dsl/template'
+
+module Aws
+  module Cfn
+    module Compiler
+      class Base
+        attr_accessor :items
+        attr_accessor :opts
+        attr_accessor :spec
+
+        require 'dldinternet/mixlib/logging'
+        include DLDInternet::Mixlib::Logging
+
+        def initialize
+          @items = {}
+          @config ||= {}
+          @config[:log_opts] = lambda{|mlll| {
+                                      :pattern      => "%#{mlll}l: %m %C\n",
+                                      :date_pattern => '%Y-%m-%d %H:%M:%S',
+                                    }
+                                  }
+          @config[:log_level] = :step
+          @logger = getLogger(@config)
+        end
+
+        def validate(compiled)
+          raise 'No Resources!?' unless compiled['Resources']
+
+          # Mappings => Resources
+          maps  = find_maps(compiled) #.select { |a| !(a =~ /^AWS::/) }
+          rscs  = compiled['Resources'].keys
+          mpgs  = compiled['Mappings'].nil? ? [] : compiled['Mappings'].keys
+          names = rscs+mpgs
+
+          unless (maps-names).empty?
+            @logger.error '!!! Unknown mappings !!!'
+            (maps-names).each do |name|
+              @logger.error "  #{name}"
+            end
+            abort!
+          end
+          @logger.step '  Mappings validated'
+
+          # Parameters => Resources => Out@logger.step
+          refs  = find_refs(compiled).select { |a,_| !(a =~ /^AWS::/) }
+          prms  = compiled['Parameters'].keys rescue []
+          # outs  = compiled['Outputs'].keys rescue []
+          names = rscs+prms
+
+          unless (refs.keys-names).empty?
+            @logger.error '!!! Unknown references !!!'
+            (refs.keys-names).each do |name|
+              @logger.error "  #{name} from #{refs[name][0]}:#{refs[name][1]}"
+            end
+            abort!
+          end
+          @logger.step '  References validated'
+        end
+
+        def save(compiled, output_file)
+          begin
+            hash = {}
+            compiled.each do |item,value|
+              unless value.nil?
+                if (not value.is_a?(Hash)) or (value.count > 0)
+                  hash[item] = value
+                end
+              end
+            end
+
+            File.open output_file, 'w' do |f|
+              f.write JSON.pretty_generate(hash, { indent: "\t", space: ' '})
+            end
+            @logger.step '  Compiled file written.'
+          rescue
+            @logger.error "!!! Could not write compiled file: #{$!}"
+            abort!
+          end
+        end
+
+        def load(spec=nil)
+          if spec
+            begin
+              abs = File.absolute_path(File.expand_path(spec))
+              unless File.exists?(abs)
+                abs = File.absolute_path(File.expand_path(File.join(@opts[:directory],spec)))
+              end
+            rescue
+              # pass
+            end
+            if File.exists?(abs)
+              raise "Unsupported specification file type: #{spec}=>#{abs}\n\tSupported types are: json,yaml,jts,yts\n" unless abs =~ /\.(json|ya?ml|jts|yts)\z/i
+
+              @logger.step "Loading specification #{abs}..."
+              spec = File.read(abs)
+
+              case File.extname(File.basename(abs)).downcase
+                when /json|jts/
+                  @spec = JSON.parse(spec)
+                when /yaml|yts/
+                  @spec = YAML.load(spec)
+                else
+                  raise "Unsupported file type for specification: #{spec}"
+              end
+              # @spec = spec
+            else
+              raise "Unable to open specification: #{abs}"
+            end
+            @dsl ||= Aws::Cfn::Dsl::Template.new(@opts[:directory])
+            %w( Mappings Parameters Resources Outputs ).each do |dir|
+              load_dir(dir,@spec)
+            end
+          else
+            raise "No specification provided"
+          end
+        end
+
+        protected
+
+        def abort!
+          @logger.fatal '!!! Aborting !!!'
+          exit
+        end
+
+        def find_refs(hash, type='Reference', parent='')
+          h = {}
+          newparent = parent
+          if hash.is_a? Hash
+            hash.keys.collect do |key|
+              if %w{Mappings Parameters Resources Outputs}.include? key
+                type = key#.gsub(/s$/, '')
+                newparent = key
+              elsif %w{Mappings Parameters Resources Outputs}.include? parent
+                newparent = key
+              end
+              if %w{Ref}.include? key
+                h = { hash[key] => [type,newparent] }
+              elsif 'Fn::GetAtt' == key
+                h = { hash[key].first => [type,newparent] }
+                # elsif %w{SourceSecurityGroupName CacheSecurityGroupNames SecurityGroupNames}.include? key
+                #   a = find_refs(hash[key],type,newparent)
+                #   h = merge(h, a, *[type,newparent])
+              else
+                a = find_refs(hash[key],type,newparent)
+                h = merge(h, a, *[type,newparent])
+              end
+            end.flatten.compact.uniq
+          elsif hash.is_a? Array
+            a = hash.map{|i| find_refs(i,type,newparent) }
+            h = merge(h, a, type, *[type,newparent])
+          end
+          h
+        end
+
+        def merge(h, a, *type)
+          if a.is_a? Hash
+            if a.size > 0
+              h.merge! a
+            end
+          else
+            a.flatten.compact.uniq.map { |i|
+              if i.is_a? Hash
+                if i.size > 0
+                  h.merge! i
+                  h
+                end
+              else
+                h[i] = type
+              end
+            }
+          end
+          h
+        end
+
+        def find_maps(hash)
+          if hash.is_a? Hash
+            tr = []
+            hash.keys.collect do |key|
+              if 'Fn::FindInMap' == key
+                hash[key].first
+              else
+                find_maps(hash[key])
+              end
+            end.flatten.compact.uniq
+          elsif hash.is_a? Array
+            hash.collect{|a| find_maps(a)}.flatten.compact.uniq
+          end
+        end
+
+        # --------------------------------------------------------------------------------
+        def get_file_set(want, path, exts=[])
+          # Dir[File.join(@opts[:directory], "#{dir}.*")] | Dir[File.join(@opts[:directory], dir.to_s, "**", "*")]
+          raise "Bad call to #{self.class.name}.getPathSet: want == nil" unless want
+          @logger.debug "Look for #{want.ai} in #{[path]} with #{exts} extensions"
+          if exts.nil?
+            exts = @config[:precedence]
+          end
+          file_regex=%r/^(\S+)\.(#{exts.join('|')})$/
+          if exts.empty?
+            file_regex=%r/^(\S+)()$/
+            exts=['']
+          end
+          regex = "^(#{want.join('|')})$"
+          set = {}
+          abs = File.expand_path(path)
+          abs = path unless @config[:expandedpaths]
+          raise "Oops! Does '#{path}' directory exist?" unless File.directory?(abs)
+          #abs = File.realpath(abs)
+          begin
+            Dir.glob("#{abs}/*").each{ |f|
+              match = File.basename(f).match(file_regex)
+              if match
+                name = match[1]
+                ext  = match[2]
+                set[ext] = {} unless set[ext]
+                @logger.trace "#{name} =~ #{regex}"
+                set[ext][name] = f if name.match(regex)
+              end
+            }
+          rescue RegexpError => e
+            raise ChopError.new "The regular expression attempting to match resources in '#{path}' is incorrect! #{e.message}"
+          end
+          @logger.debug "getPathSet set=#{set.ai}"
+          res = {}
+          # Iterate extension sets in increasing precedence order ...
+          # Survivor will be the most desireable version of the item
+          # i.e. the .rb environment, role, data bag, etc. will be preferred over the .json version
+          exts.each{ |e|
+            h = set[e]
+            if h
+              h.each{ |n,f|
+                @logger.warn "Ignoring #{File.basename(res[n])}" if res[n]
+                res[n] = f
+              }
+            else
+              @logger.info "'#{e}' set is empty! (No #{path}/*.#{e} files found using precedence #{exts})"
+            end
+          }
+          set = res
+        end
+
+        def load_dir(dir,spec=nil)
+          logStep "Loading #{dir}..."
+
+          if spec and spec[dir]
+            raise "No such directory: #{@opts[:directory]}" unless File.directory?(@opts[:directory])
+            set = []
+            if File.directory?(File.join(@opts[:directory], dir))
+              @items[dir] = {}
+              set = get_file_set([".*"], "#{@opts[:directory]}/#{dir}", @config[:precedence])
+            else
+              if File.directory?(File.join(@opts[:directory], dir.downcase))
+                @items[dir] = {}
+                set = get_file_set(['.*'], dir.downcase, @config[:precedence])
+              else
+                @logger.error "  !! error: Cannot load bricks from #{File.join(@opts[:directory], dir)}"
+                abort!
+              end
+            end
+
+            item = {}
+            spec[dir].each do |base|
+              @logger.info "\tUsing #{dir}/#{base}"
+              if set[base]
+                if item.has_key?(base)
+                  @logger.error "  !! error: Duplicate item: #{dir}/#{base}"
+                  abort!
+                end
+
+                filename = set[base]
+                unless filename =~ /\.(ru?by?|ya?ml|js(|on))\z/i
+                  @logger.info "Brick not supported/ relevant: #{filename}"
+                  next
+                end
+
+                begin
+                  @logger.step "  reading #{filename}"
+                  content = File.read(filename)
+                  next if content.size==0
+
+                  if filename =~ /\.(rb|ruby)\z/i
+                    eval "@dsl.#{content.gsub(%r'^\s+','')}"
+                    unless @dsl.dict[dir.to_sym]
+                      raise "Unable to expand #{filename} for #{dir}/#{base}"
+                    end
+                    item.merge! @dsl.dict[dir.to_sym]
+                  elsif filename =~ /\.js(|on)\z/i
+                    item.merge! JSON.parse(content)
+                  elsif filename =~ /\.ya?ml\z/i
+                    item.merge! YAML.load(content)
+                  else
+                    next
+                  end
+
+                rescue
+                  @logger.error "  !! error: #{$!}"
+                  abort!
+                end
+              else
+                @logger.error "  !! error: #{dir}/#{base} not found!"
+                abort!
+              end
+            end
+            item.keys.each { |key|
+              if @items[dir].has_key?(key)
+                @logger.error "  !! error: Duplicate item: #{dir}/#{key}"
+                abort!
+              end
+            }
+            @items[dir].merge! item
+
+            unless @items[dir].keys.count == spec[dir].count
+              @logger.error "  !! error: Suspect that a #{dir} item was missed! \nRequested: #{spec[dir]}\n    Found: #{@items[dir].keys}"
+              abort!
+            end
+          end
+
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/main.rb

@@ -0,0 +1,68 @@
+
+require 'json'
+require 'ap'
+require 'yaml'
+require 'slop'
+
+module Aws
+  module Cfn
+    module Compiler
+      class Main < Base
+
+        def run
+
+          @opts = Slop.parse(help: true) do
+            on :d, :directory=,     'The directory to look in', as: String
+            on :o, :output=,        'The JSON file to output', as: String
+            on :s, :specification=, 'The specification to use when selecting components. A JSON or YAML file or JSON object', as: String
+            on :f, :formatversion=, 'The AWS Template format version. Default 2010-09-09', as: String
+            on :p, :precedence=,    'The precedence of template component types. Default: rb,yaml,yml,json,js', as: String, default: 'rb,ruby,yaml,yml,json,js'
+            on :t, :description=,   "The AWS Template description. Default: output basename or #{File.basename(__FILE__,'.rb')}", as: String
+            on :x, :expandedpaths=, 'Show expanded paths in output', as: String, default: 'off'
+          end
+
+          unless @opts[:directory]
+            puts @opts
+            exit
+          end
+
+          @config[:precedence]    = @opts[:precedence].split(%r',+\s*').reverse
+          @config[:expandedpaths] = @opts[:expandedpaths].downcase.match %r'^(1|true|on|yes|enable|set)$'
+
+          load @opts[:specification]
+
+          desc = @opts[:output] ? File.basename(@opts[:output]).gsub(%r/\.(json|yaml)/, '') : File.basename(__FILE__,'.rb')
+          if @spec and @spec['Description']
+            desc = @spec['Description']
+          end
+          vers = '2010-09-09'
+          if @spec and @spec['AWSTemplateFormatVersion']
+            vers = @spec['AWSTemplateFormatVersion']
+          end
+          # noinspection RubyStringKeysInHashInspection
+          compiled = {
+              'AWSTemplateFormatVersion' => (@opts[:formatversion].nil? ? vers : @opts[:formatversion]),
+              'Description'              => (@opts[:description].nil? ? desc : @opts[:description]),
+              'Mappings'                 => @items['Mappings'],
+              'Parameters'               => @items['Parameters'],
+              'Resources'                => @items['Resources'],
+              'Outputs'                  => @items['Outputs'],
+          }
+
+          @logger.step'Validating compiled file...'
+
+          validate(compiled)
+
+          output_file = @opts[:output] || 'compiled.json'
+          puts
+          @logger.step"Writing compiled file to #{output_file}..."
+          save(compiled, output_file)
+
+          puts
+          @logger.step'*** Compiled Successfully ***'
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/cfn/compiler/version.rb

@@ -1,7 +1,7 @@
 module Aws
   module Cfn
     module Compiler
-      VERSION = "0.3.1"
+      VERSION = "0.5.0"
     end
   end
 end

data/lib/aws/cfn/compiler.rb

@@ -1,282 +1,4 @@
 require "aws/cfn/compiler/version"
+require "aws/cfn/compiler/base"
+require "aws/cfn/compiler/main"
 
-require 'json'
-require 'ap'
-require 'yaml'
-require 'slop'
-
-module Aws
-  module Cfn
-    module Compiler
-      class Main
-        attr_accessor :items
-        attr_accessor :opts
-        attr_accessor :spec
-
-        def initialize
-          @items = {}
-        end
-
-        def run
-
-          @opts = Slop.parse(help: true) do
-            on :d, :directory=, 'The directory to look in', as: String
-            on :o, :output=, 'The JSON file to output', as: String
-            on :s, :specification=, 'The specification to use when selecting components. A JSON or YAML file or JSON object', as: String
-            on :f, :formatversion=, 'The AWS Template format version. Default 2010-09-09', as: String
-            on :t, :description=, "The AWS Template description. Default: output basename or #{File.basename(__FILE__,'.rb')}", as: String
-          end
-
-          unless @opts[:directory]
-            puts @opts
-            exit
-          end
-
-          load @opts[:specification]
-
-          desc = @opts[:output] ? File.basename(@opts[:output]).gsub(%r/\.(json|yaml)/, '') : File.basename(__FILE__,'.rb')
-          if @spec and @spec['Description']
-            desc = @spec['Description']
-          end
-          vers = '2010-09-09'
-          if @spec and @spec['AWSTemplateFormatVersion']
-            vers = @spec['AWSTemplateFormatVersion']
-          end
-          # noinspection RubyStringKeysInHashInspection
-          compiled = {
-              'AWSTemplateFormatVersion' => (@opts[:formatversion].nil? ? vers : @opts[:formatversion]),
-              'Description'              => (@opts[:description].nil? ? desc : @opts[:description]),
-              'Mappings'                 => @items['Mappings'],
-              'Parameters'               => @items['Parameters'],
-              'Resources'                => @items['Resources'],
-              'Outputs'                  => @items['Outputs'],
-          }
-
-          puts
-          puts 'Validating compiled file...'
-
-          validate(compiled)
-
-          output_file = @opts[:output] || 'compiled.json'
-          puts
-          puts "Writing compiled file to #{output_file}..."
-          save(compiled, output_file)
-
-          puts
-          puts '*** Compiled Successfully ***'
-        end
-
-        def validate(compiled)
-          raise 'No Resources!?' unless compiled['Resources']
-
-          # Mappings => Resources
-          maps  = find_maps(compiled) #.select { |a| !(a =~ /^AWS::/) }
-          rscs  = compiled['Resources'].keys
-          mpgs  = compiled['Mappings'].nil? ? [] : compiled['Mappings'].keys
-          names = rscs+mpgs
-
-          unless (maps-names).empty?
-            puts '!!! Unknown mappings !!!'
-            (maps-names).each do |name|
-              puts "  #{name}"
-            end
-            abort!
-          end
-          puts '  Mappings validated'
-
-          # Parameters => Resources => Outputs
-          refs  = find_refs(compiled).select { |a,_| !(a =~ /^AWS::/) }
-          prms  = compiled['Parameters'].keys rescue []
-          # outs  = compiled['Outputs'].keys rescue []
-          names = rscs+prms
-
-          unless (refs.keys-names).empty?
-            puts '!!! Unknown references !!!'
-            (refs.keys-names).each do |name|
-              puts "  #{name} from #{refs[name][0]}:#{refs[name][1]}"
-            end
-            abort!
-          end
-          puts '  References validated'
-        end
-
-        def save(compiled, output_file)
-          begin
-            hash = {}
-            compiled.each do |item,value|
-              unless value.nil?
-                if (not value.is_a?(Hash)) or (value.count > 0)
-                  hash[item] = value
-                end
-              end
-            end
-
-            File.open output_file, 'w' do |f|
-              f.write JSON.pretty_generate(hash, { indent: "\t", space: ' '})
-            end
-            puts '  Compiled file written.'
-          rescue
-            puts "!!! Could not write compiled file: #{$!}"
-            abort!
-          end
-        end
-
-        def load(spec=nil)
-          if spec
-            begin
-              abs = File.absolute_path(File.expand_path(spec))
-              unless File.exists?(abs)
-                abs = File.absolute_path(File.expand_path(File.join(@opts[:directory],spec)))
-              end
-            rescue
-              # pass
-            end
-            if File.exists?(abs)
-              raise "Unsupported specification file type: #{spec}=>#{abs}\n\tSupported types are: json,yaml,jts,yts\n" unless abs =~ /\.(json|ya?ml|jts|yts)\z/i
-
-              puts "Loading specification #{abs}..."
-              spec = File.read(abs)
-
-              case File.extname(File.basename(abs)).downcase
-                when /json|jts/
-                  @spec = JSON.parse(spec)
-                when /yaml|yts/
-                  @spec = YAML.load(spec)
-                else
-                  raise "Unsupported file type for specification: #{spec}"
-              end
-              # @spec = spec
-            else
-              raise "Unable to open specification: #{abs}"
-            end
-          end
-          %w( Mappings Parameters Resources Outputs ).each do |dir|
-            load_dir(dir,@spec)
-          end
-        end
-
-        protected
-
-        def abort!
-          puts '!!! Aborting !!!'
-          exit
-        end
-
-        def find_refs(hash, type='Reference', parent='')
-          h = {}
-          newparent = parent
-          if hash.is_a? Hash
-            hash.keys.collect do |key|
-              if %w{Mappings Parameters Resources Outputs}.include? key
-                type = key#.gsub(/s$/, '')
-                newparent = key
-              elsif %w{Mappings Parameters Resources Outputs}.include? parent
-                newparent = key
-              end
-              if %w{Ref}.include? key
-                h = { hash[key] => [type,newparent] }
-              elsif 'Fn::GetAtt' == key
-                h = { hash[key].first => [type,newparent] }
-                # elsif %w{SourceSecurityGroupName CacheSecurityGroupNames SecurityGroupNames}.include? key
-                #   a = find_refs(hash[key],type,newparent)
-                #   h = merge(h, a, *[type,newparent])
-              else
-                a = find_refs(hash[key],type,newparent)
-                h = merge(h, a, *[type,newparent])
-              end
-            end.flatten.compact.uniq
-          elsif hash.is_a? Array
-            a = hash.map{|i| find_refs(i,type,newparent) }
-            h = merge(h, a, type, *[type,newparent])
-          end
-          h
-        end
-
-        def merge(h, a, *type)
-          if a.is_a? Hash
-            if a.size > 0
-              h.merge! a
-            end
-          else
-            a.flatten.compact.uniq.map { |i|
-              if i.is_a? Hash
-                if i.size > 0
-                  h.merge! i
-                  h
-                end
-              else
-                h[i] = type
-              end
-            }
-          end
-          h
-        end
-
-        def find_maps(hash)
-          if hash.is_a? Hash
-            tr = []
-            hash.keys.collect do |key|
-              if 'Fn::FindInMap' == key
-                hash[key].first
-              else
-                find_maps(hash[key])
-              end
-            end.flatten.compact.uniq
-          elsif hash.is_a? Array
-            hash.collect{|a| find_maps(a)}.flatten.compact.uniq
-          end
-        end
-
-        def load_dir(dir,spec=nil)
-          puts "Loading #{dir}..."
-          raise "No such directory: #{@opts[:directory]}" unless File.directory?(@opts[:directory])
-          set = []
-          if File.directory?(File.join(@opts[:directory], dir))
-            @items[dir] = {}
-            set = get_file_set(dir)
-          else
-            if File.directory?(File.join(@opts[:directory], dir.downcase))
-              @items[dir] = {}
-              set = get_file_set(dir.downcase)
-            end
-          end
-          set.collect do |filename|
-            next unless filename =~ /\.(json|ya?ml)\z/i
-            if spec and spec.has_key?(dir)
-              base = File.basename(filename).gsub(%r/\.(rb|yaml)/, '')
-              next     if spec[dir].nil? # Edge case ... explicitly want NONE of these!
-              next unless spec[dir].include?(base)
-              puts "\tUsing #{dir}/#{base}"
-            end
-            begin
-              puts "  reading #{filename}"
-              content = File.read(filename)
-              next if content.size==0
-
-              if filename =~ /\.json\z/i
-                item = JSON.parse(content)
-              elsif filename =~ /\.ya?ml\z/i
-                item = YAML.load(content)
-              else
-                next
-              end
-              item.keys.each { |key| raise "Duplicate item: #{key}" if @items[dir].has_key?(key) }
-              @items[dir].merge! item
-            rescue
-              puts "  !! error: #{$!}"
-              abort!
-            end
-          end
-          if spec and spec[dir]
-            raise "Suspect that a #{dir} item was missed! \nRequested: #{spec[dir]}\n    Found: #{@items[dir].keys}" unless (@items[dir].keys.count == spec[dir].count)
-          end
-        end
-
-        def get_file_set(dir)
-          Dir[File.join(@opts[:directory], "#{dir}.*")] | Dir[File.join(@opts[:directory], dir.to_s, "**", "*")]
-        end
-
-      end
-    end
-  end
-end

metadata

@@ -1,119 +1,165 @@
 --- !ruby/object:Gem::Specification
 name: aws-cfn-compiler
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.5.0
 platform: ruby
 authors:
 - PKinney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-26 00:00:00.000000000 Z
+date: 2014-06-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: psych
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-cfn-dsl
+  name: slop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.0.1
-    - - "~>"
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: 0.0.1
-    - - "~>"
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: slop
+  name: inifile
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
+  name: aws-cfn-dsl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+    - - ~>
       - !ruby/object:Gem::Version
-        version: '1.6'
-  type: :development
+        version: '0.2'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '1.6'
+        version: 0.2.0
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.2'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: dldinternet-mixlib-cli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
+        version: 0.1.0
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.0
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: dldinternet-mixlib-logging
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.1.6
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
 description: The idea is to create a folder structure to better manage pieces of a
   CloudFormation deployment. Additionally, writing in JSON is hard, so the compiler
   takes YAML files as well.
@@ -124,9 +170,9 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- ".gitignore"
-- ".ruby-gemset"
-- ".ruby-version"
+- .gitignore
+- .ruby-gemset
+- .ruby-version
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -149,6 +195,8 @@ files:
 - example/resources/subnets.json
 - example/resources/vpc.json
 - lib/aws/cfn/compiler.rb
+- lib/aws/cfn/compiler/base.rb
+- lib/aws/cfn/compiler/main.rb
 - lib/aws/cfn/compiler/version.rb
 homepage: https://github.com/dldinternet/aws-cfn-compiler
 licenses:
@@ -160,12 +208,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []