mkstack 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: aa22c1f1c57234678eefa866bc219909dc00334d91d6d4daf70be9762233d51c
+  data.tar.gz: c5a1f8095d6b34d467fafd443fbc9605b206a42ef70e65f47ec9290ed8783ca3
+SHA512:
+  metadata.gz: b0a0588f47fbc38f7efddb0175afbfacb7d40604a41094a9bc705a26fc6edc154c00bc9f2146e1a19d3326d1d7bfdbb2a6f2de9918a5f8a67ca11e06e202d286
+  data.tar.gz: b23350d9abe01bb969d9dbf617d7025f5572481e526b37a7632a6ec8fb46c7a64aebb92266166105bfd55ffd73f5187b642620d04bdf4ede746ceeb99bf696ee

data/bin/mkstack

@@ -0,0 +1,106 @@
+#!/usr/local/bin/ruby
+
+require "logger"
+require "optparse"
+require "mkstack"
+
+
+##################################################
+# Main
+
+# Define logger
+$logger = Logger.new(STDERR)
+$logger.level = Logger::WARN
+$logger.formatter = proc { |s, d, p, m|
+  "[%s] %5s [%5s] - %s\n" % [ d.strftime("%Y-%m-%d %H:%M:%S"), s, caller(4, 1).first.split(":")[1], m ]
+}
+
+
+# Default options
+options = {}
+options[:erb] = true
+options[:format] = "json"
+
+
+# Command line options
+opts = OptionParser.new do |p|
+  desc_padding = " " * (p.summary_indent.length + p.summary_width)
+
+  p.banner = "Usage: #{p.program_name} [ options ] file1 [ file2... ]"
+
+  # Help
+  p.on("-h", "--help", "Display this message") { puts opts ; exit }
+  p.separator(" ")
+
+  # Verbosity
+  p.on("-d", "--debug",   "Show debug messages")         { $logger.level = Logger::DEBUG }
+  p.on("-v", "--verbose", "Be verbose")                  { $logger.level = Logger::INFO }
+  p.on("-q", "--quiet",   "Only show errors")            { $logger.level = Logger::ERROR }
+  p.on("-s", "--silient", "Don't show any log messages") { $logger.level = Logger::FATAL }
+  p.separator(" ")
+
+  # Output
+  p.on("-o", "--output=FILE",   "Print final template to FILE") { |x| options[:save] = x }
+  p.on("%s Use '-' for stdout" % [ desc_padding ])
+  p.on("-f", "--format=FORMAT", [ "json", "yaml" ], "Print as FORMAT") { |x| options[:format] = x }
+  p.on("%s Supported formats: json (default), yaml" % [ desc_padding ])
+  p.separator(" ")
+
+  # Operations
+  p.on("--erb", "--[no-]erb", "Perform ERB processing (default is true)") { |x| options[:erb] = x }
+  p.on("--validate", "Call ValidateTemplate after merging") { options[:validate] = true }
+end
+
+files = opts.parse!.uniq
+opts.parse "--help" if files.count == 0
+
+template = MkStack::Template.new(options[:format])
+
+
+# Merge files
+files.each do |file|
+  begin
+    Dir.chdir(File.dirname(file)) { template.merge(File.basename(file), options[:erb]) }
+  rescue KeyError => e
+    $logger.warn { "#{file}: already parsed" }
+  rescue Exception => e
+    $logger.error { e.message }
+  end
+end
+
+
+# Check limits
+$logger.debug { "Checking limits" }
+
+template.sections.each do |name, section|
+  $logger.warn { "#{name} limit exceeded: (#{section.length} > #{section.limit})" } if section.exceeds_limit?
+end
+
+$logger.warn { "At least one Resources member must be defined" } if template["Resources"].length == 0
+$logger.warn { "Template too large (#{template.length} > #{template.limit})" } if template.exceeds_limit?
+
+
+# Validate template
+if options[:validate] then
+  begin
+    $logger.info { "Validating #{template.format} template" }
+
+    $logger.debug { template.validate }
+  rescue Exception => e
+    $logger.error { e.message }
+  end
+end
+
+
+# Save template
+if options[:save] then
+  begin
+    $stdout = File.new(options[:save], File::CREAT | File::WRONLY) unless options[:save] == "-"
+
+    $logger.info { "Saving #{template.format} to #{$stdout.path}" } unless $stdout == STDOUT
+
+    puts template.pp
+  rescue Exception => e
+    $logger.error { e.message }
+  end
+end

data/lib/mkstack.rb

@@ -0,0 +1,81 @@
+require_relative "mkstack/template"
+
+=begin rdoc
+
+Merge multiple CloudFormation template files into a single template.
+Each file may be in either JSON or YAML format.
+
+Get started with <i>template = MkStack::Template.new</i>
+
+== ERB
+
+By default all files are run through an ERB (Embedded RuBy) processor.
+ 
+  <% desc = "awesome" %>
+
+  AWSTemplateFormatVersion: "2010-09-09"
+  Description: My <%= desc %> CloudFormation template
+
+It is safe to leave this enabled.  If a file doesn't have any ERB tags
+it is passed through untouched.
+
+== Include
+
+MkStack searches each file for a section named <b>Include</b>, which should
+be a list of filenames.  These function the same as adding the listed
+files on the command line.
+
+=== JSON
+
+  "Include" : [
+    "foo.yaml",
+    "bar.json"
+  ]
+
+=== YAML
+
+  Include:
+    - foo.yaml
+    - bar.json
+
+== ERB and Include working together
+
+MkStack uses a single <i>binding</i> for all files.  This allows ERB
+tags defined in one file to be referenced in subsequent files.
+
+=== foo.yaml
+
+  Include:
+    - bar.json
+
+  <% tags_json = %q{
+       "Tags": [
+         { "Key" : "application", "Value" : "mkstack" }
+       ]
+     }
+  %>
+
+=== bar.json
+
+  {
+    "Resources" : {
+      "sg": {
+        "Type" : "AWS::EC2::SecurityGroup",
+        "Properties" : {
+          "GroupDescription" : { "Fn::Sub" : "Security Group for ${application}" }
+          <%= tags_json %>
+        }
+      }
+    }
+  }
+
+Note that foo.yaml is processed <i>before</i> bar.json.
+
+== See Also
+
+  MkStack::Template
+  MkStack::Section
+=end
+
+module MkStack
+end

data/lib/mkstack/section.rb

@@ -0,0 +1,32 @@
+module MkStack
+  ##################################################
+  # A CloudFormation template section
+  class Section
+    attr_reader :name, :limit
+    attr_accessor :contents
+
+    # * name: The section's name (Resources, Outputs, etc.)
+    # * type: The section's type (Hash or String)
+    # * limit: The AWS limit for this section, if any
+    def initialize(name, type, limit = nil)
+      @name = name
+      @limit = limit
+
+      @contents = type.new
+    end
+
+    # Merge or override a section snippet
+    def merge(contents)
+      raise TypeError.new("#{contents.class} != #{@contents.class}") if contents.class != @contents.class
+
+      return @contents.merge!(contents) if @contents.respond_to?(:merge!)
+      @contents = contents
+    end
+
+    # Return the length of the section's contents
+    def length; @contents.length; end
+
+    # Check if the section exceeds the AWS limit
+    def exceeds_limit?; @limit && length > @limit; end
+  end
+end

data/lib/mkstack/template.rb

@@ -0,0 +1,178 @@
+require_relative "section"
+
+require "erb"
+require "json"
+require "yaml"
+
+module MkStack
+  ##################################################
+  # A CloudFormation template
+  class Template
+    attr_reader :sections, :limit, :format
+
+    def initialize(format = "json")
+      @format = format
+
+      @sections = {
+        "AWSTemplateFormatVersion" => Section.new("AWSTemplateFormatVersion", String, nil),
+        "Description" => Section.new("Description", String, 1024),
+
+        "Conditions"  => Section.new("Conditions",  Hash, nil),
+        "Mappings"    => Section.new("Mappings",    Hash, 100),
+        "Metadata"    => Section.new("Metadata",    Hash, nil),
+        "Outputs"     => Section.new("Outputs",     Hash, 60),
+        "Parameters"  => Section.new("Parameters",  Hash, 60),
+        "Resources"   => Section.new("Resources",   Hash, nil),
+        "Transforms"  => Section.new("Transforms",  Hash, nil),
+      }
+      @limit = 51200
+
+      # Keep track of parsed files to avoid loops
+      @parsed = {}
+
+      # Save a binding so ERB can reuse it instead of creating a new one
+      # every time we load a file.  This allows ERB code in one file to
+      # be referenced in another.
+      @binding = binding
+
+      # See add_domain_types
+      @yaml_domain = "mlfs.org,2019"
+      @global_tag = "tag:#{@yaml_domain}:"
+    end
+
+    # Shorthand accessor for template sections
+    def [](section); @sections[section]; end
+
+    # Return the length of the entire template
+    def length; to_json.to_s.length; end
+
+    # Check if the template exceeds the AWS limit
+    def exceeds_limit?; limit && length > limit; end
+
+
+    #########################
+    # Merge contents of a file
+    def merge(file, erb)
+      contents = load(file, erb)
+
+      begin
+        # Try JSON
+        cfn = JSON.load(contents)
+      rescue Exception => e
+        # Try YAML
+        add_domain_types
+        cfn = YAML.load(contents)
+      end
+
+      # Merge sections that are present in the file
+      @sections.each { |name, section| section.merge(cfn[name]) if cfn[name] }
+
+      # Look for Includes and merge them
+      # Files are Included relative to the file with the Include directive
+      cfn["Include"].each do |file|
+        Dir.chdir(File.dirname(file)) { self.merge(File.basename(file), erb) }
+      end if cfn["Include"]
+    end
+
+
+    #########################
+    # Call ValidateTemplate[https://docs.aws.amazon.com/AWSCloudFormation/latest/APIReference/API_ValidateTemplate.html]
+    def validate
+      require "aws-sdk-cloudformation"
+      Aws::CloudFormation::Client.new.validate_template({ template_body: pp })
+    end
+
+
+    #########################
+    # Format contents
+    def pp
+      case @format
+      when "json"
+        to_hash.to_json
+      when "yaml"
+        # Strip enclosing quotes around tags and revert tags to their short form
+        # And keep Psych from splitting "long" lines
+        to_hash.to_yaml({ line_width: -1 }).gsub(/"(#{@global_tag}[^"]+?)"/, '\1').gsub("#{@global_tag}", "!")
+      else
+        to_hash
+      end
+    end
+
+    private
+
+    #########################
+    # Create a hash of each populated section's contents
+    def to_hash
+      h = Hash.new
+      @sections.each { |k, v| h[k] = v.contents if v.length > 0 }
+      h
+    end
+
+
+    #########################
+    # Read file and (optionally) perform ERB processing on it
+    def load(file, erb = true)
+      path = File.expand_path(file)
+      raise KeyError if @parsed.has_key?(path)
+
+      $logger.info { "Loading #{file}" } if $logger
+
+      contents = File.read(file)
+      contents = ERB.new(contents).result(@binding) if erb
+
+      @parsed[path] = true
+
+      return contents
+    end
+
+
+    #########################
+    # Define YAML domains to handle CloudFormation intrinsic
+    # functions.
+    #
+    # CloudFormation uses <b>!</b> to denote the YAML short form of
+    # intrinsic functions, which is the same prefix YAML uses for
+    # local tags.  The default handler strips undefined local tags,
+    # leaving just the value.
+    #
+    # This puts the tags back, but in global tag format.  The global
+    # tag prefix is converted back to <b>!</b> on output.
+    #
+    # Using the short form will force the output to be in YAML format.
+    def add_domain_types
+      functions = [
+        "Base64",
+        "Cidr",
+        "FindInMap",
+        "GetAtt",
+        "GetAZs",
+        "ImportValue",
+        "Join",
+        "Ref",
+        "Select",
+        "Split",
+        "Transform",
+        "And",
+        "Equals",
+        "If",
+        "Not",
+        "Or",
+      ].each do |function|
+        YAML::add_domain_type(@yaml_domain, function) do |type, val|
+          @format = "yaml"
+          "#{type} #{val}"
+        end
+      end
+
+      # The syntax for !Sub requires double-quotes around Strings
+      functions = [
+        "Sub",
+      ].each do |function|
+        YAML::add_domain_type(@yaml_domain, function) do |type, val|
+          @format = "yaml"
+          val.is_a?(String)? "#{type} \"#{val}\"" : "#{type} #{val}"
+        end
+      end
+    end
+  end
+end

data/mkstack.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |s|
+  s.name	= "mkstack"
+  s.version	= "1.0.0"
+  s.summary	= "Merge multiple CloudFormation template files into a single template"
+  s.description	= <<-EOF
+Merge multiple CloudFormation template files into a single template.  Each file may be in either JSON or YAML format.  By default all files are run through an ERB (Embedded RuBy) processor.
+EOF
+
+  s.authors	= [ "Andy Rosen" ]
+  s.email	= [ "ajr@corp.mlfs.org" ]
+  s.homepage	= "https://github.com/ajrosen/AWS/tree/master/mkstack"
+  s.licenses	= [ "GPL-3.0+" ]
+
+  s.files	= Dir[ "mkstack.gemspec", "bin/*", "lib/**/*.rb" ]
+  s.executables	= [ "mkstack" ]
+
+  s.add_runtime_dependency "aws-sdk-cloudformation", "~> 1"
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification
+name: mkstack
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Andy Rosen
+autorequire: 
+bindir: bin
+cert_chain:
+- |
+  -----BEGIN CERTIFICATE-----
+  MIIEOjCCAqKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAlMSMwIQYDVQQDDBphanIv
+  REM9Y29ycC9EQz1tbGZzL0RDPW9yZzAeFw0xOTA1MDEwMjIwNTBaFw0yMDA0MzAw
+  MjIwNTBaMCUxIzAhBgNVBAMMGmFqci9EQz1jb3JwL0RDPW1sZnMvREM9b3JnMIIB
+  ojANBgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAqQ2xCUJ4wY8WujSzYd3OGTbj
+  JDMeU44pXOTLc49Rs8ydukGfd0YBvYMzifmiVRj6depGx2+Ln28Y2mT6IB+zHq8X
+  s1lrMdFCReztJjQ7OYS16YcZ6pmLkYClnHN3VNqayk1lQEJGCr8aawMeroSB01om
+  d5wqDATnKEG3x4bnlxFJb3LHzUG1CgNuVCuNREi8zN/uYdm2MGe1fTJguy4/vzBQ
+  /FnAMt1mr3LtM6YZRGaitIlOKBV/08v7fjH31KRmSMMHPq6A+WyWKRNKnK3tHpSN
+  JbnFW7mFQGtBpfh8zY8OCQ76Aw8cb5bEIPTI+Hd4FoJLPKnexTI28endSLigOUCF
+  76kLOiVOVOjdqZ8vEdSgWVugEAzIC30xW5b8yD6N7GdT7n3ktgoZu7jZMUW3D6PQ
+  wS6BaABVsXbeVtFrzK2tQ65EwLfRluRLTfbnQ7qvMIYCwC3Ib9DTMLavCiOot1vc
+  RBWXIrDex0tt3EN0dDIxP9O+7ciDgM4zPe6BvaF/AgMBAAGjdTBzMAkGA1UdEwQC
+  MAAwCwYDVR0PBAQDAgSwMB0GA1UdDgQWBBRDowzw+kmcX6FV7T9BkQsUgmzLZjAc
+  BgNVHREEFTATgRFhanJAY29ycC5tbGZzLm9yZzAcBgNVHRIEFTATgRFhanJAY29y
+  cC5tbGZzLm9yZzANBgkqhkiG9w0BAQsFAAOCAYEAbbgfVpRCtujGFRHNYLWnq/iQ
+  vGtNs265jQKdq3SZ5HIsPP4RdiknOk2Q0BP4GDkXOhadtuuqeCVlJUczcrCKiKuP
+  Vu7iOQKpOq9bafhjvTpRPZL7uXDu0lwBrDyL9PGouBBsijTtGCc/A8cu/2HVoX+Z
+  X1pqmuJqVlgXp9ktbzPBeIdaFFT+9WzIzWCJ73oBYNZ1EP/4CRKIlmRCWzBGZtu6
+  QqNypgW/OCQVRcJGZ99myNQO5/EImQp1py6iHymkdCx2RMsdTUpcm3gUl4XaFd4J
+  pQ1HNcDdD3UCSDqlPpv2nVpOV4u1s3yojIHt0RcVXhSzqGHgmXNjKDIP0zokFBKx
+  NigI/5A3vv9PplxXMatOeirqPqDY5F4Aim4t2Dn+d+9OiPADkIBwuIuwwgRmcWhE
+  E2DV+ftSSpAT1zeHEZXx6BsvJVwtXq1kMDYibgB4lTU4lq1oHrx2x+PO2cukHlom
+  4cVT/heohjMDtNBaDMVsTjTxtNq6oi/pvyLqBGmQ
+  -----END CERTIFICATE-----
+date: 2019-05-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudformation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+description: 'Merge multiple CloudFormation template files into a single template.  Each
+  file may be in either JSON or YAML format.  By default all files are run through
+  an ERB (Embedded RuBy) processor.
+
+'
+email:
+- ajr@corp.mlfs.org
+executables:
+- mkstack
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/mkstack
+- lib/mkstack.rb
+- lib/mkstack/section.rb
+- lib/mkstack/template.rb
+- mkstack.gemspec
+homepage: https://github.com/ajrosen/AWS/tree/master/mkstack
+licenses:
+- GPL-3.0+
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Merge multiple CloudFormation template files into a single template
+test_files: []