cfndsl-pipeline 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e9d71f50ea1fafc076f9390e279f8bb3c3c3799567a16f04894708fd79727ad8
+  data.tar.gz: b75350a9198ab44e4907c07c2e9ccdb27b6f8600a6ab9ea2c1dbbf4d6fc4995f
+SHA512:
+  metadata.gz: 226f45ed1fef866683576232ce522a5a8de498562d2b8d1b0a58a79018c03ad00cf2f1a6d78f666d76d69e478de244728af7767b1ed579fdcdafc8a7e639be58
+  data.tar.gz: a97395d390c44b247521915fd42610c3257e53311c770e807ed99cfa40c70f3b8c37205008f2181284b76f443f87dc628324129957f700e3dc05cb4200cf1905

data/bin/cfndsl_pipeline

@@ -0,0 +1,75 @@
+#!/usr/bin/env ruby
+require 'optparse'
+require 'cfndsl-pipeline'
+
+USAGE = "Usage: #{File.basename(__FILE__)} -t input file -o output dir [ -b bucket | -p | -c ] [include1 include2 etc]"
+cli_options = {}
+
+pipe_options = CfnDslPipeline::Options.new
+
+op = OptionParser.new do |opts|
+  opts.banner = USAGE
+  opts.on('-t', '--template file', 'Input file') do |v|
+    cli_options[:template] = v
+  end
+
+  opts.on('-o', '--output dir', 'Output directory') do |v|
+    cli_options[:output] = v
+  end
+
+  opts.on('-b', '--bucket', 'Existing S3 bucket for cost estimation and large template syntax validation') do |v|
+    pipe_options[:validation_bucket] = v
+  end
+
+  opts.on('--disable-syntax', 'Enable syntax check') do
+    pipe_options[:validate_syntax] = false
+  end
+
+  opts.on('-p', '--params', 'Create cloudformation deploy compatible params file') do
+    pipe_options[:dump_deploy_params] = true
+  end
+
+  opts.on('--disable-nag', 'Enable cfn_nag ') do
+    pipe_options[:validate_cfn_nag] = false
+  end
+
+  opts.on('--syntax-report', 'Save template syntax report') do
+    pipe_options[:save_syntax_report] = true
+  end
+
+
+  opts.on('--audit-report', 'Save cfn_nag audit report') do
+    pipe_options[:save_audit_report] = true
+  end  
+
+  opts.on('-c', '--estimate', 'Generate URL for AWS simple cost calculator') do
+    pipe_options[:validate_cfn_nag] = true
+  end
+
+  opts.on_tail('-h', '--help', 'show this message') do
+    puts opts
+    exit
+  end
+
+  opts.on_tail('-v', '--version', 'show the version') do
+    puts CfnDsl::Pipeline::VERSION
+    exit
+  end
+end
+
+op.parse!
+
+ 
+unless cli_options[:template] && cli_options[:output]
+  puts op
+  exit 1
+end
+
+cfndsl_extras = []
+ARGV.each do |arg|
+  cfndsl_extras << [:yaml, arg]
+end if ARGV.length > 0
+
+pipeline = CfnDslPipeline::Pipeline.new(cli_options[:output], pipe_options)
+pipeline.build(cli_options[:template], cfndsl_extras)
+

data/lib/cfndsl-pipeline.rb

@@ -0,0 +1,65 @@
+#!/usr/bin/env ruby
+#
+# 
+# The MIT License
+#
+# Copyright (c) 2019 Cam Maxwell (cameron.maxwell@gmail.com)
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+# 
+
+require 'cfndsl'
+require 'fileutils'
+
+require_relative 'options'
+require_relative 'params'
+require_relative 'monkey_patches'
+require_relative 'stdout_capture'
+require_relative 'run-cfndsl'
+require_relative 'run-cfn_nag'
+require_relative 'run-syntax'
+
+module CfnDslPipeline
+  class Pipeline
+
+    attr_accessor :input_filename, :output_dir, :options, :base_name, :template, :output_filename, :output_file, :syntax_report
+
+    def initialize (output_dir, options)
+      self.input_filename = ''
+      self.output_file = nil
+      self.template = nil
+      self.options = options || nil
+      self.syntax_report = []
+      FileUtils.mkdir_p output_dir
+      abort "Could not create output directory #{output_dir}" if Dir[output_dir] == nil
+      self.output_dir = output_dir
+    end
+
+    def build(input_filename, cfndsl_extras)
+      abort "Input file #{input_filename} doesn't exist!" if !File.file?(input_filename)
+      self.input_filename = "#{input_filename}"
+      self.base_name = File.basename(input_filename, '.*')
+      self.output_filename = File.expand_path("#{self.output_dir}/#{self.base_name}.yaml")
+      exec_cfndsl cfndsl_extras
+      exec_syntax_validation if self.options.validate_syntax
+      exec_dump_params if self.options.dump_deploy_params
+      exec_cfn_nag if self.options.validate_cfn_nag
+    end
+  end
+end

data/lib/monkey_patches.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require 'cfndsl/globals'
+require 'cfndsl/version'
+PARAM_PROPS = %w[Description Default AllowedPattern AllowedValues].freeze
+
+# Automatically add Parameters for Tag values
+CfnDsl::CloudFormationTemplate.class_eval do
+  def initialize
+    return unless defined? external_parameters[:TagStandard]
+
+    # parameters for tagging standard
+    external_parameters[:TagStandard].each do |param_name, props|
+      logical_name = props['LogicalName'] || param_name
+      Parameter(logical_name) do
+        Type(props['Type'])
+        PARAM_PROPS.each do |key|
+          # puts key, props[key]
+          send(key, props[key]) if props[key]
+        end
+      end
+    end if external_parameters[:TagStandard].kind_of?(Hash)
+  end
+end
+
+module CfnDsl
+  # extends CfnDsl esource Properties to automatically substitute
+  # FnSub recuraively
+  class PropertyDefinition < JSONable
+    def initialize(value)
+      @value = fix_substitutions(value)
+    end
+
+    def fix_substitutions(val)
+      return val unless defined? val.class.to_s.downcase
+      meth = "fix_#{val.class.to_s.downcase}"
+      if respond_to?(meth.to_sym)
+        return send(meth, val)
+      end
+      val
+    end
+
+    def fix_hash(val)
+      val.transform_values! { |item| fix_substitutions item }
+    end
+
+    def fix_array(val)
+      val.map! { |item| fix_substitutions item }
+    end
+
+    def fix_string(val)
+      val.include?('${') ? FnSub(val) : val
+    end
+  end
+
+  # Automatically apply Tag standard to  CfnDsl Resources (if supplied)
+  class ResourceDefinition
+    def initialize
+      apply_tag_standard
+    end
+
+    def apply_tag_standard
+      return unless defined? external_parameters[:TagStandard]
+
+      # begin
+      external_parameters[:TagStandard].each do |tag_name, props|
+        add_tag(tag_name.to_s, Ref(props['LogicalName'] || tag_name))
+      end if external_parameters[:TagStandard].kind_of?(Hash)
+
+      # rescue
+      # end
+    end
+  end
+end

data/lib/options.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module CfnDslPipeline
+  class Options
+    attr_accessor :aws_region, :validation_bucket, :save_audit_report, :validate_syntax, :save_syntax_report, :validate_cfn_nag, :validate_output, :estimate_cost, :dump_deploy_params
+    def initialize()
+      self.aws_region='ap-southeast-2'
+      self.validation_bucket=''
+      self.validate_cfn_nag=true
+      self.validate_syntax=true
+      self.estimate_cost=false
+      self.save_syntax_report=false
+      self.dump_deploy_params=true
+      self.save_audit_report=false
+    end
+  end
+end
+

data/lib/params.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+require 'shellwords'
+
+module CfnDslPipeline
+  class Pipeline
+  	def exec_dump_params
+      param_filename = "#{self.output_dir}/#{self.base_name}.params"
+      puts "Deploy parameters written to #{param_filename}"
+      param_file = File.open(File.expand_path(param_filename), 'w')
+      self.syntax_report['parameters'].each do | param |
+        param_file.puts "#{param['parameter_key']}=#{Shellwords.escape(param['default_value'])}"
+      end
+    end
+  end
+end  

data/lib/run-cfn_nag.rb

@@ -0,0 +1,41 @@
+require 'cfn-nag'
+require 'colorize'
+
+module CfnDslPipeline
+  class Pipeline
+    def exec_cfn_nag
+      puts "Auditing template with cfn-nag..."
+      cfn_nag_config = CfnNagConfig.new(
+        print_suppression: false,
+        fail_on_warnings: true
+      )
+      cfn_nag = CfnNag.new(config: cfn_nag_config)
+      result = cfn_nag.audit(cloudformation_string: self.template)
+      if self.options.save_audit_report
+        audit_report = Capture.capture do
+          SimpleStdoutResults.new.render([{
+            filename: output_filename,
+            file_results: result
+          }])
+        end
+        audit_filename = "#{self.output_dir}/#{self.base_name}.audit"
+        File.open(File.expand_path(audit_filename), 'w').puts audit_report['stdout']
+        puts "Saved audit report to #{audit_filename}"
+        if result[:failure_count]>0
+          puts "Audit failed. #{result[:failure_count]} error(s) found     ( ಠ ʖ̯ ಠ)  ".red
+        elsif result[:violations].count>0
+          puts "Audit passed with #{result[:warning_count]} warnings.     (._.)  ".yellow
+        else
+          puts "Audit passed!        \( ﾟヮﾟ)/      ヽ(´ー｀)ノ".green
+        end        
+      else
+        ColoredStdoutResults.new.render([{
+          filename: "cfn-nag results:",
+          file_results: result
+        }]) 
+      end       
+
+
+    end
+  end
+end

data/lib/run-cfndsl.rb

@@ -0,0 +1,18 @@
+require 'cfndsl'
+require 'cfndsl/globals'
+require 'cfndsl/version'
+
+module CfnDslPipeline
+  class Pipeline
+   def exec_cfndsl(cfndsl_extras)
+      print "Generating CloudFormation template...\n"
+      model = CfnDsl.eval_file_with_extras("#{@input_filename}", cfndsl_extras)
+      @template = JSON.parse(model.to_json).to_yaml
+      File.open(@output_filename, 'w') do |file|
+        file.puts @template
+      end
+      @output_file = File.open(@output_filename)
+      puts "  #{@output_file.size} bytes written to #{@output_filename}"
+    end
+  end
+end

data/lib/run-syntax.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+require 'aws-sdk-cloudformation'
+require 'aws-sdk-s3' 
+require 'uuid'
+
+
+module CfnDslPipeline
+  class Pipeline
+    attr_accessor :cfn_client, :s3_client
+
+    def initialize
+      self.cfn_client = Aws::CloudFormation::Client.new(region: self.aws_region)
+      self.s3_client = Aws::S3::Client.new(region: self.aws_region)
+    end
+
+    def exec_syntax_validation
+      print "Validating template syntax...\n"
+      if self.options.estimate_cost || (self.output_file.size > 51200)
+        puts "Filesize is greater than 51200, or cost estimation required. Validating via S3 bucket "
+        uuid = UUID.new
+        object_name = "#{uuid.generate}"
+
+        if self.options.validation_bucket
+          bucket_name = self.options.validation_bucket
+          puts "Using existing S3 bucket #{bucket_name}..."
+          bucket = self.s3_client.bucket(self.options.validation_bucket)
+        else
+          bucket_name = "arch-code-#{uuid.generate}"
+          puts "Creating temporary S3 bucket #{bucket_name}..."
+          bucket = self.s3_client.bucket(bucket_name)
+          bucket.create 
+        end
+        upload_template(bucket, object_name)
+
+        self.syntax_report = s3_validate_syntax(bucket, object_name)
+
+        if self.options.estimate_cost
+          estimate_cost(bucket_name, object_name)
+        end
+
+        if !self.options.validation_bucket
+          puts "Deleting temporary S3 bucket..."
+          bucket.delete! 
+        end
+
+      else 
+        self.syntax_report = local_validate_syntax
+      end
+
+      save_syntax_report if self.options.save_syntax_report
+
+    end
+
+    private
+    def save_syntax_report
+      report_filename = "#{self.output_dir}/#{self.base_name}.report"
+      puts "Syntax validation report written to #{report_filename}"
+      File.open(File.expand_path(report_filename), 'w').puts self.syntax_report.to_hash.to_yaml
+    end
+
+    def upload_template(bucket, object_name)
+      puts "Uploading template to temporary S3 bucket..."
+      object = bucket.object(object_name)
+      object.upload_file(self.output_file)
+      puts "  https://s3.amazonaws.com/#{bucket_name}/#{object_name}"
+    end
+
+    def estimate_cost(bucket, object_name)
+      puts "Estimate cost of template..."
+      client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
+      costing = client.estimate_template_cost(template_url: "https://#{bucket.url}/#{object_name}")
+      puts "Cost Calculator URL is: #{costing.url}"
+    end
+
+    def s3_validate_syntax(bucket, object_name)
+      if self.options.validate_syntax
+        puts "Validating template syntax in S3 Bucket..."
+        client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
+        client.validate_template(template_url: "https://s3.amazonaws.com/#{bucket.url}/#{object_name}")
+      end
+    end
+
+    def local_validate_syntax
+      puts "Validating template syntax locally..."
+      client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
+      client.validate_template(template_body: self.template)
+    end
+  end
+end

data/lib/stdout_capture.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'stringio'
+require 'ostruct'
+
+# Mess wit stdout, capture, restore stdout
+class Capture
+  def self.capture(&block)
+    # redirect output to StringIO objects
+    stdout = StringIO.new
+    stderr = StringIO.new
+    $stdout = stdout
+    $stderr = stderr
+
+    result = block.call
+
+    # restore normal output
+    $stdout = STDOUT
+    $stderr = STDERR
+
+    OpenStruct.new result: result, stdout: stdout.string, stderr: stderr.string
+  end
+end

data/lib/version.rb

@@ -0,0 +1,3 @@
+module CfnDslPipeline
+  VERSION = "0.1.0"
+end

metadata

@@ -0,0 +1,140 @@
+--- !ruby/object:Gem::Specification
+name: cfndsl-pipeline
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Cam Maxwell
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: cfn-nag
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.35
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.4.35
+- !ruby/object:Gem::Dependency
+  name: cfndsl
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudformation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.25.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.25.0
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.46.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.46.0
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 2.3.9
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.8.1
+description: Integrated CfnDsl CloudFormation template generation pipeline that integrates
+  cfn_nag, AWS template validation, and AWS template costing (where possible), and
+  generated `aws cloudformation deploy` compatible parameters files
+email: cameron.maxwell@gmail.com
+executables:
+- cfndsl_pipeline
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/cfndsl_pipeline
+- lib/cfndsl-pipeline.rb
+- lib/monkey_patches.rb
+- lib/options.rb
+- lib/params.rb
+- lib/run-cfn_nag.rb
+- lib/run-cfndsl.rb
+- lib/run-syntax.rb
+- lib/stdout_capture.rb
+- lib/version.rb
+homepage: https://github.com/cmaxwellau/cfndsl-pipeline.git
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.1
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.10
+signing_key: 
+specification_version: 4
+summary: Integrated build pipeline for building CloudFormation with CfnDsl
+test_files: []