cfndsl-pipeline 0.1.2 → 0.1.4

data/examples/standard_tags.yaml

@@ -0,0 +1,41 @@
+---
+
+TagStandard:
+  ServiceName:
+    Default: axt
+    Type: String
+    LogicalName: pServiceName
+    Label: Service pServiceName
+  EnvironmentID:
+    Default: dev1
+    AllowedPattern: (^dev|^sit|^prod)([-])?(\d{1,2})
+    Type: String
+    LogicalName: pEnvironmentID
+    Label: Environment ID
+  CostCentre:
+    Default: CC_DEVOPS
+    Type: String
+    LogicalName: pCostCentre
+    AllowedPattern: '[a-zA-Z0-9+-=._:/@ ]*'
+    Label: Cost Centre
+  Owner:
+    Default: 'Jane Doe'
+    Type: String
+    LogicalName: pOwner
+    Label: Owner
+  ApplicationID:
+    Default: CMDB-NS2010
+    Type: String
+    LogicalName: pApplicationID
+    Label:
+  AppCategory:
+    Default: B
+    Type: String
+    AllowedPattern: '[ABCD]'
+    LogicalName: pAppCategory
+    Label: Application Category
+  SupportGroup:
+    Default: 'DevOps Support'
+    Type: String
+    LogicalName: pSupportGroup
+    Label: Support Group

data/lib/cfndsl-pipeline.rb

@@ -1,6 +1,5 @@
 #!/usr/bin/env ruby
 #
-# 
 # The MIT License
 #
 # Copyright (c) 2019 Cam Maxwell (cameron.maxwell@gmail.com)
@@ -22,44 +21,45 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
-# 
+#
 
 require 'cfndsl'
+require 'cfn-nag'
 require 'fileutils'
 
+require_relative 'monkey-patches/cfndsl_patch'
+require_relative 'monkey-patches/stdout_capture'
 require_relative 'options'
 require_relative 'params'
-require_relative 'monkey_patches'
-require_relative 'stdout_capture'
-require_relative 'run-cfndsl'
-require_relative 'run-cfn_nag'
-require_relative 'run-syntax'
+require_relative 'exec_cfndsl'
+require_relative 'exec_cfn_nag'
+require_relative 'exec_syntax'
 
 module CfnDslPipeline
+  # Main pipeline
   class Pipeline
-
     attr_accessor :input_filename, :output_dir, :options, :base_name, :template, :output_filename, :output_file, :syntax_report
 
-    def initialize (output_dir, options)
+    def initialize(output_dir, options)
       self.input_filename = ''
       self.output_file = nil
       self.template = nil
       self.options = options || nil
       self.syntax_report = []
       FileUtils.mkdir_p output_dir
-      abort "Could not create output directory #{output_dir}" if Dir[output_dir] == nil
+      abort "Could not create output directory #{output_dir}" unless Dir[output_dir]
       self.output_dir = output_dir
     end
 
     def build(input_filename, cfndsl_extras)
-      abort "Input file #{input_filename} doesn't exist!" if !File.file?(input_filename)
+      abort "Input file #{input_filename} doesn't exist!" unless File.file?(input_filename)
       self.input_filename = "#{input_filename}"
       self.base_name = File.basename(input_filename, '.*')
-      self.output_filename = File.expand_path("#{self.output_dir}/#{self.base_name}.yaml")
+      self.output_filename = File.expand_path("#{output_dir}/#{base_name}.yaml")
       exec_cfndsl cfndsl_extras
-      exec_syntax_validation if self.options.validate_syntax
-      exec_dump_params if self.options.dump_deploy_params
-      exec_cfn_nag if self.options.validate_cfn_nag
+      exec_syntax_validation if options.validate_syntax
+      exec_dump_params if options.dump_deploy_params
+      exec_cfn_nag if options.validate_cfn_nag
     end
   end
 end

data/lib/cli_options.rb

@@ -0,0 +1,124 @@
+
+require 'optparse'
+
+module CfnDslPipeline
+  # Command Line Options processing
+  class CliOptions
+    attr_accessor :output, :template, :pipeline, :cfndsl_extras, :op
+
+    USAGE = "Usage: #{File.basename(__FILE__)} input file [ -o output_dir ] [ -b bucket ] OPTIONS [ include1 include2 etc.. ]"
+
+    def initialize
+      @output = './'
+      @cfndsl_extras = []
+      @pipeline = CfnDslPipeline::Options.new
+      parse && validate
+    end
+
+    private
+
+    # rubocop:disable Metrics/AbcSize
+    # rubocop:disable Metrics/MethodLength
+    def parse
+      @op = OptionParser.new do |opts|
+        opts.banner = USAGE
+
+        opts.on('-o', '--output dir', 'Optional output directory. Default is current directory') do |dir|
+          @output = dir
+        end
+
+        opts.on('-b', '--bucket', 'Optional Existing S3 bucket for cost estimation and large template syntax validation') do |bucket|
+          pipeline.validation_bucket = bucket
+        end
+
+        opts.on('-p', '--params', 'Create cloudformation deploy compatible params file') do
+          pipeline.dump_deploy_params = true
+        end
+
+        opts.on('-s', '--syntax', 'Enable syntax check') do
+          pipeline.validate_syntax = true
+        end
+
+        opts.on('--syntax-report', 'Save template syntax report') do
+          pipeline.save_syntax_report = true
+        end
+
+        opts.on('-a', '--audit', 'Enable cfn_nag audit') do
+          pipeline.validate_cfn_nag = false
+        end
+
+        opts.on('--audit-rule-dir', 'cfn_nag audit custom rules directory') do
+          pipeline.cfn_nag[:rule_directory] = true
+        end
+
+        opts.on('--audit-report', 'Save cfn_nag audit report') do
+          pipeline.save_audit_report = true
+        end
+
+        opts.on('--audit-debug', 'Enable cfn_nag rule debug output') do
+          pipeline.debug_audit = true
+        end
+
+        opts.on('-e', '--estimate-costs', 'Generate URL for AWS simple cost calculator') do
+          pipeline.estimate_cost = true
+        end
+
+        opts.on('-r', '--aws-region', 'AWS region to use. Default: ap-southeast-2') do |region|
+          pipeline.aws_region = region
+        end
+
+        opts.on_tail('-h', '--help', 'show this message') do
+          puts opts
+          exit
+        end
+
+        opts.on_tail('-d', '--debug', 'show pipeline debug messages') do
+          pipeline.debug = true
+          exit
+        end
+
+        opts.on_tail('-v', '--version', 'Show version') do
+          puts CfnDsl::Pipeline::VERSION
+          exit
+        end
+      end
+      @op.parse!
+
+      # first non-dash parameter is the mandatory input file
+      @template = ARGV.pop
+
+      ARGV.each do |arg|
+        @cfndsl_extras << [:yaml, arg]
+      end if ARGV.length > 0
+
+      pipeline
+    end
+    # rubocop:enable Metrics/AbcSize
+    # rubocop:enable Metrics/MethodLength
+
+    def fatal(msg)
+      puts msg
+      puts @op
+      exit 1
+    end
+
+    # rubocop:disable Metrics/CyclomaticComplexity
+    # rubocop:disable Metrics/PerceivedComplexity
+    def validate
+      # Exit on invalid option combinations
+      fatal 'Error: Input template file does not exist.' unless @template && File.file?(@template)
+
+      if @pipeline.save_syntax_report
+        fatal 'Error: save syntax report is set, but syntax validation was not enabled.' unless @pipeline.validate_syntax && !@pipeline.save_syntax_report
+      end
+      # rubocop:disable  Style/GuardClause
+      if @pipeline.cfn_nag.rule_directory || @pipeline.debug_audit || @pipeline.save_audit_report
+        fatal 'Error: Audit options set, but audit was not enabled' unless @pipeline.validate_cfn_nag
+        fatal 'Error: cfn_nag rule directory does not exist' unless File.directory?(@pipeline.cfn_nag.rule_directory)
+      end
+      # rubocop:enable  Style/GuardClause
+    end
+    # rubocop:enable Metrics/CyclomaticComplexity
+    # rubocop:enable Metrics/PerceivedComplexity
+  end
+end

data/lib/exec_cfn_nag.rb

@@ -0,0 +1,58 @@
+require 'cfn-nag'
+require 'logging'
+require 'colorize'
+
+module CfnDslPipeline
+  # Add cfn_nag functions to pipeline
+  class Pipeline
+    def exec_cfn_nag
+      puts 'Auditing template with cfn-nag...'
+      configure_cfn_nag_logging
+      cfn_nag = CfnNag.new(config: options.cfn_nag)
+      result = cfn_nag.audit(cloudformation_string: template)
+      save_report result
+      display_report result
+      show_summary result
+    end
+
+    private
+
+    def configure_cfn_nag_logging
+      CfnNagLogging.configure_logging([:debug]) if options.debug_audit
+    end
+
+    def display_report(result)
+      ColoredStdoutResults.new.render([
+        {
+          filename: "#{@base_name}",
+          file_results: result
+        }
+      ])
+    end
+
+    def save_report(result)
+      return unless options.save_audit_report
+      report_data = Capture.capture do
+        SimpleStdoutResults.new.render([
+          {
+            filename: "#{@base_name}",
+            file_results: result
+          }
+        ])
+      end
+      filename = "#{output_dir}/#{base_name}.audit"
+      File.open(File.expand_path(filename), 'w').puts report_data['stdout']
+      puts "Saved audit report to #{filename}"
+    end
+
+    def show_summary(result)
+      if result[:failure_count] > 0
+        puts "Audit failed. #{result[:failure_count]} error(s) found     ( ಠ ʖ̯ ಠ)  ".red
+      elsif result[:violations].count > 0
+        puts "Audit passed with #{result[:warning_count]} warnings.     (._.)  ".yellow
+      else
+        puts 'Audit passed!        ヽ( ﾟヮﾟ)/      ヽ(´ー｀)ノ'.green
+      end
+    end
+  end
+end

data/lib/{run-cfndsl.rb → exec_cfndsl.rb}

@@ -3,16 +3,18 @@ require 'cfndsl/globals'
 require 'cfndsl/version'
 
 module CfnDslPipeline
+  #
   class Pipeline
-   def exec_cfndsl(cfndsl_extras)
-      print "Generating CloudFormation template...\n"
-      model = CfnDsl.eval_file_with_extras("#{@input_filename}", cfndsl_extras)
+    def exec_cfndsl(cfndsl_extras)
+      puts 'Generating CloudFormation template...'
+
+      model = CfnDsl.eval_file_with_extras("#{@input_filename}", cfndsl_extras, (options.debug_cfndsl ? STDOUT : nil))
       @template = JSON.parse(model.to_json).to_yaml
       File.open(@output_filename, 'w') do |file|
         file.puts @template
       end
       @output_file = File.open(@output_filename)
-      puts "  #{@output_file.size} bytes written to #{@output_filename}"
+      puts "#{@output_file.size} bytes written to #{@output_filename}"
     end
   end
-end
+end

data/lib/exec_syntax.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+require 'aws-sdk-cloudformation'
+require 'aws-sdk-s3'
+require 'uuid'
+
+module CfnDslPipeline
+  #
+  class Pipeline
+    attr_accessor :s3_client
+
+    def initialize
+      self.s3_client = Aws::S3::Client.new(region: aws_region)
+    end
+    # rubocop:disable Metrics/AbcSize
+    def exec_syntax_validation
+      puts 'Validating template syntax...'
+      if options.estimate_cost || (output_file.size > 51_200)
+        puts 'Filesize is greater than 51200, or cost estimation required. Validating via S3 bucket '
+        uuid = UUID.new
+        bucket = determine_bucket
+        object_name = uuid.generate.to_s
+        upload_template(bucket, object_name)
+        self.syntax_report = s3_validate_syntax(bucket, object_name)
+        estimate_cost(bucket_name, object_name)
+        unless options.validation_bucket
+          puts 'Deleting temporary S3 bucket...'
+          bucket.delete!
+        end
+      else
+        self.syntax_report = local_validate_syntax
+      end
+      save_syntax_report
+    end
+    # rubocop:enable Metrics/AbcSize
+
+    private
+
+    def determine_bucket
+      if options.validation_bucket
+        bucket_name = options.validation_bucket
+        puts "Using existing S3 bucket #{bucket_name}..."
+        bucket = s3_client.bucket(options.validation_bucket)
+      else
+        bucket_name = "arch-code-#{uuid.generate}"
+        puts "Creating temporary S3 bucket #{bucket_name}..."
+        bucket = s3_client.bucket(bucket_name)
+        bucket.create
+      end
+      bucket
+    end
+
+    def save_syntax_report
+      return unless options.save_syntax_report
+      report_filename = "#{output_dir}/#{base_name}.report"
+      puts "Syntax validation report written to #{report_filename}"
+      File.open(File.expand_path(report_filename), 'w').puts syntax_report.to_hash.to_yaml
+    end
+
+    def upload_template(bucket, object_name)
+      puts 'Uploading template to temporary S3 bucket...'
+      object = bucket.object(object_name)
+      object.upload_file(output_file)
+      puts "https://s3.amazonaws.com/#{bucket_name}/#{object_name}"
+    end
+
+    def estimate_cost(bucket, object_name)
+      return unless options.estimate_cost
+      puts 'Estimate cost of template...'
+      client = Aws::CloudFormation::Client.new(region: options.aws_region)
+      costing = client.estimate_template_cost(template_url: "https://#{bucket.url}/#{object_name}")
+      puts "Cost Calculator URL is: #{costing.url}"
+    end
+
+    def s3_validate_syntax(bucket, object_name)
+      return unless options.validate_syntax
+      puts 'Validating template syntax in S3 Bucket...'
+      client = Aws::CloudFormation::Client.new(region: options.aws_region)
+      client.validate_template(template_url: "https://s3.amazonaws.com/#{bucket.url}/#{object_name}")
+    end
+
+    def local_validate_syntax
+      puts 'Validating template syntax locally...'
+      client = Aws::CloudFormation::Client.new(region: options.aws_region)
+      client.validate_template(template_body: template)
+    end
+  end
+end

data/lib/{monkey_patches.rb → monkey-patches/cfndsl_patch.rb}

@@ -1,10 +1,14 @@
 # frozen_string_literal: true
-
 require 'cfndsl/globals'
 require 'cfndsl/version'
-PARAM_PROPS = %w[Description Default AllowedPattern AllowedValues].freeze
-HAS_PROPAGATABLE_TAGS = %w[CfnDsl::AWS::Types::AWS_AutoScaling_AutoScalingGroup].freeze
-HAS_MAPPED_TAGS = %w[CfnDsl::AWS::Types::AWS_Serverless_Function CfnDsl::AWS::Types::AWS_Serverless_SimpleTable CfnDsl::AWS::Types::AWS_Serverless_Application].freeze
+require 'cfndsl/external_parameters'
+require 'cfndsl/aws/cloud_formation_template'
+
+PARAM_PROPS = %w([Description Default AllowedPattern AllowedValues]).freeze
+HAS_PROPAGATABLE_TAGS = %w([CfnDsl::AWS::Types::AWS_AutoScaling_AutoScalingGroup]).freeze
+# rubocop:disable Metrics/LineLength
+HAS_MAPPED_TAGS = %w([CfnDsl::AWS::Types::AWS_Serverless_Function CfnDsl::AWS::Types::AWS_Serverless_SimpleTable CfnDsl::AWS::Types::AWS_Serverless_Application]).freeze
+# rubocop:enable Metrics/LineLength
 
 # Automatically add Parameters for Tag values
 CfnDsl::CloudFormationTemplate.class_eval do
@@ -21,11 +25,20 @@ CfnDsl::CloudFormationTemplate.class_eval do
           send(key, props[key]) if props[key]
         end
       end
-    end if external_parameters[:TagStandard].kind_of?(Hash)
+    end if external_parameters[:TagStandard].is_a?(Hash)
   end
 end
 
 module CfnDsl
+  # Add ability to reset params when being used in loops in rakefiles etc
+  class ExternalParameters
+    class << self
+      def reset
+        @parameters = self.class.defaults.clone
+      end
+    end
+  end
+
   # extends CfnDsl esource Properties to automatically substitute
   # FnSub recuraively
   class PropertyDefinition < JSONable
@@ -36,9 +49,8 @@ module CfnDsl
     def fix_substitutions(val)
       return val unless defined? val.class.to_s.downcase
       meth = "fix_#{val.class.to_s.downcase}"
-      if respond_to?(meth.to_sym)
-        return send(meth, val)
-      end
+
+      return send(meth, val) if respond_to?(meth.to_sym)
       val
     end
 
@@ -62,27 +74,31 @@ module CfnDsl
       apply_tag_standard
     end
 
+    private
+
     def apply_tag_standard
       return unless defined? external_parameters[:TagStandard]
-      return unless external_parameters[:TagStandard].kind_of?(Hash)
-
-      resource_type = self.class.to_s
+      return unless external_parameters[:TagStandard].is_a?(Hash)
+      apply_tags(external_parameters[:TagStandard]) if defined? self.Tag
+      apply_tags_map(external_parameters[:TagStandard]) if HAS_MAPPED_TAGS.include? self.class.to_s
+    end
 
-      if defined? self.Tag
-        external_parameters[:TagStandard].each do |tag_name, props|
-          send(:Tag) do
-            Key tag_name.to_s
-            Value Ref(props['LogicalName'] || tag_name)
-            PropagateAtLaunch true if HAS_PROPAGATABLE_TAGS.include? resource_type
-          end
-        end
-      elsif HAS_MAPPED_TAGS.include? resource_type
-        tag_map = {}
-        external_parameters[:TagStandard].each do |tag_name, props| 
-          tag_map[tag_name.to_s] = Ref(props['LogicalName'] || tag_name)
+    def apply_tags(tags)
+      tags.each do |tag_name, props|
+        send(:Tag) do
+          Key tag_name.to_s
+          Value Ref(props['LogicalName'] || tag_name)
+          PropagateAtLaunch true if HAS_PROPAGATABLE_TAGS.include? self.class.to_s
         end
-        Tags tag_map
       end
     end
+
+    def apply_tags_map(tags)
+      tag_map = {}
+      tags.each do |tag_name, props|
+        tag_map[tag_name.to_s] = Ref(props['LogicalName'] || tag_name)
+      end
+      Tags tag_map
+    end
   end
 end