cfndsl-pipeline 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA256:
3
+ metadata.gz: e9d71f50ea1fafc076f9390e279f8bb3c3c3799567a16f04894708fd79727ad8
4
+ data.tar.gz: b75350a9198ab44e4907c07c2e9ccdb27b6f8600a6ab9ea2c1dbbf4d6fc4995f
5
+ SHA512:
6
+ metadata.gz: 226f45ed1fef866683576232ce522a5a8de498562d2b8d1b0a58a79018c03ad00cf2f1a6d78f666d76d69e478de244728af7767b1ed579fdcdafc8a7e639be58
7
+ data.tar.gz: a97395d390c44b247521915fd42610c3257e53311c770e807ed99cfa40c70f3b8c37205008f2181284b76f443f87dc628324129957f700e3dc05cb4200cf1905
@@ -0,0 +1,75 @@
1
+ #!/usr/bin/env ruby
2
+ require 'optparse'
3
+ require 'cfndsl-pipeline'
4
+
5
+ USAGE = "Usage: #{File.basename(__FILE__)} -t input file -o output dir [ -b bucket | -p | -c ] [include1 include2 etc]"
6
+ cli_options = {}
7
+
8
+ pipe_options = CfnDslPipeline::Options.new
9
+
10
+ op = OptionParser.new do |opts|
11
+ opts.banner = USAGE
12
+ opts.on('-t', '--template file', 'Input file') do |v|
13
+ cli_options[:template] = v
14
+ end
15
+
16
+ opts.on('-o', '--output dir', 'Output directory') do |v|
17
+ cli_options[:output] = v
18
+ end
19
+
20
+ opts.on('-b', '--bucket', 'Existing S3 bucket for cost estimation and large template syntax validation') do |v|
21
+ pipe_options[:validation_bucket] = v
22
+ end
23
+
24
+ opts.on('--disable-syntax', 'Enable syntax check') do
25
+ pipe_options[:validate_syntax] = false
26
+ end
27
+
28
+ opts.on('-p', '--params', 'Create cloudformation deploy compatible params file') do
29
+ pipe_options[:dump_deploy_params] = true
30
+ end
31
+
32
+ opts.on('--disable-nag', 'Enable cfn_nag ') do
33
+ pipe_options[:validate_cfn_nag] = false
34
+ end
35
+
36
+ opts.on('--syntax-report', 'Save template syntax report') do
37
+ pipe_options[:save_syntax_report] = true
38
+ end
39
+
40
+
41
+ opts.on('--audit-report', 'Save cfn_nag audit report') do
42
+ pipe_options[:save_audit_report] = true
43
+ end
44
+
45
+ opts.on('-c', '--estimate', 'Generate URL for AWS simple cost calculator') do
46
+ pipe_options[:validate_cfn_nag] = true
47
+ end
48
+
49
+ opts.on_tail('-h', '--help', 'show this message') do
50
+ puts opts
51
+ exit
52
+ end
53
+
54
+ opts.on_tail('-v', '--version', 'show the version') do
55
+ puts CfnDsl::Pipeline::VERSION
56
+ exit
57
+ end
58
+ end
59
+
60
+ op.parse!
61
+
62
+
63
+ unless cli_options[:template] && cli_options[:output]
64
+ puts op
65
+ exit 1
66
+ end
67
+
68
+ cfndsl_extras = []
69
+ ARGV.each do |arg|
70
+ cfndsl_extras << [:yaml, arg]
71
+ end if ARGV.length > 0
72
+
73
+ pipeline = CfnDslPipeline::Pipeline.new(cli_options[:output], pipe_options)
74
+ pipeline.build(cli_options[:template], cfndsl_extras)
75
+
@@ -0,0 +1,65 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ #
4
+ # The MIT License
5
+ #
6
+ # Copyright (c) 2019 Cam Maxwell (cameron.maxwell@gmail.com)
7
+ #
8
+ # Permission is hereby granted, free of charge, to any person obtaining a copy
9
+ # of this software and associated documentation files (the "Software"), to deal
10
+ # in the Software without restriction, including without limitation the rights
11
+ # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
12
+ # copies of the Software, and to permit persons to whom the Software is
13
+ # furnished to do so, subject to the following conditions:
14
+ #
15
+ # The above copyright notice and this permission notice shall be included in
16
+ # all copies or substantial portions of the Software.
17
+ #
18
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19
+ # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20
+ # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
21
+ # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
22
+ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
24
+ # THE SOFTWARE.
25
+ #
26
+
27
+ require 'cfndsl'
28
+ require 'fileutils'
29
+
30
+ require_relative 'options'
31
+ require_relative 'params'
32
+ require_relative 'monkey_patches'
33
+ require_relative 'stdout_capture'
34
+ require_relative 'run-cfndsl'
35
+ require_relative 'run-cfn_nag'
36
+ require_relative 'run-syntax'
37
+
38
+ module CfnDslPipeline
39
+ class Pipeline
40
+
41
+ attr_accessor :input_filename, :output_dir, :options, :base_name, :template, :output_filename, :output_file, :syntax_report
42
+
43
+ def initialize (output_dir, options)
44
+ self.input_filename = ''
45
+ self.output_file = nil
46
+ self.template = nil
47
+ self.options = options || nil
48
+ self.syntax_report = []
49
+ FileUtils.mkdir_p output_dir
50
+ abort "Could not create output directory #{output_dir}" if Dir[output_dir] == nil
51
+ self.output_dir = output_dir
52
+ end
53
+
54
+ def build(input_filename, cfndsl_extras)
55
+ abort "Input file #{input_filename} doesn't exist!" if !File.file?(input_filename)
56
+ self.input_filename = "#{input_filename}"
57
+ self.base_name = File.basename(input_filename, '.*')
58
+ self.output_filename = File.expand_path("#{self.output_dir}/#{self.base_name}.yaml")
59
+ exec_cfndsl cfndsl_extras
60
+ exec_syntax_validation if self.options.validate_syntax
61
+ exec_dump_params if self.options.dump_deploy_params
62
+ exec_cfn_nag if self.options.validate_cfn_nag
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,74 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'cfndsl/globals'
4
+ require 'cfndsl/version'
5
+ PARAM_PROPS = %w[Description Default AllowedPattern AllowedValues].freeze
6
+
7
+ # Automatically add Parameters for Tag values
8
+ CfnDsl::CloudFormationTemplate.class_eval do
9
+ def initialize
10
+ return unless defined? external_parameters[:TagStandard]
11
+
12
+ # parameters for tagging standard
13
+ external_parameters[:TagStandard].each do |param_name, props|
14
+ logical_name = props['LogicalName'] || param_name
15
+ Parameter(logical_name) do
16
+ Type(props['Type'])
17
+ PARAM_PROPS.each do |key|
18
+ # puts key, props[key]
19
+ send(key, props[key]) if props[key]
20
+ end
21
+ end
22
+ end if external_parameters[:TagStandard].kind_of?(Hash)
23
+ end
24
+ end
25
+
26
+ module CfnDsl
27
+ # extends CfnDsl esource Properties to automatically substitute
28
+ # FnSub recuraively
29
+ class PropertyDefinition < JSONable
30
+ def initialize(value)
31
+ @value = fix_substitutions(value)
32
+ end
33
+
34
+ def fix_substitutions(val)
35
+ return val unless defined? val.class.to_s.downcase
36
+ meth = "fix_#{val.class.to_s.downcase}"
37
+ if respond_to?(meth.to_sym)
38
+ return send(meth, val)
39
+ end
40
+ val
41
+ end
42
+
43
+ def fix_hash(val)
44
+ val.transform_values! { |item| fix_substitutions item }
45
+ end
46
+
47
+ def fix_array(val)
48
+ val.map! { |item| fix_substitutions item }
49
+ end
50
+
51
+ def fix_string(val)
52
+ val.include?('${') ? FnSub(val) : val
53
+ end
54
+ end
55
+
56
+ # Automatically apply Tag standard to CfnDsl Resources (if supplied)
57
+ class ResourceDefinition
58
+ def initialize
59
+ apply_tag_standard
60
+ end
61
+
62
+ def apply_tag_standard
63
+ return unless defined? external_parameters[:TagStandard]
64
+
65
+ # begin
66
+ external_parameters[:TagStandard].each do |tag_name, props|
67
+ add_tag(tag_name.to_s, Ref(props['LogicalName'] || tag_name))
68
+ end if external_parameters[:TagStandard].kind_of?(Hash)
69
+
70
+ # rescue
71
+ # end
72
+ end
73
+ end
74
+ end
data/lib/options.rb ADDED
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module CfnDslPipeline
4
+ class Options
5
+ attr_accessor :aws_region, :validation_bucket, :save_audit_report, :validate_syntax, :save_syntax_report, :validate_cfn_nag, :validate_output, :estimate_cost, :dump_deploy_params
6
+ def initialize()
7
+ self.aws_region='ap-southeast-2'
8
+ self.validation_bucket=''
9
+ self.validate_cfn_nag=true
10
+ self.validate_syntax=true
11
+ self.estimate_cost=false
12
+ self.save_syntax_report=false
13
+ self.dump_deploy_params=true
14
+ self.save_audit_report=false
15
+ end
16
+ end
17
+ end
18
+
data/lib/params.rb ADDED
@@ -0,0 +1,15 @@
1
+ # frozen_string_literal: true
2
+ require 'shellwords'
3
+
4
+ module CfnDslPipeline
5
+ class Pipeline
6
+ def exec_dump_params
7
+ param_filename = "#{self.output_dir}/#{self.base_name}.params"
8
+ puts "Deploy parameters written to #{param_filename}"
9
+ param_file = File.open(File.expand_path(param_filename), 'w')
10
+ self.syntax_report['parameters'].each do | param |
11
+ param_file.puts "#{param['parameter_key']}=#{Shellwords.escape(param['default_value'])}"
12
+ end
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,41 @@
1
+ require 'cfn-nag'
2
+ require 'colorize'
3
+
4
+ module CfnDslPipeline
5
+ class Pipeline
6
+ def exec_cfn_nag
7
+ puts "Auditing template with cfn-nag..."
8
+ cfn_nag_config = CfnNagConfig.new(
9
+ print_suppression: false,
10
+ fail_on_warnings: true
11
+ )
12
+ cfn_nag = CfnNag.new(config: cfn_nag_config)
13
+ result = cfn_nag.audit(cloudformation_string: self.template)
14
+ if self.options.save_audit_report
15
+ audit_report = Capture.capture do
16
+ SimpleStdoutResults.new.render([{
17
+ filename: output_filename,
18
+ file_results: result
19
+ }])
20
+ end
21
+ audit_filename = "#{self.output_dir}/#{self.base_name}.audit"
22
+ File.open(File.expand_path(audit_filename), 'w').puts audit_report['stdout']
23
+ puts "Saved audit report to #{audit_filename}"
24
+ if result[:failure_count]>0
25
+ puts "Audit failed. #{result[:failure_count]} error(s) found ( ಠ ʖ̯ ಠ) ".red
26
+ elsif result[:violations].count>0
27
+ puts "Audit passed with #{result[:warning_count]} warnings. (._.) ".yellow
28
+ else
29
+ puts "Audit passed! \( ゚ヮ゚)/ ヽ(´ー`)ノ".green
30
+ end
31
+ else
32
+ ColoredStdoutResults.new.render([{
33
+ filename: "cfn-nag results:",
34
+ file_results: result
35
+ }])
36
+ end
37
+
38
+
39
+ end
40
+ end
41
+ end
data/lib/run-cfndsl.rb ADDED
@@ -0,0 +1,18 @@
1
+ require 'cfndsl'
2
+ require 'cfndsl/globals'
3
+ require 'cfndsl/version'
4
+
5
+ module CfnDslPipeline
6
+ class Pipeline
7
+ def exec_cfndsl(cfndsl_extras)
8
+ print "Generating CloudFormation template...\n"
9
+ model = CfnDsl.eval_file_with_extras("#{@input_filename}", cfndsl_extras)
10
+ @template = JSON.parse(model.to_json).to_yaml
11
+ File.open(@output_filename, 'w') do |file|
12
+ file.puts @template
13
+ end
14
+ @output_file = File.open(@output_filename)
15
+ puts " #{@output_file.size} bytes written to #{@output_filename}"
16
+ end
17
+ end
18
+ end
data/lib/run-syntax.rb ADDED
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+ require 'aws-sdk-cloudformation'
3
+ require 'aws-sdk-s3'
4
+ require 'uuid'
5
+
6
+
7
+ module CfnDslPipeline
8
+ class Pipeline
9
+ attr_accessor :cfn_client, :s3_client
10
+
11
+ def initialize
12
+ self.cfn_client = Aws::CloudFormation::Client.new(region: self.aws_region)
13
+ self.s3_client = Aws::S3::Client.new(region: self.aws_region)
14
+ end
15
+
16
+ def exec_syntax_validation
17
+ print "Validating template syntax...\n"
18
+ if self.options.estimate_cost || (self.output_file.size > 51200)
19
+ puts "Filesize is greater than 51200, or cost estimation required. Validating via S3 bucket "
20
+ uuid = UUID.new
21
+ object_name = "#{uuid.generate}"
22
+
23
+ if self.options.validation_bucket
24
+ bucket_name = self.options.validation_bucket
25
+ puts "Using existing S3 bucket #{bucket_name}..."
26
+ bucket = self.s3_client.bucket(self.options.validation_bucket)
27
+ else
28
+ bucket_name = "arch-code-#{uuid.generate}"
29
+ puts "Creating temporary S3 bucket #{bucket_name}..."
30
+ bucket = self.s3_client.bucket(bucket_name)
31
+ bucket.create
32
+ end
33
+ upload_template(bucket, object_name)
34
+
35
+ self.syntax_report = s3_validate_syntax(bucket, object_name)
36
+
37
+ if self.options.estimate_cost
38
+ estimate_cost(bucket_name, object_name)
39
+ end
40
+
41
+ if !self.options.validation_bucket
42
+ puts "Deleting temporary S3 bucket..."
43
+ bucket.delete!
44
+ end
45
+
46
+ else
47
+ self.syntax_report = local_validate_syntax
48
+ end
49
+
50
+ save_syntax_report if self.options.save_syntax_report
51
+
52
+ end
53
+
54
+ private
55
+ def save_syntax_report
56
+ report_filename = "#{self.output_dir}/#{self.base_name}.report"
57
+ puts "Syntax validation report written to #{report_filename}"
58
+ File.open(File.expand_path(report_filename), 'w').puts self.syntax_report.to_hash.to_yaml
59
+ end
60
+
61
+ def upload_template(bucket, object_name)
62
+ puts "Uploading template to temporary S3 bucket..."
63
+ object = bucket.object(object_name)
64
+ object.upload_file(self.output_file)
65
+ puts " https://s3.amazonaws.com/#{bucket_name}/#{object_name}"
66
+ end
67
+
68
+ def estimate_cost(bucket, object_name)
69
+ puts "Estimate cost of template..."
70
+ client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
71
+ costing = client.estimate_template_cost(template_url: "https://#{bucket.url}/#{object_name}")
72
+ puts "Cost Calculator URL is: #{costing.url}"
73
+ end
74
+
75
+ def s3_validate_syntax(bucket, object_name)
76
+ if self.options.validate_syntax
77
+ puts "Validating template syntax in S3 Bucket..."
78
+ client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
79
+ client.validate_template(template_url: "https://s3.amazonaws.com/#{bucket.url}/#{object_name}")
80
+ end
81
+ end
82
+
83
+ def local_validate_syntax
84
+ puts "Validating template syntax locally..."
85
+ client = Aws::CloudFormation::Client.new(region: self.options.aws_region)
86
+ client.validate_template(template_body: self.template)
87
+ end
88
+ end
89
+ end
@@ -0,0 +1,23 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'stringio'
4
+ require 'ostruct'
5
+
6
+ # Mess wit stdout, capture, restore stdout
7
+ class Capture
8
+ def self.capture(&block)
9
+ # redirect output to StringIO objects
10
+ stdout = StringIO.new
11
+ stderr = StringIO.new
12
+ $stdout = stdout
13
+ $stderr = stderr
14
+
15
+ result = block.call
16
+
17
+ # restore normal output
18
+ $stdout = STDOUT
19
+ $stderr = STDERR
20
+
21
+ OpenStruct.new result: result, stdout: stdout.string, stderr: stderr.string
22
+ end
23
+ end
data/lib/version.rb ADDED
@@ -0,0 +1,3 @@
1
+ module CfnDslPipeline
2
+ VERSION = "0.1.0"
3
+ end
metadata ADDED
@@ -0,0 +1,140 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: cfndsl-pipeline
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Cam Maxwell
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2019-08-13 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: cfn-nag
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '='
18
+ - !ruby/object:Gem::Version
19
+ version: 0.4.35
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - '='
25
+ - !ruby/object:Gem::Version
26
+ version: 0.4.35
27
+ - !ruby/object:Gem::Dependency
28
+ name: cfndsl
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - '='
32
+ - !ruby/object:Gem::Version
33
+ version: 0.17.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - '='
39
+ - !ruby/object:Gem::Version
40
+ version: 0.17.0
41
+ - !ruby/object:Gem::Dependency
42
+ name: aws-sdk-cloudformation
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - '='
46
+ - !ruby/object:Gem::Version
47
+ version: 1.25.0
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - '='
53
+ - !ruby/object:Gem::Version
54
+ version: 1.25.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: aws-sdk-s3
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - '='
60
+ - !ruby/object:Gem::Version
61
+ version: 1.46.0
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - '='
67
+ - !ruby/object:Gem::Version
68
+ version: 1.46.0
69
+ - !ruby/object:Gem::Dependency
70
+ name: uuid
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - '='
74
+ - !ruby/object:Gem::Version
75
+ version: 2.3.9
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - '='
81
+ - !ruby/object:Gem::Version
82
+ version: 2.3.9
83
+ - !ruby/object:Gem::Dependency
84
+ name: colorize
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - '='
88
+ - !ruby/object:Gem::Version
89
+ version: 0.8.1
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - '='
95
+ - !ruby/object:Gem::Version
96
+ version: 0.8.1
97
+ description: Integrated CfnDsl CloudFormation template generation pipeline that integrates
98
+ cfn_nag, AWS template validation, and AWS template costing (where possible), and
99
+ generated `aws cloudformation deploy` compatible parameters files
100
+ email: cameron.maxwell@gmail.com
101
+ executables:
102
+ - cfndsl_pipeline
103
+ extensions: []
104
+ extra_rdoc_files: []
105
+ files:
106
+ - bin/cfndsl_pipeline
107
+ - lib/cfndsl-pipeline.rb
108
+ - lib/monkey_patches.rb
109
+ - lib/options.rb
110
+ - lib/params.rb
111
+ - lib/run-cfn_nag.rb
112
+ - lib/run-cfndsl.rb
113
+ - lib/run-syntax.rb
114
+ - lib/stdout_capture.rb
115
+ - lib/version.rb
116
+ homepage: https://github.com/cmaxwellau/cfndsl-pipeline.git
117
+ licenses:
118
+ - MIT
119
+ metadata: {}
120
+ post_install_message:
121
+ rdoc_options: []
122
+ require_paths:
123
+ - lib
124
+ required_ruby_version: !ruby/object:Gem::Requirement
125
+ requirements:
126
+ - - ">="
127
+ - !ruby/object:Gem::Version
128
+ version: 2.4.1
129
+ required_rubygems_version: !ruby/object:Gem::Requirement
130
+ requirements:
131
+ - - ">="
132
+ - !ruby/object:Gem::Version
133
+ version: '0'
134
+ requirements: []
135
+ rubyforge_project:
136
+ rubygems_version: 2.7.10
137
+ signing_key:
138
+ specification_version: 4
139
+ summary: Integrated build pipeline for building CloudFormation with CfnDsl
140
+ test_files: []