cfnpp 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/cfnpp +396 -0
- data/lib/cfnpp/replacer.rb +83 -0
- data/lib/cfnpp/templateresult.rb +28 -0
- data/lib/cfnpp/transform.rb +273 -0
- data/lib/cfnpp/uploader.rb +85 -0
- metadata +116 -0
data/bin/cfnpp
ADDED
@@ -0,0 +1,396 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
unless Kernel.respond_to?(:require_relative)
|
4
|
+
module Kernel
|
5
|
+
def require_relative(path)
|
6
|
+
require File.join(File.dirname(caller[0]), path.to_str)
|
7
|
+
end
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
require 'rubygems'
|
12
|
+
require 'bundler/setup'
|
13
|
+
require 'aws-sdk'
|
14
|
+
require 'securerandom'
|
15
|
+
require 'json'
|
16
|
+
require 'cfnpp/transform'
|
17
|
+
require 'cfnpp/uploader'
|
18
|
+
require 'awesome_print'
|
19
|
+
require 'yaml'
|
20
|
+
require 'dogapi'
|
21
|
+
require 'ploy/yamlreader'
|
22
|
+
require 'optparse'
|
23
|
+
|
24
|
+
def main()
|
25
|
+
usage = <<eot
|
26
|
+
Usage: ./launch create|update|print STACK_TYPE ENVIRONMENT_TYPE [ENVIRONMENT_NAME]
|
27
|
+
STACK_TYPE found in stacks, i.e manta-site
|
28
|
+
ENVIRONMENT_TYPE is e.g. smoketest, used as a key into configuration
|
29
|
+
ENVIRONMENT_NAME is your unique name, i.e stsmith
|
30
|
+
Launch a cloudformation template.
|
31
|
+
eot
|
32
|
+
|
33
|
+
cli_opts = {}
|
34
|
+
optparse = OptionParser.new do|opts|
|
35
|
+
opts.on('-h', '--help', 'Display this screen') do
|
36
|
+
puts usage
|
37
|
+
exit
|
38
|
+
end
|
39
|
+
opts.on('-d', '--deployconfig URL', 'stackhub URL for deployconfig') do |url|
|
40
|
+
cli_opts[:deployconfig] = url
|
41
|
+
end
|
42
|
+
opts.on('-F', '--force', 'Force stack update and disable cancel-bad-rolling-updates from executing. (Affects only this update)') do |url|
|
43
|
+
cli_opts[:force] = 1
|
44
|
+
end
|
45
|
+
opts.on('-b', '--basepath PATH', 'base path for stacks; defaults to ./stacks') do |basepath|
|
46
|
+
cli_opts[:basepath] = basepath
|
47
|
+
end
|
48
|
+
opts.on('-f', '--file PATH', 'use PATH instead of $basepath/main.yml for main template') do |filepath|
|
49
|
+
cli_opts[:filepath] = filepath
|
50
|
+
end
|
51
|
+
opts.on('-c', '--conf PATH', 'use PATH instead of $basepath/conf/$environment_type for conf file') do |confpath|
|
52
|
+
cli_opts[:confpath] = confpath
|
53
|
+
end
|
54
|
+
end
|
55
|
+
optparse.parse!
|
56
|
+
|
57
|
+
if cli_opts[:deployconfig]
|
58
|
+
yr = Ploy::YamlReader.new
|
59
|
+
conf = yr.from_http(cli_opts[:deployconfig])
|
60
|
+
if (conf['locked'])
|
61
|
+
puts "skipping autoupdate: deployconfig is locked"
|
62
|
+
exit
|
63
|
+
end
|
64
|
+
unless (conf['stack_autoupdate'])
|
65
|
+
puts "skipping autoupdate: autoupdate is false"
|
66
|
+
exit
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
whoami = `whoami`
|
71
|
+
whoami.chomp!
|
72
|
+
|
73
|
+
case whoami
|
74
|
+
when "judd"
|
75
|
+
whoami="jmontgomery"
|
76
|
+
when "juddmontgomery"
|
77
|
+
whoami="jmontgomery"
|
78
|
+
end
|
79
|
+
|
80
|
+
basepath = cli_opts[:basepath] || "stacks"
|
81
|
+
|
82
|
+
action = ARGV.shift
|
83
|
+
stack_type = ARGV.shift
|
84
|
+
template_path = "#{basepath}/#{stack_type}/main.yml"
|
85
|
+
template_path = cli_opts[:filepath] if cli_opts[:filepath]
|
86
|
+
environment_type = ARGV.shift
|
87
|
+
environment_name = ARGV.shift || whoami
|
88
|
+
|
89
|
+
cfn = AWS::CloudFormation.new
|
90
|
+
stack_name = "#{stack_type}--#{environment_name}--#{environment_type}"
|
91
|
+
conf_path = cli_opts[:confpath] || "#{basepath}/#{stack_type}/conf/#{environment_type}.yml"
|
92
|
+
|
93
|
+
stack = cfn.stacks[stack_name]
|
94
|
+
if stack.exists?
|
95
|
+
status = stack.status
|
96
|
+
if status =~ /_IN_PROGRESS$/
|
97
|
+
puts "can't update stack (#{stack_name}) while status is (#{status})"
|
98
|
+
exit 0
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
allowed_actions = ['create', 'update', 'print', 'validate']
|
103
|
+
if !action || !allowed_actions.include?(action) || !template_path || !File.file?(template_path) || !File.file?(conf_path)
|
104
|
+
if action && ! allowed_actions.include?(action)
|
105
|
+
puts "unknown action (#{action})"
|
106
|
+
end
|
107
|
+
|
108
|
+
if conf_path
|
109
|
+
if conf_path && !File.file?(conf_path)
|
110
|
+
puts "ERROR: file (#{conf_path}) not found for stack (#{environment_type})"
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
if stack_type
|
115
|
+
if template_path && !File.file?(template_path)
|
116
|
+
puts "ERROR: file (#{template_path}) not found for stack (#{stack_type})"
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if environment_type && !/(^smoketest$)/.match(environment_type)
|
121
|
+
puts "ERROR: unknown environment_type (#{environment_type})"
|
122
|
+
end
|
123
|
+
|
124
|
+
puts usage
|
125
|
+
exit 1
|
126
|
+
end
|
127
|
+
|
128
|
+
conf_yaml = File.read(conf_path)
|
129
|
+
opts = YAML::load(conf_yaml)
|
130
|
+
|
131
|
+
timestamp = Time.now.utc.strftime("%Y-%m-%dT%H.%M.%S.%LZ")
|
132
|
+
template_info = get_template_info()
|
133
|
+
opts['LastUpdateTimestamp'] = timestamp
|
134
|
+
opts['TemplateGitRevision'] = template_info[:gitrev]
|
135
|
+
opts['TemplateGitStatus'] = template_info[:gitstatus]
|
136
|
+
opts['TemplateSource'] = template_info[:source]
|
137
|
+
opts['LaunchInstanceId'] = template_info[:instanceid]
|
138
|
+
opts['StackType'] = stack_type
|
139
|
+
opts['StackName'] = stack_name
|
140
|
+
opts['EnvironmentName'] = environment_name
|
141
|
+
opts['EnvironmentType'] = environment_type
|
142
|
+
|
143
|
+
if cli_opts[:force]
|
144
|
+
opts['Force'] = 'true'
|
145
|
+
else
|
146
|
+
opts['Force'] = 'true'
|
147
|
+
end
|
148
|
+
|
149
|
+
uploader = CfnPP::Uploader.new('manta-cloudformation', stack_name)
|
150
|
+
|
151
|
+
template_result = CfnPP::Transform.load_file(template_path, opts, "main", "https://manta-cloudformation.s3.amazonaws.com/#{uploader.s3_path}")
|
152
|
+
|
153
|
+
if action == "print" #hack. TODO: dispatch actions sanely
|
154
|
+
puts JSON.pretty_generate(template_result.data)
|
155
|
+
exit 0
|
156
|
+
end
|
157
|
+
|
158
|
+
upload_results = do_upload(uploader, template_result, opts)
|
159
|
+
|
160
|
+
if action == 'validate'
|
161
|
+
puts 'uploading and validating only; no updates applied'
|
162
|
+
exit 0
|
163
|
+
end
|
164
|
+
|
165
|
+
if action == 'create'
|
166
|
+
puts "creating stack (#{stack_name}) from template (#{template_path}) with config (#{conf_path})"
|
167
|
+
stack = cfn.stacks.create(stack_name,
|
168
|
+
upload_results[:url],
|
169
|
+
:parameters => upload_results[:opts_parameters],
|
170
|
+
:capabilities => ['CAPABILITY_IAM'])
|
171
|
+
end
|
172
|
+
|
173
|
+
if action == 'update'
|
174
|
+
current_resource_count = 0
|
175
|
+
current_resource_count = stack.resources.count()
|
176
|
+
stack.resources.each_batch do |batch|
|
177
|
+
current_resource_count += batch.count
|
178
|
+
end
|
179
|
+
puts "updating stack (#{stack_name}) from template (#{template_path}) with config (#{conf_path})"
|
180
|
+
puts "stack currently contains (#{current_resource_count}) resources"
|
181
|
+
|
182
|
+
# Send an event to Datadog:
|
183
|
+
# disabled while decoupling cfnpp; TODO: enable
|
184
|
+
if false
|
185
|
+
api_key = ''
|
186
|
+
dog = Dogapi::Client.new(api_key)
|
187
|
+
dd_event_title = "stack update"
|
188
|
+
dd_event_text = "#{stack_name} stack was updated by #{whoami}"
|
189
|
+
dd_event_tags = "deployment, stackname:#{stack_name}"
|
190
|
+
dog.emit_event(Dogapi::Event.new("#{dd_event_text}", :msg_title => "#{dd_event_title}", :tags => "#{dd_event_tags}"))
|
191
|
+
end
|
192
|
+
|
193
|
+
# Update the stack
|
194
|
+
stack.update(:template => upload_results[:url],
|
195
|
+
:parameters => upload_results[:opts_parameters],
|
196
|
+
:capabilities => ['CAPABILITY_IAM'])
|
197
|
+
end
|
198
|
+
|
199
|
+
after_update(stack, timestamp)
|
200
|
+
end
|
201
|
+
|
202
|
+
def do_upload(uploader, template_result, opts)
|
203
|
+
|
204
|
+
template_result.substacks.each do |substack_result|
|
205
|
+
do_upload(uploader, substack_result, opts)
|
206
|
+
end
|
207
|
+
|
208
|
+
upload_res = uploader.upload_template(template_result, opts)
|
209
|
+
|
210
|
+
if upload_res[:error]
|
211
|
+
puts "error for #{upload_res[:url]}"
|
212
|
+
if upload_res[:error] == 'validation_error'
|
213
|
+
vs = upload_res[:validation_status]
|
214
|
+
puts "validation error code: #{vs[:code]}"
|
215
|
+
puts "validation error message: #{vs[:message]}"
|
216
|
+
elsif upload_res[:error] == 'already_updated'
|
217
|
+
prev_opts = upload_res[:prev_opts]
|
218
|
+
last_update_timestamp = prev_opts['LastUpdateTimestamp']
|
219
|
+
template_git_revision = prev_opts['TemplateGitRevision']
|
220
|
+
instanceid = prev_opts['LaunchInstanceId']
|
221
|
+
puts "Update to git revision #{template_git_revision} already attempted at #{last_update_timestamp} from instance (#{instanceid})."
|
222
|
+
end
|
223
|
+
exit 1
|
224
|
+
else
|
225
|
+
puts "uploaded #{upload_res[:url]}"
|
226
|
+
end
|
227
|
+
|
228
|
+
return upload_res
|
229
|
+
|
230
|
+
end
|
231
|
+
|
232
|
+
def after_update(stack, timestamp)
|
233
|
+
stack_name = stack.name
|
234
|
+
#cmd = "watch -n 10 cfn-describe-stack-events #{stack_name}"
|
235
|
+
#cmd = "watch -n 10 bin/get_stack_events.rb #{stack_name}"
|
236
|
+
#puts cmd
|
237
|
+
if STDIN.tty? && STDOUT.tty?
|
238
|
+
puts "STDIN and STDOUT are ttys!"
|
239
|
+
|
240
|
+
begin
|
241
|
+
stackEvents = Set.new()
|
242
|
+
while true
|
243
|
+
get_stack_events( stack.name).reverse_each do |event|
|
244
|
+
if stackEvents.add?( event)
|
245
|
+
puts event
|
246
|
+
end
|
247
|
+
end
|
248
|
+
get_sub_stacks( stack.name).each do |stack|
|
249
|
+
get_stack_events( stack).reverse.each do |event|
|
250
|
+
if stackEvents.add?(event)
|
251
|
+
puts "NESTED: #{event}"
|
252
|
+
end
|
253
|
+
end
|
254
|
+
end
|
255
|
+
sleep 5
|
256
|
+
end
|
257
|
+
rescue SystemExit, Interrupt
|
258
|
+
exit 0
|
259
|
+
end
|
260
|
+
end
|
261
|
+
|
262
|
+
# This is all working, tested code, just not sure it belongs here. --stsmith
|
263
|
+
#stack_timestamp = ''
|
264
|
+
#status = ''
|
265
|
+
#begin
|
266
|
+
# loop do
|
267
|
+
# # During an in-progress update, the parameters report the new values.
|
268
|
+
# # If rollback happens, the parameters revert to their previous values.
|
269
|
+
# # This behavior makes sense and is helpful.
|
270
|
+
# stack_timestamp = stack.parameters['LastUpdateTimestamp']
|
271
|
+
# status = stack.status
|
272
|
+
# puts "#{stack_name} #{status} #{stack_timestamp}"
|
273
|
+
# break unless status =~ /(CREATE|UPDATE)_IN_PROGRESS$/ && stack_timestamp == timestamp
|
274
|
+
# sleep 5
|
275
|
+
# end
|
276
|
+
#
|
277
|
+
#rescue AWS::CloudFormation::Errors::Throttling
|
278
|
+
# puts "API request throttled."
|
279
|
+
# retry
|
280
|
+
#end
|
281
|
+
#
|
282
|
+
#exit_status = 1
|
283
|
+
#if status =~ /(CREATE|UPDATE)_COMPLETE/ && stack_timestamp == timestamp
|
284
|
+
# exit_status = 0
|
285
|
+
#end
|
286
|
+
#exit exit_status
|
287
|
+
end
|
288
|
+
|
289
|
+
def get_sub_stacks( stack_name)
|
290
|
+
stacks = Array.new()
|
291
|
+
AWS.memoize do
|
292
|
+
cfm = AWS::CloudFormation.new()
|
293
|
+
stack = cfm.stacks[stack_name]
|
294
|
+
stack.resource_summaries.each do |resource|
|
295
|
+
if resource[:resource_type] == 'AWS::CloudFormation::Stack'
|
296
|
+
stacks.push( resource[:physical_resource_id] )
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
return stacks
|
301
|
+
end
|
302
|
+
|
303
|
+
def get_stack_events( stack_name)
|
304
|
+
|
305
|
+
events = Array.new()
|
306
|
+
|
307
|
+
AWS.memoize do
|
308
|
+
cfm = AWS::CloudFormation.new()
|
309
|
+
stack = cfm.stacks[stack_name]
|
310
|
+
|
311
|
+
table = {
|
312
|
+
'columns' => [ 'timestamp', 'resource_type', 'logical_resource_id', 'resource_status', 'resource_status_reason' ],
|
313
|
+
'column_maxlengths' => {},
|
314
|
+
'rows' => []
|
315
|
+
}
|
316
|
+
|
317
|
+
stack.events.each do |event|
|
318
|
+
row = {}
|
319
|
+
table['columns'].each do |column_name|
|
320
|
+
maxlength = table['column_maxlengths'][column_name] || 0
|
321
|
+
value = sprintf("%s", event.send(column_name))
|
322
|
+
row[column_name] = value
|
323
|
+
maxlength = (value.length > maxlength) ? value.length : maxlength
|
324
|
+
table['column_maxlengths'][column_name] = maxlength
|
325
|
+
end
|
326
|
+
table['rows'].push(row)
|
327
|
+
end
|
328
|
+
|
329
|
+
header_row = {}
|
330
|
+
table['columns'].each do |column_name|
|
331
|
+
header_row[column_name] = column_name
|
332
|
+
end
|
333
|
+
|
334
|
+
table['rows'].unshift(header_row)
|
335
|
+
table['rows'].each do |row|
|
336
|
+
row_text = ''
|
337
|
+
last_column_name = table['columns'].last
|
338
|
+
table['columns'].each do |column_name|
|
339
|
+
if column_name != last_column_name
|
340
|
+
maxlength = table['column_maxlengths'][column_name] || 0
|
341
|
+
row_text += sprintf("%-*s", maxlength, row[column_name])
|
342
|
+
row_text += ' '
|
343
|
+
else
|
344
|
+
row_text += sprintf("%s", row[column_name])
|
345
|
+
end
|
346
|
+
end
|
347
|
+
#puts row_text
|
348
|
+
events.push( row_text)
|
349
|
+
end
|
350
|
+
end
|
351
|
+
return events
|
352
|
+
end
|
353
|
+
|
354
|
+
|
355
|
+
def get_template_info()
|
356
|
+
gitrev = ''
|
357
|
+
gitstatus = ''
|
358
|
+
source = '' # ploy or git
|
359
|
+
|
360
|
+
pwd = Dir.pwd
|
361
|
+
|
362
|
+
# Try from ploy package.
|
363
|
+
package = `dpkg -S #{pwd} 2>/dev/null`
|
364
|
+
if package && package =~ /^cloudformation-tools:/
|
365
|
+
gitrev = `dpkg-query -W -f='${gitrev}' cloudformation-tools`
|
366
|
+
end
|
367
|
+
|
368
|
+
#If stin and stdout or tty then someone is running this from command line.
|
369
|
+
if STDIN.tty? && STDOUT.tty?
|
370
|
+
source = 'cmd'
|
371
|
+
else
|
372
|
+
source = 'ploy'
|
373
|
+
end
|
374
|
+
|
375
|
+
# Try from git.
|
376
|
+
if gitrev == ''
|
377
|
+
gitrev = `/usr/bin/git log --format='%H' -n 1`
|
378
|
+
gitrev.chomp!
|
379
|
+
if gitrev
|
380
|
+
source = 'git'
|
381
|
+
gitstatus = `git status --porcelain`
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
instanceid = `/bin/netcat 169.254.169.254 80 -w 1 -q 0 </dev/null && /usr/bin/ec2metadata --instance-id`
|
386
|
+
instanceid.chomp!
|
387
|
+
|
388
|
+
template = {
|
389
|
+
:gitrev => gitrev,
|
390
|
+
:gitstatus => gitstatus,
|
391
|
+
:source => source,
|
392
|
+
:instanceid => instanceid
|
393
|
+
}
|
394
|
+
end
|
395
|
+
|
396
|
+
main()
|
@@ -0,0 +1,83 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'yaml'
|
3
|
+
require 'erb'
|
4
|
+
require 'pp'
|
5
|
+
|
6
|
+
module CfnPP
|
7
|
+
# Implements logic for having textual templates that get turned
|
8
|
+
# into CloudFormation Fn::Join invocations. This makes userdata scripts
|
9
|
+
# and other things much easier to read.
|
10
|
+
#
|
11
|
+
# The templates are ERB syntax, with three functions available for
|
12
|
+
# inserting CloudFormation references: +cfn_raw+, +cfn_ref+ and +cfn_getatt+.
|
13
|
+
#
|
14
|
+
# == Example Template
|
15
|
+
# #!/bin/bash
|
16
|
+
# yum update -y aws-cfn-bootstrap
|
17
|
+
# /opt/aws/bin/cfn-init -s <%= cfn_ref("AWS::StackId") %> -r LaunchConfig --region <%= cfn_ref("AWS::Region") %>
|
18
|
+
# /opt/aws/bin/cfn-signal -e $? <%= cfn_ref("WaitHandle") %>
|
19
|
+
# # Setup correct file ownership
|
20
|
+
# chown -R apache:apache /var/www/html/wordpress
|
21
|
+
#
|
22
|
+
# == Example use of CfnPP::Replacer
|
23
|
+
# template_text = "..."
|
24
|
+
# cfn_hash = CfnReplacer.new.process(template_text)
|
25
|
+
class Replacer
|
26
|
+
# Turns input text into a Hash appropriate for inserting into a
|
27
|
+
# CloudFormation structure.
|
28
|
+
def initialize(text, vars = {}, opts = {})
|
29
|
+
@text = text
|
30
|
+
@r_vars = vars
|
31
|
+
@opts = opts
|
32
|
+
end
|
33
|
+
|
34
|
+
def process()
|
35
|
+
tmpl = ERB.new @text
|
36
|
+
joinparts = []
|
37
|
+
tmpl.result(self.get_binding).split('@@@').each do |chunk|
|
38
|
+
if chunk.match(/^\{/)
|
39
|
+
chunk = JSON.parse(chunk)
|
40
|
+
end
|
41
|
+
joinparts.push(chunk)
|
42
|
+
end
|
43
|
+
return { "Fn::Join" => [ '', joinparts ] }
|
44
|
+
end
|
45
|
+
|
46
|
+
def process_basic()
|
47
|
+
tmpl = ERB.new @text
|
48
|
+
return tmpl.result(self.get_binding)
|
49
|
+
end
|
50
|
+
|
51
|
+
def cfn_render(path)
|
52
|
+
txt = File.read File.join(File.dirname('.'), path)
|
53
|
+
tmpl = ERB.new txt
|
54
|
+
return tmpl.result(self.get_binding)
|
55
|
+
end
|
56
|
+
|
57
|
+
# for local refs (instead of cfn_ref)
|
58
|
+
def cfn_cfnpp_ref(s)
|
59
|
+
return @opts[s]
|
60
|
+
end
|
61
|
+
|
62
|
+
# called in the template to include any arbitrary CloudFormation code.
|
63
|
+
def cfn_raw(h)
|
64
|
+
txt = JSON.dump h
|
65
|
+
return "@@@#{txt}@@@"
|
66
|
+
end
|
67
|
+
|
68
|
+
# shortcut for inserting a cfn "Ref"; just pass the "Ref" value
|
69
|
+
def cfn_ref(s)
|
70
|
+
return cfn_raw({ "Ref" => s })
|
71
|
+
end
|
72
|
+
|
73
|
+
# shortcut for inserting a cfn "Fn::GetAtt"; just pass a two element
|
74
|
+
# array of the key and value to get
|
75
|
+
def cfn_getatt(k,v)
|
76
|
+
return cfn_raw({ "Fn::GetAtt" => [k, v]})
|
77
|
+
end
|
78
|
+
|
79
|
+
def get_binding
|
80
|
+
binding
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
require 'digest'
|
2
|
+
|
3
|
+
module CfnPP
|
4
|
+
class TemplateResult
|
5
|
+
attr_accessor :substacks
|
6
|
+
attr_accessor :data
|
7
|
+
|
8
|
+
def initialize(name, data, stack_url_base="", substacks = [])
|
9
|
+
@base_name = name
|
10
|
+
@data = data
|
11
|
+
@stack_url_base = stack_url_base
|
12
|
+
@substacks = substacks
|
13
|
+
end
|
14
|
+
|
15
|
+
def url
|
16
|
+
return "#{@stack_url_base}/#{name}/template.json"
|
17
|
+
end
|
18
|
+
|
19
|
+
def name
|
20
|
+
return "#{@base_name}-#{checksum}"
|
21
|
+
end
|
22
|
+
|
23
|
+
def checksum
|
24
|
+
return Digest::SHA256.hexdigest(@data.to_json)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
@@ -0,0 +1,273 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'yaml'
|
3
|
+
require_relative 'replacer'
|
4
|
+
require_relative 'templateresult'
|
5
|
+
require 'set'
|
6
|
+
require 'erb'
|
7
|
+
|
8
|
+
# CfnPP is a module with various methods to make working with CloudFormation
|
9
|
+
# templates easier. Particularly, it has facilities for writing templates in
|
10
|
+
# YAML, with added features for modularizing templates, easily writing
|
11
|
+
# user-data blocks, etc.
|
12
|
+
module CfnPP
|
13
|
+
# This class has methods to read in cloudformation templates in YAML
|
14
|
+
# format with some Manta-specific extensions to make things easier.
|
15
|
+
class Transform
|
16
|
+
# returns a ruby hash, from a file at +path+
|
17
|
+
#
|
18
|
+
# This is the easiest way to load things. It takes care of
|
19
|
+
# setting reasonable file base for includes, etc., and gives
|
20
|
+
# you back a hash ready for use.
|
21
|
+
def self.load_file(path, opts = {}, name = "main", stack_url_base="http://example.com")
|
22
|
+
return self.load_yaml(File.read(path), path, opts, name, stack_url_base)
|
23
|
+
end
|
24
|
+
|
25
|
+
# returns a ruby hash, from unparsed YAML input text.
|
26
|
+
#
|
27
|
+
def self.load_yaml(yaml_txt, filebase=".", opts={}, name = "main", stack_url_base="http://example.com")
|
28
|
+
h = YAML::load(yaml_txt)
|
29
|
+
return self.new(h, filebase, opts, name, stack_url_base).as_template_result
|
30
|
+
end
|
31
|
+
|
32
|
+
# CfnPP::Transform is initialized with a hash and an optional file base
|
33
|
+
# parameter. The file base will default to ".", which may or may
|
34
|
+
# not be what you want.
|
35
|
+
def initialize(in_hash, filebase=".", opts={}, name="main", stack_url_base="http://example.com")
|
36
|
+
@name = name
|
37
|
+
@opts = opts
|
38
|
+
@filebase = filebase
|
39
|
+
@stack_url_base = stack_url_base
|
40
|
+
@in_hash = { :root => in_hash }
|
41
|
+
@tops = self.class.stdtops()
|
42
|
+
trans_hash(@in_hash)
|
43
|
+
@in_hash = @in_hash[:root]
|
44
|
+
@substacks = grab_stacks(@in_hash)
|
45
|
+
lift
|
46
|
+
@in_hash = apply_opts(@in_hash, opts)
|
47
|
+
prune(@in_hash)
|
48
|
+
end
|
49
|
+
|
50
|
+
# Return the parsed, processed CfnPP YAML file as a ruby hash
|
51
|
+
def as_hash
|
52
|
+
return @in_hash
|
53
|
+
end
|
54
|
+
|
55
|
+
def as_template_result
|
56
|
+
return CfnPP::TemplateResult.new(@name, @in_hash, @stack_url_base, @substacks)
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
|
61
|
+
# which keys always get lifted to the top
|
62
|
+
def self.stdtops
|
63
|
+
return Set.new ["Parameters", "Mappings", "Resources", "Outputs", "Conditions"]
|
64
|
+
end
|
65
|
+
|
66
|
+
# classic recursion!
|
67
|
+
def apply_opts(thing, opts)
|
68
|
+
if thing.is_a? Hash
|
69
|
+
if thing.has_key? 'CfnPPRef'
|
70
|
+
if not opts.has_key? thing['CfnPPRef']
|
71
|
+
raise "missing value for '#{thing['CfnPPRef']}'"
|
72
|
+
end
|
73
|
+
return "#{opts[thing['CfnPPRef']] || ''}"
|
74
|
+
else
|
75
|
+
r = {}
|
76
|
+
thing.each { |k,v| r[k] = apply_opts(v, opts) }
|
77
|
+
return r
|
78
|
+
end
|
79
|
+
elsif thing.is_a? Array
|
80
|
+
return thing.collect { |t| apply_opts(t, opts) }
|
81
|
+
else
|
82
|
+
return thing
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
# give the contents of a named file; handle any directory weirdness
|
87
|
+
# here
|
88
|
+
def read_ext(name)
|
89
|
+
return File.read File.join(File.dirname(@filebase), name)
|
90
|
+
end
|
91
|
+
|
92
|
+
# how the results of a CfnPPTemplate are put back into the tree
|
93
|
+
def sub_merge(h, key, v)
|
94
|
+
if v.is_a? Hash then
|
95
|
+
h[key].merge! v
|
96
|
+
else
|
97
|
+
h[key] = v
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
# sniff the type of filter based on the string contents
|
102
|
+
def auto_filter(txt)
|
103
|
+
if txt =~ /^---/
|
104
|
+
return "erb-yaml"
|
105
|
+
else
|
106
|
+
return "replacer"
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
# given the multiple ways a template can be specified, create
|
111
|
+
# a single predictable Hash to be used elsewhere
|
112
|
+
def norm_tmplspec(v)
|
113
|
+
if v.is_a? String
|
114
|
+
rec = {
|
115
|
+
"filter" => "",
|
116
|
+
"txt" => "",
|
117
|
+
"vars" => {},
|
118
|
+
}
|
119
|
+
if File.exists? File.join(File.dirname(@filebase), v)
|
120
|
+
rec["txt"] = read_ext v
|
121
|
+
else
|
122
|
+
rec["txt"] = v
|
123
|
+
end
|
124
|
+
rec["filter"] = auto_filter(rec["txt"])
|
125
|
+
return rec
|
126
|
+
elsif v.is_a? Hash
|
127
|
+
if v["path"]
|
128
|
+
v["txt"] = read_ext v["path"]
|
129
|
+
end
|
130
|
+
if not v.has_key? "filter"
|
131
|
+
v["filter"] = auto_filter(v["txt"])
|
132
|
+
end
|
133
|
+
return v
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Apply the given filter to the text content, with given
|
138
|
+
# vars.
|
139
|
+
# the overloading of Replacer here is super weird and
|
140
|
+
# ugly. Sorry. Will fix... soon?
|
141
|
+
def proc_tmplspec(rec)
|
142
|
+
replacer = Replacer.new(rec["txt"], rec["vars"], @opts)
|
143
|
+
if rec["filter"] == 'replacer'
|
144
|
+
return replacer.process
|
145
|
+
elsif rec["filter"] == 'erb-yaml'
|
146
|
+
res = replacer.process_basic
|
147
|
+
#puts "TEXT: #{res}"
|
148
|
+
#puts "#{res}"
|
149
|
+
#ERB.new rec["txt"]
|
150
|
+
return YAML::load(res)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
# trans/trans_hash/trans_array walk the tree
|
155
|
+
def trans(e)
|
156
|
+
if e.is_a? Hash
|
157
|
+
trans_hash(e)
|
158
|
+
end
|
159
|
+
if e.is_a? Array
|
160
|
+
trans_array(e)
|
161
|
+
end
|
162
|
+
end
|
163
|
+
|
164
|
+
def trans_hash(h)
|
165
|
+
h.keys.each do |key|
|
166
|
+
if (h[key].is_a? Hash) and (h[key].has_key? "CfnPPTemplate")
|
167
|
+
rec = norm_tmplspec h[key]["CfnPPTemplate"]
|
168
|
+
v = proc_tmplspec(rec)
|
169
|
+
trans(v)
|
170
|
+
sub_merge(h, key, v)
|
171
|
+
elsif (h[key].is_a? Hash) and (h[key].has_key? "CfnPPStack")
|
172
|
+
rec = h[key]["CfnPPStack"]
|
173
|
+
if rec.has_key? "inline"
|
174
|
+
inline = rec["inline"]
|
175
|
+
name = rec["name"]
|
176
|
+
rec.delete("inline")
|
177
|
+
sub_params = rec["params"]
|
178
|
+
rec["result"] = self.class.new(inline, @filebase, sub_params.merge(@opts), name, @stack_url_base).as_template_result
|
179
|
+
rec["Resources"] = {} if not rec["Resources"]
|
180
|
+
rec["Resources"][name] = {
|
181
|
+
"Type" => "AWS::CloudFormation::Stack",
|
182
|
+
"Properties" => {
|
183
|
+
"TemplateURL" => rec["result"].url,
|
184
|
+
"Parameters" => sub_params,
|
185
|
+
"TimeoutInMinutes" => 60
|
186
|
+
}
|
187
|
+
}
|
188
|
+
end
|
189
|
+
else
|
190
|
+
trans(h[key])
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
|
195
|
+
def trans_array(a)
|
196
|
+
a.each do |e|
|
197
|
+
trans(e)
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# recursively remove any keys we want cleaned up. due to lifting they
|
202
|
+
# can leave junk laying around
|
203
|
+
def prune(h)
|
204
|
+
prunes = ["CfnPPTemplate", "MantaTemplateInclude", "MantaTemplate", "MantaInclude", "CfnPPSection", "CfnPPStack"]
|
205
|
+
if h.is_a? Hash then
|
206
|
+
h.keys.each do |k|
|
207
|
+
if prunes.include? k then
|
208
|
+
h.delete(k)
|
209
|
+
else
|
210
|
+
prune(h[k])
|
211
|
+
end
|
212
|
+
end
|
213
|
+
elsif h.is_a? Array then
|
214
|
+
h.each { |e| prune(e) }
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
# return all of the embedded stack objects. super, super
|
219
|
+
# ugly
|
220
|
+
def grab_stacks(h)
|
221
|
+
stacks = []
|
222
|
+
if h.is_a? Hash
|
223
|
+
h.keys.each do |k|
|
224
|
+
if k == "CfnPPStack" and h[k].has_key? "result"
|
225
|
+
stacks.push(h[k]["result"])
|
226
|
+
else
|
227
|
+
stacks.concat(grab_stacks(h[k]))
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
return stacks
|
232
|
+
end
|
233
|
+
|
234
|
+
# given some defined top keys, find them everywhere, cut them out,
|
235
|
+
# and put them back in at the top level. Weirdly fiddly code.
|
236
|
+
def lift
|
237
|
+
def lifter(h, tops, store)
|
238
|
+
# this guard is a super ugly hacky
|
239
|
+
if h.has_key? 'Type' and h['Type'] == 'AWS::CloudFormation::Stack'
|
240
|
+
return
|
241
|
+
end
|
242
|
+
h.keys.each do |key|
|
243
|
+
if h[key].is_a? Hash
|
244
|
+
lifter(h[key], tops, store)
|
245
|
+
elsif h[key].is_a? Array
|
246
|
+
h[key].each do |e|
|
247
|
+
if e.is_a? Hash
|
248
|
+
lifter(e, tops, store)
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
if tops.include? key
|
253
|
+
if (not store[key])
|
254
|
+
store[key] = []
|
255
|
+
end
|
256
|
+
store[key].push h.delete(key)
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
h = @in_hash
|
262
|
+
store = {}
|
263
|
+
lifter(h, @tops, store)
|
264
|
+
store.keys.each do |k|
|
265
|
+
n = {}
|
266
|
+
store[k].each do |se|
|
267
|
+
n = n.merge se
|
268
|
+
end
|
269
|
+
h[k] = n
|
270
|
+
end
|
271
|
+
end
|
272
|
+
end
|
273
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
require 'json'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
module CfnPP
|
6
|
+
class Uploader
|
7
|
+
def initialize(bucketname, stackname)
|
8
|
+
@bucket = AWS::S3.new.buckets[bucketname]
|
9
|
+
@stackname = stackname
|
10
|
+
@timestamp = new_timestamp
|
11
|
+
@s3_path = "testing/stacks/#{@stackname}/#{@timestamp}"
|
12
|
+
@cfn = AWS::CloudFormation.new
|
13
|
+
end
|
14
|
+
|
15
|
+
def new_timestamp
|
16
|
+
return Time.now.utc.strftime("%Y-%m-%dT%H.%M.%S.%LZ")
|
17
|
+
end
|
18
|
+
|
19
|
+
def s3_path
|
20
|
+
return @s3_path
|
21
|
+
end
|
22
|
+
|
23
|
+
def upload_template(template_result, opts)
|
24
|
+
r = { :url => nil, :error => nil, :validation_status => nil }
|
25
|
+
|
26
|
+
already_updated = ploy_update_guard(template_result.name, opts)
|
27
|
+
if already_updated
|
28
|
+
r[:err] = 'already_updated'
|
29
|
+
r[:prev_opts] = already_updated
|
30
|
+
return r
|
31
|
+
end
|
32
|
+
|
33
|
+
obj = @bucket.objects.create("#{@s3_path}/#{template_result.name}/template.json", template_result.data.to_json)
|
34
|
+
r[:url] = obj.public_url
|
35
|
+
r[:validation_status] = @cfn.validate_template(r[:url])
|
36
|
+
if r[:validation_status].has_key? :code #error condition
|
37
|
+
r[:error] = 'validation_error'
|
38
|
+
else
|
39
|
+
tp = r[:validation_status].fetch(:parameters, [])
|
40
|
+
r[:opts_parameters] = opts_parameters(tp, opts)
|
41
|
+
upload_parameters(template_result.name, r[:opts_parameters])
|
42
|
+
end
|
43
|
+
return r
|
44
|
+
end
|
45
|
+
|
46
|
+
def opts_parameters(params, opts)
|
47
|
+
opts_parameters = {}
|
48
|
+
params.each do |param|
|
49
|
+
key = param[:parameter_key]
|
50
|
+
opts_parameters[key] = opts[key] if opts.has_key? key
|
51
|
+
end
|
52
|
+
return opts_parameters
|
53
|
+
end
|
54
|
+
|
55
|
+
def upload_parameters(name, opts_parameters)
|
56
|
+
return @bucket.objects.create("#{@s3_path}/#{name}/parameters.yml", opts_parameters.to_yaml)
|
57
|
+
end
|
58
|
+
|
59
|
+
def ploy_update_guard(name, opts)
|
60
|
+
if opts['TemplateSource'] == 'ploy'
|
61
|
+
return find_previous_ploy_update(cfn_bucket, opts)
|
62
|
+
end
|
63
|
+
return nil
|
64
|
+
end
|
65
|
+
|
66
|
+
def find_previous_ploy_update(name, opts)
|
67
|
+
stack_name = opts['StackName']
|
68
|
+
launch_list = @bucket.objects.with_prefix("stacks/#{stack_name}").sort do |a,b|
|
69
|
+
a.key <=> b.key
|
70
|
+
end
|
71
|
+
launch_list.each do |o|
|
72
|
+
if o.key =~ /\/parameters.yml$/
|
73
|
+
prev_opts = YAML::load(o.read)
|
74
|
+
if prev_opts['TemplateSource'] == opts['TemplateSource']
|
75
|
+
if prev_opts['TemplateGitRevision'] == opts['TemplateGitRevision']
|
76
|
+
return prev_opts
|
77
|
+
end
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
return nil
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
end
|
metadata
ADDED
@@ -0,0 +1,116 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cfnpp
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.2.0
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Michael Bruce
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2014-03-27 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: aws-sdk
|
16
|
+
requirement: &21160100 !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: *21160100
|
25
|
+
- !ruby/object:Gem::Dependency
|
26
|
+
name: json
|
27
|
+
requirement: &21157980 !ruby/object:Gem::Requirement
|
28
|
+
none: false
|
29
|
+
requirements:
|
30
|
+
- - ! '>='
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '0'
|
33
|
+
type: :runtime
|
34
|
+
prerelease: false
|
35
|
+
version_requirements: *21157980
|
36
|
+
- !ruby/object:Gem::Dependency
|
37
|
+
name: awesome_print
|
38
|
+
requirement: &21157160 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ! '>='
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
version: '0'
|
44
|
+
type: :runtime
|
45
|
+
prerelease: false
|
46
|
+
version_requirements: *21157160
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: travis
|
49
|
+
requirement: &21156220 !ruby/object:Gem::Requirement
|
50
|
+
none: false
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
type: :runtime
|
56
|
+
prerelease: false
|
57
|
+
version_requirements: *21156220
|
58
|
+
- !ruby/object:Gem::Dependency
|
59
|
+
name: dogapi
|
60
|
+
requirement: &21155240 !ruby/object:Gem::Requirement
|
61
|
+
none: false
|
62
|
+
requirements:
|
63
|
+
- - ! '>='
|
64
|
+
- !ruby/object:Gem::Version
|
65
|
+
version: '0'
|
66
|
+
type: :runtime
|
67
|
+
prerelease: false
|
68
|
+
version_requirements: *21155240
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: ploy
|
71
|
+
requirement: &21154400 !ruby/object:Gem::Requirement
|
72
|
+
none: false
|
73
|
+
requirements:
|
74
|
+
- - ! '>='
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: *21154400
|
80
|
+
description: ! '["cfnpp", ["Michael Bruce"]]'
|
81
|
+
email: mbruce@manta.com
|
82
|
+
executables:
|
83
|
+
- cfnpp
|
84
|
+
extensions: []
|
85
|
+
extra_rdoc_files: []
|
86
|
+
files:
|
87
|
+
- lib/cfnpp/uploader.rb
|
88
|
+
- lib/cfnpp/transform.rb
|
89
|
+
- lib/cfnpp/templateresult.rb
|
90
|
+
- lib/cfnpp/replacer.rb
|
91
|
+
- bin/cfnpp
|
92
|
+
homepage:
|
93
|
+
licenses: []
|
94
|
+
post_install_message:
|
95
|
+
rdoc_options: []
|
96
|
+
require_paths:
|
97
|
+
- lib
|
98
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
99
|
+
none: false
|
100
|
+
requirements:
|
101
|
+
- - ! '>='
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
requirements: []
|
111
|
+
rubyforge_project:
|
112
|
+
rubygems_version: 1.8.11
|
113
|
+
signing_key:
|
114
|
+
specification_version: 3
|
115
|
+
summary: cfnpp
|
116
|
+
test_files: []
|