sfn 2.2.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/bin/command-config-generator +44 -0
- data/bin/sfn +1 -5
- data/docs/README.md +2 -1
- data/docs/callbacks.md +18 -0
- data/docs/command-config.md +1276 -0
- data/docs/configuration.md +0 -105
- data/docs/overview.md +1 -0
- data/lib/sfn.rb +1 -0
- data/lib/sfn/api_provider.rb +9 -0
- data/lib/sfn/api_provider/google.rb +84 -0
- data/lib/sfn/command.rb +22 -0
- data/lib/sfn/command/create.rb +33 -45
- data/lib/sfn/command/describe.rb +1 -1
- data/lib/sfn/command/destroy.rb +7 -1
- data/lib/sfn/command/diff.rb +4 -5
- data/lib/sfn/command/graph.rb +1 -3
- data/lib/sfn/command/print.rb +3 -4
- data/lib/sfn/command/update.rb +89 -87
- data/lib/sfn/command/validate.rb +24 -11
- data/lib/sfn/command_module/base.rb +1 -12
- data/lib/sfn/command_module/stack.rb +45 -56
- data/lib/sfn/command_module/template.rb +74 -33
- data/lib/sfn/config/print.rb +5 -0
- data/lib/sfn/config/update.rb +4 -0
- data/lib/sfn/config/validate.rb +4 -0
- data/lib/sfn/monkey_patch/stack.rb +153 -83
- data/lib/sfn/monkey_patch/stack/azure.rb +23 -0
- data/lib/sfn/monkey_patch/stack/google.rb +129 -0
- data/lib/sfn/planner/aws.rb +111 -72
- data/lib/sfn/utils/json.rb +3 -0
- data/lib/sfn/utils/stack_parameter_scrubber.rb +20 -23
- data/lib/sfn/utils/stack_parameter_validator.rb +162 -157
- data/lib/sfn/version.rb +1 -1
- data/sfn.gemspec +3 -2
- metadata +34 -8
@@ -15,6 +15,23 @@ module Sfn
|
|
15
15
|
|
16
16
|
module InstanceMethods
|
17
17
|
|
18
|
+
# Extract template content based on type
|
19
|
+
#
|
20
|
+
# @param thing [SparkleFormation, Hash]
|
21
|
+
# @param scrub [Truthy, Falsey] scrub nested templates
|
22
|
+
# @return [Hash]
|
23
|
+
def template_content(thing, scrub=false)
|
24
|
+
if(thing.is_a?(SparkleFormation))
|
25
|
+
if(scrub)
|
26
|
+
dump_stack_for_storage(thing)
|
27
|
+
else
|
28
|
+
config[:sparkle_dump] ? thing.sparkle_dump : thing.dump
|
29
|
+
end
|
30
|
+
else
|
31
|
+
thing
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
18
35
|
# Request compile time parameter value
|
19
36
|
#
|
20
37
|
# @param p_name [String, Symbol] name of parameter
|
@@ -70,7 +87,7 @@ module Sfn
|
|
70
87
|
ui.fatal "Failed to receive allowed parameter! (Parameter: #{p_name})"
|
71
88
|
exit 1
|
72
89
|
else
|
73
|
-
ui.error "Invalid value provided for parameter
|
90
|
+
ui.error "Invalid value provided for parameter `#{p_name}`. Must be type: `#{p_config[:type].to_s.capitalize}`"
|
74
91
|
end
|
75
92
|
end
|
76
93
|
end
|
@@ -100,12 +117,19 @@ module Sfn
|
|
100
117
|
# @return [SparkleFormation::SparkleCollection]
|
101
118
|
def sparkle_collection
|
102
119
|
memoize(:sparkle_collection) do
|
103
|
-
collection = SparkleFormation::SparkleCollection.new
|
120
|
+
collection = SparkleFormation::SparkleCollection.new(
|
121
|
+
:provider => config.get(:credentials, :provider)
|
122
|
+
)
|
104
123
|
begin
|
105
124
|
if(config[:base_directory])
|
106
|
-
root_pack = SparkleFormation::SparklePack.new(
|
125
|
+
root_pack = SparkleFormation::SparklePack.new(
|
126
|
+
:root => config[:base_directory],
|
127
|
+
:provider => config.get(:credentials, :provider)
|
128
|
+
)
|
107
129
|
else
|
108
|
-
root_pack = SparkleFormation::SparklePack.new
|
130
|
+
root_pack = SparkleFormation::SparklePack.new(
|
131
|
+
:provider => config.get(:credentials, :provider)
|
132
|
+
)
|
109
133
|
end
|
110
134
|
collection.set_root(root_pack)
|
111
135
|
rescue Errno::ENOENT
|
@@ -175,9 +199,6 @@ module Sfn
|
|
175
199
|
sf.stack_resource_types.push(s_type)
|
176
200
|
end
|
177
201
|
end
|
178
|
-
if(sf.nested? && !sf.isolated_nests?)
|
179
|
-
raise TypeError.new('Template does not contain isolated stack nesting! Sfn does not support mixed mixed resources within root stack!')
|
180
|
-
end
|
181
202
|
run_callbacks_for(:template, :stack_name => arguments.first, :sparkle_stack => sf)
|
182
203
|
if(sf.nested? && config[:apply_nesting])
|
183
204
|
validate_nesting_bucket!
|
@@ -190,12 +211,13 @@ module Sfn
|
|
190
211
|
when :shallow
|
191
212
|
process_nested_stack_shallow(sf, c_stack)
|
192
213
|
when :none
|
193
|
-
sf
|
214
|
+
sf
|
194
215
|
else
|
195
216
|
raise ArgumentError.new "Unknown nesting style requested: #{config[:apply_nesting].inspect}!"
|
196
217
|
end
|
218
|
+
sf
|
197
219
|
else
|
198
|
-
sf
|
220
|
+
sf
|
199
221
|
end
|
200
222
|
else
|
201
223
|
template = _from_json(File.read(config[:file]))
|
@@ -249,20 +271,13 @@ module Sfn
|
|
249
271
|
#
|
250
272
|
# @param sf [SparkleFormation] stack
|
251
273
|
# @param c_stack [Miasma::Models::Orchestration::Stack] existing stack
|
252
|
-
# @return [
|
274
|
+
# @return [SparkleFormation::SparkleStruct] compiled structure
|
253
275
|
def process_nested_stack_deep(sf, c_stack=nil)
|
254
276
|
sf.apply_nesting(:deep) do |stack_name, stack, resource|
|
255
277
|
run_callbacks_for(:template, :stack_name => stack_name, :sparkle_stack => stack)
|
256
|
-
|
257
278
|
stack_resource = resource._dump
|
258
|
-
|
259
|
-
if(stack.parent)
|
260
|
-
current_parameters = stack.parent.compile.resources.set!(stack_name).properties.parameters
|
261
|
-
current_parameters = current_parameters.nil? ? Smash.new : current_parameters._dump
|
262
|
-
else
|
263
|
-
current_parameters = Smash.new
|
264
|
-
end
|
265
279
|
current_stack = c_stack ? c_stack.nested_stacks.detect{|s| s.data[:logical_id] == stack_name} : nil
|
280
|
+
current_parameters = extract_current_nested_template_parameters(stack, stack_name, current_stack)
|
266
281
|
if(current_stack && current_stack.data[:parent_stack])
|
267
282
|
current_parameters.merge!(
|
268
283
|
current_stack.data[:parent_stack].template.fetch(
|
@@ -283,21 +298,7 @@ module Sfn
|
|
283
298
|
:current_parameters => current_parameters
|
284
299
|
)
|
285
300
|
)
|
286
|
-
|
287
|
-
bucket = provider.connection.api_for(:storage).buckets.get(
|
288
|
-
config[:nesting_bucket]
|
289
|
-
)
|
290
|
-
unless(bucket)
|
291
|
-
raise "Failed to locate configured bucket for stack template storage (#{config[:nesting_bucket]})!"
|
292
|
-
end
|
293
|
-
file = bucket.files.build
|
294
|
-
file.name = "#{full_stack_name}.json"
|
295
|
-
file.content_type = 'text/json'
|
296
|
-
file.body = MultiJson.dump(Sfn::Utils::StackParameterScrubber.scrub!(stack_definition))
|
297
|
-
file.save
|
298
|
-
result.merge!(
|
299
|
-
:url => file.url
|
300
|
-
)
|
301
|
+
store_template(full_stack_name, stack, result)
|
301
302
|
else
|
302
303
|
result = Smash.new(
|
303
304
|
:url => "http://example.com/bucket/#{full_stack_name}.json"
|
@@ -309,6 +310,45 @@ module Sfn
|
|
309
310
|
end
|
310
311
|
end
|
311
312
|
|
313
|
+
# Extract currently defined parameters for nested template
|
314
|
+
#
|
315
|
+
# @param stack [SparkleFormation]
|
316
|
+
# @param stack_name [String]
|
317
|
+
# @param c_stack [Miasma::Models::Orchestration::Stack]
|
318
|
+
# @return [Hash]
|
319
|
+
def extract_current_nested_template_parameters(stack, stack_name, c_stack=nil)
|
320
|
+
if(stack.parent)
|
321
|
+
current_parameters = stack.parent.compile.resources.set!(stack_name).properties.parameters
|
322
|
+
current_parameters.nil? ? Smash.new : current_parameters._dump
|
323
|
+
else
|
324
|
+
Smash.new
|
325
|
+
end
|
326
|
+
end
|
327
|
+
|
328
|
+
# Store template in remote bucket and update given result hash
|
329
|
+
#
|
330
|
+
# @param full_stack_name [String] unique resource name for template
|
331
|
+
# @param stack [SparkleFormation] template instance
|
332
|
+
# @param result [Hash]
|
333
|
+
# @return [Hash]
|
334
|
+
def store_template(full_stack_name, stack, result)
|
335
|
+
stack_definition = stack.is_a?(SparkleFormation) ? dump_stack_for_storage(stack) : stack
|
336
|
+
bucket = provider.connection.api_for(:storage).buckets.get(
|
337
|
+
config[:nesting_bucket]
|
338
|
+
)
|
339
|
+
unless(bucket)
|
340
|
+
raise "Failed to locate configured bucket for stack template storage (#{config[:nesting_bucket]})!"
|
341
|
+
end
|
342
|
+
file = bucket.files.build
|
343
|
+
file.name = "#{full_stack_name}.json"
|
344
|
+
file.content_type = 'text/json'
|
345
|
+
file.body = MultiJson.dump(parameter_scrub!(stack_definition))
|
346
|
+
file.save
|
347
|
+
result.merge!(
|
348
|
+
:url => file.url
|
349
|
+
)
|
350
|
+
end
|
351
|
+
|
312
352
|
# Remove internally used `Stack` property from Stack resources and
|
313
353
|
# and generate compiled Hash
|
314
354
|
#
|
@@ -522,6 +562,7 @@ module Sfn
|
|
522
562
|
extend Sfn::CommandModule::Template::ClassMethods
|
523
563
|
include Sfn::CommandModule::Template::InstanceMethods
|
524
564
|
include Sfn::Utils::PathSelector
|
565
|
+
include Sfn::Utils::StackParameterScrubber
|
525
566
|
end
|
526
567
|
end
|
527
568
|
|
data/lib/sfn/config/print.rb
CHANGED
data/lib/sfn/config/update.rb
CHANGED
@@ -11,6 +11,10 @@ module Sfn
|
|
11
11
|
:description => 'Apply outputs from stack to input parameters',
|
12
12
|
:short_flag => 'A'
|
13
13
|
)
|
14
|
+
attribute(
|
15
|
+
:apply_mapping, Smash,
|
16
|
+
:description => 'Customize apply stack mapping as [StackName__]OutputName:ParameterName'
|
17
|
+
)
|
14
18
|
attribute(
|
15
19
|
:parameter, Smash,
|
16
20
|
:multiple => true,
|
data/lib/sfn/config/validate.rb
CHANGED
@@ -7,7 +7,12 @@ module Sfn
|
|
7
7
|
# Expand stack model functionality
|
8
8
|
module Stack
|
9
9
|
|
10
|
+
autoload :Azure, 'sfn/monkey_patch/stack/azure'
|
11
|
+
autoload :Google, 'sfn/monkey_patch/stack/google'
|
12
|
+
|
10
13
|
include Bogo::AnimalStrings
|
14
|
+
include Azure
|
15
|
+
include Google
|
11
16
|
|
12
17
|
## Status helpers
|
13
18
|
|
@@ -135,58 +140,85 @@ module Sfn
|
|
135
140
|
# @param min [Integer] lowest allowed return value (defaults 5)
|
136
141
|
# @return [Integer] percent complete (0..100)
|
137
142
|
def percent_complete(min = 5)
|
138
|
-
if(
|
139
|
-
|
140
|
-
total_complete = resources.all.find_all do |resource|
|
141
|
-
resource.status.downcase.end_with?('complete')
|
142
|
-
end.size
|
143
|
-
result = ((total_complete.to_f / total_resources) * 100).to_i
|
144
|
-
result > min.to_i ? result : min
|
143
|
+
if(self.respond_to?("percent_complete_#{api.provider}"))
|
144
|
+
self.send("percent_complete_#{api.provider}", min)
|
145
145
|
else
|
146
|
-
|
146
|
+
if(in_progress?)
|
147
|
+
total_resources = template.fetch('Resources', []).size
|
148
|
+
total_complete = resources.all.find_all do |resource|
|
149
|
+
resource.status.downcase.end_with?('complete')
|
150
|
+
end.size
|
151
|
+
result = ((total_complete.to_f / total_resources) * 100).to_i
|
152
|
+
result > min.to_i ? result : min
|
153
|
+
else
|
154
|
+
100
|
155
|
+
end
|
147
156
|
end
|
148
157
|
end
|
149
158
|
|
150
159
|
# Apply stack outputs to current stack parameters
|
151
160
|
#
|
161
|
+
# @param opts [Hash]
|
162
|
+
# @option opts [String] :parameter_key key used for parameters block
|
163
|
+
# @option opts [String] :default_key key used within parameter for default value
|
164
|
+
# @option opts [Hash] :mapping custom output -> parameter name mapping
|
152
165
|
# @param remote_stack [Miasma::Orchestration::Stack]
|
153
166
|
# @return [self]
|
154
167
|
# @note setting `DisableApply` within parameter hash will
|
155
168
|
# prevent parameters being overridden
|
156
|
-
def apply_stack(remote_stack, ignore_params=nil)
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
unless(val['DisableApply'])
|
163
|
-
[snake(key), key]
|
164
|
-
end
|
165
|
-
end.compact
|
166
|
-
]
|
167
|
-
if(ignore_params)
|
168
|
-
valid_parameters = Hash[
|
169
|
-
valid_parameters.map do |snake_param, camel_param|
|
170
|
-
unless(ignore_params.include?(camel_param))
|
171
|
-
[snake_param, camel_param]
|
172
|
-
end
|
173
|
-
end.compact
|
174
|
-
]
|
169
|
+
def apply_stack(remote_stack, opts={}, ignore_params=nil)
|
170
|
+
if(self.respond_to?("apply_stack_#{api.provider}"))
|
171
|
+
self.send("apply_stack_#{api.provider}", remote_stack, opts, ignore_params)
|
172
|
+
else
|
173
|
+
unless(opts[:mapping])
|
174
|
+
opts[:mapping] = {}
|
175
175
|
end
|
176
|
-
if(
|
177
|
-
|
178
|
-
|
179
|
-
|
176
|
+
if(opts[:parameter_key])
|
177
|
+
stack_parameters = template[opts[:parameter_key]]
|
178
|
+
default_key = opts.fetch(
|
179
|
+
:default_key,
|
180
|
+
opts[:parameter_key].to_s[0,1].match(/[a-z]/) ? 'default' : 'Default'
|
181
|
+
)
|
182
|
+
else
|
183
|
+
if(template['Parameters'])
|
184
|
+
default_key = 'Default'
|
185
|
+
stack_parameters = template['Parameters']
|
186
|
+
else
|
187
|
+
default_key = 'default'
|
188
|
+
stack_parameters = template['parameters']
|
189
|
+
end
|
190
|
+
end
|
191
|
+
if(stack_parameters)
|
192
|
+
valid_parameters = stack_parameters.find_all do |key, val|
|
193
|
+
!val['DisableApply'] && !val['disable_apply']
|
194
|
+
end.map(&:first)
|
195
|
+
if(ignore_params)
|
196
|
+
valid_parameters.reject! do |key|
|
197
|
+
ignore_params.include?(key)
|
180
198
|
end
|
181
199
|
end
|
182
|
-
else
|
183
200
|
remote_stack.outputs.each do |output|
|
184
|
-
|
185
|
-
|
201
|
+
o_key = output.key.downcase.tr('_', '')
|
202
|
+
p_key = valid_parameters.detect do |v_param|
|
203
|
+
v_param.downcase.tr('_', '') == o_key
|
204
|
+
end
|
205
|
+
unless(p_key)
|
206
|
+
map_key = opts[:mapping].keys.detect do |map_key|
|
207
|
+
map_key.downcase.tr('_', '') == o_key
|
208
|
+
end
|
209
|
+
if(map_key)
|
210
|
+
p_key = valid_parameters.detect do |v_param|
|
211
|
+
v_param.downcase.tr('_', '') == opts[:mapping][map_key].downcase.tr('_', '')
|
212
|
+
end
|
213
|
+
end
|
214
|
+
end
|
215
|
+
if(p_key)
|
216
|
+
parameters.merge!(p_key => output.value)
|
186
217
|
end
|
187
218
|
end
|
188
219
|
end
|
189
220
|
end
|
221
|
+
self
|
190
222
|
end
|
191
223
|
|
192
224
|
# Return all stacks contained within this stack
|
@@ -194,39 +226,47 @@ module Sfn
|
|
194
226
|
# @param recurse [TrueClass, FalseClass] recurse to fetch _all_ stacks
|
195
227
|
# @return [Array<Miasma::Models::Orchestration::Stack>]
|
196
228
|
def nested_stacks(recurse=true)
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
if(n_stack)
|
213
|
-
n_stack.data[:logical_id] = resource.name
|
214
|
-
n_stack.data[:parent_stack] = self
|
215
|
-
n_stack.api.data[:stack_types] = api.data[:stack_types]
|
216
|
-
if(recurse)
|
217
|
-
[n_stack] + n_stack.nested_stacks(recurse)
|
229
|
+
if(self.respond_to?("nested_stacks_#{api.provider}"))
|
230
|
+
self.send("nested_stacks_#{api.provider}", recurse)
|
231
|
+
else
|
232
|
+
resources.reload.all.map do |resource|
|
233
|
+
if(api.data.fetch(:stack_types, []).include?(resource.type))
|
234
|
+
# Custom remote load support
|
235
|
+
if(resource.type == 'Custom::JackalStack')
|
236
|
+
location, stack_id = resource.id.to_s.split('-', 2)
|
237
|
+
if(l_conf = api.data[:locations][location])
|
238
|
+
n_stack = Miasma.api(
|
239
|
+
:type => :orchestration,
|
240
|
+
:provider => l_conf[:provider],
|
241
|
+
:credentials => l_conf
|
242
|
+
).stacks.get(stack_id)
|
243
|
+
end
|
218
244
|
else
|
219
|
-
n_stack
|
245
|
+
n_stack = resource.expand
|
246
|
+
end
|
247
|
+
if(n_stack)
|
248
|
+
n_stack.data[:logical_id] = resource.name
|
249
|
+
n_stack.data[:parent_stack] = self
|
250
|
+
n_stack.api.data[:stack_types] = api.data[:stack_types]
|
251
|
+
if(recurse)
|
252
|
+
[n_stack] + n_stack.nested_stacks(recurse)
|
253
|
+
else
|
254
|
+
n_stack
|
255
|
+
end
|
220
256
|
end
|
221
257
|
end
|
222
|
-
end
|
223
|
-
end
|
258
|
+
end.flatten.compact
|
259
|
+
end
|
224
260
|
end
|
225
261
|
|
226
262
|
# @return [TrueClass, FalseClass] stack contains nested stacks
|
227
263
|
def nested?
|
228
|
-
|
229
|
-
|
264
|
+
if(self.respond_to?("nested_#{api.provider}?"))
|
265
|
+
self.send("nested_#{api.provider}?")
|
266
|
+
else
|
267
|
+
!!resources.detect do |resource|
|
268
|
+
api.data.fetch(:stack_types, []).include?(resource.type)
|
269
|
+
end
|
230
270
|
end
|
231
271
|
end
|
232
272
|
|
@@ -234,22 +274,26 @@ module Sfn
|
|
234
274
|
#
|
235
275
|
# @return [Smash, NilClass]
|
236
276
|
def policy
|
237
|
-
if(self.api.provider
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
277
|
+
if(self.respond_to?("policy_#{api.provider}"))
|
278
|
+
self.send("policy_#{api.provider}")
|
279
|
+
else
|
280
|
+
if(self.api.provider == :aws) # cause this is the only one
|
281
|
+
begin
|
282
|
+
result = self.api.request(
|
283
|
+
:path => '/',
|
284
|
+
:form => Smash.new(
|
285
|
+
'Action' => 'GetStackPolicy',
|
286
|
+
'StackName' => self.id
|
287
|
+
)
|
244
288
|
)
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
252
|
-
|
289
|
+
serialized_policy = result.get(:body, 'GetStackPolicyResult', 'StackPolicyBody')
|
290
|
+
MultiJson.load(serialized_policy).to_smash
|
291
|
+
rescue Miasma::Error::ApiError::RequestError => e
|
292
|
+
if(e.response.code == 404)
|
293
|
+
nil
|
294
|
+
else
|
295
|
+
raise
|
296
|
+
end
|
253
297
|
end
|
254
298
|
end
|
255
299
|
end
|
@@ -263,14 +307,40 @@ module Sfn
|
|
263
307
|
# contain any direct values for parameters (which is what we
|
264
308
|
# are testing for)
|
265
309
|
def nesting_style
|
266
|
-
if(
|
267
|
-
self.
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
end
|
273
|
-
|
310
|
+
if(self.respond_to?("nesting_style_#{api.provider}"))
|
311
|
+
self.send("nesting_style_#{api.provider}")
|
312
|
+
else
|
313
|
+
if(nested?)
|
314
|
+
self.template['Resources'].find_all do |t_resource|
|
315
|
+
t_resource['Type'] == self.api.class.const_get(:RESOURCE_MAPPING).key(self.class)
|
316
|
+
end.detect do |t_resource|
|
317
|
+
t_resource['Properties'].fetch('Parameters', {}).values.detect do |t_value|
|
318
|
+
!t_value.is_a?(Hash)
|
319
|
+
end
|
320
|
+
end ? :deep : :shallow
|
321
|
+
end
|
322
|
+
end
|
323
|
+
end
|
324
|
+
|
325
|
+
# Reformat template data structure to SparkleFormation style structure
|
326
|
+
#
|
327
|
+
# @return [Hash]
|
328
|
+
def sparkleish_template(*args)
|
329
|
+
if(self.respond_to?("sparkleish_template_#{api.provider}"))
|
330
|
+
self.send("sparkleish_template_#{api.provider}", *args)
|
331
|
+
else
|
332
|
+
template
|
333
|
+
end
|
334
|
+
end
|
335
|
+
|
336
|
+
# Provide easy parameters override
|
337
|
+
#
|
338
|
+
# @return [Hash]
|
339
|
+
def root_parameters
|
340
|
+
if(self.respond_to?("root_parameters_#{api.provider}"))
|
341
|
+
self.send("root_parameters_#{api.provider}")
|
342
|
+
else
|
343
|
+
parameters
|
274
344
|
end
|
275
345
|
end
|
276
346
|
|