cloud-mu 3.1.3 → 3.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +21 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +4 -4
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +147 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +158 -111
- data/modules/mu/adoption.rb +404 -71
- data/modules/mu/cleanup.rb +221 -306
- data/modules/mu/cloud.rb +129 -1633
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +10 -9
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +5 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +19 -10
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +10 -21
- data/modules/mu/config/ref.rb +411 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +98 -71
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +71 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +43 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +410 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +218 -2612
- data/modules/mu/mommacat/daemon.rb +403 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +380 -122
- data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
- data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
- data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
- data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
- data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
- data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
- data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
- data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
- data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
- data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
- data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +68 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
- data/modules/mu/{clouds → providers}/google/database.rb +11 -21
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +140 -168
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
- data/modules/mu/{clouds → providers}/google/role.rb +94 -58
- data/modules/mu/{clouds → providers}/google/server.rb +243 -156
- data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +240 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
- data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -47,7 +47,11 @@ module MU
|
|
47
47
|
}
|
48
48
|
end
|
49
49
|
|
50
|
+
type_map = {}
|
51
|
+
|
50
52
|
@config['attributes'].each { |attr|
|
53
|
+
type_map[attr['name']] = attr['type']
|
54
|
+
|
51
55
|
params[:attribute_definitions] << {
|
52
56
|
:attribute_name => attr['name'],
|
53
57
|
:attribute_type => attr['type']
|
@@ -67,6 +71,11 @@ module MU
|
|
67
71
|
}
|
68
72
|
end
|
69
73
|
}
|
74
|
+
# apparently the HASH key always has to be before RANGE, so sort it
|
75
|
+
# lexically by that field and call it a day
|
76
|
+
params[:key_schema].sort! { |a, b|
|
77
|
+
a[:key_type] <=> b[:key_type]
|
78
|
+
}
|
70
79
|
|
71
80
|
if @config['secondary_indexes']
|
72
81
|
@config['secondary_indexes'].each { |idx|
|
@@ -99,7 +108,11 @@ module MU
|
|
99
108
|
}
|
100
109
|
end
|
101
110
|
|
102
|
-
|
111
|
+
if @tags
|
112
|
+
params[:tags] = @tags.each_key.map { |k| { :key => k, :value => @tags[k] } }
|
113
|
+
end
|
114
|
+
|
115
|
+
MU.log "Creating DynamoDB table #{@mu_name}", MU::NOTICE, details: params
|
103
116
|
|
104
117
|
resp = MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).create_table(params)
|
105
118
|
@cloud_id = @mu_name
|
@@ -109,8 +122,24 @@ module MU
|
|
109
122
|
sleep 5 if resp.table.table_status == "CREATING"
|
110
123
|
end while resp.table.table_status == "CREATING"
|
111
124
|
|
112
|
-
|
113
125
|
tagTable if !@config['scrub_mu_isms']
|
126
|
+
|
127
|
+
if @config['populate'] and !@config['populate'].empty?
|
128
|
+
MU.log "Preloading #{@mu_name} with #{@config['populate'].size.to_s} items"
|
129
|
+
items_to_write = @config['populate'].dup
|
130
|
+
begin
|
131
|
+
batch = items_to_write.slice!(0, (items_to_write.length >= 25 ? 25 : items_to_write.length))
|
132
|
+
begin
|
133
|
+
MU::Cloud::AWS.dynamo(credentials: @config['credentials'], region: @config['region']).batch_write_item(
|
134
|
+
request_items: {
|
135
|
+
@cloud_id => batch.map { |i| { put_request: { item: i } } }
|
136
|
+
}
|
137
|
+
)
|
138
|
+
rescue ::Aws::DynamoDB::Errors::ValidationException => e
|
139
|
+
MU.log e.message, MU::ERR, details: item
|
140
|
+
end
|
141
|
+
end while !items_to_write.empty?
|
142
|
+
end
|
114
143
|
end
|
115
144
|
|
116
145
|
# Apply tags to this DynamoDB table
|
@@ -143,6 +172,7 @@ module MU
|
|
143
172
|
# Called automatically by {MU::Deploy#createResources}
|
144
173
|
def groom
|
145
174
|
tagTable if !@config['scrub_mu_isms']
|
175
|
+
MU.log "NoSQL Table #{@config['name']}: #{@cloud_id}", MU::SUMMARY
|
146
176
|
end
|
147
177
|
|
148
178
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
@@ -163,7 +193,9 @@ module MU
|
|
163
193
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
164
194
|
# @param region [String]: The cloud provider region
|
165
195
|
# @return [void]
|
166
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
196
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
197
|
+
MU.log "AWS::NoSQLDb.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
198
|
+
|
167
199
|
resp = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tables
|
168
200
|
if resp and resp.table_names
|
169
201
|
resp.table_names.each { |table|
|
@@ -178,16 +210,23 @@ module MU
|
|
178
210
|
begin
|
179
211
|
tags = MU::Cloud::AWS.dynamo(credentials: credentials, region: region).list_tags_of_resource(resource_arn: desc.table_arn)
|
180
212
|
if tags and tags.tags
|
213
|
+
deploy_match = false
|
214
|
+
master_match = false
|
181
215
|
tags.tags.each { |tag|
|
182
|
-
if tag.key == "MU-ID" and tag.value ==
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
end
|
216
|
+
if tag.key == "MU-ID" and tag.value == deploy_id
|
217
|
+
deploy_match = true
|
218
|
+
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
219
|
+
master_match = true
|
187
220
|
end
|
188
221
|
}
|
222
|
+
if deploy_match and (master_match or ignoremaster)
|
223
|
+
MU.log "Deleting DynamoDB table #{desc.table_name}"
|
224
|
+
if !noop
|
225
|
+
MU::Cloud::AWS.dynamo(credentials: credentials, region: region).delete_table(table_name: desc.table_name)
|
226
|
+
end
|
227
|
+
end
|
189
228
|
end
|
190
|
-
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
229
|
+
rescue Aws::DynamoDB::Errors::ResourceNotFoundException
|
191
230
|
end
|
192
231
|
|
193
232
|
}
|
@@ -205,7 +244,8 @@ module MU
|
|
205
244
|
# Return the metadata for this user cofiguration
|
206
245
|
# @return [Hash]
|
207
246
|
def notify
|
208
|
-
|
247
|
+
return nil if !@cloud_id or !cloud_desc(use_cache: false)
|
248
|
+
MU.structToHash(cloud_desc, stringify_keys: true)
|
209
249
|
end
|
210
250
|
|
211
251
|
# Locate an existing DynamoDB table
|
@@ -235,14 +275,74 @@ module MU
|
|
235
275
|
found
|
236
276
|
end
|
237
277
|
|
278
|
+
# Reverse-map our cloud description into a runnable config hash.
|
279
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
280
|
+
# calculate our own accordingly based on what's live in the cloud.
|
281
|
+
def toKitten(**_args)
|
282
|
+
bok = {
|
283
|
+
"cloud" => "AWS",
|
284
|
+
"credentials" => @config['credentials'],
|
285
|
+
"cloud_id" => @cloud_id,
|
286
|
+
"region" => @config['region']
|
287
|
+
}
|
288
|
+
|
289
|
+
if !cloud_desc
|
290
|
+
MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
|
291
|
+
return nil
|
292
|
+
end
|
293
|
+
bok['name'] = cloud_desc.table_name
|
294
|
+
bok['read_capacity'] = cloud_desc.provisioned_throughput.read_capacity_units
|
295
|
+
bok['write_capacity'] = cloud_desc.provisioned_throughput.write_capacity_units
|
296
|
+
|
297
|
+
|
298
|
+
cloud_desc.attribute_definitions.each { |attr|
|
299
|
+
bok['attributes'] ||= []
|
300
|
+
newattr = {
|
301
|
+
"name" => attr.attribute_name,
|
302
|
+
"type" => attr.attribute_type
|
303
|
+
}
|
304
|
+
if cloud_desc.key_schema
|
305
|
+
cloud_desc.key_schema.each { |key|
|
306
|
+
next if key.attribute_name == attr.attribute_name
|
307
|
+
if key.key_type == "RANGE"
|
308
|
+
newattr["primary_partition"] = true
|
309
|
+
elsif key.key_type == "HASH"
|
310
|
+
newattr["primary_sort"] = true
|
311
|
+
end
|
312
|
+
}
|
313
|
+
end
|
314
|
+
bok['attributes'] << newattr
|
315
|
+
}
|
316
|
+
|
317
|
+
if cloud_desc.stream_specification and cloud_desc.stream_specification.stream_enabled
|
318
|
+
|
319
|
+
bok['stream'] = cloud_desc.stream_specification.stream_view_type
|
320
|
+
# cloud_desc.latest_stream_arn
|
321
|
+
# MU::Cloud::AWS.dynamostream(credentials: @credentials, region: @config['region']).list_streams
|
322
|
+
end
|
323
|
+
|
324
|
+
bok["populate"] = MU::Cloud::AWS.dynamo(credentials: @credentials, region: @config['region']).scan(
|
325
|
+
table_name: @cloud_id
|
326
|
+
).items
|
327
|
+
|
328
|
+
bok
|
329
|
+
end
|
330
|
+
|
238
331
|
# Cloud-specific configuration properties.
|
239
|
-
# @param
|
332
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
240
333
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
241
|
-
def self.schema(
|
334
|
+
def self.schema(_config)
|
242
335
|
toplevel_required = ["attributes"]
|
243
336
|
|
244
337
|
|
245
338
|
schema = {
|
339
|
+
"populate" => {
|
340
|
+
"type" => "array",
|
341
|
+
"items" => {
|
342
|
+
"type" => "object",
|
343
|
+
"description" => "Key-value pairs, compatible with the +attributes+ schema, with which to populate this +table+ during its initial creation."
|
344
|
+
}
|
345
|
+
},
|
246
346
|
"attributes" => {
|
247
347
|
"type" => "array",
|
248
348
|
"minItems" => 1,
|
@@ -360,9 +460,9 @@ module MU
|
|
360
460
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::nosqldbs}, bare and unvalidated.
|
361
461
|
|
362
462
|
# @param db [Hash]: The resource to process and validate
|
363
|
-
# @param
|
463
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
364
464
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
365
|
-
def self.validateConfig(db,
|
465
|
+
def self.validateConfig(db, _configurator)
|
366
466
|
ok = true
|
367
467
|
|
368
468
|
partition = nil
|
@@ -399,8 +499,6 @@ module MU
|
|
399
499
|
ok
|
400
500
|
end
|
401
501
|
|
402
|
-
private
|
403
|
-
|
404
502
|
end
|
405
503
|
end
|
406
504
|
end
|
@@ -27,8 +27,8 @@ module MU
|
|
27
27
|
|
28
28
|
# Called automatically by {MU::Deploy#createResources}
|
29
29
|
def create
|
30
|
-
MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @mu_name)
|
31
30
|
@cloud_id = @mu_name
|
31
|
+
MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials']).create_topic(name: @cloud_id)
|
32
32
|
MU.log "Created SNS topic #{@mu_name}"
|
33
33
|
end
|
34
34
|
|
@@ -36,17 +36,48 @@ module MU
|
|
36
36
|
def groom
|
37
37
|
if @config['subscriptions']
|
38
38
|
@config['subscriptions'].each { |sub|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
39
|
+
if sub['resource'] and !sub['endpoint']
|
40
|
+
endpoint_obj = nil
|
41
|
+
MU.retrier([], max: 5, wait: 9, loop_if: Proc.new { endpoint_obj.nil? }) {
|
42
|
+
endpoint_obj = MU::Config::Ref.get(sub['resource']).kitten(@deploy)
|
43
|
+
}
|
44
|
+
sub['endpoint'] = endpoint_obj.arn
|
45
|
+
end
|
46
|
+
subscribe(sub['endpoint'], sub['type'])
|
46
47
|
}
|
47
48
|
end
|
48
49
|
end
|
49
50
|
|
51
|
+
# Subscribe something to this SNS topic
|
52
|
+
# @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
|
53
|
+
# @param protocol [String]: The protocol being subscribed
|
54
|
+
def subscribe(endpoint, protocol)
|
55
|
+
self.class.subscribe(arn, endpoint, protocol, region: @config['region'], credentials: @credentials)
|
56
|
+
end
|
57
|
+
|
58
|
+
# Subscribe something to an SNS topic
|
59
|
+
# @param cloud_id [String]: The short name or ARN of an existing SNS topic
|
60
|
+
# @param endpoint [String]: The address, identifier, or ARN of the resource being subscribed
|
61
|
+
# @param protocol [String]: The protocol being subscribed
|
62
|
+
# @param region [String]: The region of the target SNS topic
|
63
|
+
# @param credentials [String]:
|
64
|
+
def self.subscribe(cloud_id, endpoint, protocol, region: nil, credentials: nil)
|
65
|
+
topic = find(cloud_id: cloud_id, region: region, credentials: credentials).values.first
|
66
|
+
if !topic
|
67
|
+
raise MuError, "Failed to find SNS Topic #{cloud_id} in #{region}"
|
68
|
+
end
|
69
|
+
arn = topic["TopicArn"]
|
70
|
+
|
71
|
+
resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
|
72
|
+
|
73
|
+
resp.each { |subscription|
|
74
|
+
return subscription if subscription.protocol == protocol and subscription.endpoint == endpoint
|
75
|
+
}
|
76
|
+
|
77
|
+
MU.log "Subscribing #{endpoint} (#{protocol}) to SNS topic #{arn}", MU::NOTICE
|
78
|
+
MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
|
79
|
+
end
|
80
|
+
|
50
81
|
# Does this resource type exist as a global (cloud-wide) artifact, or
|
51
82
|
# is it localized to a region/zone?
|
52
83
|
# @return [Boolean]
|
@@ -65,13 +96,18 @@ module MU
|
|
65
96
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
66
97
|
# @param region [String]: The cloud provider region
|
67
98
|
# @return [void]
|
68
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
99
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
100
|
+
MU.log "AWS::Notifier.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
101
|
+
MU.log "Placeholder: AWS Notifier artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
102
|
+
|
69
103
|
MU::Cloud::AWS.sns(region: region, credentials: credentials).list_topics.topics.each { |topic|
|
70
|
-
if topic.topic_arn.match(
|
104
|
+
if topic.topic_arn.match(deploy_id)
|
71
105
|
# We don't have a way to tag our SNS topics, so we will delete any topic that has the MU-ID in its ARN.
|
72
106
|
# This may fail to find notifier groups in some cases (eg. cache_cluster) so we might want to delete from each API as well.
|
73
|
-
MU
|
74
|
-
|
107
|
+
MU.log "Deleting SNS topic: #{topic.topic_arn}"
|
108
|
+
if !noop
|
109
|
+
MU::Cloud::AWS.sns(region: region, credentials: credentials).delete_topic(topic_arn: topic.topic_arn)
|
110
|
+
end
|
75
111
|
end
|
76
112
|
}
|
77
113
|
end
|
@@ -86,6 +122,7 @@ module MU
|
|
86
122
|
# Return the metadata for this user cofiguration
|
87
123
|
# @return [Hash]
|
88
124
|
def notify
|
125
|
+
return nil if !@cloud_id or !cloud_desc(use_cache: false)
|
89
126
|
desc = MU::Cloud::AWS.sns(region: @config["region"], credentials: @config["credentials"]).get_topic_attributes(topic_arn: arn).attributes
|
90
127
|
MU.structToHash(desc)
|
91
128
|
end
|
@@ -96,9 +133,16 @@ module MU
|
|
96
133
|
found = {}
|
97
134
|
|
98
135
|
if args[:cloud_id]
|
99
|
-
arn =
|
100
|
-
|
101
|
-
|
136
|
+
arn = if args[:cloud_id].match(/^arn:/)
|
137
|
+
args[:cloud_id]
|
138
|
+
else
|
139
|
+
"arn:"+(MU::Cloud::AWS.isGovCloud?(args[:region]) ? "aws-us-gov" : "aws")+":sns:"+args[:region]+":"+MU::Cloud::AWS.credToAcct(args[:credentials])+":"+args[:cloud_id]
|
140
|
+
end
|
141
|
+
begin
|
142
|
+
desc = MU::Cloud::AWS.sns(region: args[:region], credentials: args[:credentials]).get_topic_attributes(topic_arn: arn).attributes
|
143
|
+
found[args[:cloud_id]] = desc if desc
|
144
|
+
rescue ::Aws::SNS::Errors::NotFound
|
145
|
+
end
|
102
146
|
else
|
103
147
|
next_token = nil
|
104
148
|
begin
|
@@ -115,21 +159,72 @@ module MU
|
|
115
159
|
found
|
116
160
|
end
|
117
161
|
|
162
|
+
# Reverse-map our cloud description into a runnable config hash.
|
163
|
+
# We assume that any values we have in +@config+ are placeholders, and
|
164
|
+
# calculate our own accordingly based on what's live in the cloud.
|
165
|
+
def toKitten(**_args)
|
166
|
+
bok = {
|
167
|
+
"cloud" => "AWS",
|
168
|
+
"credentials" => @config['credentials'],
|
169
|
+
"cloud_id" => @cloud_id,
|
170
|
+
"region" => @config['region']
|
171
|
+
}
|
172
|
+
|
173
|
+
if !cloud_desc
|
174
|
+
MU.log "toKitten failed to load a cloud_desc from #{@cloud_id}", MU::ERR, details: @config
|
175
|
+
return nil
|
176
|
+
end
|
177
|
+
|
178
|
+
bok['name'] = cloud_desc["DisplayName"].empty? ? @cloud_id : cloud_desc["DisplayName"]
|
179
|
+
svcmap = {
|
180
|
+
"lambda" => "functions",
|
181
|
+
"sqs" => "msg_queues"
|
182
|
+
}
|
183
|
+
MU::Cloud::AWS.sns(region: @config['region'], credentials: @credentials).list_subscriptions_by_topic(topic_arn: cloud_desc["TopicArn"]).subscriptions.each { |sub|
|
184
|
+
bok['subscriptions'] ||= []
|
185
|
+
|
186
|
+
bok['subscriptions'] << if sub.endpoint.match(/^arn:[^:]+:(sqs|lambda):([^:]+):(\d+):.*?([^:\/]+)$/)
|
187
|
+
_wholestring, service, region, account, id = Regexp.last_match.to_a
|
188
|
+
{
|
189
|
+
"type" => sub.protocol,
|
190
|
+
"resource" => MU::Config::Ref.get(
|
191
|
+
type: svcmap[service],
|
192
|
+
region: region,
|
193
|
+
credentials: @credentials,
|
194
|
+
id: id,
|
195
|
+
cloud: "AWS",
|
196
|
+
habitat: MU::Config::Ref.get(
|
197
|
+
id: account,
|
198
|
+
cloud: "AWS",
|
199
|
+
credentials: @credentials
|
200
|
+
)
|
201
|
+
)
|
202
|
+
}
|
203
|
+
else
|
204
|
+
{
|
205
|
+
"type" => sub.protocol,
|
206
|
+
"endpoint" => sub.endpoint
|
207
|
+
}
|
208
|
+
end
|
209
|
+
}
|
210
|
+
|
211
|
+
bok
|
212
|
+
end
|
213
|
+
|
118
214
|
# Cloud-specific configuration properties.
|
119
|
-
# @param
|
215
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
120
216
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
121
|
-
def self.schema(
|
217
|
+
def self.schema(_config)
|
122
218
|
toplevel_required = []
|
123
219
|
schema = {
|
124
220
|
"subscriptions" => {
|
125
221
|
"type" => "array",
|
126
222
|
"items" => {
|
127
223
|
"type" => "object",
|
128
|
-
"required" => ["endpoint"],
|
129
224
|
"properties" => {
|
130
225
|
"type" => {
|
131
226
|
"type" => "string",
|
132
|
-
"description" => "",
|
227
|
+
"description" => "Type of endpoint or resource which should receive notifications. If not specified, will attempt to auto-detect.",
|
133
228
|
"enum" => ["http", "https", "email", "email-json", "sms", "sqs", "application", "lambda"]
|
134
229
|
}
|
135
230
|
}
|
@@ -150,19 +245,35 @@ module MU
|
|
150
245
|
|
151
246
|
if notifier['subscriptions']
|
152
247
|
notifier['subscriptions'].each { |sub|
|
248
|
+
if sub['resource'] and configurator.haveLitterMate?(sub['resource']['name'], sub['resource']['type'])
|
249
|
+
sub['resource']['cloud'] = "AWS"
|
250
|
+
MU::Config.addDependency(notifier, sub['resource']['name'], sub['resource']['type'])
|
251
|
+
end
|
153
252
|
if !sub["type"]
|
154
|
-
if sub[
|
155
|
-
sub[
|
156
|
-
|
157
|
-
sub[
|
158
|
-
|
159
|
-
|
160
|
-
elsif sub[
|
161
|
-
sub["
|
162
|
-
|
163
|
-
sub["
|
164
|
-
|
165
|
-
|
253
|
+
sub['type'] = if sub['resource']
|
254
|
+
if sub['resource']['type'] == "functions"
|
255
|
+
"lambda"
|
256
|
+
elsif sub['resource']['type'] == "msg_queues"
|
257
|
+
"sqs"
|
258
|
+
end
|
259
|
+
elsif sub['endpoint']
|
260
|
+
if sub["endpoint"].match(/^http:/i)
|
261
|
+
"http"
|
262
|
+
elsif sub["endpoint"].match(/^https:/i)
|
263
|
+
"https"
|
264
|
+
elsif sub["endpoint"].match(/:sqs:/i)
|
265
|
+
"sqs"
|
266
|
+
elsif sub["endpoint"].match(/:lambda:/i)
|
267
|
+
"lambda"
|
268
|
+
elsif sub["endpoint"].match(/^\+?[\d\-]+$/)
|
269
|
+
"sms"
|
270
|
+
elsif sub["endpoint"].match(/\A[\w+\-.]+@[a-z\d\-]+(\.[a-z]+)*\.[a-z]+\z/i)
|
271
|
+
"email"
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
if !sub['type']
|
276
|
+
MU.log "Notifier #{notifier['name']} subscription did not specify a type, and I'm unable to guess one", MU::ERR, details: sub
|
166
277
|
ok = false
|
167
278
|
end
|
168
279
|
end
|
@@ -173,40 +284,6 @@ module MU
|
|
173
284
|
end
|
174
285
|
|
175
286
|
|
176
|
-
# Subscribe to a notifier group. This can either be an email address, SQS queue, application endpoint, etc...
|
177
|
-
# Will create the subscription only if it doesn't already exist.
|
178
|
-
# @param arn [String]: The cloud provider's identifier of the notifier group.
|
179
|
-
# @param protocol [String]: The type of the subscription (eg. email,https, etc..).
|
180
|
-
# @param endpoint [String]: The endpoint of the subscription. This will depend on the 'protocol' (as an example if protocol is email, endpoint will be the email address) ..
|
181
|
-
# @param region [String]: The cloud provider region.
|
182
|
-
def self.subscribe(arn: nil, protocol: nil, endpoint: nil, region: MU.curRegion, credentials: nil)
|
183
|
-
retries = 0
|
184
|
-
begin
|
185
|
-
resp = MU::Cloud::AWS.sns(region: region, credentials: credentials).list_subscriptions_by_topic(topic_arn: arn).subscriptions
|
186
|
-
rescue Aws::SNS::Errors::NotFound
|
187
|
-
if retries < 5
|
188
|
-
MU.log "Couldn't find topic #{arn}, retrying several times in case of a lagging resource"
|
189
|
-
retries += 1
|
190
|
-
sleep 30
|
191
|
-
retry
|
192
|
-
else
|
193
|
-
raise MuError, "Couldn't find topic #{arn}, giving up"
|
194
|
-
end
|
195
|
-
end
|
196
|
-
|
197
|
-
already_subscribed = false
|
198
|
-
if resp && !resp.empty?
|
199
|
-
resp.each { |subscription|
|
200
|
-
already_subscribed = true if subscription.protocol == protocol && subscription.endpoint == endpoint
|
201
|
-
}
|
202
|
-
end
|
203
|
-
|
204
|
-
unless already_subscribed
|
205
|
-
MU::Cloud::AWS.sns(region: region, credentials: credentials).subscribe(topic_arn: arn, protocol: protocol, endpoint: endpoint)
|
206
|
-
MU.log "Subscribed #{endpoint} to SNS topic #{arn}"
|
207
|
-
end
|
208
|
-
end
|
209
|
-
|
210
287
|
# Test if a notifier group exists
|
211
288
|
# Create a new notifier group. Will check if the group exists before creating it.
|
212
289
|
# @param topic_name [String]: The cloud provider's name for the notifier group.
|