cloud-mu 3.1.2 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +10 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +135 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +165 -111
- data/modules/mu/adoption.rb +401 -68
- data/modules/mu/cleanup.rb +199 -306
- data/modules/mu/cloud.rb +100 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +46 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +920 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +165 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +4 -4
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +83 -104
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +6 -6
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +4 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +3 -3
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +3 -3
- data/modules/mu/config/ref.rb +365 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +97 -70
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +70 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +83 -60
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +30 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +389 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +217 -2612
- data/modules/mu/mommacat/daemon.rb +397 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +271 -112
- data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
- data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
- data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
- data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
- data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
- data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
- data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
- data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
- data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
- data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
- data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +67 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
- data/modules/mu/{clouds → providers}/google/database.rb +10 -20
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +139 -167
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
- data/modules/mu/{clouds → providers}/google/role.rb +92 -58
- data/modules/mu/{clouds → providers}/google/server.rb +242 -155
- data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +11 -0
- data/modules/tests/centos7.yaml +11 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -0
- data/modules/tests/super_simple_bok.yml +1 -3
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +232 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
@@ -125,6 +125,8 @@ module MU
|
|
125
125
|
# @param region [String]: The cloud provider region
|
126
126
|
# @return [void]
|
127
127
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
128
|
+
MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
129
|
+
MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
128
130
|
alarms = []
|
129
131
|
# We don't have a way to tag alarms, so we try to delete them by the deploy ID.
|
130
132
|
# This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well.
|
@@ -252,9 +254,9 @@ module MU
|
|
252
254
|
end
|
253
255
|
|
254
256
|
# Cloud-specific configuration properties.
|
255
|
-
# @param
|
257
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
256
258
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
257
|
-
def self.schema(
|
259
|
+
def self.schema(_config)
|
258
260
|
toplevel_required = []
|
259
261
|
schema = {}
|
260
262
|
[toplevel_required, schema]
|
@@ -319,7 +321,7 @@ module MU
|
|
319
321
|
if !depclass.nil?
|
320
322
|
dimension["depclass"] = depclass
|
321
323
|
if !dimension["name"].nil? and !dimension["name"].empty?
|
322
|
-
alarm
|
324
|
+
MU::Config.addDependency(alarm, dimension["name"], depclass)
|
323
325
|
end
|
324
326
|
end
|
325
327
|
}
|
@@ -33,7 +33,7 @@ module MU
|
|
33
33
|
bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
|
34
34
|
|
35
35
|
MU.log "Creating S3 bucket #{bucket_name}"
|
36
|
-
|
36
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
|
37
37
|
acl: @config['acl'],
|
38
38
|
bucket: bucket_name
|
39
39
|
)
|
@@ -90,7 +90,6 @@ module MU
|
|
90
90
|
tagBucket if !@config['scrub_mu_isms']
|
91
91
|
|
92
92
|
current = cloud_desc
|
93
|
-
|
94
93
|
if @config['policies']
|
95
94
|
@config['policies'].each { |pol|
|
96
95
|
pol['grant_to'] ||= [
|
@@ -98,9 +97,9 @@ module MU
|
|
98
97
|
]
|
99
98
|
}
|
100
99
|
|
101
|
-
policy_docs = MU::Cloud
|
100
|
+
policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true)
|
102
101
|
policy_docs.each { |doc|
|
103
|
-
MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first
|
102
|
+
MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first)
|
104
103
|
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
|
105
104
|
bucket: @cloud_id,
|
106
105
|
policy: JSON.generate(doc.values.first)
|
@@ -177,10 +176,6 @@ module MU
|
|
177
176
|
end
|
178
177
|
|
179
178
|
begin
|
180
|
-
puts data
|
181
|
-
puts acl
|
182
|
-
puts bucket
|
183
|
-
puts path
|
184
179
|
MU.log "Writing #{path} to S3 bucket #{bucket}"
|
185
180
|
MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(
|
186
181
|
acl: acl,
|
@@ -213,6 +208,7 @@ puts path
|
|
213
208
|
# @param region [String]: The cloud provider region
|
214
209
|
# @return [void]
|
215
210
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
211
|
+
MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
216
212
|
|
217
213
|
resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets
|
218
214
|
if resp and resp.buckets
|
@@ -243,15 +239,21 @@ puts path
|
|
243
239
|
|
244
240
|
begin
|
245
241
|
tags = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_tagging(bucket: bucket.name).tag_set
|
242
|
+
deploy_match = false
|
243
|
+
master_match = false
|
246
244
|
tags.each { |tag|
|
247
245
|
if tag.key == "MU-ID" and tag.value == MU.deploy_id
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
end
|
252
|
-
break
|
246
|
+
deploy_match = true
|
247
|
+
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
248
|
+
master_match = true
|
253
249
|
end
|
254
250
|
}
|
251
|
+
if deploy_match and (ignoremaster or master_match)
|
252
|
+
MU.log "Deleting S3 Bucket #{bucket.name}"
|
253
|
+
if !noop
|
254
|
+
MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
|
255
|
+
end
|
256
|
+
end
|
255
257
|
rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect
|
256
258
|
next
|
257
259
|
end
|
@@ -302,12 +304,12 @@ puts path
|
|
302
304
|
end
|
303
305
|
|
304
306
|
# Cloud-specific configuration properties.
|
305
|
-
# @param
|
307
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
306
308
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
307
|
-
def self.schema(
|
309
|
+
def self.schema(_config)
|
308
310
|
toplevel_required = []
|
309
311
|
schema = {
|
310
|
-
"policies" => MU::Cloud
|
312
|
+
"policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema,
|
311
313
|
"acl" => {
|
312
314
|
"type" => "string",
|
313
315
|
"enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
|
@@ -325,15 +327,15 @@ puts path
|
|
325
327
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
|
326
328
|
|
327
329
|
# @param bucket [Hash]: The resource to process and validate
|
328
|
-
# @param
|
330
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
329
331
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
330
|
-
def self.validateConfig(bucket,
|
332
|
+
def self.validateConfig(bucket, _configurator)
|
331
333
|
ok = true
|
332
334
|
|
333
335
|
if bucket['policies']
|
334
336
|
bucket['policies'].each { |pol|
|
335
337
|
if !pol['permissions'] or pol['permissions'].empty?
|
336
|
-
pol['permissions'] = ["s3:GetObject"]
|
338
|
+
pol['permissions'] = ["s3:GetObject", "s3:ListBucket"]
|
337
339
|
end
|
338
340
|
}
|
339
341
|
end
|
@@ -341,11 +343,13 @@ puts path
|
|
341
343
|
ok
|
342
344
|
end
|
343
345
|
|
344
|
-
private
|
345
|
-
|
346
346
|
# AWS doesn't really implement a useful describe_ method for S3 buckets;
|
347
347
|
# instead we run the million little individual API calls to construct
|
348
348
|
# an approximation for our uses
|
349
|
+
# @param bucket [String]:
|
350
|
+
# @param minimal [Boolean]:
|
351
|
+
# @param credentials [String]:
|
352
|
+
# @param region [String]:
|
349
353
|
def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil)
|
350
354
|
@@region_cache = {}
|
351
355
|
@@region_cache_semaphore = Mutex.new
|
@@ -372,7 +376,7 @@ puts path
|
|
372
376
|
}
|
373
377
|
end
|
374
378
|
|
375
|
-
rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
|
379
|
+
rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
|
376
380
|
desc[method] = nil
|
377
381
|
next
|
378
382
|
end
|
@@ -170,7 +170,7 @@ module MU
|
|
170
170
|
# config_struct[:preferred_cache_cluster_a_zs] = @config["preferred_cache_cluster_azs"]
|
171
171
|
|
172
172
|
MU.log "Creating cache replication group #{@config['identifier']}"
|
173
|
-
|
173
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
|
174
174
|
|
175
175
|
wait_start_time = Time.now
|
176
176
|
retries = 0
|
@@ -180,7 +180,7 @@ module MU
|
|
180
180
|
waiter.before_attempt do |attempts|
|
181
181
|
MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
|
182
182
|
end
|
183
|
-
waiter.before_wait do |
|
183
|
+
waiter.before_wait do |_attempts, r|
|
184
184
|
throw :success if r.replication_groups.first.status == "available"
|
185
185
|
throw :failure if Time.now - wait_start_time > 1800
|
186
186
|
end
|
@@ -199,7 +199,7 @@ module MU
|
|
199
199
|
addStandardTags(member, "cluster", region: @config['region'])
|
200
200
|
}
|
201
201
|
|
202
|
-
MU::Cloud
|
202
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
|
203
203
|
name: resp.replication_group_id,
|
204
204
|
target: "#{resp.node_groups.first.primary_endpoint.address}.",
|
205
205
|
cloudclass: MU::Cloud::CacheCluster,
|
@@ -207,7 +207,7 @@ module MU
|
|
207
207
|
)
|
208
208
|
|
209
209
|
resp.node_groups.first.node_group_members.each { |member|
|
210
|
-
MU::Cloud
|
210
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
|
211
211
|
name: member.cache_cluster_id,
|
212
212
|
target: "#{member.read_endpoint.address}.",
|
213
213
|
cloudclass: MU::Cloud::CacheCluster,
|
@@ -228,7 +228,7 @@ module MU
|
|
228
228
|
|
229
229
|
MU.log "Creating cache cluster #{@config['identifier']}"
|
230
230
|
begin
|
231
|
-
|
231
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
|
232
232
|
rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
|
233
233
|
if e.message.match(/security group (sg-[^\s]+)/)
|
234
234
|
bad_sg = Regexp.last_match[1]
|
@@ -248,7 +248,7 @@ module MU
|
|
248
248
|
waiter.before_attempt do |attempts|
|
249
249
|
MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
|
250
250
|
end
|
251
|
-
waiter.before_wait do |
|
251
|
+
waiter.before_wait do |_attempts, r|
|
252
252
|
throw :success if r.cache_clusters.first.cache_cluster_status == "available"
|
253
253
|
throw :failure if Time.now - wait_start_time > 1800
|
254
254
|
end
|
@@ -270,7 +270,7 @@ module MU
|
|
270
270
|
def createSubnetGroup
|
271
271
|
subnet_ids = []
|
272
272
|
if @config["vpc"] && !@config["vpc"].empty?
|
273
|
-
raise MuError
|
273
|
+
raise MuError.new "Didn't find the VPC specified for #{@mu_name}", details: @config["vpc"].to_h unless @vpc
|
274
274
|
|
275
275
|
vpc_id = @vpc.cloud_id
|
276
276
|
|
@@ -283,7 +283,7 @@ module MU
|
|
283
283
|
else
|
284
284
|
@config["vpc"]["subnets"].each { |subnet|
|
285
285
|
subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
|
286
|
-
raise MuError
|
286
|
+
raise MuError.new "Couldn't find a live subnet matching #{subnet} in #{@vpc}", details: @vpc.subnets if subnet_obj.nil?
|
287
287
|
subnet_ids << subnet_obj.cloud_id
|
288
288
|
}
|
289
289
|
end
|
@@ -317,7 +317,7 @@ module MU
|
|
317
317
|
"vpc_id" => vpc_id,
|
318
318
|
"subnets" => mu_subnets
|
319
319
|
}
|
320
|
-
|
320
|
+
|
321
321
|
MU.log "Using default VPC for cache cluster #{@config['identifier']}"
|
322
322
|
end
|
323
323
|
end
|
@@ -327,30 +327,13 @@ module MU
|
|
327
327
|
else
|
328
328
|
MU.log "Creating subnet group #{@config["subnet_group_name"]} for cache cluster #{@config['identifier']}"
|
329
329
|
|
330
|
-
|
330
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
|
331
331
|
cache_subnet_group_name: @config["subnet_group_name"],
|
332
332
|
cache_subnet_group_description: @config["subnet_group_name"],
|
333
333
|
subnet_ids: subnet_ids
|
334
334
|
)
|
335
335
|
|
336
|
-
|
337
|
-
# Adding just for consistency, but do we really need this for cache clusters? I guess Nagios and such..
|
338
|
-
if @config["vpc"]["nat_host_name"] || @config["vpc"]["nat_host_id"] || @config["vpc"]["nat_host_tag"] || @config["vpc"]["nat_host_ip"]
|
339
|
-
nat = @nat
|
340
|
-
if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
|
341
|
-
MU.log "Using NAT Gateway, not modifying security groups"
|
342
|
-
else
|
343
|
-
nat_name, nat_conf, nat_deploydata = @nat.describe
|
344
|
-
@deploy.kittens['firewall_rules'].each_pair { |name, acl|
|
345
|
-
# XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
|
346
|
-
if acl.config["admin"]
|
347
|
-
acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
|
348
|
-
acl.addRule([nat_deploydata["private_ip_address"]], proto: "udp")
|
349
|
-
break
|
350
|
-
end
|
351
|
-
}
|
352
|
-
end
|
353
|
-
end
|
336
|
+
allowBastionAccess
|
354
337
|
|
355
338
|
if @dependencies.has_key?('firewall_rule')
|
356
339
|
@config["security_group_ids"] = []
|
@@ -364,7 +347,7 @@ module MU
|
|
364
347
|
# Create a Cache Cluster parameter group.
|
365
348
|
def createParameterGroup
|
366
349
|
MU.log "Creating a cache cluster parameter group #{@config["parameter_group_name"]}"
|
367
|
-
|
350
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
|
368
351
|
cache_parameter_group_name: @config["parameter_group_name"],
|
369
352
|
cache_parameter_group_family: @config["parameter_group_family"],
|
370
353
|
description: "Parameter group for #{@config["parameter_group_family"]}"
|
@@ -404,7 +387,7 @@ module MU
|
|
404
387
|
def self.getCacheClusterById(cc_id, region: MU.curRegion, credentials: nil)
|
405
388
|
begin
|
406
389
|
MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters(cache_cluster_id: cc_id).cache_clusters.first
|
407
|
-
rescue Aws::ElastiCache::Errors::CacheClusterNotFound
|
390
|
+
rescue Aws::ElastiCache::Errors::CacheClusterNotFound
|
408
391
|
nil
|
409
392
|
end
|
410
393
|
end
|
@@ -430,7 +413,7 @@ module MU
|
|
430
413
|
}
|
431
414
|
end
|
432
415
|
# XXX this should be a call to @deploy.nameKitten
|
433
|
-
MU::Cloud
|
416
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
|
434
417
|
|
435
418
|
deploy_struct = {
|
436
419
|
"identifier" => repl_group.replication_group_id,
|
@@ -532,7 +515,7 @@ module MU
|
|
532
515
|
|
533
516
|
attempts = 0
|
534
517
|
begin
|
535
|
-
|
518
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
|
536
519
|
cache_cluster_id: @config["identifier"],
|
537
520
|
snapshot_name: snap_id
|
538
521
|
)
|
@@ -669,7 +652,7 @@ module MU
|
|
669
652
|
threads << Thread.new(replication_group) { |myrepl_group|
|
670
653
|
MU.dupGlobals(parent_thread_id)
|
671
654
|
Thread.abort_on_exception = true
|
672
|
-
|
655
|
+
terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
|
673
656
|
}
|
674
657
|
}
|
675
658
|
end
|
@@ -681,7 +664,7 @@ module MU
|
|
681
664
|
threads << Thread.new(cluster) { |mycluster|
|
682
665
|
MU.dupGlobals(parent_thread_id)
|
683
666
|
Thread.abort_on_exception = true
|
684
|
-
|
667
|
+
terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
|
685
668
|
}
|
686
669
|
}
|
687
670
|
end
|
@@ -694,35 +677,16 @@ module MU
|
|
694
677
|
end
|
695
678
|
|
696
679
|
# Cloud-specific configuration properties.
|
697
|
-
# @param
|
680
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
698
681
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
699
|
-
def self.schema(
|
682
|
+
def self.schema(_config)
|
700
683
|
toplevel_required = []
|
701
684
|
schema = {
|
702
685
|
"create_replication_group" => {
|
703
686
|
"type" => "boolean",
|
704
687
|
"description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
|
705
688
|
},
|
706
|
-
"ingress_rules" =>
|
707
|
-
"items" => {
|
708
|
-
"properties" => {
|
709
|
-
"sgs" => {
|
710
|
-
"type" => "array",
|
711
|
-
"items" => {
|
712
|
-
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
713
|
-
"type" => "string"
|
714
|
-
}
|
715
|
-
},
|
716
|
-
"lbs" => {
|
717
|
-
"type" => "array",
|
718
|
-
"items" => {
|
719
|
-
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
720
|
-
"type" => "string"
|
721
|
-
}
|
722
|
-
}
|
723
|
-
}
|
724
|
-
}
|
725
|
-
}
|
689
|
+
"ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
|
726
690
|
}
|
727
691
|
[toplevel_required, schema]
|
728
692
|
end
|
@@ -804,9 +768,8 @@ module MU
|
|
804
768
|
# @param noop [Boolean]: If true, will only print what would be done.
|
805
769
|
# @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
|
806
770
|
# @param region [String]: The cloud provider's region in which to operate.
|
807
|
-
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
808
771
|
# @return [void]
|
809
|
-
def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion,
|
772
|
+
def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
|
810
773
|
raise MuError, "terminate_cache_cluster requires a non-nil cache cluster descriptor" if cluster.nil? || cluster.empty?
|
811
774
|
|
812
775
|
cluster_id = cluster.cache_cluster_id
|
@@ -824,7 +787,7 @@ module MU
|
|
824
787
|
end
|
825
788
|
|
826
789
|
# The API is broken, cluster.cache_nodes is returnning an empty array, and the only URL we can get is the config one with cluster.configuration_endpoint.address.
|
827
|
-
# MU::Cloud
|
790
|
+
# MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
|
828
791
|
|
829
792
|
if %w{deleting deleted}.include?(cluster.cache_cluster_status)
|
830
793
|
MU.log "#{cluster_id} has already been terminated", MU::WARN
|
@@ -874,7 +837,7 @@ module MU
|
|
874
837
|
waiter.before_attempt do |attempts|
|
875
838
|
MU.log "Waiting for cache cluster #{cluster_id} to delete..", MU::NOTICE if attempts % 10 == 0
|
876
839
|
end
|
877
|
-
waiter.before_wait do |
|
840
|
+
waiter.before_wait do |_attempts, resp|
|
878
841
|
throw :success if resp.cache_clusters.first.cache_cluster_status == "deleted"
|
879
842
|
throw :failure if Time.now - wait_start_time > 1800
|
880
843
|
end
|
@@ -893,19 +856,19 @@ module MU
|
|
893
856
|
MU.log "#{cluster_id} has been terminated"
|
894
857
|
|
895
858
|
unless noop
|
896
|
-
|
897
|
-
|
859
|
+
delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
|
860
|
+
delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
|
898
861
|
end
|
899
862
|
end
|
863
|
+
private_class_method :terminate_cache_cluster
|
900
864
|
|
901
865
|
# Remove a Cache Cluster Replication Group and associated artifacts
|
902
866
|
# @param repl_group [OpenStruct]: The cloud provider's description of the Cache Cluster artifact.
|
903
867
|
# @param noop [Boolean]: If true, will only print what would be done.
|
904
868
|
# @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
|
905
869
|
# @param region [String]: The cloud provider's region in which to operate.
|
906
|
-
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
907
870
|
# @return [void]
|
908
|
-
def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion,
|
871
|
+
def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
|
909
872
|
raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
|
910
873
|
|
911
874
|
repl_group_id = repl_group.replication_group_id
|
@@ -926,10 +889,10 @@ module MU
|
|
926
889
|
end
|
927
890
|
|
928
891
|
# What's the likelihood of having more than one node group? maybe iterate over node_groups instead of assuming there is only one?
|
929
|
-
MU::Cloud
|
892
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
|
930
893
|
# Assuming we also created DNS records for each of our cluster's read endpoint.
|
931
894
|
repl_group.node_groups.first.node_group_members.each { |member|
|
932
|
-
MU::Cloud
|
895
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
|
933
896
|
}
|
934
897
|
|
935
898
|
if %w{deleting deleted}.include?(repl_group.status)
|
@@ -983,7 +946,7 @@ module MU
|
|
983
946
|
waiter.before_attempt do |attempts|
|
984
947
|
MU.log "Waiting for #{repl_group_id} to delete..", MU::NOTICE if attempts % 10 == 0
|
985
948
|
end
|
986
|
-
waiter.before_wait do |
|
949
|
+
waiter.before_wait do |_attempts, resp|
|
987
950
|
throw :success if resp.replication_groups.first.status == "deleted"
|
988
951
|
throw :failure if Time.now - wait_start_time > 1800
|
989
952
|
end
|
@@ -1005,6 +968,7 @@ module MU
|
|
1005
968
|
MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region) if parameter_group && !parameter_group.start_with?("default")
|
1006
969
|
end
|
1007
970
|
end
|
971
|
+
private_class_method :terminate_replication_group
|
1008
972
|
|
1009
973
|
# Remove a Cache Cluster Subnet Group.
|
1010
974
|
# @param subnet_group_id [string]: The cloud provider's ID of the cache cluster subnet group.
|
@@ -1026,6 +990,7 @@ module MU
|
|
1026
990
|
MU.log "Subnet group #{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
1027
991
|
end
|
1028
992
|
end
|
993
|
+
private_class_method :delete_subnet_group
|
1029
994
|
|
1030
995
|
# Remove a Cache Cluster Parameter Group.
|
1031
996
|
# @param parameter_group_id [string]: The cloud provider's ID of the cache cluster parameter group.
|
@@ -1049,6 +1014,7 @@ module MU
|
|
1049
1014
|
MU.log "Parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
1050
1015
|
end
|
1051
1016
|
end
|
1017
|
+
private_class_method :delete_parameter_group
|
1052
1018
|
end
|
1053
1019
|
end
|
1054
1020
|
end
|