cloud-mu 3.1.2 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +10 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +135 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +165 -111
- data/modules/mu/adoption.rb +401 -68
- data/modules/mu/cleanup.rb +199 -306
- data/modules/mu/cloud.rb +100 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +46 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +920 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +165 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +4 -4
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +83 -104
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +6 -6
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +4 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +3 -3
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +3 -3
- data/modules/mu/config/ref.rb +365 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +97 -70
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +70 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +83 -60
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +30 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +389 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +217 -2612
- data/modules/mu/mommacat/daemon.rb +397 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +271 -112
- data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
- data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
- data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
- data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
- data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
- data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
- data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
- data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
- data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
- data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
- data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +67 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
- data/modules/mu/{clouds → providers}/google/database.rb +10 -20
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +139 -167
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
- data/modules/mu/{clouds → providers}/google/role.rb +92 -58
- data/modules/mu/{clouds → providers}/google/server.rb +242 -155
- data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +11 -0
- data/modules/tests/centos7.yaml +11 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -0
- data/modules/tests/super_simple_bok.yml +1 -3
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +232 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
|
@@ -125,6 +125,8 @@ module MU
|
|
|
125
125
|
# @param region [String]: The cloud provider region
|
|
126
126
|
# @return [void]
|
|
127
127
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
128
|
+
MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
129
|
+
MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
|
128
130
|
alarms = []
|
|
129
131
|
# We don't have a way to tag alarms, so we try to delete them by the deploy ID.
|
|
130
132
|
# This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well.
|
|
@@ -252,9 +254,9 @@ module MU
|
|
|
252
254
|
end
|
|
253
255
|
|
|
254
256
|
# Cloud-specific configuration properties.
|
|
255
|
-
# @param
|
|
257
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
|
256
258
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
|
257
|
-
def self.schema(
|
|
259
|
+
def self.schema(_config)
|
|
258
260
|
toplevel_required = []
|
|
259
261
|
schema = {}
|
|
260
262
|
[toplevel_required, schema]
|
|
@@ -319,7 +321,7 @@ module MU
|
|
|
319
321
|
if !depclass.nil?
|
|
320
322
|
dimension["depclass"] = depclass
|
|
321
323
|
if !dimension["name"].nil? and !dimension["name"].empty?
|
|
322
|
-
alarm
|
|
324
|
+
MU::Config.addDependency(alarm, dimension["name"], depclass)
|
|
323
325
|
end
|
|
324
326
|
end
|
|
325
327
|
}
|
|
@@ -33,7 +33,7 @@ module MU
|
|
|
33
33
|
bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
|
|
34
34
|
|
|
35
35
|
MU.log "Creating S3 bucket #{bucket_name}"
|
|
36
|
-
|
|
36
|
+
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
|
|
37
37
|
acl: @config['acl'],
|
|
38
38
|
bucket: bucket_name
|
|
39
39
|
)
|
|
@@ -90,7 +90,6 @@ module MU
|
|
|
90
90
|
tagBucket if !@config['scrub_mu_isms']
|
|
91
91
|
|
|
92
92
|
current = cloud_desc
|
|
93
|
-
|
|
94
93
|
if @config['policies']
|
|
95
94
|
@config['policies'].each { |pol|
|
|
96
95
|
pol['grant_to'] ||= [
|
|
@@ -98,9 +97,9 @@ module MU
|
|
|
98
97
|
]
|
|
99
98
|
}
|
|
100
99
|
|
|
101
|
-
policy_docs = MU::Cloud
|
|
100
|
+
policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true)
|
|
102
101
|
policy_docs.each { |doc|
|
|
103
|
-
MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first
|
|
102
|
+
MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first)
|
|
104
103
|
MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
|
|
105
104
|
bucket: @cloud_id,
|
|
106
105
|
policy: JSON.generate(doc.values.first)
|
|
@@ -177,10 +176,6 @@ module MU
|
|
|
177
176
|
end
|
|
178
177
|
|
|
179
178
|
begin
|
|
180
|
-
puts data
|
|
181
|
-
puts acl
|
|
182
|
-
puts bucket
|
|
183
|
-
puts path
|
|
184
179
|
MU.log "Writing #{path} to S3 bucket #{bucket}"
|
|
185
180
|
MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(
|
|
186
181
|
acl: acl,
|
|
@@ -213,6 +208,7 @@ puts path
|
|
|
213
208
|
# @param region [String]: The cloud provider region
|
|
214
209
|
# @return [void]
|
|
215
210
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
211
|
+
MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
|
216
212
|
|
|
217
213
|
resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets
|
|
218
214
|
if resp and resp.buckets
|
|
@@ -243,15 +239,21 @@ puts path
|
|
|
243
239
|
|
|
244
240
|
begin
|
|
245
241
|
tags = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_tagging(bucket: bucket.name).tag_set
|
|
242
|
+
deploy_match = false
|
|
243
|
+
master_match = false
|
|
246
244
|
tags.each { |tag|
|
|
247
245
|
if tag.key == "MU-ID" and tag.value == MU.deploy_id
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
end
|
|
252
|
-
break
|
|
246
|
+
deploy_match = true
|
|
247
|
+
elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
|
|
248
|
+
master_match = true
|
|
253
249
|
end
|
|
254
250
|
}
|
|
251
|
+
if deploy_match and (ignoremaster or master_match)
|
|
252
|
+
MU.log "Deleting S3 Bucket #{bucket.name}"
|
|
253
|
+
if !noop
|
|
254
|
+
MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
|
|
255
|
+
end
|
|
256
|
+
end
|
|
255
257
|
rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect
|
|
256
258
|
next
|
|
257
259
|
end
|
|
@@ -302,12 +304,12 @@ puts path
|
|
|
302
304
|
end
|
|
303
305
|
|
|
304
306
|
# Cloud-specific configuration properties.
|
|
305
|
-
# @param
|
|
307
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
|
306
308
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
|
307
|
-
def self.schema(
|
|
309
|
+
def self.schema(_config)
|
|
308
310
|
toplevel_required = []
|
|
309
311
|
schema = {
|
|
310
|
-
"policies" => MU::Cloud
|
|
312
|
+
"policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema,
|
|
311
313
|
"acl" => {
|
|
312
314
|
"type" => "string",
|
|
313
315
|
"enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
|
|
@@ -325,15 +327,15 @@ puts path
|
|
|
325
327
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
|
|
326
328
|
|
|
327
329
|
# @param bucket [Hash]: The resource to process and validate
|
|
328
|
-
# @param
|
|
330
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
|
329
331
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
|
330
|
-
def self.validateConfig(bucket,
|
|
332
|
+
def self.validateConfig(bucket, _configurator)
|
|
331
333
|
ok = true
|
|
332
334
|
|
|
333
335
|
if bucket['policies']
|
|
334
336
|
bucket['policies'].each { |pol|
|
|
335
337
|
if !pol['permissions'] or pol['permissions'].empty?
|
|
336
|
-
pol['permissions'] = ["s3:GetObject"]
|
|
338
|
+
pol['permissions'] = ["s3:GetObject", "s3:ListBucket"]
|
|
337
339
|
end
|
|
338
340
|
}
|
|
339
341
|
end
|
|
@@ -341,11 +343,13 @@ puts path
|
|
|
341
343
|
ok
|
|
342
344
|
end
|
|
343
345
|
|
|
344
|
-
private
|
|
345
|
-
|
|
346
346
|
# AWS doesn't really implement a useful describe_ method for S3 buckets;
|
|
347
347
|
# instead we run the million little individual API calls to construct
|
|
348
348
|
# an approximation for our uses
|
|
349
|
+
# @param bucket [String]:
|
|
350
|
+
# @param minimal [Boolean]:
|
|
351
|
+
# @param credentials [String]:
|
|
352
|
+
# @param region [String]:
|
|
349
353
|
def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil)
|
|
350
354
|
@@region_cache = {}
|
|
351
355
|
@@region_cache_semaphore = Mutex.new
|
|
@@ -372,7 +376,7 @@ puts path
|
|
|
372
376
|
}
|
|
373
377
|
end
|
|
374
378
|
|
|
375
|
-
rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
|
|
379
|
+
rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
|
|
376
380
|
desc[method] = nil
|
|
377
381
|
next
|
|
378
382
|
end
|
|
@@ -170,7 +170,7 @@ module MU
|
|
|
170
170
|
# config_struct[:preferred_cache_cluster_a_zs] = @config["preferred_cache_cluster_azs"]
|
|
171
171
|
|
|
172
172
|
MU.log "Creating cache replication group #{@config['identifier']}"
|
|
173
|
-
|
|
173
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
|
|
174
174
|
|
|
175
175
|
wait_start_time = Time.now
|
|
176
176
|
retries = 0
|
|
@@ -180,7 +180,7 @@ module MU
|
|
|
180
180
|
waiter.before_attempt do |attempts|
|
|
181
181
|
MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
|
|
182
182
|
end
|
|
183
|
-
waiter.before_wait do |
|
|
183
|
+
waiter.before_wait do |_attempts, r|
|
|
184
184
|
throw :success if r.replication_groups.first.status == "available"
|
|
185
185
|
throw :failure if Time.now - wait_start_time > 1800
|
|
186
186
|
end
|
|
@@ -199,7 +199,7 @@ module MU
|
|
|
199
199
|
addStandardTags(member, "cluster", region: @config['region'])
|
|
200
200
|
}
|
|
201
201
|
|
|
202
|
-
MU::Cloud
|
|
202
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
|
|
203
203
|
name: resp.replication_group_id,
|
|
204
204
|
target: "#{resp.node_groups.first.primary_endpoint.address}.",
|
|
205
205
|
cloudclass: MU::Cloud::CacheCluster,
|
|
@@ -207,7 +207,7 @@ module MU
|
|
|
207
207
|
)
|
|
208
208
|
|
|
209
209
|
resp.node_groups.first.node_group_members.each { |member|
|
|
210
|
-
MU::Cloud
|
|
210
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
|
|
211
211
|
name: member.cache_cluster_id,
|
|
212
212
|
target: "#{member.read_endpoint.address}.",
|
|
213
213
|
cloudclass: MU::Cloud::CacheCluster,
|
|
@@ -228,7 +228,7 @@ module MU
|
|
|
228
228
|
|
|
229
229
|
MU.log "Creating cache cluster #{@config['identifier']}"
|
|
230
230
|
begin
|
|
231
|
-
|
|
231
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
|
|
232
232
|
rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
|
|
233
233
|
if e.message.match(/security group (sg-[^\s]+)/)
|
|
234
234
|
bad_sg = Regexp.last_match[1]
|
|
@@ -248,7 +248,7 @@ module MU
|
|
|
248
248
|
waiter.before_attempt do |attempts|
|
|
249
249
|
MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
|
|
250
250
|
end
|
|
251
|
-
waiter.before_wait do |
|
|
251
|
+
waiter.before_wait do |_attempts, r|
|
|
252
252
|
throw :success if r.cache_clusters.first.cache_cluster_status == "available"
|
|
253
253
|
throw :failure if Time.now - wait_start_time > 1800
|
|
254
254
|
end
|
|
@@ -270,7 +270,7 @@ module MU
|
|
|
270
270
|
def createSubnetGroup
|
|
271
271
|
subnet_ids = []
|
|
272
272
|
if @config["vpc"] && !@config["vpc"].empty?
|
|
273
|
-
raise MuError
|
|
273
|
+
raise MuError.new "Didn't find the VPC specified for #{@mu_name}", details: @config["vpc"].to_h unless @vpc
|
|
274
274
|
|
|
275
275
|
vpc_id = @vpc.cloud_id
|
|
276
276
|
|
|
@@ -283,7 +283,7 @@ module MU
|
|
|
283
283
|
else
|
|
284
284
|
@config["vpc"]["subnets"].each { |subnet|
|
|
285
285
|
subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
|
|
286
|
-
raise MuError
|
|
286
|
+
raise MuError.new "Couldn't find a live subnet matching #{subnet} in #{@vpc}", details: @vpc.subnets if subnet_obj.nil?
|
|
287
287
|
subnet_ids << subnet_obj.cloud_id
|
|
288
288
|
}
|
|
289
289
|
end
|
|
@@ -317,7 +317,7 @@ module MU
|
|
|
317
317
|
"vpc_id" => vpc_id,
|
|
318
318
|
"subnets" => mu_subnets
|
|
319
319
|
}
|
|
320
|
-
|
|
320
|
+
|
|
321
321
|
MU.log "Using default VPC for cache cluster #{@config['identifier']}"
|
|
322
322
|
end
|
|
323
323
|
end
|
|
@@ -327,30 +327,13 @@ module MU
|
|
|
327
327
|
else
|
|
328
328
|
MU.log "Creating subnet group #{@config["subnet_group_name"]} for cache cluster #{@config['identifier']}"
|
|
329
329
|
|
|
330
|
-
|
|
330
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
|
|
331
331
|
cache_subnet_group_name: @config["subnet_group_name"],
|
|
332
332
|
cache_subnet_group_description: @config["subnet_group_name"],
|
|
333
333
|
subnet_ids: subnet_ids
|
|
334
334
|
)
|
|
335
335
|
|
|
336
|
-
|
|
337
|
-
# Adding just for consistency, but do we really need this for cache clusters? I guess Nagios and such..
|
|
338
|
-
if @config["vpc"]["nat_host_name"] || @config["vpc"]["nat_host_id"] || @config["vpc"]["nat_host_tag"] || @config["vpc"]["nat_host_ip"]
|
|
339
|
-
nat = @nat
|
|
340
|
-
if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
|
|
341
|
-
MU.log "Using NAT Gateway, not modifying security groups"
|
|
342
|
-
else
|
|
343
|
-
nat_name, nat_conf, nat_deploydata = @nat.describe
|
|
344
|
-
@deploy.kittens['firewall_rules'].each_pair { |name, acl|
|
|
345
|
-
# XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
|
|
346
|
-
if acl.config["admin"]
|
|
347
|
-
acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
|
|
348
|
-
acl.addRule([nat_deploydata["private_ip_address"]], proto: "udp")
|
|
349
|
-
break
|
|
350
|
-
end
|
|
351
|
-
}
|
|
352
|
-
end
|
|
353
|
-
end
|
|
336
|
+
allowBastionAccess
|
|
354
337
|
|
|
355
338
|
if @dependencies.has_key?('firewall_rule')
|
|
356
339
|
@config["security_group_ids"] = []
|
|
@@ -364,7 +347,7 @@ module MU
|
|
|
364
347
|
# Create a Cache Cluster parameter group.
|
|
365
348
|
def createParameterGroup
|
|
366
349
|
MU.log "Creating a cache cluster parameter group #{@config["parameter_group_name"]}"
|
|
367
|
-
|
|
350
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
|
|
368
351
|
cache_parameter_group_name: @config["parameter_group_name"],
|
|
369
352
|
cache_parameter_group_family: @config["parameter_group_family"],
|
|
370
353
|
description: "Parameter group for #{@config["parameter_group_family"]}"
|
|
@@ -404,7 +387,7 @@ module MU
|
|
|
404
387
|
def self.getCacheClusterById(cc_id, region: MU.curRegion, credentials: nil)
|
|
405
388
|
begin
|
|
406
389
|
MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters(cache_cluster_id: cc_id).cache_clusters.first
|
|
407
|
-
rescue Aws::ElastiCache::Errors::CacheClusterNotFound
|
|
390
|
+
rescue Aws::ElastiCache::Errors::CacheClusterNotFound
|
|
408
391
|
nil
|
|
409
392
|
end
|
|
410
393
|
end
|
|
@@ -430,7 +413,7 @@ module MU
|
|
|
430
413
|
}
|
|
431
414
|
end
|
|
432
415
|
# XXX this should be a call to @deploy.nameKitten
|
|
433
|
-
MU::Cloud
|
|
416
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
|
|
434
417
|
|
|
435
418
|
deploy_struct = {
|
|
436
419
|
"identifier" => repl_group.replication_group_id,
|
|
@@ -532,7 +515,7 @@ module MU
|
|
|
532
515
|
|
|
533
516
|
attempts = 0
|
|
534
517
|
begin
|
|
535
|
-
|
|
518
|
+
MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
|
|
536
519
|
cache_cluster_id: @config["identifier"],
|
|
537
520
|
snapshot_name: snap_id
|
|
538
521
|
)
|
|
@@ -669,7 +652,7 @@ module MU
|
|
|
669
652
|
threads << Thread.new(replication_group) { |myrepl_group|
|
|
670
653
|
MU.dupGlobals(parent_thread_id)
|
|
671
654
|
Thread.abort_on_exception = true
|
|
672
|
-
|
|
655
|
+
terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
|
|
673
656
|
}
|
|
674
657
|
}
|
|
675
658
|
end
|
|
@@ -681,7 +664,7 @@ module MU
|
|
|
681
664
|
threads << Thread.new(cluster) { |mycluster|
|
|
682
665
|
MU.dupGlobals(parent_thread_id)
|
|
683
666
|
Thread.abort_on_exception = true
|
|
684
|
-
|
|
667
|
+
terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
|
|
685
668
|
}
|
|
686
669
|
}
|
|
687
670
|
end
|
|
@@ -694,35 +677,16 @@ module MU
|
|
|
694
677
|
end
|
|
695
678
|
|
|
696
679
|
# Cloud-specific configuration properties.
|
|
697
|
-
# @param
|
|
680
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
|
698
681
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
|
699
|
-
def self.schema(
|
|
682
|
+
def self.schema(_config)
|
|
700
683
|
toplevel_required = []
|
|
701
684
|
schema = {
|
|
702
685
|
"create_replication_group" => {
|
|
703
686
|
"type" => "boolean",
|
|
704
687
|
"description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
|
|
705
688
|
},
|
|
706
|
-
"ingress_rules" =>
|
|
707
|
-
"items" => {
|
|
708
|
-
"properties" => {
|
|
709
|
-
"sgs" => {
|
|
710
|
-
"type" => "array",
|
|
711
|
-
"items" => {
|
|
712
|
-
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
|
713
|
-
"type" => "string"
|
|
714
|
-
}
|
|
715
|
-
},
|
|
716
|
-
"lbs" => {
|
|
717
|
-
"type" => "array",
|
|
718
|
-
"items" => {
|
|
719
|
-
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
|
720
|
-
"type" => "string"
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
}
|
|
725
|
-
}
|
|
689
|
+
"ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
|
|
726
690
|
}
|
|
727
691
|
[toplevel_required, schema]
|
|
728
692
|
end
|
|
@@ -804,9 +768,8 @@ module MU
|
|
|
804
768
|
# @param noop [Boolean]: If true, will only print what would be done.
|
|
805
769
|
# @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
|
|
806
770
|
# @param region [String]: The cloud provider's region in which to operate.
|
|
807
|
-
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
|
808
771
|
# @return [void]
|
|
809
|
-
def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion,
|
|
772
|
+
def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
|
|
810
773
|
raise MuError, "terminate_cache_cluster requires a non-nil cache cluster descriptor" if cluster.nil? || cluster.empty?
|
|
811
774
|
|
|
812
775
|
cluster_id = cluster.cache_cluster_id
|
|
@@ -824,7 +787,7 @@ module MU
|
|
|
824
787
|
end
|
|
825
788
|
|
|
826
789
|
# The API is broken, cluster.cache_nodes is returnning an empty array, and the only URL we can get is the config one with cluster.configuration_endpoint.address.
|
|
827
|
-
# MU::Cloud
|
|
790
|
+
# MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
|
|
828
791
|
|
|
829
792
|
if %w{deleting deleted}.include?(cluster.cache_cluster_status)
|
|
830
793
|
MU.log "#{cluster_id} has already been terminated", MU::WARN
|
|
@@ -874,7 +837,7 @@ module MU
|
|
|
874
837
|
waiter.before_attempt do |attempts|
|
|
875
838
|
MU.log "Waiting for cache cluster #{cluster_id} to delete..", MU::NOTICE if attempts % 10 == 0
|
|
876
839
|
end
|
|
877
|
-
waiter.before_wait do |
|
|
840
|
+
waiter.before_wait do |_attempts, resp|
|
|
878
841
|
throw :success if resp.cache_clusters.first.cache_cluster_status == "deleted"
|
|
879
842
|
throw :failure if Time.now - wait_start_time > 1800
|
|
880
843
|
end
|
|
@@ -893,19 +856,19 @@ module MU
|
|
|
893
856
|
MU.log "#{cluster_id} has been terminated"
|
|
894
857
|
|
|
895
858
|
unless noop
|
|
896
|
-
|
|
897
|
-
|
|
859
|
+
delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
|
|
860
|
+
delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
|
|
898
861
|
end
|
|
899
862
|
end
|
|
863
|
+
private_class_method :terminate_cache_cluster
|
|
900
864
|
|
|
901
865
|
# Remove a Cache Cluster Replication Group and associated artifacts
|
|
902
866
|
# @param repl_group [OpenStruct]: The cloud provider's description of the Cache Cluster artifact.
|
|
903
867
|
# @param noop [Boolean]: If true, will only print what would be done.
|
|
904
868
|
# @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
|
|
905
869
|
# @param region [String]: The cloud provider's region in which to operate.
|
|
906
|
-
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
|
907
870
|
# @return [void]
|
|
908
|
-
def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion,
|
|
871
|
+
def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
|
|
909
872
|
raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
|
|
910
873
|
|
|
911
874
|
repl_group_id = repl_group.replication_group_id
|
|
@@ -926,10 +889,10 @@ module MU
|
|
|
926
889
|
end
|
|
927
890
|
|
|
928
891
|
# What's the likelihood of having more than one node group? maybe iterate over node_groups instead of assuming there is only one?
|
|
929
|
-
MU::Cloud
|
|
892
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
|
|
930
893
|
# Assuming we also created DNS records for each of our cluster's read endpoint.
|
|
931
894
|
repl_group.node_groups.first.node_group_members.each { |member|
|
|
932
|
-
MU::Cloud
|
|
895
|
+
MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
|
|
933
896
|
}
|
|
934
897
|
|
|
935
898
|
if %w{deleting deleted}.include?(repl_group.status)
|
|
@@ -983,7 +946,7 @@ module MU
|
|
|
983
946
|
waiter.before_attempt do |attempts|
|
|
984
947
|
MU.log "Waiting for #{repl_group_id} to delete..", MU::NOTICE if attempts % 10 == 0
|
|
985
948
|
end
|
|
986
|
-
waiter.before_wait do |
|
|
949
|
+
waiter.before_wait do |_attempts, resp|
|
|
987
950
|
throw :success if resp.replication_groups.first.status == "deleted"
|
|
988
951
|
throw :failure if Time.now - wait_start_time > 1800
|
|
989
952
|
end
|
|
@@ -1005,6 +968,7 @@ module MU
|
|
|
1005
968
|
MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region) if parameter_group && !parameter_group.start_with?("default")
|
|
1006
969
|
end
|
|
1007
970
|
end
|
|
971
|
+
private_class_method :terminate_replication_group
|
|
1008
972
|
|
|
1009
973
|
# Remove a Cache Cluster Subnet Group.
|
|
1010
974
|
# @param subnet_group_id [string]: The cloud provider's ID of the cache cluster subnet group.
|
|
@@ -1026,6 +990,7 @@ module MU
|
|
|
1026
990
|
MU.log "Subnet group #{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
|
1027
991
|
end
|
|
1028
992
|
end
|
|
993
|
+
private_class_method :delete_subnet_group
|
|
1029
994
|
|
|
1030
995
|
# Remove a Cache Cluster Parameter Group.
|
|
1031
996
|
# @param parameter_group_id [string]: The cloud provider's ID of the cache cluster parameter group.
|
|
@@ -1049,6 +1014,7 @@ module MU
|
|
|
1049
1014
|
MU.log "Parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
|
1050
1015
|
end
|
|
1051
1016
|
end
|
|
1017
|
+
private_class_method :delete_parameter_group
|
|
1052
1018
|
end
|
|
1053
1019
|
end
|
|
1054
1020
|
end
|