cloud-mu 3.1.3 → 3.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +10 -2
- data/bin/mu-adopt +5 -1
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-run-tests +112 -27
- data/cloud-mu.gemspec +20 -20
- data/cookbooks/mu-tools/libraries/helper.rb +2 -1
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +5 -15
- data/modules/mu.rb +10 -14
- data/modules/mu/adoption.rb +20 -14
- data/modules/mu/cleanup.rb +13 -9
- data/modules/mu/cloud.rb +26 -26
- data/modules/mu/clouds/aws.rb +100 -59
- data/modules/mu/clouds/aws/alarm.rb +4 -2
- data/modules/mu/clouds/aws/bucket.rb +25 -21
- data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
- data/modules/mu/clouds/aws/collection.rb +21 -20
- data/modules/mu/clouds/aws/container_cluster.rb +47 -26
- data/modules/mu/clouds/aws/database.rb +57 -68
- data/modules/mu/clouds/aws/dnszone.rb +14 -14
- data/modules/mu/clouds/aws/endpoint.rb +20 -16
- data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
- data/modules/mu/clouds/aws/folder.rb +7 -7
- data/modules/mu/clouds/aws/function.rb +15 -12
- data/modules/mu/clouds/aws/group.rb +14 -10
- data/modules/mu/clouds/aws/habitat.rb +16 -13
- data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
- data/modules/mu/clouds/aws/log.rb +13 -10
- data/modules/mu/clouds/aws/msg_queue.rb +15 -8
- data/modules/mu/clouds/aws/nosqldb.rb +18 -11
- data/modules/mu/clouds/aws/notifier.rb +11 -6
- data/modules/mu/clouds/aws/role.rb +87 -70
- data/modules/mu/clouds/aws/search_domain.rb +30 -19
- data/modules/mu/clouds/aws/server.rb +102 -72
- data/modules/mu/clouds/aws/server_pool.rb +47 -28
- data/modules/mu/clouds/aws/storage_pool.rb +5 -6
- data/modules/mu/clouds/aws/user.rb +13 -10
- data/modules/mu/clouds/aws/vpc.rb +135 -121
- data/modules/mu/clouds/azure.rb +16 -9
- data/modules/mu/clouds/azure/container_cluster.rb +2 -3
- data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
- data/modules/mu/clouds/azure/habitat.rb +8 -6
- data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
- data/modules/mu/clouds/azure/role.rb +8 -10
- data/modules/mu/clouds/azure/server.rb +65 -25
- data/modules/mu/clouds/azure/user.rb +5 -7
- data/modules/mu/clouds/azure/vpc.rb +12 -15
- data/modules/mu/clouds/cloudformation.rb +8 -7
- data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
- data/modules/mu/clouds/google.rb +39 -24
- data/modules/mu/clouds/google/bucket.rb +9 -11
- data/modules/mu/clouds/google/container_cluster.rb +27 -42
- data/modules/mu/clouds/google/database.rb +6 -9
- data/modules/mu/clouds/google/firewall_rule.rb +11 -10
- data/modules/mu/clouds/google/folder.rb +16 -9
- data/modules/mu/clouds/google/function.rb +127 -161
- data/modules/mu/clouds/google/group.rb +21 -18
- data/modules/mu/clouds/google/habitat.rb +18 -15
- data/modules/mu/clouds/google/loadbalancer.rb +14 -16
- data/modules/mu/clouds/google/role.rb +48 -31
- data/modules/mu/clouds/google/server.rb +105 -105
- data/modules/mu/clouds/google/server_pool.rb +12 -31
- data/modules/mu/clouds/google/user.rb +67 -13
- data/modules/mu/clouds/google/vpc.rb +58 -65
- data/modules/mu/config.rb +89 -1738
- data/modules/mu/config/bucket.rb +3 -3
- data/modules/mu/config/collection.rb +3 -3
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/dnszone.rb +5 -5
- data/modules/mu/config/doc_helpers.rb +517 -0
- data/modules/mu/config/endpoint.rb +3 -3
- data/modules/mu/config/firewall_rule.rb +118 -3
- data/modules/mu/config/folder.rb +3 -3
- data/modules/mu/config/function.rb +2 -2
- data/modules/mu/config/group.rb +3 -3
- data/modules/mu/config/habitat.rb +3 -3
- data/modules/mu/config/loadbalancer.rb +3 -3
- data/modules/mu/config/log.rb +3 -3
- data/modules/mu/config/msg_queue.rb +3 -3
- data/modules/mu/config/nosqldb.rb +3 -3
- data/modules/mu/config/notifier.rb +2 -2
- data/modules/mu/config/ref.rb +333 -0
- data/modules/mu/config/role.rb +3 -3
- data/modules/mu/config/schema_helpers.rb +508 -0
- data/modules/mu/config/search_domain.rb +3 -3
- data/modules/mu/config/server.rb +86 -58
- data/modules/mu/config/server_pool.rb +2 -2
- data/modules/mu/config/tail.rb +189 -0
- data/modules/mu/config/user.rb +3 -3
- data/modules/mu/config/vpc.rb +44 -4
- data/modules/mu/defaults/Google.yaml +2 -2
- data/modules/mu/deploy.rb +13 -10
- data/modules/mu/groomer.rb +1 -1
- data/modules/mu/groomers/ansible.rb +69 -24
- data/modules/mu/groomers/chef.rb +52 -44
- data/modules/mu/logger.rb +17 -14
- data/modules/mu/master.rb +317 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -2
- data/modules/mu/mommacat.rb +85 -1766
- data/modules/mu/mommacat/daemon.rb +394 -0
- data/modules/mu/mommacat/naming.rb +366 -0
- data/modules/mu/mommacat/storage.rb +689 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- metadata +112 -102
@@ -181,8 +181,8 @@ module MU
|
|
181
181
|
main_sg = @deploy.findLitterMate(type: "firewall_rules", name: "server_pool#{@config['name']}workers")
|
182
182
|
tagme << main_sg.cloud_id if main_sg
|
183
183
|
MU.log "Applying kubernetes.io tags to VPC resources", details: tagme
|
184
|
-
MU::Cloud::AWS.createTag("kubernetes.io/cluster/#{@mu_name}", "shared",
|
185
|
-
MU::Cloud::AWS.createTag("kubernetes.io/cluster/elb", @mu_name,
|
184
|
+
MU::Cloud::AWS.createTag(tagme, "kubernetes.io/cluster/#{@mu_name}", "shared", credentials: @config['credentials'])
|
185
|
+
MU::Cloud::AWS.createTag(tagme_elb, "kubernetes.io/cluster/elb", @mu_name, credentials: @config['credentials'])
|
186
186
|
|
187
187
|
if @config['flavor'] == "Fargate"
|
188
188
|
fargate_subnets = []
|
@@ -268,7 +268,18 @@ module MU
|
|
268
268
|
authmap_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{eks_auth}"}
|
269
269
|
MU.log "Configuring Kubernetes <=> IAM mapping for worker nodes", MU::NOTICE, details: authmap_cmd
|
270
270
|
# maybe guard this mess
|
271
|
-
|
271
|
+
retries = 0
|
272
|
+
begin
|
273
|
+
puts %x{#{authmap_cmd}}
|
274
|
+
if $?.exitstatus != 0
|
275
|
+
if retries >= 10
|
276
|
+
raise MuError, "Failed to apply #{authmap_cmd}"
|
277
|
+
end
|
278
|
+
sleep 10
|
279
|
+
retries += 1
|
280
|
+
end
|
281
|
+
end while $?.exitstatus != 0
|
282
|
+
|
272
283
|
end
|
273
284
|
|
274
285
|
# and this one
|
@@ -309,21 +320,22 @@ module MU
|
|
309
320
|
end
|
310
321
|
end
|
311
322
|
|
312
|
-
|
323
|
+
threads = []
|
324
|
+
serverpool.listNodes.each { |mynode|
|
313
325
|
resources = resource_lookup[node.cloud_desc.instance_type]
|
314
|
-
|
326
|
+
threads << Thread.new(mynode) { |node|
|
315
327
|
ident_doc = nil
|
316
328
|
ident_doc_sig = nil
|
317
329
|
if !node.windows?
|
318
330
|
session = node.getSSHSession(10, 30)
|
319
331
|
ident_doc = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/")
|
320
332
|
ident_doc_sig = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/signature/")
|
321
|
-
else
|
322
|
-
begin
|
323
|
-
session = node.getWinRMSession(1, 60)
|
324
|
-
rescue
|
325
|
-
session = node.getSSHSession(1, 60)
|
326
|
-
end
|
333
|
+
# else
|
334
|
+
# begin
|
335
|
+
# session = node.getWinRMSession(1, 60)
|
336
|
+
# rescue StandardError # XXX
|
337
|
+
# session = node.getSSHSession(1, 60)
|
338
|
+
# end
|
327
339
|
end
|
328
340
|
MU.log "Identity document for #{node}", MU::DEBUG, details: ident_doc
|
329
341
|
MU.log "Identity document signature for #{node}", MU::DEBUG, details: ident_doc_sig
|
@@ -354,6 +366,9 @@ module MU
|
|
354
366
|
|
355
367
|
}
|
356
368
|
}
|
369
|
+
threads.each { |t|
|
370
|
+
t.join
|
371
|
+
}
|
357
372
|
end
|
358
373
|
|
359
374
|
if @config['flavor'] != "EKS" and @config['containers']
|
@@ -387,7 +402,6 @@ module MU
|
|
387
402
|
# Reorganize things so that we have services and task definitions
|
388
403
|
# mapped to the set of containers they must contain
|
389
404
|
tasks = {}
|
390
|
-
created_generic_loggroup = false
|
391
405
|
|
392
406
|
@config['containers'].each { |c|
|
393
407
|
service_name = c['service'] ? @mu_name+"-"+c['service'].upcase : @mu_name
|
@@ -653,7 +667,7 @@ module MU
|
|
653
667
|
listme = services.slice!(0, (services.length >= 10 ? 10 : services.length))
|
654
668
|
if services.size > 0
|
655
669
|
tasks_defined.concat(
|
656
|
-
|
670
|
+
MU::Cloud::AWS.ecs(region: region, credentials: credentials).describe_services(
|
657
671
|
cluster: cluster,
|
658
672
|
services: listme
|
659
673
|
).services.map { |s| s.task_definition }
|
@@ -693,7 +707,6 @@ module MU
|
|
693
707
|
cluster: cluster,
|
694
708
|
tasks: task_ids
|
695
709
|
).tasks.each { |t|
|
696
|
-
task_name = t.task_definition_arn.sub(/^.*?:task-definition\/([^\/:]+)$/, '\1')
|
697
710
|
t.containers.each { |c|
|
698
711
|
containers[c.name] ||= {}
|
699
712
|
containers[c.name][t.desired_status] ||= {
|
@@ -736,10 +749,12 @@ MU.log c.name, MU::NOTICE, details: t
|
|
736
749
|
to_return
|
737
750
|
end
|
738
751
|
|
752
|
+
@cloud_desc_cache = nil
|
739
753
|
# Return the cloud layer descriptor for this EKS/ECS/Fargate cluster
|
740
754
|
# @return [OpenStruct]
|
741
|
-
def cloud_desc
|
742
|
-
if @
|
755
|
+
def cloud_desc(use_cache: true)
|
756
|
+
return @cloud_desc_cache if @cloud_desc_cache and use_cache
|
757
|
+
@cloud_desc_cache = if @config['flavor'] == "EKS" or
|
743
758
|
(@config['flavor'] == "Fargate" and !@config['containers'])
|
744
759
|
resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_cluster(
|
745
760
|
name: @cloud_id
|
@@ -751,6 +766,7 @@ MU.log c.name, MU::NOTICE, details: t
|
|
751
766
|
)
|
752
767
|
resp.clusters.first
|
753
768
|
end
|
769
|
+
@cloud_desc_cache
|
754
770
|
end
|
755
771
|
|
756
772
|
# Canonical Amazon Resource Number for this resource
|
@@ -857,8 +873,10 @@ MU.log c.name, MU::NOTICE, details: t
|
|
857
873
|
# @param region [String]: The cloud provider region
|
858
874
|
# @return [void]
|
859
875
|
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
860
|
-
|
876
|
+
MU.log "AWS::ContainerCluster.cleanup: need to support flags['known']", MU::DEBUG, details: flags
|
877
|
+
MU.log "Placeholder: AWS ContainerCluster artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
|
861
878
|
|
879
|
+
resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters
|
862
880
|
|
863
881
|
if resp and resp.cluster_arns and resp.cluster_arns.size > 0
|
864
882
|
resp.cluster_arns.each { |arn|
|
@@ -902,10 +920,10 @@ MU.log c.name, MU::NOTICE, details: t
|
|
902
920
|
if !noop
|
903
921
|
# TODO de-register container instances
|
904
922
|
begin
|
905
|
-
|
906
|
-
|
907
|
-
|
908
|
-
rescue Aws::ECS::Errors::ClusterContainsTasksException
|
923
|
+
MU::Cloud::AWS.ecs(credentials: credentials, region: region).delete_cluster(
|
924
|
+
cluster: cluster
|
925
|
+
)
|
926
|
+
rescue Aws::ECS::Errors::ClusterContainsTasksException
|
909
927
|
sleep 5
|
910
928
|
retry
|
911
929
|
end
|
@@ -1059,9 +1077,9 @@ MU.log c.name, MU::NOTICE, details: t
|
|
1059
1077
|
end
|
1060
1078
|
|
1061
1079
|
# Cloud-specific configuration properties.
|
1062
|
-
# @param
|
1080
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
1063
1081
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
1064
|
-
def self.schema(
|
1082
|
+
def self.schema(_config)
|
1065
1083
|
toplevel_required = []
|
1066
1084
|
|
1067
1085
|
schema = {
|
@@ -1950,7 +1968,7 @@ MU.log c.name, MU::NOTICE, details: t
|
|
1950
1968
|
"name" => cluster['name']
|
1951
1969
|
}
|
1952
1970
|
]
|
1953
|
-
worker_pool["run_list"] = ["mu-tools::eks"]
|
1971
|
+
worker_pool["run_list"] = ["recipe[mu-tools::eks]"]
|
1954
1972
|
worker_pool["run_list"].concat(cluster["run_list"]) if cluster["run_list"]
|
1955
1973
|
MU::Config::Server.common_properties.keys.each { |k|
|
1956
1974
|
if cluster[k] and !worker_pool[k]
|
@@ -1998,8 +2016,11 @@ MU.log c.name, MU::NOTICE, details: t
|
|
1998
2016
|
ok
|
1999
2017
|
end
|
2000
2018
|
|
2001
|
-
|
2002
|
-
|
2019
|
+
# Delete a Fargate profile, needed both for cleanup and regroom updates
|
2020
|
+
# @param profile [String]:
|
2021
|
+
# @param cluster [String]:
|
2022
|
+
# @param region [String]:
|
2023
|
+
# @param credentials [String]:
|
2003
2024
|
def self.purge_fargate_profile(profile, cluster, region, credentials)
|
2004
2025
|
check = begin
|
2005
2026
|
MU::Cloud::AWS.eks(region: region, credentials: credentials).delete_fargate_profile(
|
@@ -331,22 +331,22 @@ module MU
|
|
331
331
|
begin
|
332
332
|
if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
|
333
333
|
MU.log "Creating database instance #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
|
334
|
-
|
334
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_from_db_snapshot(config)
|
335
335
|
elsif @config["creation_style"] == "point_in_time"
|
336
336
|
MU.log "Creating database instance #{@config['identifier']} based on point in time backup #{@config['restore_time']} of #{@config['source_identifier']}"
|
337
|
-
|
337
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_to_point_in_time(point_in_time_config)
|
338
338
|
elsif @config["read_replica_of"]
|
339
339
|
MU.log "Creating read replica database instance #{@config['identifier']} for #{@config['source_identifier']}"
|
340
340
|
begin
|
341
|
-
|
341
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
|
342
342
|
rescue Aws::RDS::Errors::DBSubnetGroupNotAllowedFault => e
|
343
343
|
MU.log "Being forced to use source database's subnet group: #{e.message}", MU::WARN
|
344
344
|
read_replica_struct.delete(:db_subnet_group_name)
|
345
|
-
|
345
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
|
346
346
|
end
|
347
347
|
elsif @config["creation_style"] == "new"
|
348
348
|
MU.log "Creating pristine database instance #{@config['identifier']} (#{@config['name']}) in #{@config['region']}"
|
349
|
-
|
349
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance(config)
|
350
350
|
end
|
351
351
|
rescue Aws::RDS::Errors::InvalidParameterValue => e
|
352
352
|
if attempts < 5
|
@@ -369,7 +369,7 @@ module MU
|
|
369
369
|
waiter.before_attempt do |w_attempts|
|
370
370
|
MU.log "Waiting for RDS database #{@config['identifier']} to be ready...", MU::NOTICE if w_attempts % 10 == 0
|
371
371
|
end
|
372
|
-
waiter.before_wait do |
|
372
|
+
waiter.before_wait do |_attempts, r|
|
373
373
|
throw :success if r.db_instances.first.db_instance_status == "available"
|
374
374
|
throw :failure if Time.now - wait_start_time > 3600
|
375
375
|
end
|
@@ -438,7 +438,7 @@ module MU
|
|
438
438
|
waiter.before_attempt do |w_attempts|
|
439
439
|
MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if w_attempts % 10 == 0
|
440
440
|
end
|
441
|
-
waiter.before_wait do |
|
441
|
+
waiter.before_wait do |_attempts, r|
|
442
442
|
throw :success if r.db_instances.first.db_instance_status == "available"
|
443
443
|
throw :failure if Time.now - wait_start_time > 2400
|
444
444
|
end
|
@@ -528,17 +528,16 @@ module MU
|
|
528
528
|
|
529
529
|
attempts = 0
|
530
530
|
begin
|
531
|
-
|
532
|
-
|
533
|
-
|
534
|
-
|
535
|
-
|
536
|
-
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
541
|
-
end
|
531
|
+
if @config["creation_style"] == "new"
|
532
|
+
MU.log "Creating new database cluster #{@config['identifier']}"
|
533
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster(cluster_config_struct)
|
534
|
+
elsif %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
|
535
|
+
MU.log "Creating new database cluster #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
|
536
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_from_snapshot(cluster_config_struct)
|
537
|
+
elsif @config["creation_style"] == "point_in_time"
|
538
|
+
MU.log "Creating new database cluster #{@config['identifier']} from point in time backup #{@config["restore_time"]} of #{@config["source_identifier"]}"
|
539
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_to_point_in_time(cluster_config_struct)
|
540
|
+
end
|
542
541
|
rescue Aws::RDS::Errors::InvalidParameterValue => e
|
543
542
|
if attempts < 5
|
544
543
|
MU.log "Got #{e.inspect} while creating database cluster #{@config['identifier']}, will retry a few times in case of transient errors.", MU::WARN, details: cluster_config_struct
|
@@ -651,7 +650,6 @@ module MU
|
|
651
650
|
}
|
652
651
|
# Default VPC has only public subnets by default so setting publicly_accessible = true
|
653
652
|
@config["publicly_accessible"] = true
|
654
|
-
using_default_vpc = true
|
655
653
|
MU.log "Using default VPC for cache cluster #{@config['identifier']}"
|
656
654
|
end
|
657
655
|
end
|
@@ -693,8 +691,8 @@ module MU
|
|
693
691
|
if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
|
694
692
|
MU.log "Using NAT Gateway, not modifying security groups"
|
695
693
|
else
|
696
|
-
|
697
|
-
@deploy.kittens['firewall_rules'].
|
694
|
+
_nat_name, _nat_conf, nat_deploydata = @nat.describe
|
695
|
+
@deploy.kittens['firewall_rules'].values.each { |acl|
|
698
696
|
# XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
|
699
697
|
if acl.config["admin"]
|
700
698
|
acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
|
@@ -763,7 +761,7 @@ module MU
|
|
763
761
|
MU::Cloud::AWS.rds(region: region).describe_db_cluster_parameter_groups(db_cluster_parameter_group_name: param_group_id).db_cluster_parameter_groups.first
|
764
762
|
# rescue DBClusterParameterGroupNotFound => e
|
765
763
|
# Of course the API will return DBParameterGroupNotFound instead of the documented DBClusterParameterGroupNotFound error.
|
766
|
-
rescue Aws::RDS::Errors::DBParameterGroupNotFound
|
764
|
+
rescue Aws::RDS::Errors::DBParameterGroupNotFound
|
767
765
|
#we're fine returning nil
|
768
766
|
end
|
769
767
|
|
@@ -773,7 +771,7 @@ module MU
|
|
773
771
|
# @return [OpenStruct]
|
774
772
|
def self.getDBParameterGroup(param_group_id, region: MU.curRegion)
|
775
773
|
MU::Cloud::AWS.rds(region: region).describe_db_parameter_groups(db_parameter_group_name: param_group_id).db_parameter_groups.first
|
776
|
-
rescue Aws::RDS::Errors::DBParameterGroupNotFound
|
774
|
+
rescue Aws::RDS::Errors::DBParameterGroupNotFound
|
777
775
|
#we're fine returning nil
|
778
776
|
end
|
779
777
|
|
@@ -783,7 +781,7 @@ module MU
|
|
783
781
|
# @return [OpenStruct]
|
784
782
|
def self.getSubnetGroup(subnet_id, region: MU.curRegion)
|
785
783
|
MU::Cloud::AWS.rds(region: region).describe_db_subnet_groups(db_subnet_group_name: subnet_id).db_subnet_groups.first
|
786
|
-
rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault
|
784
|
+
rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault
|
787
785
|
#we're fine returning nil
|
788
786
|
end
|
789
787
|
|
@@ -815,11 +813,10 @@ module MU
|
|
815
813
|
|
816
814
|
#Setting up connection params
|
817
815
|
ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
|
818
|
-
keypairname,
|
816
|
+
keypairname, _ssh_private_key, _ssh_public_key = @deploy.SSHKey
|
819
817
|
if is_private and @vpc
|
820
818
|
if @config['vpc']['nat_host_name']
|
821
819
|
begin
|
822
|
-
proxy_cmd = "ssh -q -o StrictHostKeyChecking=no -W %h:%p #{nat_ssh_user}@#{nat_host_name}"
|
823
820
|
gateway = Net::SSH::Gateway.new(
|
824
821
|
@config['vpc']['nat_host_name'],
|
825
822
|
@config['vpc']['nat_ssh_user'],
|
@@ -953,7 +950,7 @@ module MU
|
|
953
950
|
db_security_group_name: rds_sg.db_security_group_name,
|
954
951
|
cidrip: cidr
|
955
952
|
)
|
956
|
-
rescue Aws::RDS::Errors::AuthorizationAlreadyExists
|
953
|
+
rescue Aws::RDS::Errors::AuthorizationAlreadyExists
|
957
954
|
MU.log "CIDR #{cidr} already in database instance #{@cloud_id} security group", MU::WARN
|
958
955
|
end
|
959
956
|
}
|
@@ -975,7 +972,7 @@ module MU
|
|
975
972
|
def self.getDatabaseById(db_id, region: MU.curRegion, credentials: nil)
|
976
973
|
raise MuError, "You must provide a db_id" if db_id.nil?
|
977
974
|
MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_instances(db_instance_identifier: db_id).db_instances.first
|
978
|
-
rescue Aws::RDS::Errors::DBInstanceNotFound
|
975
|
+
rescue Aws::RDS::Errors::DBInstanceNotFound
|
979
976
|
# We're fine with this returning nil when searching for a database instance the doesn't exist.
|
980
977
|
end
|
981
978
|
|
@@ -985,7 +982,7 @@ module MU
|
|
985
982
|
# @return [OpenStruct]
|
986
983
|
def self.getDatabaseClusterById(db_cluster_id, region: MU.curRegion, credentials: nil)
|
987
984
|
MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_clusters(db_cluster_identifier: db_cluster_id).db_clusters.first
|
988
|
-
rescue Aws::RDS::Errors::DBClusterNotFoundFault
|
985
|
+
rescue Aws::RDS::Errors::DBClusterNotFoundFault
|
989
986
|
# We're fine with this returning nil when searching for a database cluster the doesn't exist.
|
990
987
|
end
|
991
988
|
|
@@ -1125,20 +1122,19 @@ module MU
|
|
1125
1122
|
|
1126
1123
|
attempts = 0
|
1127
1124
|
begin
|
1128
|
-
|
1129
|
-
|
1130
|
-
|
1131
|
-
|
1132
|
-
|
1133
|
-
|
1134
|
-
|
1135
|
-
|
1136
|
-
|
1137
|
-
|
1138
|
-
|
1139
|
-
|
1140
|
-
|
1141
|
-
end
|
1125
|
+
if @config["create_cluster"]
|
1126
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster_snapshot(
|
1127
|
+
db_cluster_snapshot_identifier: snap_id,
|
1128
|
+
db_cluster_identifier: @config["identifier"],
|
1129
|
+
tags: allTags
|
1130
|
+
)
|
1131
|
+
else
|
1132
|
+
MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_snapshot(
|
1133
|
+
db_snapshot_identifier: snap_id,
|
1134
|
+
db_instance_identifier: @config["identifier"],
|
1135
|
+
tags: allTags
|
1136
|
+
)
|
1137
|
+
end
|
1142
1138
|
rescue Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault => e
|
1143
1139
|
raise MuError, e.inspect if attempts >= 10
|
1144
1140
|
attempts += 1
|
@@ -1215,7 +1211,6 @@ module MU
|
|
1215
1211
|
threads = []
|
1216
1212
|
|
1217
1213
|
resp.db_instances.each { |db|
|
1218
|
-
db_id = db.db_instance_identifier
|
1219
1214
|
arn = MU::Cloud::AWS::Database.getARN(db.db_instance_identifier, "db", "rds", region: region, credentials: credentials)
|
1220
1215
|
tags = MU::Cloud::AWS.rds(credentials: credentials, region: region).list_tags_for_resource(resource_name: arn).tag_list
|
1221
1216
|
|
@@ -1241,7 +1236,7 @@ module MU
|
|
1241
1236
|
threads << Thread.new(db) { |mydb|
|
1242
1237
|
MU.dupGlobals(parent_thread_id)
|
1243
1238
|
Thread.abort_on_exception = true
|
1244
|
-
|
1239
|
+
terminate_rds_instance(mydb, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: db.db_instance_identifier, mu_name: db.db_instance_identifier.upcase, credentials: credentials)
|
1245
1240
|
}
|
1246
1241
|
end
|
1247
1242
|
}
|
@@ -1281,7 +1276,7 @@ module MU
|
|
1281
1276
|
threads << Thread.new(cluster) { |mydbcluster|
|
1282
1277
|
MU.dupGlobals(parent_thread_id)
|
1283
1278
|
Thread.abort_on_exception = true
|
1284
|
-
|
1279
|
+
terminate_rds_cluster(mydbcluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: cluster_id, mu_name: cluster_id.upcase, credentials: credentials)
|
1285
1280
|
}
|
1286
1281
|
end
|
1287
1282
|
}
|
@@ -1317,10 +1312,10 @@ module MU
|
|
1317
1312
|
|
1318
1313
|
if delete
|
1319
1314
|
parent_thread_id = Thread.current.object_id
|
1320
|
-
threads << Thread.new(
|
1315
|
+
threads << Thread.new(sub_group_id) { |mysubgroup|
|
1321
1316
|
MU.dupGlobals(parent_thread_id)
|
1322
1317
|
Thread.abort_on_exception = true
|
1323
|
-
|
1318
|
+
delete_subnet_group(mysubgroup, region: region) unless noop
|
1324
1319
|
}
|
1325
1320
|
end
|
1326
1321
|
}
|
@@ -1350,10 +1345,10 @@ module MU
|
|
1350
1345
|
|
1351
1346
|
if delete
|
1352
1347
|
parent_thread_id = Thread.current.object_id
|
1353
|
-
threads << Thread.new(
|
1348
|
+
threads << Thread.new(param_group_id) { |myparamgroup|
|
1354
1349
|
MU.dupGlobals(parent_thread_id)
|
1355
1350
|
Thread.abort_on_exception = true
|
1356
|
-
|
1351
|
+
delete_db_parameter_group(myparamgroup, region: region) unless noop
|
1357
1352
|
}
|
1358
1353
|
end
|
1359
1354
|
}
|
@@ -1383,10 +1378,10 @@ module MU
|
|
1383
1378
|
|
1384
1379
|
if delete
|
1385
1380
|
parent_thread_id = Thread.current.object_id
|
1386
|
-
threads << Thread.new(
|
1381
|
+
threads << Thread.new(param_group_id) { |myparamgroup|
|
1387
1382
|
MU.dupGlobals(parent_thread_id)
|
1388
1383
|
Thread.abort_on_exception = true
|
1389
|
-
|
1384
|
+
delete_db_cluster_parameter_group(myparamgroup, region: region) unless noop
|
1390
1385
|
}
|
1391
1386
|
end
|
1392
1387
|
}
|
@@ -1398,9 +1393,9 @@ module MU
|
|
1398
1393
|
end
|
1399
1394
|
|
1400
1395
|
# Cloud-specific configuration properties.
|
1401
|
-
# @param
|
1396
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
1402
1397
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
1403
|
-
def self.schema(
|
1398
|
+
def self.schema(_config)
|
1404
1399
|
toplevel_required = []
|
1405
1400
|
rds_parameters_primitive = {
|
1406
1401
|
"type" => "array",
|
@@ -1512,9 +1507,9 @@ module MU
|
|
1512
1507
|
|
1513
1508
|
# Cloud-specific pre-processing of {MU::Config::BasketofKittens::databases}, bare and unvalidated.
|
1514
1509
|
# @param db [Hash]: The resource to process and validate
|
1515
|
-
# @param
|
1510
|
+
# @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
|
1516
1511
|
# @return [Boolean]: True if validation succeeded, False otherwise
|
1517
|
-
def self.validateConfig(db,
|
1512
|
+
def self.validateConfig(db, _configurator)
|
1518
1513
|
ok = true
|
1519
1514
|
|
1520
1515
|
if db['creation_style'] == "existing_snapshot" and
|
@@ -1623,7 +1618,7 @@ module MU
|
|
1623
1618
|
MU::Cloud::AWS.rds(region: db['region']).describe_db_instances(
|
1624
1619
|
db_instance_identifier: db['identifier']
|
1625
1620
|
)
|
1626
|
-
rescue Aws::RDS::Errors::DBInstanceNotFound
|
1621
|
+
rescue Aws::RDS::Errors::DBInstanceNotFound
|
1627
1622
|
MU.log "Source database #{db['identifier']} was specified for #{db['name']}, but no such database exists in #{db['region']}", MU::ERR
|
1628
1623
|
ok = false
|
1629
1624
|
end
|
@@ -1716,13 +1711,6 @@ module MU
|
|
1716
1711
|
mu_name: mu_name
|
1717
1712
|
).first
|
1718
1713
|
|
1719
|
-
subnet_group = nil
|
1720
|
-
begin
|
1721
|
-
subnet_group = db.db_subnet_group.db_subnet_group_name if db.db_subnet_group
|
1722
|
-
rescue NoMethodError
|
1723
|
-
# ignorable for non-VPC databases
|
1724
|
-
end
|
1725
|
-
|
1726
1714
|
rdssecgroups = Array.new
|
1727
1715
|
begin
|
1728
1716
|
secgroup = MU::Cloud::AWS.rds(region: region).describe_db_security_groups(db_security_group_name: db_id)
|
@@ -1731,7 +1719,6 @@ module MU
|
|
1731
1719
|
end
|
1732
1720
|
|
1733
1721
|
rdssecgroups << db_id if !secgroup.nil?
|
1734
|
-
parameter_group = db.db_parameter_groups.first.db_parameter_group_name
|
1735
1722
|
|
1736
1723
|
# We can use an AWS waiter for this.
|
1737
1724
|
unless db.db_instance_status == "available"
|
@@ -1823,6 +1810,7 @@ module MU
|
|
1823
1810
|
groomclass.deleteSecret(vault: db_id.upcase) if !noop
|
1824
1811
|
MU.log "#{db_id} has been terminated"
|
1825
1812
|
end
|
1813
|
+
private_class_method :terminate_rds_instance
|
1826
1814
|
|
1827
1815
|
# Remove an RDS database cluster and associated artifacts
|
1828
1816
|
# @param cluster [OpenStruct]: The cloud provider's description of the database artifact
|
@@ -1841,9 +1829,6 @@ module MU
|
|
1841
1829
|
mu_name: mu_name
|
1842
1830
|
).first
|
1843
1831
|
|
1844
|
-
subnet_group = cluster.db_subnet_group
|
1845
|
-
cluster_parameter_group = cluster.db_cluster_parameter_group
|
1846
|
-
|
1847
1832
|
# We can use an AWS waiter for this.
|
1848
1833
|
unless cluster.status == "available"
|
1849
1834
|
loop do
|
@@ -1914,6 +1899,7 @@ module MU
|
|
1914
1899
|
|
1915
1900
|
MU.log "#{cluster_id} has been terminated"
|
1916
1901
|
end
|
1902
|
+
private_class_method :terminate_rds_cluster
|
1917
1903
|
|
1918
1904
|
# Remove a database subnet group.
|
1919
1905
|
# @param subnet_group_id [string]: The cloud provider's ID of the database subnet group.
|
@@ -1935,6 +1921,7 @@ module MU
|
|
1935
1921
|
MU.log "#{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
1936
1922
|
end
|
1937
1923
|
end
|
1924
|
+
private_class_method :delete_subnet_group
|
1938
1925
|
|
1939
1926
|
# Remove a database parameter group.
|
1940
1927
|
# @param parameter_group_id [string]: The cloud provider's ID of the database parameter group.
|
@@ -1956,6 +1943,7 @@ module MU
|
|
1956
1943
|
MU.log "DB parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
1957
1944
|
end
|
1958
1945
|
end
|
1946
|
+
private_class_method :delete_db_parameter_group
|
1959
1947
|
|
1960
1948
|
# Remove a database cluster parameter group.
|
1961
1949
|
# @param parameter_group_id [string]: The cloud provider's ID of the database cluster parameter group.
|
@@ -1978,6 +1966,7 @@ module MU
|
|
1978
1966
|
MU.log "Cluster parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
|
1979
1967
|
end
|
1980
1968
|
end
|
1969
|
+
private_class_method :delete_db_cluster_parameter_group
|
1981
1970
|
|
1982
1971
|
end #class
|
1983
1972
|
end #class
|