cloud-mu 3.1.3 → 3.1.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +10 -2
  3. data/bin/mu-adopt +5 -1
  4. data/bin/mu-load-config.rb +2 -3
  5. data/bin/mu-run-tests +112 -27
  6. data/cloud-mu.gemspec +20 -20
  7. data/cookbooks/mu-tools/libraries/helper.rb +2 -1
  8. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  9. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  10. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  11. data/extras/image-generators/Google/centos6.yaml +1 -0
  12. data/extras/image-generators/Google/centos7.yaml +1 -1
  13. data/modules/mommacat.ru +5 -15
  14. data/modules/mu.rb +10 -14
  15. data/modules/mu/adoption.rb +20 -14
  16. data/modules/mu/cleanup.rb +13 -9
  17. data/modules/mu/cloud.rb +26 -26
  18. data/modules/mu/clouds/aws.rb +100 -59
  19. data/modules/mu/clouds/aws/alarm.rb +4 -2
  20. data/modules/mu/clouds/aws/bucket.rb +25 -21
  21. data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
  22. data/modules/mu/clouds/aws/collection.rb +21 -20
  23. data/modules/mu/clouds/aws/container_cluster.rb +47 -26
  24. data/modules/mu/clouds/aws/database.rb +57 -68
  25. data/modules/mu/clouds/aws/dnszone.rb +14 -14
  26. data/modules/mu/clouds/aws/endpoint.rb +20 -16
  27. data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
  28. data/modules/mu/clouds/aws/folder.rb +7 -7
  29. data/modules/mu/clouds/aws/function.rb +15 -12
  30. data/modules/mu/clouds/aws/group.rb +14 -10
  31. data/modules/mu/clouds/aws/habitat.rb +16 -13
  32. data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
  33. data/modules/mu/clouds/aws/log.rb +13 -10
  34. data/modules/mu/clouds/aws/msg_queue.rb +15 -8
  35. data/modules/mu/clouds/aws/nosqldb.rb +18 -11
  36. data/modules/mu/clouds/aws/notifier.rb +11 -6
  37. data/modules/mu/clouds/aws/role.rb +87 -70
  38. data/modules/mu/clouds/aws/search_domain.rb +30 -19
  39. data/modules/mu/clouds/aws/server.rb +102 -72
  40. data/modules/mu/clouds/aws/server_pool.rb +47 -28
  41. data/modules/mu/clouds/aws/storage_pool.rb +5 -6
  42. data/modules/mu/clouds/aws/user.rb +13 -10
  43. data/modules/mu/clouds/aws/vpc.rb +135 -121
  44. data/modules/mu/clouds/azure.rb +16 -9
  45. data/modules/mu/clouds/azure/container_cluster.rb +2 -3
  46. data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
  47. data/modules/mu/clouds/azure/habitat.rb +8 -6
  48. data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
  49. data/modules/mu/clouds/azure/role.rb +8 -10
  50. data/modules/mu/clouds/azure/server.rb +65 -25
  51. data/modules/mu/clouds/azure/user.rb +5 -7
  52. data/modules/mu/clouds/azure/vpc.rb +12 -15
  53. data/modules/mu/clouds/cloudformation.rb +8 -7
  54. data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
  55. data/modules/mu/clouds/google.rb +39 -24
  56. data/modules/mu/clouds/google/bucket.rb +9 -11
  57. data/modules/mu/clouds/google/container_cluster.rb +27 -42
  58. data/modules/mu/clouds/google/database.rb +6 -9
  59. data/modules/mu/clouds/google/firewall_rule.rb +11 -10
  60. data/modules/mu/clouds/google/folder.rb +16 -9
  61. data/modules/mu/clouds/google/function.rb +127 -161
  62. data/modules/mu/clouds/google/group.rb +21 -18
  63. data/modules/mu/clouds/google/habitat.rb +18 -15
  64. data/modules/mu/clouds/google/loadbalancer.rb +14 -16
  65. data/modules/mu/clouds/google/role.rb +48 -31
  66. data/modules/mu/clouds/google/server.rb +105 -105
  67. data/modules/mu/clouds/google/server_pool.rb +12 -31
  68. data/modules/mu/clouds/google/user.rb +67 -13
  69. data/modules/mu/clouds/google/vpc.rb +58 -65
  70. data/modules/mu/config.rb +89 -1738
  71. data/modules/mu/config/bucket.rb +3 -3
  72. data/modules/mu/config/collection.rb +3 -3
  73. data/modules/mu/config/container_cluster.rb +2 -2
  74. data/modules/mu/config/dnszone.rb +5 -5
  75. data/modules/mu/config/doc_helpers.rb +517 -0
  76. data/modules/mu/config/endpoint.rb +3 -3
  77. data/modules/mu/config/firewall_rule.rb +118 -3
  78. data/modules/mu/config/folder.rb +3 -3
  79. data/modules/mu/config/function.rb +2 -2
  80. data/modules/mu/config/group.rb +3 -3
  81. data/modules/mu/config/habitat.rb +3 -3
  82. data/modules/mu/config/loadbalancer.rb +3 -3
  83. data/modules/mu/config/log.rb +3 -3
  84. data/modules/mu/config/msg_queue.rb +3 -3
  85. data/modules/mu/config/nosqldb.rb +3 -3
  86. data/modules/mu/config/notifier.rb +2 -2
  87. data/modules/mu/config/ref.rb +333 -0
  88. data/modules/mu/config/role.rb +3 -3
  89. data/modules/mu/config/schema_helpers.rb +508 -0
  90. data/modules/mu/config/search_domain.rb +3 -3
  91. data/modules/mu/config/server.rb +86 -58
  92. data/modules/mu/config/server_pool.rb +2 -2
  93. data/modules/mu/config/tail.rb +189 -0
  94. data/modules/mu/config/user.rb +3 -3
  95. data/modules/mu/config/vpc.rb +44 -4
  96. data/modules/mu/defaults/Google.yaml +2 -2
  97. data/modules/mu/deploy.rb +13 -10
  98. data/modules/mu/groomer.rb +1 -1
  99. data/modules/mu/groomers/ansible.rb +69 -24
  100. data/modules/mu/groomers/chef.rb +52 -44
  101. data/modules/mu/logger.rb +17 -14
  102. data/modules/mu/master.rb +317 -2
  103. data/modules/mu/master/chef.rb +3 -4
  104. data/modules/mu/master/ldap.rb +3 -3
  105. data/modules/mu/master/ssl.rb +12 -2
  106. data/modules/mu/mommacat.rb +85 -1766
  107. data/modules/mu/mommacat/daemon.rb +394 -0
  108. data/modules/mu/mommacat/naming.rb +366 -0
  109. data/modules/mu/mommacat/storage.rb +689 -0
  110. data/modules/tests/bucket.yml +4 -0
  111. data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
  112. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  113. data/modules/tests/regrooms/bucket.yml +19 -0
  114. metadata +112 -102
@@ -181,8 +181,8 @@ module MU
181
181
  main_sg = @deploy.findLitterMate(type: "firewall_rules", name: "server_pool#{@config['name']}workers")
182
182
  tagme << main_sg.cloud_id if main_sg
183
183
  MU.log "Applying kubernetes.io tags to VPC resources", details: tagme
184
- MU::Cloud::AWS.createTag("kubernetes.io/cluster/#{@mu_name}", "shared", tagme, credentials: @config['credentials'])
185
- MU::Cloud::AWS.createTag("kubernetes.io/cluster/elb", @mu_name, tagme_elb, credentials: @config['credentials'])
184
+ MU::Cloud::AWS.createTag(tagme, "kubernetes.io/cluster/#{@mu_name}", "shared", credentials: @config['credentials'])
185
+ MU::Cloud::AWS.createTag(tagme_elb, "kubernetes.io/cluster/elb", @mu_name, credentials: @config['credentials'])
186
186
 
187
187
  if @config['flavor'] == "Fargate"
188
188
  fargate_subnets = []
@@ -268,7 +268,18 @@ module MU
268
268
  authmap_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{eks_auth}"}
269
269
  MU.log "Configuring Kubernetes <=> IAM mapping for worker nodes", MU::NOTICE, details: authmap_cmd
270
270
  # maybe guard this mess
271
- %x{#{authmap_cmd}}
271
+ retries = 0
272
+ begin
273
+ puts %x{#{authmap_cmd}}
274
+ if $?.exitstatus != 0
275
+ if retries >= 10
276
+ raise MuError, "Failed to apply #{authmap_cmd}"
277
+ end
278
+ sleep 10
279
+ retries += 1
280
+ end
281
+ end while $?.exitstatus != 0
282
+
272
283
  end
273
284
 
274
285
  # and this one
@@ -309,21 +320,22 @@ module MU
309
320
  end
310
321
  end
311
322
 
312
- serverpool.listNodes.each { |node|
323
+ threads = []
324
+ serverpool.listNodes.each { |mynode|
313
325
  resources = resource_lookup[node.cloud_desc.instance_type]
314
- t = Thread.new {
326
+ threads << Thread.new(mynode) { |node|
315
327
  ident_doc = nil
316
328
  ident_doc_sig = nil
317
329
  if !node.windows?
318
330
  session = node.getSSHSession(10, 30)
319
331
  ident_doc = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/")
320
332
  ident_doc_sig = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/signature/")
321
- else
322
- begin
323
- session = node.getWinRMSession(1, 60)
324
- rescue Exception # XXX
325
- session = node.getSSHSession(1, 60)
326
- end
333
+ # else
334
+ # begin
335
+ # session = node.getWinRMSession(1, 60)
336
+ # rescue StandardError # XXX
337
+ # session = node.getSSHSession(1, 60)
338
+ # end
327
339
  end
328
340
  MU.log "Identity document for #{node}", MU::DEBUG, details: ident_doc
329
341
  MU.log "Identity document signature for #{node}", MU::DEBUG, details: ident_doc_sig
@@ -354,6 +366,9 @@ module MU
354
366
 
355
367
  }
356
368
  }
369
+ threads.each { |t|
370
+ t.join
371
+ }
357
372
  end
358
373
 
359
374
  if @config['flavor'] != "EKS" and @config['containers']
@@ -387,7 +402,6 @@ module MU
387
402
  # Reorganize things so that we have services and task definitions
388
403
  # mapped to the set of containers they must contain
389
404
  tasks = {}
390
- created_generic_loggroup = false
391
405
 
392
406
  @config['containers'].each { |c|
393
407
  service_name = c['service'] ? @mu_name+"-"+c['service'].upcase : @mu_name
@@ -653,7 +667,7 @@ module MU
653
667
  listme = services.slice!(0, (services.length >= 10 ? 10 : services.length))
654
668
  if services.size > 0
655
669
  tasks_defined.concat(
656
- tasks = MU::Cloud::AWS.ecs(region: region, credentials: credentials).describe_services(
670
+ MU::Cloud::AWS.ecs(region: region, credentials: credentials).describe_services(
657
671
  cluster: cluster,
658
672
  services: listme
659
673
  ).services.map { |s| s.task_definition }
@@ -693,7 +707,6 @@ module MU
693
707
  cluster: cluster,
694
708
  tasks: task_ids
695
709
  ).tasks.each { |t|
696
- task_name = t.task_definition_arn.sub(/^.*?:task-definition\/([^\/:]+)$/, '\1')
697
710
  t.containers.each { |c|
698
711
  containers[c.name] ||= {}
699
712
  containers[c.name][t.desired_status] ||= {
@@ -736,10 +749,12 @@ MU.log c.name, MU::NOTICE, details: t
736
749
  to_return
737
750
  end
738
751
 
752
+ @cloud_desc_cache = nil
739
753
  # Return the cloud layer descriptor for this EKS/ECS/Fargate cluster
740
754
  # @return [OpenStruct]
741
- def cloud_desc
742
- if @config['flavor'] == "EKS" or
755
+ def cloud_desc(use_cache: true)
756
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
757
+ @cloud_desc_cache = if @config['flavor'] == "EKS" or
743
758
  (@config['flavor'] == "Fargate" and !@config['containers'])
744
759
  resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_cluster(
745
760
  name: @cloud_id
@@ -751,6 +766,7 @@ MU.log c.name, MU::NOTICE, details: t
751
766
  )
752
767
  resp.clusters.first
753
768
  end
769
+ @cloud_desc_cache
754
770
  end
755
771
 
756
772
  # Canonical Amazon Resource Number for this resource
@@ -857,8 +873,10 @@ MU.log c.name, MU::NOTICE, details: t
857
873
  # @param region [String]: The cloud provider region
858
874
  # @return [void]
859
875
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
860
- resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters
876
+ MU.log "AWS::ContainerCluster.cleanup: need to support flags['known']", MU::DEBUG, details: flags
877
+ MU.log "Placeholder: AWS ContainerCluster artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
861
878
 
879
+ resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters
862
880
 
863
881
  if resp and resp.cluster_arns and resp.cluster_arns.size > 0
864
882
  resp.cluster_arns.each { |arn|
@@ -902,10 +920,10 @@ MU.log c.name, MU::NOTICE, details: t
902
920
  if !noop
903
921
  # TODO de-register container instances
904
922
  begin
905
- deletion = MU::Cloud::AWS.ecs(credentials: credentials, region: region).delete_cluster(
906
- cluster: cluster
907
- )
908
- rescue Aws::ECS::Errors::ClusterContainsTasksException => e
923
+ MU::Cloud::AWS.ecs(credentials: credentials, region: region).delete_cluster(
924
+ cluster: cluster
925
+ )
926
+ rescue Aws::ECS::Errors::ClusterContainsTasksException
909
927
  sleep 5
910
928
  retry
911
929
  end
@@ -1059,9 +1077,9 @@ MU.log c.name, MU::NOTICE, details: t
1059
1077
  end
1060
1078
 
1061
1079
  # Cloud-specific configuration properties.
1062
- # @param config [MU::Config]: The calling MU::Config object
1080
+ # @param _config [MU::Config]: The calling MU::Config object
1063
1081
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
1064
- def self.schema(config)
1082
+ def self.schema(_config)
1065
1083
  toplevel_required = []
1066
1084
 
1067
1085
  schema = {
@@ -1950,7 +1968,7 @@ MU.log c.name, MU::NOTICE, details: t
1950
1968
  "name" => cluster['name']
1951
1969
  }
1952
1970
  ]
1953
- worker_pool["run_list"] = ["mu-tools::eks"]
1971
+ worker_pool["run_list"] = ["recipe[mu-tools::eks]"]
1954
1972
  worker_pool["run_list"].concat(cluster["run_list"]) if cluster["run_list"]
1955
1973
  MU::Config::Server.common_properties.keys.each { |k|
1956
1974
  if cluster[k] and !worker_pool[k]
@@ -1998,8 +2016,11 @@ MU.log c.name, MU::NOTICE, details: t
1998
2016
  ok
1999
2017
  end
2000
2018
 
2001
- private
2002
-
2019
+ # Delete a Fargate profile, needed both for cleanup and regroom updates
2020
+ # @param profile [String]:
2021
+ # @param cluster [String]:
2022
+ # @param region [String]:
2023
+ # @param credentials [String]:
2003
2024
  def self.purge_fargate_profile(profile, cluster, region, credentials)
2004
2025
  check = begin
2005
2026
  MU::Cloud::AWS.eks(region: region, credentials: credentials).delete_fargate_profile(
@@ -331,22 +331,22 @@ module MU
331
331
  begin
332
332
  if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
333
333
  MU.log "Creating database instance #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
334
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_from_db_snapshot(config)
334
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_from_db_snapshot(config)
335
335
  elsif @config["creation_style"] == "point_in_time"
336
336
  MU.log "Creating database instance #{@config['identifier']} based on point in time backup #{@config['restore_time']} of #{@config['source_identifier']}"
337
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_to_point_in_time(point_in_time_config)
337
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_to_point_in_time(point_in_time_config)
338
338
  elsif @config["read_replica_of"]
339
339
  MU.log "Creating read replica database instance #{@config['identifier']} for #{@config['source_identifier']}"
340
340
  begin
341
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
341
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
342
342
  rescue Aws::RDS::Errors::DBSubnetGroupNotAllowedFault => e
343
343
  MU.log "Being forced to use source database's subnet group: #{e.message}", MU::WARN
344
344
  read_replica_struct.delete(:db_subnet_group_name)
345
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
345
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
346
346
  end
347
347
  elsif @config["creation_style"] == "new"
348
348
  MU.log "Creating pristine database instance #{@config['identifier']} (#{@config['name']}) in #{@config['region']}"
349
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance(config)
349
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance(config)
350
350
  end
351
351
  rescue Aws::RDS::Errors::InvalidParameterValue => e
352
352
  if attempts < 5
@@ -369,7 +369,7 @@ module MU
369
369
  waiter.before_attempt do |w_attempts|
370
370
  MU.log "Waiting for RDS database #{@config['identifier']} to be ready...", MU::NOTICE if w_attempts % 10 == 0
371
371
  end
372
- waiter.before_wait do |w_attempts, r|
372
+ waiter.before_wait do |_attempts, r|
373
373
  throw :success if r.db_instances.first.db_instance_status == "available"
374
374
  throw :failure if Time.now - wait_start_time > 3600
375
375
  end
@@ -438,7 +438,7 @@ module MU
438
438
  waiter.before_attempt do |w_attempts|
439
439
  MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if w_attempts % 10 == 0
440
440
  end
441
- waiter.before_wait do |w_attempts, r|
441
+ waiter.before_wait do |_attempts, r|
442
442
  throw :success if r.db_instances.first.db_instance_status == "available"
443
443
  throw :failure if Time.now - wait_start_time > 2400
444
444
  end
@@ -528,17 +528,16 @@ module MU
528
528
 
529
529
  attempts = 0
530
530
  begin
531
- resp =
532
- if @config["creation_style"] == "new"
533
- MU.log "Creating new database cluster #{@config['identifier']}"
534
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster(cluster_config_struct)
535
- elsif %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
536
- MU.log "Creating new database cluster #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
537
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_from_snapshot(cluster_config_struct)
538
- elsif @config["creation_style"] == "point_in_time"
539
- MU.log "Creating new database cluster #{@config['identifier']} from point in time backup #{@config["restore_time"]} of #{@config["source_identifier"]}"
540
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_to_point_in_time(cluster_config_struct)
541
- end
531
+ if @config["creation_style"] == "new"
532
+ MU.log "Creating new database cluster #{@config['identifier']}"
533
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster(cluster_config_struct)
534
+ elsif %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
535
+ MU.log "Creating new database cluster #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
536
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_from_snapshot(cluster_config_struct)
537
+ elsif @config["creation_style"] == "point_in_time"
538
+ MU.log "Creating new database cluster #{@config['identifier']} from point in time backup #{@config["restore_time"]} of #{@config["source_identifier"]}"
539
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_to_point_in_time(cluster_config_struct)
540
+ end
542
541
  rescue Aws::RDS::Errors::InvalidParameterValue => e
543
542
  if attempts < 5
544
543
  MU.log "Got #{e.inspect} while creating database cluster #{@config['identifier']}, will retry a few times in case of transient errors.", MU::WARN, details: cluster_config_struct
@@ -651,7 +650,6 @@ module MU
651
650
  }
652
651
  # Default VPC has only public subnets by default so setting publicly_accessible = true
653
652
  @config["publicly_accessible"] = true
654
- using_default_vpc = true
655
653
  MU.log "Using default VPC for cache cluster #{@config['identifier']}"
656
654
  end
657
655
  end
@@ -693,8 +691,8 @@ module MU
693
691
  if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
694
692
  MU.log "Using NAT Gateway, not modifying security groups"
695
693
  else
696
- nat_name, nat_conf, nat_deploydata = @nat.describe
697
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
694
+ _nat_name, _nat_conf, nat_deploydata = @nat.describe
695
+ @deploy.kittens['firewall_rules'].values.each { |acl|
698
696
  # XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
699
697
  if acl.config["admin"]
700
698
  acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
@@ -763,7 +761,7 @@ module MU
763
761
  MU::Cloud::AWS.rds(region: region).describe_db_cluster_parameter_groups(db_cluster_parameter_group_name: param_group_id).db_cluster_parameter_groups.first
764
762
  # rescue DBClusterParameterGroupNotFound => e
765
763
  # Of course the API will return DBParameterGroupNotFound instead of the documented DBClusterParameterGroupNotFound error.
766
- rescue Aws::RDS::Errors::DBParameterGroupNotFound => e
764
+ rescue Aws::RDS::Errors::DBParameterGroupNotFound
767
765
  #we're fine returning nil
768
766
  end
769
767
 
@@ -773,7 +771,7 @@ module MU
773
771
  # @return [OpenStruct]
774
772
  def self.getDBParameterGroup(param_group_id, region: MU.curRegion)
775
773
  MU::Cloud::AWS.rds(region: region).describe_db_parameter_groups(db_parameter_group_name: param_group_id).db_parameter_groups.first
776
- rescue Aws::RDS::Errors::DBParameterGroupNotFound => e
774
+ rescue Aws::RDS::Errors::DBParameterGroupNotFound
777
775
  #we're fine returning nil
778
776
  end
779
777
 
@@ -783,7 +781,7 @@ module MU
783
781
  # @return [OpenStruct]
784
782
  def self.getSubnetGroup(subnet_id, region: MU.curRegion)
785
783
  MU::Cloud::AWS.rds(region: region).describe_db_subnet_groups(db_subnet_group_name: subnet_id).db_subnet_groups.first
786
- rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault => e
784
+ rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault
787
785
  #we're fine returning nil
788
786
  end
789
787
 
@@ -815,11 +813,10 @@ module MU
815
813
 
816
814
  #Setting up connection params
817
815
  ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
818
- keypairname, ssh_private_key, ssh_public_key = @deploy.SSHKey
816
+ keypairname, _ssh_private_key, _ssh_public_key = @deploy.SSHKey
819
817
  if is_private and @vpc
820
818
  if @config['vpc']['nat_host_name']
821
819
  begin
822
- proxy_cmd = "ssh -q -o StrictHostKeyChecking=no -W %h:%p #{nat_ssh_user}@#{nat_host_name}"
823
820
  gateway = Net::SSH::Gateway.new(
824
821
  @config['vpc']['nat_host_name'],
825
822
  @config['vpc']['nat_ssh_user'],
@@ -953,7 +950,7 @@ module MU
953
950
  db_security_group_name: rds_sg.db_security_group_name,
954
951
  cidrip: cidr
955
952
  )
956
- rescue Aws::RDS::Errors::AuthorizationAlreadyExists => e
953
+ rescue Aws::RDS::Errors::AuthorizationAlreadyExists
957
954
  MU.log "CIDR #{cidr} already in database instance #{@cloud_id} security group", MU::WARN
958
955
  end
959
956
  }
@@ -975,7 +972,7 @@ module MU
975
972
  def self.getDatabaseById(db_id, region: MU.curRegion, credentials: nil)
976
973
  raise MuError, "You must provide a db_id" if db_id.nil?
977
974
  MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_instances(db_instance_identifier: db_id).db_instances.first
978
- rescue Aws::RDS::Errors::DBInstanceNotFound => e
975
+ rescue Aws::RDS::Errors::DBInstanceNotFound
979
976
  # We're fine with this returning nil when searching for a database instance the doesn't exist.
980
977
  end
981
978
 
@@ -985,7 +982,7 @@ module MU
985
982
  # @return [OpenStruct]
986
983
  def self.getDatabaseClusterById(db_cluster_id, region: MU.curRegion, credentials: nil)
987
984
  MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_clusters(db_cluster_identifier: db_cluster_id).db_clusters.first
988
- rescue Aws::RDS::Errors::DBClusterNotFoundFault => e
985
+ rescue Aws::RDS::Errors::DBClusterNotFoundFault
989
986
  # We're fine with this returning nil when searching for a database cluster the doesn't exist.
990
987
  end
991
988
 
@@ -1125,20 +1122,19 @@ module MU
1125
1122
 
1126
1123
  attempts = 0
1127
1124
  begin
1128
- snapshot =
1129
- if @config["create_cluster"]
1130
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster_snapshot(
1131
- db_cluster_snapshot_identifier: snap_id,
1132
- db_cluster_identifier: @config["identifier"],
1133
- tags: allTags
1134
- )
1135
- else
1136
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_snapshot(
1137
- db_snapshot_identifier: snap_id,
1138
- db_instance_identifier: @config["identifier"],
1139
- tags: allTags
1140
- )
1141
- end
1125
+ if @config["create_cluster"]
1126
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster_snapshot(
1127
+ db_cluster_snapshot_identifier: snap_id,
1128
+ db_cluster_identifier: @config["identifier"],
1129
+ tags: allTags
1130
+ )
1131
+ else
1132
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_snapshot(
1133
+ db_snapshot_identifier: snap_id,
1134
+ db_instance_identifier: @config["identifier"],
1135
+ tags: allTags
1136
+ )
1137
+ end
1142
1138
  rescue Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault => e
1143
1139
  raise MuError, e.inspect if attempts >= 10
1144
1140
  attempts += 1
@@ -1215,7 +1211,6 @@ module MU
1215
1211
  threads = []
1216
1212
 
1217
1213
  resp.db_instances.each { |db|
1218
- db_id = db.db_instance_identifier
1219
1214
  arn = MU::Cloud::AWS::Database.getARN(db.db_instance_identifier, "db", "rds", region: region, credentials: credentials)
1220
1215
  tags = MU::Cloud::AWS.rds(credentials: credentials, region: region).list_tags_for_resource(resource_name: arn).tag_list
1221
1216
 
@@ -1241,7 +1236,7 @@ module MU
1241
1236
  threads << Thread.new(db) { |mydb|
1242
1237
  MU.dupGlobals(parent_thread_id)
1243
1238
  Thread.abort_on_exception = true
1244
- MU::Cloud::AWS::Database.terminate_rds_instance(mydb, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: db.db_instance_identifier, mu_name: db.db_instance_identifier.upcase, credentials: credentials)
1239
+ terminate_rds_instance(mydb, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: db.db_instance_identifier, mu_name: db.db_instance_identifier.upcase, credentials: credentials)
1245
1240
  }
1246
1241
  end
1247
1242
  }
@@ -1281,7 +1276,7 @@ module MU
1281
1276
  threads << Thread.new(cluster) { |mydbcluster|
1282
1277
  MU.dupGlobals(parent_thread_id)
1283
1278
  Thread.abort_on_exception = true
1284
- MU::Cloud::AWS::Database.terminate_rds_cluster(mydbcluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: cluster_id, mu_name: cluster_id.upcase, credentials: credentials)
1279
+ terminate_rds_cluster(mydbcluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: cluster_id, mu_name: cluster_id.upcase, credentials: credentials)
1285
1280
  }
1286
1281
  end
1287
1282
  }
@@ -1317,10 +1312,10 @@ module MU
1317
1312
 
1318
1313
  if delete
1319
1314
  parent_thread_id = Thread.current.object_id
1320
- threads << Thread.new(sub_group) { |mysubgroup|
1315
+ threads << Thread.new(sub_group_id) { |mysubgroup|
1321
1316
  MU.dupGlobals(parent_thread_id)
1322
1317
  Thread.abort_on_exception = true
1323
- MU::Cloud::AWS::Database.delete_subnet_group(sub_group_id, region: region) unless noop
1318
+ delete_subnet_group(mysubgroup, region: region) unless noop
1324
1319
  }
1325
1320
  end
1326
1321
  }
@@ -1350,10 +1345,10 @@ module MU
1350
1345
 
1351
1346
  if delete
1352
1347
  parent_thread_id = Thread.current.object_id
1353
- threads << Thread.new(param_group) { |myparamgroup|
1348
+ threads << Thread.new(param_group_id) { |myparamgroup|
1354
1349
  MU.dupGlobals(parent_thread_id)
1355
1350
  Thread.abort_on_exception = true
1356
- MU::Cloud::AWS::Database.delete_db_parameter_group(param_group_id, region: region) unless noop
1351
+ delete_db_parameter_group(myparamgroup, region: region) unless noop
1357
1352
  }
1358
1353
  end
1359
1354
  }
@@ -1383,10 +1378,10 @@ module MU
1383
1378
 
1384
1379
  if delete
1385
1380
  parent_thread_id = Thread.current.object_id
1386
- threads << Thread.new(param_group) { |myparamgroup|
1381
+ threads << Thread.new(param_group_id) { |myparamgroup|
1387
1382
  MU.dupGlobals(parent_thread_id)
1388
1383
  Thread.abort_on_exception = true
1389
- MU::Cloud::AWS::Database.delete_db_cluster_parameter_group(param_group_id, region: region) unless noop
1384
+ delete_db_cluster_parameter_group(myparamgroup, region: region) unless noop
1390
1385
  }
1391
1386
  end
1392
1387
  }
@@ -1398,9 +1393,9 @@ module MU
1398
1393
  end
1399
1394
 
1400
1395
  # Cloud-specific configuration properties.
1401
- # @param config [MU::Config]: The calling MU::Config object
1396
+ # @param _config [MU::Config]: The calling MU::Config object
1402
1397
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
1403
- def self.schema(config)
1398
+ def self.schema(_config)
1404
1399
  toplevel_required = []
1405
1400
  rds_parameters_primitive = {
1406
1401
  "type" => "array",
@@ -1512,9 +1507,9 @@ module MU
1512
1507
 
1513
1508
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::databases}, bare and unvalidated.
1514
1509
  # @param db [Hash]: The resource to process and validate
1515
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1510
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1516
1511
  # @return [Boolean]: True if validation succeeded, False otherwise
1517
- def self.validateConfig(db, configurator)
1512
+ def self.validateConfig(db, _configurator)
1518
1513
  ok = true
1519
1514
 
1520
1515
  if db['creation_style'] == "existing_snapshot" and
@@ -1623,7 +1618,7 @@ module MU
1623
1618
  MU::Cloud::AWS.rds(region: db['region']).describe_db_instances(
1624
1619
  db_instance_identifier: db['identifier']
1625
1620
  )
1626
- rescue Aws::RDS::Errors::DBInstanceNotFound => e
1621
+ rescue Aws::RDS::Errors::DBInstanceNotFound
1627
1622
  MU.log "Source database #{db['identifier']} was specified for #{db['name']}, but no such database exists in #{db['region']}", MU::ERR
1628
1623
  ok = false
1629
1624
  end
@@ -1716,13 +1711,6 @@ module MU
1716
1711
  mu_name: mu_name
1717
1712
  ).first
1718
1713
 
1719
- subnet_group = nil
1720
- begin
1721
- subnet_group = db.db_subnet_group.db_subnet_group_name if db.db_subnet_group
1722
- rescue NoMethodError
1723
- # ignorable for non-VPC databases
1724
- end
1725
-
1726
1714
  rdssecgroups = Array.new
1727
1715
  begin
1728
1716
  secgroup = MU::Cloud::AWS.rds(region: region).describe_db_security_groups(db_security_group_name: db_id)
@@ -1731,7 +1719,6 @@ module MU
1731
1719
  end
1732
1720
 
1733
1721
  rdssecgroups << db_id if !secgroup.nil?
1734
- parameter_group = db.db_parameter_groups.first.db_parameter_group_name
1735
1722
 
1736
1723
  # We can use an AWS waiter for this.
1737
1724
  unless db.db_instance_status == "available"
@@ -1823,6 +1810,7 @@ module MU
1823
1810
  groomclass.deleteSecret(vault: db_id.upcase) if !noop
1824
1811
  MU.log "#{db_id} has been terminated"
1825
1812
  end
1813
+ private_class_method :terminate_rds_instance
1826
1814
 
1827
1815
  # Remove an RDS database cluster and associated artifacts
1828
1816
  # @param cluster [OpenStruct]: The cloud provider's description of the database artifact
@@ -1841,9 +1829,6 @@ module MU
1841
1829
  mu_name: mu_name
1842
1830
  ).first
1843
1831
 
1844
- subnet_group = cluster.db_subnet_group
1845
- cluster_parameter_group = cluster.db_cluster_parameter_group
1846
-
1847
1832
  # We can use an AWS waiter for this.
1848
1833
  unless cluster.status == "available"
1849
1834
  loop do
@@ -1914,6 +1899,7 @@ module MU
1914
1899
 
1915
1900
  MU.log "#{cluster_id} has been terminated"
1916
1901
  end
1902
+ private_class_method :terminate_rds_cluster
1917
1903
 
1918
1904
  # Remove a database subnet group.
1919
1905
  # @param subnet_group_id [string]: The cloud provider's ID of the database subnet group.
@@ -1935,6 +1921,7 @@ module MU
1935
1921
  MU.log "#{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1936
1922
  end
1937
1923
  end
1924
+ private_class_method :delete_subnet_group
1938
1925
 
1939
1926
  # Remove a database parameter group.
1940
1927
  # @param parameter_group_id [string]: The cloud provider's ID of the database parameter group.
@@ -1956,6 +1943,7 @@ module MU
1956
1943
  MU.log "DB parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1957
1944
  end
1958
1945
  end
1946
+ private_class_method :delete_db_parameter_group
1959
1947
 
1960
1948
  # Remove a database cluster parameter group.
1961
1949
  # @param parameter_group_id [string]: The cloud provider's ID of the database cluster parameter group.
@@ -1978,6 +1966,7 @@ module MU
1978
1966
  MU.log "Cluster parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1979
1967
  end
1980
1968
  end
1969
+ private_class_method :delete_db_cluster_parameter_group
1981
1970
 
1982
1971
  end #class
1983
1972
  end #class