cloud-mu 3.1.3 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +10 -2
  3. data/bin/mu-adopt +5 -1
  4. data/bin/mu-load-config.rb +2 -3
  5. data/bin/mu-run-tests +112 -27
  6. data/cloud-mu.gemspec +20 -20
  7. data/cookbooks/mu-tools/libraries/helper.rb +2 -1
  8. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  9. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  10. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  11. data/extras/image-generators/Google/centos6.yaml +1 -0
  12. data/extras/image-generators/Google/centos7.yaml +1 -1
  13. data/modules/mommacat.ru +5 -15
  14. data/modules/mu.rb +10 -14
  15. data/modules/mu/adoption.rb +20 -14
  16. data/modules/mu/cleanup.rb +13 -9
  17. data/modules/mu/cloud.rb +26 -26
  18. data/modules/mu/clouds/aws.rb +100 -59
  19. data/modules/mu/clouds/aws/alarm.rb +4 -2
  20. data/modules/mu/clouds/aws/bucket.rb +25 -21
  21. data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
  22. data/modules/mu/clouds/aws/collection.rb +21 -20
  23. data/modules/mu/clouds/aws/container_cluster.rb +47 -26
  24. data/modules/mu/clouds/aws/database.rb +57 -68
  25. data/modules/mu/clouds/aws/dnszone.rb +14 -14
  26. data/modules/mu/clouds/aws/endpoint.rb +20 -16
  27. data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
  28. data/modules/mu/clouds/aws/folder.rb +7 -7
  29. data/modules/mu/clouds/aws/function.rb +15 -12
  30. data/modules/mu/clouds/aws/group.rb +14 -10
  31. data/modules/mu/clouds/aws/habitat.rb +16 -13
  32. data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
  33. data/modules/mu/clouds/aws/log.rb +13 -10
  34. data/modules/mu/clouds/aws/msg_queue.rb +15 -8
  35. data/modules/mu/clouds/aws/nosqldb.rb +18 -11
  36. data/modules/mu/clouds/aws/notifier.rb +11 -6
  37. data/modules/mu/clouds/aws/role.rb +87 -70
  38. data/modules/mu/clouds/aws/search_domain.rb +30 -19
  39. data/modules/mu/clouds/aws/server.rb +102 -72
  40. data/modules/mu/clouds/aws/server_pool.rb +47 -28
  41. data/modules/mu/clouds/aws/storage_pool.rb +5 -6
  42. data/modules/mu/clouds/aws/user.rb +13 -10
  43. data/modules/mu/clouds/aws/vpc.rb +135 -121
  44. data/modules/mu/clouds/azure.rb +16 -9
  45. data/modules/mu/clouds/azure/container_cluster.rb +2 -3
  46. data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
  47. data/modules/mu/clouds/azure/habitat.rb +8 -6
  48. data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
  49. data/modules/mu/clouds/azure/role.rb +8 -10
  50. data/modules/mu/clouds/azure/server.rb +65 -25
  51. data/modules/mu/clouds/azure/user.rb +5 -7
  52. data/modules/mu/clouds/azure/vpc.rb +12 -15
  53. data/modules/mu/clouds/cloudformation.rb +8 -7
  54. data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
  55. data/modules/mu/clouds/google.rb +39 -24
  56. data/modules/mu/clouds/google/bucket.rb +9 -11
  57. data/modules/mu/clouds/google/container_cluster.rb +27 -42
  58. data/modules/mu/clouds/google/database.rb +6 -9
  59. data/modules/mu/clouds/google/firewall_rule.rb +11 -10
  60. data/modules/mu/clouds/google/folder.rb +16 -9
  61. data/modules/mu/clouds/google/function.rb +127 -161
  62. data/modules/mu/clouds/google/group.rb +21 -18
  63. data/modules/mu/clouds/google/habitat.rb +18 -15
  64. data/modules/mu/clouds/google/loadbalancer.rb +14 -16
  65. data/modules/mu/clouds/google/role.rb +48 -31
  66. data/modules/mu/clouds/google/server.rb +105 -105
  67. data/modules/mu/clouds/google/server_pool.rb +12 -31
  68. data/modules/mu/clouds/google/user.rb +67 -13
  69. data/modules/mu/clouds/google/vpc.rb +58 -65
  70. data/modules/mu/config.rb +89 -1738
  71. data/modules/mu/config/bucket.rb +3 -3
  72. data/modules/mu/config/collection.rb +3 -3
  73. data/modules/mu/config/container_cluster.rb +2 -2
  74. data/modules/mu/config/dnszone.rb +5 -5
  75. data/modules/mu/config/doc_helpers.rb +517 -0
  76. data/modules/mu/config/endpoint.rb +3 -3
  77. data/modules/mu/config/firewall_rule.rb +118 -3
  78. data/modules/mu/config/folder.rb +3 -3
  79. data/modules/mu/config/function.rb +2 -2
  80. data/modules/mu/config/group.rb +3 -3
  81. data/modules/mu/config/habitat.rb +3 -3
  82. data/modules/mu/config/loadbalancer.rb +3 -3
  83. data/modules/mu/config/log.rb +3 -3
  84. data/modules/mu/config/msg_queue.rb +3 -3
  85. data/modules/mu/config/nosqldb.rb +3 -3
  86. data/modules/mu/config/notifier.rb +2 -2
  87. data/modules/mu/config/ref.rb +333 -0
  88. data/modules/mu/config/role.rb +3 -3
  89. data/modules/mu/config/schema_helpers.rb +508 -0
  90. data/modules/mu/config/search_domain.rb +3 -3
  91. data/modules/mu/config/server.rb +86 -58
  92. data/modules/mu/config/server_pool.rb +2 -2
  93. data/modules/mu/config/tail.rb +189 -0
  94. data/modules/mu/config/user.rb +3 -3
  95. data/modules/mu/config/vpc.rb +44 -4
  96. data/modules/mu/defaults/Google.yaml +2 -2
  97. data/modules/mu/deploy.rb +13 -10
  98. data/modules/mu/groomer.rb +1 -1
  99. data/modules/mu/groomers/ansible.rb +69 -24
  100. data/modules/mu/groomers/chef.rb +52 -44
  101. data/modules/mu/logger.rb +17 -14
  102. data/modules/mu/master.rb +317 -2
  103. data/modules/mu/master/chef.rb +3 -4
  104. data/modules/mu/master/ldap.rb +3 -3
  105. data/modules/mu/master/ssl.rb +12 -2
  106. data/modules/mu/mommacat.rb +85 -1766
  107. data/modules/mu/mommacat/daemon.rb +394 -0
  108. data/modules/mu/mommacat/naming.rb +366 -0
  109. data/modules/mu/mommacat/storage.rb +689 -0
  110. data/modules/tests/bucket.yml +4 -0
  111. data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
  112. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  113. data/modules/tests/regrooms/bucket.yml +19 -0
  114. metadata +112 -102
@@ -181,8 +181,8 @@ module MU
181
181
  main_sg = @deploy.findLitterMate(type: "firewall_rules", name: "server_pool#{@config['name']}workers")
182
182
  tagme << main_sg.cloud_id if main_sg
183
183
  MU.log "Applying kubernetes.io tags to VPC resources", details: tagme
184
- MU::Cloud::AWS.createTag("kubernetes.io/cluster/#{@mu_name}", "shared", tagme, credentials: @config['credentials'])
185
- MU::Cloud::AWS.createTag("kubernetes.io/cluster/elb", @mu_name, tagme_elb, credentials: @config['credentials'])
184
+ MU::Cloud::AWS.createTag(tagme, "kubernetes.io/cluster/#{@mu_name}", "shared", credentials: @config['credentials'])
185
+ MU::Cloud::AWS.createTag(tagme_elb, "kubernetes.io/cluster/elb", @mu_name, credentials: @config['credentials'])
186
186
 
187
187
  if @config['flavor'] == "Fargate"
188
188
  fargate_subnets = []
@@ -268,7 +268,18 @@ module MU
268
268
  authmap_cmd = %Q{#{MU::Master.kubectl} --kubeconfig "#{kube_conf}" apply -f "#{eks_auth}"}
269
269
  MU.log "Configuring Kubernetes <=> IAM mapping for worker nodes", MU::NOTICE, details: authmap_cmd
270
270
  # maybe guard this mess
271
- %x{#{authmap_cmd}}
271
+ retries = 0
272
+ begin
273
+ puts %x{#{authmap_cmd}}
274
+ if $?.exitstatus != 0
275
+ if retries >= 10
276
+ raise MuError, "Failed to apply #{authmap_cmd}"
277
+ end
278
+ sleep 10
279
+ retries += 1
280
+ end
281
+ end while $?.exitstatus != 0
282
+
272
283
  end
273
284
 
274
285
  # and this one
@@ -309,21 +320,22 @@ module MU
309
320
  end
310
321
  end
311
322
 
312
- serverpool.listNodes.each { |node|
323
+ threads = []
324
+ serverpool.listNodes.each { |mynode|
313
325
  resources = resource_lookup[node.cloud_desc.instance_type]
314
- t = Thread.new {
326
+ threads << Thread.new(mynode) { |node|
315
327
  ident_doc = nil
316
328
  ident_doc_sig = nil
317
329
  if !node.windows?
318
330
  session = node.getSSHSession(10, 30)
319
331
  ident_doc = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/document/")
320
332
  ident_doc_sig = session.exec!("curl -s http://169.254.169.254/latest/dynamic/instance-identity/signature/")
321
- else
322
- begin
323
- session = node.getWinRMSession(1, 60)
324
- rescue Exception # XXX
325
- session = node.getSSHSession(1, 60)
326
- end
333
+ # else
334
+ # begin
335
+ # session = node.getWinRMSession(1, 60)
336
+ # rescue StandardError # XXX
337
+ # session = node.getSSHSession(1, 60)
338
+ # end
327
339
  end
328
340
  MU.log "Identity document for #{node}", MU::DEBUG, details: ident_doc
329
341
  MU.log "Identity document signature for #{node}", MU::DEBUG, details: ident_doc_sig
@@ -354,6 +366,9 @@ module MU
354
366
 
355
367
  }
356
368
  }
369
+ threads.each { |t|
370
+ t.join
371
+ }
357
372
  end
358
373
 
359
374
  if @config['flavor'] != "EKS" and @config['containers']
@@ -387,7 +402,6 @@ module MU
387
402
  # Reorganize things so that we have services and task definitions
388
403
  # mapped to the set of containers they must contain
389
404
  tasks = {}
390
- created_generic_loggroup = false
391
405
 
392
406
  @config['containers'].each { |c|
393
407
  service_name = c['service'] ? @mu_name+"-"+c['service'].upcase : @mu_name
@@ -653,7 +667,7 @@ module MU
653
667
  listme = services.slice!(0, (services.length >= 10 ? 10 : services.length))
654
668
  if services.size > 0
655
669
  tasks_defined.concat(
656
- tasks = MU::Cloud::AWS.ecs(region: region, credentials: credentials).describe_services(
670
+ MU::Cloud::AWS.ecs(region: region, credentials: credentials).describe_services(
657
671
  cluster: cluster,
658
672
  services: listme
659
673
  ).services.map { |s| s.task_definition }
@@ -693,7 +707,6 @@ module MU
693
707
  cluster: cluster,
694
708
  tasks: task_ids
695
709
  ).tasks.each { |t|
696
- task_name = t.task_definition_arn.sub(/^.*?:task-definition\/([^\/:]+)$/, '\1')
697
710
  t.containers.each { |c|
698
711
  containers[c.name] ||= {}
699
712
  containers[c.name][t.desired_status] ||= {
@@ -736,10 +749,12 @@ MU.log c.name, MU::NOTICE, details: t
736
749
  to_return
737
750
  end
738
751
 
752
+ @cloud_desc_cache = nil
739
753
  # Return the cloud layer descriptor for this EKS/ECS/Fargate cluster
740
754
  # @return [OpenStruct]
741
- def cloud_desc
742
- if @config['flavor'] == "EKS" or
755
+ def cloud_desc(use_cache: true)
756
+ return @cloud_desc_cache if @cloud_desc_cache and use_cache
757
+ @cloud_desc_cache = if @config['flavor'] == "EKS" or
743
758
  (@config['flavor'] == "Fargate" and !@config['containers'])
744
759
  resp = MU::Cloud::AWS.eks(region: @config['region'], credentials: @config['credentials']).describe_cluster(
745
760
  name: @cloud_id
@@ -751,6 +766,7 @@ MU.log c.name, MU::NOTICE, details: t
751
766
  )
752
767
  resp.clusters.first
753
768
  end
769
+ @cloud_desc_cache
754
770
  end
755
771
 
756
772
  # Canonical Amazon Resource Number for this resource
@@ -857,8 +873,10 @@ MU.log c.name, MU::NOTICE, details: t
857
873
  # @param region [String]: The cloud provider region
858
874
  # @return [void]
859
875
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
860
- resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters
876
+ MU.log "AWS::ContainerCluster.cleanup: need to support flags['known']", MU::DEBUG, details: flags
877
+ MU.log "Placeholder: AWS ContainerCluster artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
861
878
 
879
+ resp = MU::Cloud::AWS.ecs(credentials: credentials, region: region).list_clusters
862
880
 
863
881
  if resp and resp.cluster_arns and resp.cluster_arns.size > 0
864
882
  resp.cluster_arns.each { |arn|
@@ -902,10 +920,10 @@ MU.log c.name, MU::NOTICE, details: t
902
920
  if !noop
903
921
  # TODO de-register container instances
904
922
  begin
905
- deletion = MU::Cloud::AWS.ecs(credentials: credentials, region: region).delete_cluster(
906
- cluster: cluster
907
- )
908
- rescue Aws::ECS::Errors::ClusterContainsTasksException => e
923
+ MU::Cloud::AWS.ecs(credentials: credentials, region: region).delete_cluster(
924
+ cluster: cluster
925
+ )
926
+ rescue Aws::ECS::Errors::ClusterContainsTasksException
909
927
  sleep 5
910
928
  retry
911
929
  end
@@ -1059,9 +1077,9 @@ MU.log c.name, MU::NOTICE, details: t
1059
1077
  end
1060
1078
 
1061
1079
  # Cloud-specific configuration properties.
1062
- # @param config [MU::Config]: The calling MU::Config object
1080
+ # @param _config [MU::Config]: The calling MU::Config object
1063
1081
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
1064
- def self.schema(config)
1082
+ def self.schema(_config)
1065
1083
  toplevel_required = []
1066
1084
 
1067
1085
  schema = {
@@ -1950,7 +1968,7 @@ MU.log c.name, MU::NOTICE, details: t
1950
1968
  "name" => cluster['name']
1951
1969
  }
1952
1970
  ]
1953
- worker_pool["run_list"] = ["mu-tools::eks"]
1971
+ worker_pool["run_list"] = ["recipe[mu-tools::eks]"]
1954
1972
  worker_pool["run_list"].concat(cluster["run_list"]) if cluster["run_list"]
1955
1973
  MU::Config::Server.common_properties.keys.each { |k|
1956
1974
  if cluster[k] and !worker_pool[k]
@@ -1998,8 +2016,11 @@ MU.log c.name, MU::NOTICE, details: t
1998
2016
  ok
1999
2017
  end
2000
2018
 
2001
- private
2002
-
2019
+ # Delete a Fargate profile, needed both for cleanup and regroom updates
2020
+ # @param profile [String]:
2021
+ # @param cluster [String]:
2022
+ # @param region [String]:
2023
+ # @param credentials [String]:
2003
2024
  def self.purge_fargate_profile(profile, cluster, region, credentials)
2004
2025
  check = begin
2005
2026
  MU::Cloud::AWS.eks(region: region, credentials: credentials).delete_fargate_profile(
@@ -331,22 +331,22 @@ module MU
331
331
  begin
332
332
  if %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
333
333
  MU.log "Creating database instance #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
334
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_from_db_snapshot(config)
334
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_from_db_snapshot(config)
335
335
  elsif @config["creation_style"] == "point_in_time"
336
336
  MU.log "Creating database instance #{@config['identifier']} based on point in time backup #{@config['restore_time']} of #{@config['source_identifier']}"
337
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_to_point_in_time(point_in_time_config)
337
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_instance_to_point_in_time(point_in_time_config)
338
338
  elsif @config["read_replica_of"]
339
339
  MU.log "Creating read replica database instance #{@config['identifier']} for #{@config['source_identifier']}"
340
340
  begin
341
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
341
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
342
342
  rescue Aws::RDS::Errors::DBSubnetGroupNotAllowedFault => e
343
343
  MU.log "Being forced to use source database's subnet group: #{e.message}", MU::WARN
344
344
  read_replica_struct.delete(:db_subnet_group_name)
345
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
345
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance_read_replica(read_replica_struct)
346
346
  end
347
347
  elsif @config["creation_style"] == "new"
348
348
  MU.log "Creating pristine database instance #{@config['identifier']} (#{@config['name']}) in #{@config['region']}"
349
- resp = MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance(config)
349
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_instance(config)
350
350
  end
351
351
  rescue Aws::RDS::Errors::InvalidParameterValue => e
352
352
  if attempts < 5
@@ -369,7 +369,7 @@ module MU
369
369
  waiter.before_attempt do |w_attempts|
370
370
  MU.log "Waiting for RDS database #{@config['identifier']} to be ready...", MU::NOTICE if w_attempts % 10 == 0
371
371
  end
372
- waiter.before_wait do |w_attempts, r|
372
+ waiter.before_wait do |_attempts, r|
373
373
  throw :success if r.db_instances.first.db_instance_status == "available"
374
374
  throw :failure if Time.now - wait_start_time > 3600
375
375
  end
@@ -438,7 +438,7 @@ module MU
438
438
  waiter.before_attempt do |w_attempts|
439
439
  MU.log "Waiting for RDS database #{@config['identifier'] } to be ready..", MU::NOTICE if w_attempts % 10 == 0
440
440
  end
441
- waiter.before_wait do |w_attempts, r|
441
+ waiter.before_wait do |_attempts, r|
442
442
  throw :success if r.db_instances.first.db_instance_status == "available"
443
443
  throw :failure if Time.now - wait_start_time > 2400
444
444
  end
@@ -528,17 +528,16 @@ module MU
528
528
 
529
529
  attempts = 0
530
530
  begin
531
- resp =
532
- if @config["creation_style"] == "new"
533
- MU.log "Creating new database cluster #{@config['identifier']}"
534
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster(cluster_config_struct)
535
- elsif %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
536
- MU.log "Creating new database cluster #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
537
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_from_snapshot(cluster_config_struct)
538
- elsif @config["creation_style"] == "point_in_time"
539
- MU.log "Creating new database cluster #{@config['identifier']} from point in time backup #{@config["restore_time"]} of #{@config["source_identifier"]}"
540
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_to_point_in_time(cluster_config_struct)
541
- end
531
+ if @config["creation_style"] == "new"
532
+ MU.log "Creating new database cluster #{@config['identifier']}"
533
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster(cluster_config_struct)
534
+ elsif %w{existing_snapshot new_snapshot}.include?(@config["creation_style"])
535
+ MU.log "Creating new database cluster #{@config['identifier']} from snapshot #{@config["snapshot_id"]}"
536
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_from_snapshot(cluster_config_struct)
537
+ elsif @config["creation_style"] == "point_in_time"
538
+ MU.log "Creating new database cluster #{@config['identifier']} from point in time backup #{@config["restore_time"]} of #{@config["source_identifier"]}"
539
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).restore_db_cluster_to_point_in_time(cluster_config_struct)
540
+ end
542
541
  rescue Aws::RDS::Errors::InvalidParameterValue => e
543
542
  if attempts < 5
544
543
  MU.log "Got #{e.inspect} while creating database cluster #{@config['identifier']}, will retry a few times in case of transient errors.", MU::WARN, details: cluster_config_struct
@@ -651,7 +650,6 @@ module MU
651
650
  }
652
651
  # Default VPC has only public subnets by default so setting publicly_accessible = true
653
652
  @config["publicly_accessible"] = true
654
- using_default_vpc = true
655
653
  MU.log "Using default VPC for cache cluster #{@config['identifier']}"
656
654
  end
657
655
  end
@@ -693,8 +691,8 @@ module MU
693
691
  if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
694
692
  MU.log "Using NAT Gateway, not modifying security groups"
695
693
  else
696
- nat_name, nat_conf, nat_deploydata = @nat.describe
697
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
694
+ _nat_name, _nat_conf, nat_deploydata = @nat.describe
695
+ @deploy.kittens['firewall_rules'].values.each { |acl|
698
696
  # XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
699
697
  if acl.config["admin"]
700
698
  acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
@@ -763,7 +761,7 @@ module MU
763
761
  MU::Cloud::AWS.rds(region: region).describe_db_cluster_parameter_groups(db_cluster_parameter_group_name: param_group_id).db_cluster_parameter_groups.first
764
762
  # rescue DBClusterParameterGroupNotFound => e
765
763
  # Of course the API will return DBParameterGroupNotFound instead of the documented DBClusterParameterGroupNotFound error.
766
- rescue Aws::RDS::Errors::DBParameterGroupNotFound => e
764
+ rescue Aws::RDS::Errors::DBParameterGroupNotFound
767
765
  #we're fine returning nil
768
766
  end
769
767
 
@@ -773,7 +771,7 @@ module MU
773
771
  # @return [OpenStruct]
774
772
  def self.getDBParameterGroup(param_group_id, region: MU.curRegion)
775
773
  MU::Cloud::AWS.rds(region: region).describe_db_parameter_groups(db_parameter_group_name: param_group_id).db_parameter_groups.first
776
- rescue Aws::RDS::Errors::DBParameterGroupNotFound => e
774
+ rescue Aws::RDS::Errors::DBParameterGroupNotFound
777
775
  #we're fine returning nil
778
776
  end
779
777
 
@@ -783,7 +781,7 @@ module MU
783
781
  # @return [OpenStruct]
784
782
  def self.getSubnetGroup(subnet_id, region: MU.curRegion)
785
783
  MU::Cloud::AWS.rds(region: region).describe_db_subnet_groups(db_subnet_group_name: subnet_id).db_subnet_groups.first
786
- rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault => e
784
+ rescue Aws::RDS::Errors::DBSubnetGroupNotFoundFault
787
785
  #we're fine returning nil
788
786
  end
789
787
 
@@ -815,11 +813,10 @@ module MU
815
813
 
816
814
  #Setting up connection params
817
815
  ssh_keydir = Etc.getpwuid(Process.uid).dir+"/.ssh"
818
- keypairname, ssh_private_key, ssh_public_key = @deploy.SSHKey
816
+ keypairname, _ssh_private_key, _ssh_public_key = @deploy.SSHKey
819
817
  if is_private and @vpc
820
818
  if @config['vpc']['nat_host_name']
821
819
  begin
822
- proxy_cmd = "ssh -q -o StrictHostKeyChecking=no -W %h:%p #{nat_ssh_user}@#{nat_host_name}"
823
820
  gateway = Net::SSH::Gateway.new(
824
821
  @config['vpc']['nat_host_name'],
825
822
  @config['vpc']['nat_ssh_user'],
@@ -953,7 +950,7 @@ module MU
953
950
  db_security_group_name: rds_sg.db_security_group_name,
954
951
  cidrip: cidr
955
952
  )
956
- rescue Aws::RDS::Errors::AuthorizationAlreadyExists => e
953
+ rescue Aws::RDS::Errors::AuthorizationAlreadyExists
957
954
  MU.log "CIDR #{cidr} already in database instance #{@cloud_id} security group", MU::WARN
958
955
  end
959
956
  }
@@ -975,7 +972,7 @@ module MU
975
972
  def self.getDatabaseById(db_id, region: MU.curRegion, credentials: nil)
976
973
  raise MuError, "You must provide a db_id" if db_id.nil?
977
974
  MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_instances(db_instance_identifier: db_id).db_instances.first
978
- rescue Aws::RDS::Errors::DBInstanceNotFound => e
975
+ rescue Aws::RDS::Errors::DBInstanceNotFound
979
976
  # We're fine with this returning nil when searching for a database instance the doesn't exist.
980
977
  end
981
978
 
@@ -985,7 +982,7 @@ module MU
985
982
  # @return [OpenStruct]
986
983
  def self.getDatabaseClusterById(db_cluster_id, region: MU.curRegion, credentials: nil)
987
984
  MU::Cloud::AWS.rds(region: region, credentials: credentials).describe_db_clusters(db_cluster_identifier: db_cluster_id).db_clusters.first
988
- rescue Aws::RDS::Errors::DBClusterNotFoundFault => e
985
+ rescue Aws::RDS::Errors::DBClusterNotFoundFault
989
986
  # We're fine with this returning nil when searching for a database cluster the doesn't exist.
990
987
  end
991
988
 
@@ -1125,20 +1122,19 @@ module MU
1125
1122
 
1126
1123
  attempts = 0
1127
1124
  begin
1128
- snapshot =
1129
- if @config["create_cluster"]
1130
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster_snapshot(
1131
- db_cluster_snapshot_identifier: snap_id,
1132
- db_cluster_identifier: @config["identifier"],
1133
- tags: allTags
1134
- )
1135
- else
1136
- MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_snapshot(
1137
- db_snapshot_identifier: snap_id,
1138
- db_instance_identifier: @config["identifier"],
1139
- tags: allTags
1140
- )
1141
- end
1125
+ if @config["create_cluster"]
1126
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_cluster_snapshot(
1127
+ db_cluster_snapshot_identifier: snap_id,
1128
+ db_cluster_identifier: @config["identifier"],
1129
+ tags: allTags
1130
+ )
1131
+ else
1132
+ MU::Cloud::AWS.rds(region: @config['region'], credentials: @config['credentials']).create_db_snapshot(
1133
+ db_snapshot_identifier: snap_id,
1134
+ db_instance_identifier: @config["identifier"],
1135
+ tags: allTags
1136
+ )
1137
+ end
1142
1138
  rescue Aws::RDS::Errors::InvalidDBInstanceState, Aws::RDS::Errors::InvalidDBClusterStateFault => e
1143
1139
  raise MuError, e.inspect if attempts >= 10
1144
1140
  attempts += 1
@@ -1215,7 +1211,6 @@ module MU
1215
1211
  threads = []
1216
1212
 
1217
1213
  resp.db_instances.each { |db|
1218
- db_id = db.db_instance_identifier
1219
1214
  arn = MU::Cloud::AWS::Database.getARN(db.db_instance_identifier, "db", "rds", region: region, credentials: credentials)
1220
1215
  tags = MU::Cloud::AWS.rds(credentials: credentials, region: region).list_tags_for_resource(resource_name: arn).tag_list
1221
1216
 
@@ -1241,7 +1236,7 @@ module MU
1241
1236
  threads << Thread.new(db) { |mydb|
1242
1237
  MU.dupGlobals(parent_thread_id)
1243
1238
  Thread.abort_on_exception = true
1244
- MU::Cloud::AWS::Database.terminate_rds_instance(mydb, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: db.db_instance_identifier, mu_name: db.db_instance_identifier.upcase, credentials: credentials)
1239
+ terminate_rds_instance(mydb, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: db.db_instance_identifier, mu_name: db.db_instance_identifier.upcase, credentials: credentials)
1245
1240
  }
1246
1241
  end
1247
1242
  }
@@ -1281,7 +1276,7 @@ module MU
1281
1276
  threads << Thread.new(cluster) { |mydbcluster|
1282
1277
  MU.dupGlobals(parent_thread_id)
1283
1278
  Thread.abort_on_exception = true
1284
- MU::Cloud::AWS::Database.terminate_rds_cluster(mydbcluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: cluster_id, mu_name: cluster_id.upcase, credentials: credentials)
1279
+ terminate_rds_cluster(mydbcluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: cluster_id, mu_name: cluster_id.upcase, credentials: credentials)
1285
1280
  }
1286
1281
  end
1287
1282
  }
@@ -1317,10 +1312,10 @@ module MU
1317
1312
 
1318
1313
  if delete
1319
1314
  parent_thread_id = Thread.current.object_id
1320
- threads << Thread.new(sub_group) { |mysubgroup|
1315
+ threads << Thread.new(sub_group_id) { |mysubgroup|
1321
1316
  MU.dupGlobals(parent_thread_id)
1322
1317
  Thread.abort_on_exception = true
1323
- MU::Cloud::AWS::Database.delete_subnet_group(sub_group_id, region: region) unless noop
1318
+ delete_subnet_group(mysubgroup, region: region) unless noop
1324
1319
  }
1325
1320
  end
1326
1321
  }
@@ -1350,10 +1345,10 @@ module MU
1350
1345
 
1351
1346
  if delete
1352
1347
  parent_thread_id = Thread.current.object_id
1353
- threads << Thread.new(param_group) { |myparamgroup|
1348
+ threads << Thread.new(param_group_id) { |myparamgroup|
1354
1349
  MU.dupGlobals(parent_thread_id)
1355
1350
  Thread.abort_on_exception = true
1356
- MU::Cloud::AWS::Database.delete_db_parameter_group(param_group_id, region: region) unless noop
1351
+ delete_db_parameter_group(myparamgroup, region: region) unless noop
1357
1352
  }
1358
1353
  end
1359
1354
  }
@@ -1383,10 +1378,10 @@ module MU
1383
1378
 
1384
1379
  if delete
1385
1380
  parent_thread_id = Thread.current.object_id
1386
- threads << Thread.new(param_group) { |myparamgroup|
1381
+ threads << Thread.new(param_group_id) { |myparamgroup|
1387
1382
  MU.dupGlobals(parent_thread_id)
1388
1383
  Thread.abort_on_exception = true
1389
- MU::Cloud::AWS::Database.delete_db_cluster_parameter_group(param_group_id, region: region) unless noop
1384
+ delete_db_cluster_parameter_group(myparamgroup, region: region) unless noop
1390
1385
  }
1391
1386
  end
1392
1387
  }
@@ -1398,9 +1393,9 @@ module MU
1398
1393
  end
1399
1394
 
1400
1395
  # Cloud-specific configuration properties.
1401
- # @param config [MU::Config]: The calling MU::Config object
1396
+ # @param _config [MU::Config]: The calling MU::Config object
1402
1397
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
1403
- def self.schema(config)
1398
+ def self.schema(_config)
1404
1399
  toplevel_required = []
1405
1400
  rds_parameters_primitive = {
1406
1401
  "type" => "array",
@@ -1512,9 +1507,9 @@ module MU
1512
1507
 
1513
1508
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::databases}, bare and unvalidated.
1514
1509
  # @param db [Hash]: The resource to process and validate
1515
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1510
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
1516
1511
  # @return [Boolean]: True if validation succeeded, False otherwise
1517
- def self.validateConfig(db, configurator)
1512
+ def self.validateConfig(db, _configurator)
1518
1513
  ok = true
1519
1514
 
1520
1515
  if db['creation_style'] == "existing_snapshot" and
@@ -1623,7 +1618,7 @@ module MU
1623
1618
  MU::Cloud::AWS.rds(region: db['region']).describe_db_instances(
1624
1619
  db_instance_identifier: db['identifier']
1625
1620
  )
1626
- rescue Aws::RDS::Errors::DBInstanceNotFound => e
1621
+ rescue Aws::RDS::Errors::DBInstanceNotFound
1627
1622
  MU.log "Source database #{db['identifier']} was specified for #{db['name']}, but no such database exists in #{db['region']}", MU::ERR
1628
1623
  ok = false
1629
1624
  end
@@ -1716,13 +1711,6 @@ module MU
1716
1711
  mu_name: mu_name
1717
1712
  ).first
1718
1713
 
1719
- subnet_group = nil
1720
- begin
1721
- subnet_group = db.db_subnet_group.db_subnet_group_name if db.db_subnet_group
1722
- rescue NoMethodError
1723
- # ignorable for non-VPC databases
1724
- end
1725
-
1726
1714
  rdssecgroups = Array.new
1727
1715
  begin
1728
1716
  secgroup = MU::Cloud::AWS.rds(region: region).describe_db_security_groups(db_security_group_name: db_id)
@@ -1731,7 +1719,6 @@ module MU
1731
1719
  end
1732
1720
 
1733
1721
  rdssecgroups << db_id if !secgroup.nil?
1734
- parameter_group = db.db_parameter_groups.first.db_parameter_group_name
1735
1722
 
1736
1723
  # We can use an AWS waiter for this.
1737
1724
  unless db.db_instance_status == "available"
@@ -1823,6 +1810,7 @@ module MU
1823
1810
  groomclass.deleteSecret(vault: db_id.upcase) if !noop
1824
1811
  MU.log "#{db_id} has been terminated"
1825
1812
  end
1813
+ private_class_method :terminate_rds_instance
1826
1814
 
1827
1815
  # Remove an RDS database cluster and associated artifacts
1828
1816
  # @param cluster [OpenStruct]: The cloud provider's description of the database artifact
@@ -1841,9 +1829,6 @@ module MU
1841
1829
  mu_name: mu_name
1842
1830
  ).first
1843
1831
 
1844
- subnet_group = cluster.db_subnet_group
1845
- cluster_parameter_group = cluster.db_cluster_parameter_group
1846
-
1847
1832
  # We can use an AWS waiter for this.
1848
1833
  unless cluster.status == "available"
1849
1834
  loop do
@@ -1914,6 +1899,7 @@ module MU
1914
1899
 
1915
1900
  MU.log "#{cluster_id} has been terminated"
1916
1901
  end
1902
+ private_class_method :terminate_rds_cluster
1917
1903
 
1918
1904
  # Remove a database subnet group.
1919
1905
  # @param subnet_group_id [string]: The cloud provider's ID of the database subnet group.
@@ -1935,6 +1921,7 @@ module MU
1935
1921
  MU.log "#{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1936
1922
  end
1937
1923
  end
1924
+ private_class_method :delete_subnet_group
1938
1925
 
1939
1926
  # Remove a database parameter group.
1940
1927
  # @param parameter_group_id [string]: The cloud provider's ID of the database parameter group.
@@ -1956,6 +1943,7 @@ module MU
1956
1943
  MU.log "DB parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1957
1944
  end
1958
1945
  end
1946
+ private_class_method :delete_db_parameter_group
1959
1947
 
1960
1948
  # Remove a database cluster parameter group.
1961
1949
  # @param parameter_group_id [string]: The cloud provider's ID of the database cluster parameter group.
@@ -1978,6 +1966,7 @@ module MU
1978
1966
  MU.log "Cluster parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1979
1967
  end
1980
1968
  end
1969
+ private_class_method :delete_db_cluster_parameter_group
1981
1970
 
1982
1971
  end #class
1983
1972
  end #class