cloud-mu 3.1.2 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +10 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -3
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +135 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +165 -111
  45. data/modules/mu/adoption.rb +401 -68
  46. data/modules/mu/cleanup.rb +199 -306
  47. data/modules/mu/cloud.rb +100 -1632
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +46 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +920 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +165 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +4 -4
  61. data/modules/mu/config/cache_cluster.rb +1 -1
  62. data/modules/mu/config/collection.rb +4 -4
  63. data/modules/mu/config/container_cluster.rb +9 -4
  64. data/modules/mu/config/database.rb +83 -104
  65. data/modules/mu/config/database.yml +1 -2
  66. data/modules/mu/config/dnszone.rb +6 -6
  67. data/modules/mu/config/doc_helpers.rb +516 -0
  68. data/modules/mu/config/endpoint.rb +4 -4
  69. data/modules/mu/config/firewall_rule.rb +103 -4
  70. data/modules/mu/config/folder.rb +4 -4
  71. data/modules/mu/config/function.rb +3 -3
  72. data/modules/mu/config/group.rb +4 -4
  73. data/modules/mu/config/habitat.rb +4 -4
  74. data/modules/mu/config/loadbalancer.rb +60 -14
  75. data/modules/mu/config/log.rb +4 -4
  76. data/modules/mu/config/msg_queue.rb +4 -4
  77. data/modules/mu/config/nosqldb.rb +4 -4
  78. data/modules/mu/config/notifier.rb +3 -3
  79. data/modules/mu/config/ref.rb +365 -0
  80. data/modules/mu/config/role.rb +4 -4
  81. data/modules/mu/config/schema_helpers.rb +509 -0
  82. data/modules/mu/config/search_domain.rb +4 -4
  83. data/modules/mu/config/server.rb +97 -70
  84. data/modules/mu/config/server.yml +1 -0
  85. data/modules/mu/config/server_pool.rb +5 -9
  86. data/modules/mu/config/storage_pool.rb +1 -1
  87. data/modules/mu/config/tail.rb +200 -0
  88. data/modules/mu/config/user.rb +4 -4
  89. data/modules/mu/config/vpc.rb +70 -27
  90. data/modules/mu/config/vpc.yml +0 -1
  91. data/modules/mu/defaults/AWS.yaml +83 -60
  92. data/modules/mu/defaults/Azure.yaml +1 -0
  93. data/modules/mu/defaults/Google.yaml +3 -2
  94. data/modules/mu/deploy.rb +30 -26
  95. data/modules/mu/groomer.rb +17 -2
  96. data/modules/mu/groomers/ansible.rb +188 -41
  97. data/modules/mu/groomers/chef.rb +116 -55
  98. data/modules/mu/logger.rb +127 -148
  99. data/modules/mu/master.rb +389 -2
  100. data/modules/mu/master/chef.rb +3 -4
  101. data/modules/mu/master/ldap.rb +3 -3
  102. data/modules/mu/master/ssl.rb +12 -3
  103. data/modules/mu/mommacat.rb +217 -2612
  104. data/modules/mu/mommacat/daemon.rb +397 -0
  105. data/modules/mu/mommacat/naming.rb +473 -0
  106. data/modules/mu/mommacat/search.rb +495 -0
  107. data/modules/mu/mommacat/storage.rb +722 -0
  108. data/modules/mu/{clouds → providers}/README.md +1 -1
  109. data/modules/mu/{clouds → providers}/aws.rb +271 -112
  110. data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
  111. data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
  112. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
  113. data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
  114. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
  115. data/modules/mu/providers/aws/database.rb +1744 -0
  116. data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
  117. data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
  118. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
  119. data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
  120. data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
  121. data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
  122. data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
  123. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
  124. data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
  125. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
  126. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
  127. data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
  128. data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
  129. data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
  130. data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
  131. data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
  132. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
  133. data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
  134. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  135. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  136. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  137. data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
  138. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  139. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  140. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  141. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  142. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  143. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  144. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  145. data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
  146. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  147. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  148. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  149. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  150. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  151. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  152. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  153. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  156. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  160. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  161. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  162. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  163. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  164. data/modules/mu/{clouds → providers}/google.rb +67 -30
  165. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  166. data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
  167. data/modules/mu/{clouds → providers}/google/database.rb +10 -20
  168. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  169. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  170. data/modules/mu/{clouds → providers}/google/function.rb +139 -167
  171. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  172. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  173. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
  174. data/modules/mu/{clouds → providers}/google/role.rb +92 -58
  175. data/modules/mu/{clouds → providers}/google/server.rb +242 -155
  176. data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
  177. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  178. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  179. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  180. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  181. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  182. data/modules/tests/bucket.yml +4 -0
  183. data/modules/tests/centos6.yaml +11 -0
  184. data/modules/tests/centos7.yaml +11 -0
  185. data/modules/tests/centos8.yaml +12 -0
  186. data/modules/tests/ecs.yaml +23 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/rds.yaml +108 -0
  189. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  190. data/modules/tests/regrooms/bucket.yml +19 -0
  191. data/modules/tests/regrooms/rds.yaml +123 -0
  192. data/modules/tests/server-with-scrub-muisms.yaml +1 -0
  193. data/modules/tests/super_simple_bok.yml +1 -3
  194. data/modules/tests/win2k12.yaml +17 -5
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +232 -154
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1985
@@ -125,6 +125,8 @@ module MU
125
125
  # @param region [String]: The cloud provider region
126
126
  # @return [void]
127
127
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
128
+ MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags
129
+ MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
128
130
  alarms = []
129
131
  # We don't have a way to tag alarms, so we try to delete them by the deploy ID.
130
132
  # This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well.
@@ -252,9 +254,9 @@ module MU
252
254
  end
253
255
 
254
256
  # Cloud-specific configuration properties.
255
- # @param config [MU::Config]: The calling MU::Config object
257
+ # @param _config [MU::Config]: The calling MU::Config object
256
258
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
257
- def self.schema(config)
259
+ def self.schema(_config)
258
260
  toplevel_required = []
259
261
  schema = {}
260
262
  [toplevel_required, schema]
@@ -319,7 +321,7 @@ module MU
319
321
  if !depclass.nil?
320
322
  dimension["depclass"] = depclass
321
323
  if !dimension["name"].nil? and !dimension["name"].empty?
322
- alarm["dependencies"] << { "name" => dimension["name"], "type" => depclass }
324
+ MU::Config.addDependency(alarm, dimension["name"], depclass)
323
325
  end
324
326
  end
325
327
  }
@@ -33,7 +33,7 @@ module MU
33
33
  bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
34
34
 
35
35
  MU.log "Creating S3 bucket #{bucket_name}"
36
- resp = MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
36
+ MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
37
37
  acl: @config['acl'],
38
38
  bucket: bucket_name
39
39
  )
@@ -90,7 +90,6 @@ module MU
90
90
  tagBucket if !@config['scrub_mu_isms']
91
91
 
92
92
  current = cloud_desc
93
-
94
93
  if @config['policies']
95
94
  @config['policies'].each { |pol|
96
95
  pol['grant_to'] ||= [
@@ -98,9 +97,9 @@ module MU
98
97
  ]
99
98
  }
100
99
 
101
- policy_docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
100
+ policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true)
102
101
  policy_docs.each { |doc|
103
- MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first
102
+ MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first)
104
103
  MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
105
104
  bucket: @cloud_id,
106
105
  policy: JSON.generate(doc.values.first)
@@ -177,10 +176,6 @@ module MU
177
176
  end
178
177
 
179
178
  begin
180
- puts data
181
- puts acl
182
- puts bucket
183
- puts path
184
179
  MU.log "Writing #{path} to S3 bucket #{bucket}"
185
180
  MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(
186
181
  acl: acl,
@@ -213,6 +208,7 @@ puts path
213
208
  # @param region [String]: The cloud provider region
214
209
  # @return [void]
215
210
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
211
+ MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags
216
212
 
217
213
  resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets
218
214
  if resp and resp.buckets
@@ -243,15 +239,21 @@ puts path
243
239
 
244
240
  begin
245
241
  tags = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_tagging(bucket: bucket.name).tag_set
242
+ deploy_match = false
243
+ master_match = false
246
244
  tags.each { |tag|
247
245
  if tag.key == "MU-ID" and tag.value == MU.deploy_id
248
- MU.log "Deleting S3 Bucket #{bucket.name}"
249
- if !noop
250
- MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
251
- end
252
- break
246
+ deploy_match = true
247
+ elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
248
+ master_match = true
253
249
  end
254
250
  }
251
+ if deploy_match and (ignoremaster or master_match)
252
+ MU.log "Deleting S3 Bucket #{bucket.name}"
253
+ if !noop
254
+ MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
255
+ end
256
+ end
255
257
  rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect
256
258
  next
257
259
  end
@@ -302,12 +304,12 @@ puts path
302
304
  end
303
305
 
304
306
  # Cloud-specific configuration properties.
305
- # @param config [MU::Config]: The calling MU::Config object
307
+ # @param _config [MU::Config]: The calling MU::Config object
306
308
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
307
- def self.schema(config)
309
+ def self.schema(_config)
308
310
  toplevel_required = []
309
311
  schema = {
310
- "policies" => MU::Cloud::AWS::Role.condition_schema,
312
+ "policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema,
311
313
  "acl" => {
312
314
  "type" => "string",
313
315
  "enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
@@ -325,15 +327,15 @@ puts path
325
327
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
326
328
 
327
329
  # @param bucket [Hash]: The resource to process and validate
328
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
330
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
329
331
  # @return [Boolean]: True if validation succeeded, False otherwise
330
- def self.validateConfig(bucket, configurator)
332
+ def self.validateConfig(bucket, _configurator)
331
333
  ok = true
332
334
 
333
335
  if bucket['policies']
334
336
  bucket['policies'].each { |pol|
335
337
  if !pol['permissions'] or pol['permissions'].empty?
336
- pol['permissions'] = ["s3:GetObject"]
338
+ pol['permissions'] = ["s3:GetObject", "s3:ListBucket"]
337
339
  end
338
340
  }
339
341
  end
@@ -341,11 +343,13 @@ puts path
341
343
  ok
342
344
  end
343
345
 
344
- private
345
-
346
346
  # AWS doesn't really implement a useful describe_ method for S3 buckets;
347
347
  # instead we run the million little individual API calls to construct
348
348
  # an approximation for our uses
349
+ # @param bucket [String]:
350
+ # @param minimal [Boolean]:
351
+ # @param credentials [String]:
352
+ # @param region [String]:
349
353
  def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil)
350
354
  @@region_cache = {}
351
355
  @@region_cache_semaphore = Mutex.new
@@ -372,7 +376,7 @@ puts path
372
376
  }
373
377
  end
374
378
 
375
- rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration => e
379
+ rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
376
380
  desc[method] = nil
377
381
  next
378
382
  end
@@ -170,7 +170,7 @@ module MU
170
170
  # config_struct[:preferred_cache_cluster_a_zs] = @config["preferred_cache_cluster_azs"]
171
171
 
172
172
  MU.log "Creating cache replication group #{@config['identifier']}"
173
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
173
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
174
174
 
175
175
  wait_start_time = Time.now
176
176
  retries = 0
@@ -180,7 +180,7 @@ module MU
180
180
  waiter.before_attempt do |attempts|
181
181
  MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
182
182
  end
183
- waiter.before_wait do |attempts, r|
183
+ waiter.before_wait do |_attempts, r|
184
184
  throw :success if r.replication_groups.first.status == "available"
185
185
  throw :failure if Time.now - wait_start_time > 1800
186
186
  end
@@ -199,7 +199,7 @@ module MU
199
199
  addStandardTags(member, "cluster", region: @config['region'])
200
200
  }
201
201
 
202
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
202
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
203
203
  name: resp.replication_group_id,
204
204
  target: "#{resp.node_groups.first.primary_endpoint.address}.",
205
205
  cloudclass: MU::Cloud::CacheCluster,
@@ -207,7 +207,7 @@ module MU
207
207
  )
208
208
 
209
209
  resp.node_groups.first.node_group_members.each { |member|
210
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
210
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
211
211
  name: member.cache_cluster_id,
212
212
  target: "#{member.read_endpoint.address}.",
213
213
  cloudclass: MU::Cloud::CacheCluster,
@@ -228,7 +228,7 @@ module MU
228
228
 
229
229
  MU.log "Creating cache cluster #{@config['identifier']}"
230
230
  begin
231
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
231
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
232
232
  rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
233
233
  if e.message.match(/security group (sg-[^\s]+)/)
234
234
  bad_sg = Regexp.last_match[1]
@@ -248,7 +248,7 @@ module MU
248
248
  waiter.before_attempt do |attempts|
249
249
  MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
250
250
  end
251
- waiter.before_wait do |attempts, r|
251
+ waiter.before_wait do |_attempts, r|
252
252
  throw :success if r.cache_clusters.first.cache_cluster_status == "available"
253
253
  throw :failure if Time.now - wait_start_time > 1800
254
254
  end
@@ -270,7 +270,7 @@ module MU
270
270
  def createSubnetGroup
271
271
  subnet_ids = []
272
272
  if @config["vpc"] && !@config["vpc"].empty?
273
- raise MuError, "Didn't find the VPC specified in #{@config["vpc"]}" unless @vpc
273
+ raise MuError.new "Didn't find the VPC specified for #{@mu_name}", details: @config["vpc"].to_h unless @vpc
274
274
 
275
275
  vpc_id = @vpc.cloud_id
276
276
 
@@ -283,7 +283,7 @@ module MU
283
283
  else
284
284
  @config["vpc"]["subnets"].each { |subnet|
285
285
  subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
286
- raise MuError, "Couldn't find a live subnet matching #{subnet} in #{@vpc} (#{@vpc.subnets})" if subnet_obj.nil?
286
+ raise MuError.new "Couldn't find a live subnet matching #{subnet} in #{@vpc}", details: @vpc.subnets if subnet_obj.nil?
287
287
  subnet_ids << subnet_obj.cloud_id
288
288
  }
289
289
  end
@@ -317,7 +317,7 @@ module MU
317
317
  "vpc_id" => vpc_id,
318
318
  "subnets" => mu_subnets
319
319
  }
320
- using_default_vpc = true
320
+
321
321
  MU.log "Using default VPC for cache cluster #{@config['identifier']}"
322
322
  end
323
323
  end
@@ -327,30 +327,13 @@ module MU
327
327
  else
328
328
  MU.log "Creating subnet group #{@config["subnet_group_name"]} for cache cluster #{@config['identifier']}"
329
329
 
330
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
330
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
331
331
  cache_subnet_group_name: @config["subnet_group_name"],
332
332
  cache_subnet_group_description: @config["subnet_group_name"],
333
333
  subnet_ids: subnet_ids
334
334
  )
335
335
 
336
- # Find NAT and create holes in security groups.
337
- # Adding just for consistency, but do we really need this for cache clusters? I guess Nagios and such..
338
- if @config["vpc"]["nat_host_name"] || @config["vpc"]["nat_host_id"] || @config["vpc"]["nat_host_tag"] || @config["vpc"]["nat_host_ip"]
339
- nat = @nat
340
- if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
341
- MU.log "Using NAT Gateway, not modifying security groups"
342
- else
343
- nat_name, nat_conf, nat_deploydata = @nat.describe
344
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
345
- # XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
346
- if acl.config["admin"]
347
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
348
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "udp")
349
- break
350
- end
351
- }
352
- end
353
- end
336
+ allowBastionAccess
354
337
 
355
338
  if @dependencies.has_key?('firewall_rule')
356
339
  @config["security_group_ids"] = []
@@ -364,7 +347,7 @@ module MU
364
347
  # Create a Cache Cluster parameter group.
365
348
  def createParameterGroup
366
349
  MU.log "Creating a cache cluster parameter group #{@config["parameter_group_name"]}"
367
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
350
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
368
351
  cache_parameter_group_name: @config["parameter_group_name"],
369
352
  cache_parameter_group_family: @config["parameter_group_family"],
370
353
  description: "Parameter group for #{@config["parameter_group_family"]}"
@@ -404,7 +387,7 @@ module MU
404
387
  def self.getCacheClusterById(cc_id, region: MU.curRegion, credentials: nil)
405
388
  begin
406
389
  MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters(cache_cluster_id: cc_id).cache_clusters.first
407
- rescue Aws::ElastiCache::Errors::CacheClusterNotFound => e
390
+ rescue Aws::ElastiCache::Errors::CacheClusterNotFound
408
391
  nil
409
392
  end
410
393
  end
@@ -430,7 +413,7 @@ module MU
430
413
  }
431
414
  end
432
415
  # XXX this should be a call to @deploy.nameKitten
433
- MU::Cloud::AWS::DNSZone.createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
416
+ MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
434
417
 
435
418
  deploy_struct = {
436
419
  "identifier" => repl_group.replication_group_id,
@@ -532,7 +515,7 @@ module MU
532
515
 
533
516
  attempts = 0
534
517
  begin
535
- snapshot = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
518
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
536
519
  cache_cluster_id: @config["identifier"],
537
520
  snapshot_name: snap_id
538
521
  )
@@ -669,7 +652,7 @@ module MU
669
652
  threads << Thread.new(replication_group) { |myrepl_group|
670
653
  MU.dupGlobals(parent_thread_id)
671
654
  Thread.abort_on_exception = true
672
- MU::Cloud::AWS::CacheCluster.terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: myrepl_group.replication_group_id, credentials: credentials)
655
+ terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
673
656
  }
674
657
  }
675
658
  end
@@ -681,7 +664,7 @@ module MU
681
664
  threads << Thread.new(cluster) { |mycluster|
682
665
  MU.dupGlobals(parent_thread_id)
683
666
  Thread.abort_on_exception = true
684
- MU::Cloud::AWS::CacheCluster.terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: mycluster.cache_cluster_id, credentials: credentials)
667
+ terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
685
668
  }
686
669
  }
687
670
  end
@@ -694,35 +677,16 @@ module MU
694
677
  end
695
678
 
696
679
  # Cloud-specific configuration properties.
697
- # @param config [MU::Config]: The calling MU::Config object
680
+ # @param _config [MU::Config]: The calling MU::Config object
698
681
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
699
- def self.schema(config)
682
+ def self.schema(_config)
700
683
  toplevel_required = []
701
684
  schema = {
702
685
  "create_replication_group" => {
703
686
  "type" => "boolean",
704
687
  "description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
705
688
  },
706
- "ingress_rules" => {
707
- "items" => {
708
- "properties" => {
709
- "sgs" => {
710
- "type" => "array",
711
- "items" => {
712
- "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
713
- "type" => "string"
714
- }
715
- },
716
- "lbs" => {
717
- "type" => "array",
718
- "items" => {
719
- "description" => "AWS Load Balancers which will have this rule applied to their traffic",
720
- "type" => "string"
721
- }
722
- }
723
- }
724
- }
725
- }
689
+ "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
726
690
  }
727
691
  [toplevel_required, schema]
728
692
  end
@@ -804,9 +768,8 @@ module MU
804
768
  # @param noop [Boolean]: If true, will only print what would be done.
805
769
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
806
770
  # @param region [String]: The cloud provider's region in which to operate.
807
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
808
771
  # @return [void]
809
- def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
772
+ def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
810
773
  raise MuError, "terminate_cache_cluster requires a non-nil cache cluster descriptor" if cluster.nil? || cluster.empty?
811
774
 
812
775
  cluster_id = cluster.cache_cluster_id
@@ -824,7 +787,7 @@ module MU
824
787
  end
825
788
 
826
789
  # The API is broken, cluster.cache_nodes is returnning an empty array, and the only URL we can get is the config one with cluster.configuration_endpoint.address.
827
- # MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
790
+ # MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
828
791
 
829
792
  if %w{deleting deleted}.include?(cluster.cache_cluster_status)
830
793
  MU.log "#{cluster_id} has already been terminated", MU::WARN
@@ -874,7 +837,7 @@ module MU
874
837
  waiter.before_attempt do |attempts|
875
838
  MU.log "Waiting for cache cluster #{cluster_id} to delete..", MU::NOTICE if attempts % 10 == 0
876
839
  end
877
- waiter.before_wait do |attempts, resp|
840
+ waiter.before_wait do |_attempts, resp|
878
841
  throw :success if resp.cache_clusters.first.cache_cluster_status == "deleted"
879
842
  throw :failure if Time.now - wait_start_time > 1800
880
843
  end
@@ -893,19 +856,19 @@ module MU
893
856
  MU.log "#{cluster_id} has been terminated"
894
857
 
895
858
  unless noop
896
- MU::Cloud::AWS::CacheCluster.delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
897
- MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
859
+ delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
860
+ delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
898
861
  end
899
862
  end
863
+ private_class_method :terminate_cache_cluster
900
864
 
901
865
  # Remove a Cache Cluster Replication Group and associated artifacts
902
866
  # @param repl_group [OpenStruct]: The cloud provider's description of the Cache Cluster artifact.
903
867
  # @param noop [Boolean]: If true, will only print what would be done.
904
868
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
905
869
  # @param region [String]: The cloud provider's region in which to operate.
906
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
907
870
  # @return [void]
908
- def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
871
+ def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
909
872
  raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
910
873
 
911
874
  repl_group_id = repl_group.replication_group_id
@@ -926,10 +889,10 @@ module MU
926
889
  end
927
890
 
928
891
  # What's the likelihood of having more than one node group? maybe iterate over node_groups instead of assuming there is only one?
929
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
892
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
930
893
  # Assuming we also created DNS records for each of our cluster's read endpoint.
931
894
  repl_group.node_groups.first.node_group_members.each { |member|
932
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
895
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
933
896
  }
934
897
 
935
898
  if %w{deleting deleted}.include?(repl_group.status)
@@ -983,7 +946,7 @@ module MU
983
946
  waiter.before_attempt do |attempts|
984
947
  MU.log "Waiting for #{repl_group_id} to delete..", MU::NOTICE if attempts % 10 == 0
985
948
  end
986
- waiter.before_wait do |attempts, resp|
949
+ waiter.before_wait do |_attempts, resp|
987
950
  throw :success if resp.replication_groups.first.status == "deleted"
988
951
  throw :failure if Time.now - wait_start_time > 1800
989
952
  end
@@ -1005,6 +968,7 @@ module MU
1005
968
  MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region) if parameter_group && !parameter_group.start_with?("default")
1006
969
  end
1007
970
  end
971
+ private_class_method :terminate_replication_group
1008
972
 
1009
973
  # Remove a Cache Cluster Subnet Group.
1010
974
  # @param subnet_group_id [string]: The cloud provider's ID of the cache cluster subnet group.
@@ -1026,6 +990,7 @@ module MU
1026
990
  MU.log "Subnet group #{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1027
991
  end
1028
992
  end
993
+ private_class_method :delete_subnet_group
1029
994
 
1030
995
  # Remove a Cache Cluster Parameter Group.
1031
996
  # @param parameter_group_id [string]: The cloud provider's ID of the cache cluster parameter group.
@@ -1049,6 +1014,7 @@ module MU
1049
1014
  MU.log "Parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1050
1015
  end
1051
1016
  end
1017
+ private_class_method :delete_parameter_group
1052
1018
  end
1053
1019
  end
1054
1020
  end