cloud-mu 3.1.2 → 3.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (201) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +10 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +2 -3
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +135 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +165 -111
  45. data/modules/mu/adoption.rb +401 -68
  46. data/modules/mu/cleanup.rb +199 -306
  47. data/modules/mu/cloud.rb +100 -1632
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +46 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +920 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +165 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +4 -4
  61. data/modules/mu/config/cache_cluster.rb +1 -1
  62. data/modules/mu/config/collection.rb +4 -4
  63. data/modules/mu/config/container_cluster.rb +9 -4
  64. data/modules/mu/config/database.rb +83 -104
  65. data/modules/mu/config/database.yml +1 -2
  66. data/modules/mu/config/dnszone.rb +6 -6
  67. data/modules/mu/config/doc_helpers.rb +516 -0
  68. data/modules/mu/config/endpoint.rb +4 -4
  69. data/modules/mu/config/firewall_rule.rb +103 -4
  70. data/modules/mu/config/folder.rb +4 -4
  71. data/modules/mu/config/function.rb +3 -3
  72. data/modules/mu/config/group.rb +4 -4
  73. data/modules/mu/config/habitat.rb +4 -4
  74. data/modules/mu/config/loadbalancer.rb +60 -14
  75. data/modules/mu/config/log.rb +4 -4
  76. data/modules/mu/config/msg_queue.rb +4 -4
  77. data/modules/mu/config/nosqldb.rb +4 -4
  78. data/modules/mu/config/notifier.rb +3 -3
  79. data/modules/mu/config/ref.rb +365 -0
  80. data/modules/mu/config/role.rb +4 -4
  81. data/modules/mu/config/schema_helpers.rb +509 -0
  82. data/modules/mu/config/search_domain.rb +4 -4
  83. data/modules/mu/config/server.rb +97 -70
  84. data/modules/mu/config/server.yml +1 -0
  85. data/modules/mu/config/server_pool.rb +5 -9
  86. data/modules/mu/config/storage_pool.rb +1 -1
  87. data/modules/mu/config/tail.rb +200 -0
  88. data/modules/mu/config/user.rb +4 -4
  89. data/modules/mu/config/vpc.rb +70 -27
  90. data/modules/mu/config/vpc.yml +0 -1
  91. data/modules/mu/defaults/AWS.yaml +83 -60
  92. data/modules/mu/defaults/Azure.yaml +1 -0
  93. data/modules/mu/defaults/Google.yaml +3 -2
  94. data/modules/mu/deploy.rb +30 -26
  95. data/modules/mu/groomer.rb +17 -2
  96. data/modules/mu/groomers/ansible.rb +188 -41
  97. data/modules/mu/groomers/chef.rb +116 -55
  98. data/modules/mu/logger.rb +127 -148
  99. data/modules/mu/master.rb +389 -2
  100. data/modules/mu/master/chef.rb +3 -4
  101. data/modules/mu/master/ldap.rb +3 -3
  102. data/modules/mu/master/ssl.rb +12 -3
  103. data/modules/mu/mommacat.rb +217 -2612
  104. data/modules/mu/mommacat/daemon.rb +397 -0
  105. data/modules/mu/mommacat/naming.rb +473 -0
  106. data/modules/mu/mommacat/search.rb +495 -0
  107. data/modules/mu/mommacat/storage.rb +722 -0
  108. data/modules/mu/{clouds → providers}/README.md +1 -1
  109. data/modules/mu/{clouds → providers}/aws.rb +271 -112
  110. data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
  111. data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
  112. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
  113. data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
  114. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
  115. data/modules/mu/providers/aws/database.rb +1744 -0
  116. data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
  117. data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
  118. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
  119. data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
  120. data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
  121. data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
  122. data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
  123. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
  124. data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
  125. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
  126. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
  127. data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
  128. data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
  129. data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
  130. data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
  131. data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
  132. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
  133. data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
  134. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  135. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
  136. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  137. data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
  138. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  139. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  140. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  141. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  142. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  143. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  144. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  145. data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
  146. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  147. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  148. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  149. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  150. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  151. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  152. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  153. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  154. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  155. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  156. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  160. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  161. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  162. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  163. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  164. data/modules/mu/{clouds → providers}/google.rb +67 -30
  165. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  166. data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
  167. data/modules/mu/{clouds → providers}/google/database.rb +10 -20
  168. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  169. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  170. data/modules/mu/{clouds → providers}/google/function.rb +139 -167
  171. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  172. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  173. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
  174. data/modules/mu/{clouds → providers}/google/role.rb +92 -58
  175. data/modules/mu/{clouds → providers}/google/server.rb +242 -155
  176. data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
  177. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  178. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  179. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  180. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  181. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  182. data/modules/tests/bucket.yml +4 -0
  183. data/modules/tests/centos6.yaml +11 -0
  184. data/modules/tests/centos7.yaml +11 -0
  185. data/modules/tests/centos8.yaml +12 -0
  186. data/modules/tests/ecs.yaml +23 -0
  187. data/modules/tests/includes-and-params.yaml +2 -1
  188. data/modules/tests/rds.yaml +108 -0
  189. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  190. data/modules/tests/regrooms/bucket.yml +19 -0
  191. data/modules/tests/regrooms/rds.yaml +123 -0
  192. data/modules/tests/server-with-scrub-muisms.yaml +1 -0
  193. data/modules/tests/super_simple_bok.yml +1 -3
  194. data/modules/tests/win2k12.yaml +17 -5
  195. data/modules/tests/win2k16.yaml +25 -0
  196. data/modules/tests/win2k19.yaml +25 -0
  197. data/requirements.txt +1 -0
  198. data/spec/mu/clouds/azure_spec.rb +2 -2
  199. metadata +232 -154
  200. data/extras/image-generators/AWS/windows.yaml +0 -18
  201. data/modules/mu/clouds/aws/database.rb +0 -1985
@@ -125,6 +125,8 @@ module MU
125
125
  # @param region [String]: The cloud provider region
126
126
  # @return [void]
127
127
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
128
+ MU.log "AWS::Alarm.cleanup: need to support flags['known']", MU::DEBUG, details: flags
129
+ MU.log "Placeholder: AWS Alarm artifacts do not support tags, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: ignoremaster
128
130
  alarms = []
129
131
  # We don't have a way to tag alarms, so we try to delete them by the deploy ID.
130
132
  # This can miss alarms in some cases (eg. cache_cluster) so we might want to delete alarms from each API as well.
@@ -252,9 +254,9 @@ module MU
252
254
  end
253
255
 
254
256
  # Cloud-specific configuration properties.
255
- # @param config [MU::Config]: The calling MU::Config object
257
+ # @param _config [MU::Config]: The calling MU::Config object
256
258
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
257
- def self.schema(config)
259
+ def self.schema(_config)
258
260
  toplevel_required = []
259
261
  schema = {}
260
262
  [toplevel_required, schema]
@@ -319,7 +321,7 @@ module MU
319
321
  if !depclass.nil?
320
322
  dimension["depclass"] = depclass
321
323
  if !dimension["name"].nil? and !dimension["name"].empty?
322
- alarm["dependencies"] << { "name" => dimension["name"], "type" => depclass }
324
+ MU::Config.addDependency(alarm, dimension["name"], depclass)
323
325
  end
324
326
  end
325
327
  }
@@ -33,7 +33,7 @@ module MU
33
33
  bucket_name = @deploy.getResourceName(@config["name"], max_length: 63).downcase
34
34
 
35
35
  MU.log "Creating S3 bucket #{bucket_name}"
36
- resp = MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
36
+ MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).create_bucket(
37
37
  acl: @config['acl'],
38
38
  bucket: bucket_name
39
39
  )
@@ -90,7 +90,6 @@ module MU
90
90
  tagBucket if !@config['scrub_mu_isms']
91
91
 
92
92
  current = cloud_desc
93
-
94
93
  if @config['policies']
95
94
  @config['policies'].each { |pol|
96
95
  pol['grant_to'] ||= [
@@ -98,9 +97,9 @@ module MU
98
97
  ]
99
98
  }
100
99
 
101
- policy_docs = MU::Cloud::AWS::Role.genPolicyDocument(@config['policies'], deploy_obj: @deploy)
100
+ policy_docs = MU::Cloud.resourceClass("AWS", "Role").genPolicyDocument(@config['policies'], deploy_obj: @deploy, bucket_style: true)
102
101
  policy_docs.each { |doc|
103
- MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: doc.values.first
102
+ MU.log "Applying S3 bucket policy #{doc.keys.first} to bucket #{@cloud_id}", MU::NOTICE, details: JSON.pretty_generate(doc.values.first)
104
103
  MU::Cloud::AWS.s3(credentials: @config['credentials'], region: @config['region']).put_bucket_policy(
105
104
  bucket: @cloud_id,
106
105
  policy: JSON.generate(doc.values.first)
@@ -177,10 +176,6 @@ module MU
177
176
  end
178
177
 
179
178
  begin
180
- puts data
181
- puts acl
182
- puts bucket
183
- puts path
184
179
  MU.log "Writing #{path} to S3 bucket #{bucket}"
185
180
  MU::Cloud::AWS.s3(region: region, credentials: credentials).put_object(
186
181
  acl: acl,
@@ -213,6 +208,7 @@ puts path
213
208
  # @param region [String]: The cloud provider region
214
209
  # @return [void]
215
210
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
211
+ MU.log "AWS::Bucket.cleanup: need to support flags['known']", MU::DEBUG, details: flags
216
212
 
217
213
  resp = MU::Cloud::AWS.s3(credentials: credentials, region: region).list_buckets
218
214
  if resp and resp.buckets
@@ -243,15 +239,21 @@ puts path
243
239
 
244
240
  begin
245
241
  tags = MU::Cloud::AWS.s3(credentials: credentials, region: region).get_bucket_tagging(bucket: bucket.name).tag_set
242
+ deploy_match = false
243
+ master_match = false
246
244
  tags.each { |tag|
247
245
  if tag.key == "MU-ID" and tag.value == MU.deploy_id
248
- MU.log "Deleting S3 Bucket #{bucket.name}"
249
- if !noop
250
- MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
251
- end
252
- break
246
+ deploy_match = true
247
+ elsif tag.key == "MU-MASTER-IP" and tag.value == MU.mu_public_ip
248
+ master_match = true
253
249
  end
254
250
  }
251
+ if deploy_match and (ignoremaster or master_match)
252
+ MU.log "Deleting S3 Bucket #{bucket.name}"
253
+ if !noop
254
+ MU::Cloud::AWS.s3(credentials: credentials, region: region).delete_bucket(bucket: bucket.name)
255
+ end
256
+ end
255
257
  rescue Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::PermanentRedirect
256
258
  next
257
259
  end
@@ -302,12 +304,12 @@ puts path
302
304
  end
303
305
 
304
306
  # Cloud-specific configuration properties.
305
- # @param config [MU::Config]: The calling MU::Config object
307
+ # @param _config [MU::Config]: The calling MU::Config object
306
308
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
307
- def self.schema(config)
309
+ def self.schema(_config)
308
310
  toplevel_required = []
309
311
  schema = {
310
- "policies" => MU::Cloud::AWS::Role.condition_schema,
312
+ "policies" => MU::Cloud.resourceClass("AWS", "Role").condition_schema,
311
313
  "acl" => {
312
314
  "type" => "string",
313
315
  "enum" => ["private", "public-read", "public-read-write", "authenticated-read"],
@@ -325,15 +327,15 @@ puts path
325
327
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
326
328
 
327
329
  # @param bucket [Hash]: The resource to process and validate
328
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
330
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
329
331
  # @return [Boolean]: True if validation succeeded, False otherwise
330
- def self.validateConfig(bucket, configurator)
332
+ def self.validateConfig(bucket, _configurator)
331
333
  ok = true
332
334
 
333
335
  if bucket['policies']
334
336
  bucket['policies'].each { |pol|
335
337
  if !pol['permissions'] or pol['permissions'].empty?
336
- pol['permissions'] = ["s3:GetObject"]
338
+ pol['permissions'] = ["s3:GetObject", "s3:ListBucket"]
337
339
  end
338
340
  }
339
341
  end
@@ -341,11 +343,13 @@ puts path
341
343
  ok
342
344
  end
343
345
 
344
- private
345
-
346
346
  # AWS doesn't really implement a useful describe_ method for S3 buckets;
347
347
  # instead we run the million little individual API calls to construct
348
348
  # an approximation for our uses
349
+ # @param bucket [String]:
350
+ # @param minimal [Boolean]:
351
+ # @param credentials [String]:
352
+ # @param region [String]:
349
353
  def self.describe_bucket(bucket, minimal: false, credentials: nil, region: nil)
350
354
  @@region_cache = {}
351
355
  @@region_cache_semaphore = Mutex.new
@@ -372,7 +376,7 @@ puts path
372
376
  }
373
377
  end
374
378
 
375
- rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration => e
379
+ rescue Aws::S3::Errors::NoSuchCORSConfiguration, Aws::S3::Errors::ServerSideEncryptionConfigurationNotFoundError, Aws::S3::Errors::NoSuchLifecycleConfiguration, Aws::S3::Errors::NoSuchBucketPolicy, Aws::S3::Errors::ReplicationConfigurationNotFoundError, Aws::S3::Errors::NoSuchTagSet, Aws::S3::Errors::NoSuchWebsiteConfiguration
376
380
  desc[method] = nil
377
381
  next
378
382
  end
@@ -170,7 +170,7 @@ module MU
170
170
  # config_struct[:preferred_cache_cluster_a_zs] = @config["preferred_cache_cluster_azs"]
171
171
 
172
172
  MU.log "Creating cache replication group #{@config['identifier']}"
173
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
173
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_replication_group(config_struct).replication_group
174
174
 
175
175
  wait_start_time = Time.now
176
176
  retries = 0
@@ -180,7 +180,7 @@ module MU
180
180
  waiter.before_attempt do |attempts|
181
181
  MU.log "Waiting for cache replication group #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
182
182
  end
183
- waiter.before_wait do |attempts, r|
183
+ waiter.before_wait do |_attempts, r|
184
184
  throw :success if r.replication_groups.first.status == "available"
185
185
  throw :failure if Time.now - wait_start_time > 1800
186
186
  end
@@ -199,7 +199,7 @@ module MU
199
199
  addStandardTags(member, "cluster", region: @config['region'])
200
200
  }
201
201
 
202
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
202
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
203
203
  name: resp.replication_group_id,
204
204
  target: "#{resp.node_groups.first.primary_endpoint.address}.",
205
205
  cloudclass: MU::Cloud::CacheCluster,
@@ -207,7 +207,7 @@ module MU
207
207
  )
208
208
 
209
209
  resp.node_groups.first.node_group_members.each { |member|
210
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(
210
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(
211
211
  name: member.cache_cluster_id,
212
212
  target: "#{member.read_endpoint.address}.",
213
213
  cloudclass: MU::Cloud::CacheCluster,
@@ -228,7 +228,7 @@ module MU
228
228
 
229
229
  MU.log "Creating cache cluster #{@config['identifier']}"
230
230
  begin
231
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
231
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_cluster(config_struct).cache_cluster
232
232
  rescue ::Aws::ElastiCache::Errors::InvalidParameterValue => e
233
233
  if e.message.match(/security group (sg-[^\s]+)/)
234
234
  bad_sg = Regexp.last_match[1]
@@ -248,7 +248,7 @@ module MU
248
248
  waiter.before_attempt do |attempts|
249
249
  MU.log "Waiting for cache cluster #{@config['identifier']} to become available", MU::NOTICE if attempts % 5 == 0
250
250
  end
251
- waiter.before_wait do |attempts, r|
251
+ waiter.before_wait do |_attempts, r|
252
252
  throw :success if r.cache_clusters.first.cache_cluster_status == "available"
253
253
  throw :failure if Time.now - wait_start_time > 1800
254
254
  end
@@ -270,7 +270,7 @@ module MU
270
270
  def createSubnetGroup
271
271
  subnet_ids = []
272
272
  if @config["vpc"] && !@config["vpc"].empty?
273
- raise MuError, "Didn't find the VPC specified in #{@config["vpc"]}" unless @vpc
273
+ raise MuError.new "Didn't find the VPC specified for #{@mu_name}", details: @config["vpc"].to_h unless @vpc
274
274
 
275
275
  vpc_id = @vpc.cloud_id
276
276
 
@@ -283,7 +283,7 @@ module MU
283
283
  else
284
284
  @config["vpc"]["subnets"].each { |subnet|
285
285
  subnet_obj = @vpc.getSubnet(cloud_id: subnet["subnet_id"].to_s, name: subnet["subnet_name"].to_s)
286
- raise MuError, "Couldn't find a live subnet matching #{subnet} in #{@vpc} (#{@vpc.subnets})" if subnet_obj.nil?
286
+ raise MuError.new "Couldn't find a live subnet matching #{subnet} in #{@vpc}", details: @vpc.subnets if subnet_obj.nil?
287
287
  subnet_ids << subnet_obj.cloud_id
288
288
  }
289
289
  end
@@ -317,7 +317,7 @@ module MU
317
317
  "vpc_id" => vpc_id,
318
318
  "subnets" => mu_subnets
319
319
  }
320
- using_default_vpc = true
320
+
321
321
  MU.log "Using default VPC for cache cluster #{@config['identifier']}"
322
322
  end
323
323
  end
@@ -327,30 +327,13 @@ module MU
327
327
  else
328
328
  MU.log "Creating subnet group #{@config["subnet_group_name"]} for cache cluster #{@config['identifier']}"
329
329
 
330
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
330
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_subnet_group(
331
331
  cache_subnet_group_name: @config["subnet_group_name"],
332
332
  cache_subnet_group_description: @config["subnet_group_name"],
333
333
  subnet_ids: subnet_ids
334
334
  )
335
335
 
336
- # Find NAT and create holes in security groups.
337
- # Adding just for consistency, but do we really need this for cache clusters? I guess Nagios and such..
338
- if @config["vpc"]["nat_host_name"] || @config["vpc"]["nat_host_id"] || @config["vpc"]["nat_host_tag"] || @config["vpc"]["nat_host_ip"]
339
- nat = @nat
340
- if nat.is_a?(Struct) && nat.nat_gateway_id && nat.nat_gateway_id.start_with?("nat-")
341
- MU.log "Using NAT Gateway, not modifying security groups"
342
- else
343
- nat_name, nat_conf, nat_deploydata = @nat.describe
344
- @deploy.kittens['firewall_rules'].each_pair { |name, acl|
345
- # XXX if a user doesn't set up dependencies correctly, this can die horribly on a NAT that's still in mid-creation. Fix this... possibly in the config parser.
346
- if acl.config["admin"]
347
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "tcp")
348
- acl.addRule([nat_deploydata["private_ip_address"]], proto: "udp")
349
- break
350
- end
351
- }
352
- end
353
- end
336
+ allowBastionAccess
354
337
 
355
338
  if @dependencies.has_key?('firewall_rule')
356
339
  @config["security_group_ids"] = []
@@ -364,7 +347,7 @@ module MU
364
347
  # Create a Cache Cluster parameter group.
365
348
  def createParameterGroup
366
349
  MU.log "Creating a cache cluster parameter group #{@config["parameter_group_name"]}"
367
- resp = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
350
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_cache_parameter_group(
368
351
  cache_parameter_group_name: @config["parameter_group_name"],
369
352
  cache_parameter_group_family: @config["parameter_group_family"],
370
353
  description: "Parameter group for #{@config["parameter_group_family"]}"
@@ -404,7 +387,7 @@ module MU
404
387
  def self.getCacheClusterById(cc_id, region: MU.curRegion, credentials: nil)
405
388
  begin
406
389
  MU::Cloud::AWS.elasticache(region: region, credentials: credentials).describe_cache_clusters(cache_cluster_id: cc_id).cache_clusters.first
407
- rescue Aws::ElastiCache::Errors::CacheClusterNotFound => e
390
+ rescue Aws::ElastiCache::Errors::CacheClusterNotFound
408
391
  nil
409
392
  end
410
393
  end
@@ -430,7 +413,7 @@ module MU
430
413
  }
431
414
  end
432
415
  # XXX this should be a call to @deploy.nameKitten
433
- MU::Cloud::AWS::DNSZone.createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
416
+ MU::Cloud.resourceClass("AWS", "DNSZone").createRecordsFromConfig(@config['dns_records'], target: repl_group.node_groups.first.primary_endpoint.address)
434
417
 
435
418
  deploy_struct = {
436
419
  "identifier" => repl_group.replication_group_id,
@@ -532,7 +515,7 @@ module MU
532
515
 
533
516
  attempts = 0
534
517
  begin
535
- snapshot = MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
518
+ MU::Cloud::AWS.elasticache(region: @config['region'], credentials: @config['credentials']).create_snapshot(
536
519
  cache_cluster_id: @config["identifier"],
537
520
  snapshot_name: snap_id
538
521
  )
@@ -669,7 +652,7 @@ module MU
669
652
  threads << Thread.new(replication_group) { |myrepl_group|
670
653
  MU.dupGlobals(parent_thread_id)
671
654
  Thread.abort_on_exception = true
672
- MU::Cloud::AWS::CacheCluster.terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: myrepl_group.replication_group_id, credentials: credentials)
655
+ terminate_replication_group(myrepl_group, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
673
656
  }
674
657
  }
675
658
  end
@@ -681,7 +664,7 @@ module MU
681
664
  threads << Thread.new(cluster) { |mycluster|
682
665
  MU.dupGlobals(parent_thread_id)
683
666
  Thread.abort_on_exception = true
684
- MU::Cloud::AWS::CacheCluster.terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, deploy_id: MU.deploy_id, cloud_id: mycluster.cache_cluster_id, credentials: credentials)
667
+ terminate_cache_cluster(mycluster, noop: noop, skipsnapshots: skipsnapshots, region: region, credentials: credentials)
685
668
  }
686
669
  }
687
670
  end
@@ -694,35 +677,16 @@ module MU
694
677
  end
695
678
 
696
679
  # Cloud-specific configuration properties.
697
- # @param config [MU::Config]: The calling MU::Config object
680
+ # @param _config [MU::Config]: The calling MU::Config object
698
681
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
699
- def self.schema(config)
682
+ def self.schema(_config)
700
683
  toplevel_required = []
701
684
  schema = {
702
685
  "create_replication_group" => {
703
686
  "type" => "boolean",
704
687
  "description" => "Create a replication group; will be set automatically if +engine+ is +redis+ and +node_count+ is greated than one."
705
688
  },
706
- "ingress_rules" => {
707
- "items" => {
708
- "properties" => {
709
- "sgs" => {
710
- "type" => "array",
711
- "items" => {
712
- "description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
713
- "type" => "string"
714
- }
715
- },
716
- "lbs" => {
717
- "type" => "array",
718
- "items" => {
719
- "description" => "AWS Load Balancers which will have this rule applied to their traffic",
720
- "type" => "string"
721
- }
722
- }
723
- }
724
- }
725
- }
689
+ "ingress_rules" => MU::Cloud.resourceClass("AWS", "FirewallRule").ingressRuleAddtlSchema
726
690
  }
727
691
  [toplevel_required, schema]
728
692
  end
@@ -804,9 +768,8 @@ module MU
804
768
  # @param noop [Boolean]: If true, will only print what would be done.
805
769
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
806
770
  # @param region [String]: The cloud provider's region in which to operate.
807
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
808
771
  # @return [void]
809
- def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
772
+ def self.terminate_cache_cluster(cluster, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
810
773
  raise MuError, "terminate_cache_cluster requires a non-nil cache cluster descriptor" if cluster.nil? || cluster.empty?
811
774
 
812
775
  cluster_id = cluster.cache_cluster_id
@@ -824,7 +787,7 @@ module MU
824
787
  end
825
788
 
826
789
  # The API is broken, cluster.cache_nodes is returnning an empty array, and the only URL we can get is the config one with cluster.configuration_endpoint.address.
827
- # MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
790
+ # MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: cluster_id, target: , cloudclass: MU::Cloud::CacheCluster, delete: true)
828
791
 
829
792
  if %w{deleting deleted}.include?(cluster.cache_cluster_status)
830
793
  MU.log "#{cluster_id} has already been terminated", MU::WARN
@@ -874,7 +837,7 @@ module MU
874
837
  waiter.before_attempt do |attempts|
875
838
  MU.log "Waiting for cache cluster #{cluster_id} to delete..", MU::NOTICE if attempts % 10 == 0
876
839
  end
877
- waiter.before_wait do |attempts, resp|
840
+ waiter.before_wait do |_attempts, resp|
878
841
  throw :success if resp.cache_clusters.first.cache_cluster_status == "deleted"
879
842
  throw :failure if Time.now - wait_start_time > 1800
880
843
  end
@@ -893,19 +856,19 @@ module MU
893
856
  MU.log "#{cluster_id} has been terminated"
894
857
 
895
858
  unless noop
896
- MU::Cloud::AWS::CacheCluster.delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
897
- MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
859
+ delete_subnet_group(subnet_group, region: region, credentials: credentials) if subnet_group
860
+ delete_parameter_group(parameter_group, region: region, credentials: credentials) if parameter_group && !parameter_group.start_with?("default")
898
861
  end
899
862
  end
863
+ private_class_method :terminate_cache_cluster
900
864
 
901
865
  # Remove a Cache Cluster Replication Group and associated artifacts
902
866
  # @param repl_group [OpenStruct]: The cloud provider's description of the Cache Cluster artifact.
903
867
  # @param noop [Boolean]: If true, will only print what would be done.
904
868
  # @param skipsnapshots [Boolean]: If true, will not create a last snapshot before terminating the Cache Cluster.
905
869
  # @param region [String]: The cloud provider's region in which to operate.
906
- # @param cloud_id [String]: The cloud provider's identifier for this resource.
907
870
  # @return [void]
908
- def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, deploy_id: MU.deploy_id, mu_name: nil, cloud_id: nil, credentials: nil)
871
+ def self.terminate_replication_group(repl_group, noop: false, skipsnapshots: false, region: MU.curRegion, credentials: nil)
909
872
  raise MuError, "terminate_replication_group requires a non-nil cache replication group descriptor" if repl_group.nil? || repl_group.empty?
910
873
 
911
874
  repl_group_id = repl_group.replication_group_id
@@ -926,10 +889,10 @@ module MU
926
889
  end
927
890
 
928
891
  # What's the likelihood of having more than one node group? maybe iterate over node_groups instead of assuming there is only one?
929
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
892
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: repl_group_id, target: repl_group.node_groups.first.primary_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
930
893
  # Assuming we also created DNS records for each of our cluster's read endpoint.
931
894
  repl_group.node_groups.first.node_group_members.each { |member|
932
- MU::Cloud::AWS::DNSZone.genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
895
+ MU::Cloud.resourceClass("AWS", "DNSZone").genericMuDNSEntry(name: member.cache_cluster_id, target: member.read_endpoint.address, cloudclass: MU::Cloud::CacheCluster, delete: true)
933
896
  }
934
897
 
935
898
  if %w{deleting deleted}.include?(repl_group.status)
@@ -983,7 +946,7 @@ module MU
983
946
  waiter.before_attempt do |attempts|
984
947
  MU.log "Waiting for #{repl_group_id} to delete..", MU::NOTICE if attempts % 10 == 0
985
948
  end
986
- waiter.before_wait do |attempts, resp|
949
+ waiter.before_wait do |_attempts, resp|
987
950
  throw :success if resp.replication_groups.first.status == "deleted"
988
951
  throw :failure if Time.now - wait_start_time > 1800
989
952
  end
@@ -1005,6 +968,7 @@ module MU
1005
968
  MU::Cloud::AWS::CacheCluster.delete_parameter_group(parameter_group, region: region) if parameter_group && !parameter_group.start_with?("default")
1006
969
  end
1007
970
  end
971
+ private_class_method :terminate_replication_group
1008
972
 
1009
973
  # Remove a Cache Cluster Subnet Group.
1010
974
  # @param subnet_group_id [string]: The cloud provider's ID of the cache cluster subnet group.
@@ -1026,6 +990,7 @@ module MU
1026
990
  MU.log "Subnet group #{subnet_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1027
991
  end
1028
992
  end
993
+ private_class_method :delete_subnet_group
1029
994
 
1030
995
  # Remove a Cache Cluster Parameter Group.
1031
996
  # @param parameter_group_id [string]: The cloud provider's ID of the cache cluster parameter group.
@@ -1049,6 +1014,7 @@ module MU
1049
1014
  MU.log "Parameter group #{parameter_group_id} is not in a removable state after several retries, giving up. #{e.inspect}", MU::ERR
1050
1015
  end
1051
1016
  end
1017
+ private_class_method :delete_parameter_group
1052
1018
  end
1053
1019
  end
1054
1020
  end