cloud-mu 3.1.2 → 3.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +10 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +135 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +165 -111
- data/modules/mu/adoption.rb +401 -68
- data/modules/mu/cleanup.rb +199 -306
- data/modules/mu/cloud.rb +100 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +46 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +920 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +165 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +4 -4
- data/modules/mu/config/cache_cluster.rb +1 -1
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +83 -104
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +6 -6
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +4 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +3 -3
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +3 -3
- data/modules/mu/config/ref.rb +365 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +97 -70
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +70 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +83 -60
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +30 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +389 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +217 -2612
- data/modules/mu/mommacat/daemon.rb +397 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +271 -112
- data/modules/mu/{clouds → providers}/aws/alarm.rb +5 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +26 -22
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +33 -67
- data/modules/mu/{clouds → providers}/aws/collection.rb +24 -23
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +681 -721
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +64 -63
- data/modules/mu/{clouds → providers}/aws/endpoint.rb +22 -27
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +214 -244
- data/modules/mu/{clouds → providers}/aws/folder.rb +7 -7
- data/modules/mu/{clouds → providers}/aws/function.rb +17 -22
- data/modules/mu/{clouds → providers}/aws/group.rb +23 -23
- data/modules/mu/{clouds → providers}/aws/habitat.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +57 -48
- data/modules/mu/{clouds → providers}/aws/log.rb +15 -12
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +17 -16
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +18 -11
- data/modules/mu/{clouds → providers}/aws/notifier.rb +11 -6
- data/modules/mu/{clouds → providers}/aws/role.rb +112 -86
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +39 -33
- data/modules/mu/{clouds → providers}/aws/server.rb +835 -1133
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +56 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +24 -42
- data/modules/mu/{clouds → providers}/aws/user.rb +21 -22
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +523 -929
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +95 -48
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +67 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +84 -77
- data/modules/mu/{clouds → providers}/google/database.rb +10 -20
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +139 -167
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +18 -20
- data/modules/mu/{clouds → providers}/google/role.rb +92 -58
- data/modules/mu/{clouds → providers}/google/server.rb +242 -155
- data/modules/mu/{clouds → providers}/google/server_pool.rb +25 -44
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +11 -0
- data/modules/tests/centos7.yaml +11 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -0
- data/modules/tests/super_simple_bok.yml +1 -3
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +232 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
@@ -89,7 +89,7 @@ Looking elsewhere in `cloud.rb` let's see what all we have to do:
|
|
89
89
|
generic_instance_methods = [:create, :notify, :mu_name, :cloud_id, :config]
|
90
90
|
```
|
91
91
|
|
92
|
-
Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/
|
92
|
+
Just the basics, for now. Here's what that will look like in the AWS layer, in the file `modules/mu/providers/aws/function.rb`:
|
93
93
|
|
94
94
|
```
|
95
95
|
module MU
|
@@ -33,13 +33,25 @@ module MU
|
|
33
33
|
module AdditionalResourceMethods
|
34
34
|
end
|
35
35
|
|
36
|
+
# Is this a "real" cloud provider, or a stub like CloudFormation?
|
37
|
+
def self.virtual?
|
38
|
+
false
|
39
|
+
end
|
40
|
+
|
41
|
+
# List all AWS projects available to our credentials
|
42
|
+
def self.listHabitats(credentials = nil, use_cache: true)
|
43
|
+
cfg = credConfig(credentials)
|
44
|
+
return [] if !cfg or !cfg['account_number']
|
45
|
+
[cfg['account_number']]
|
46
|
+
end
|
47
|
+
|
36
48
|
# A hook that is always called just before any of the instance method of
|
37
49
|
# our resource implementations gets invoked, so that we can ensure that
|
38
50
|
# repetitive setup tasks (like resolving +:resource_group+ for Azure
|
39
51
|
# resources) have always been done.
|
40
52
|
# @param cloudobj [MU::Cloud]
|
41
|
-
# @param
|
42
|
-
def self.resourceInitHook(cloudobj,
|
53
|
+
# @param _deploy [MU::MommaCat]
|
54
|
+
def self.resourceInitHook(cloudobj, _deploy)
|
43
55
|
class << self
|
44
56
|
attr_reader :cloudformation_data
|
45
57
|
end
|
@@ -63,7 +75,6 @@ module MU
|
|
63
75
|
return nil
|
64
76
|
end
|
65
77
|
|
66
|
-
loaded = false
|
67
78
|
cred_obj = nil
|
68
79
|
if cred_cfg['access_key'] and cred_cfg['access_secret'] and
|
69
80
|
# access key and secret just sitting in mu.yaml
|
@@ -137,11 +148,22 @@ module MU
|
|
137
148
|
# assume we've got an IAM profile and hope for the best
|
138
149
|
ENV.delete('AWS_ACCESS_KEY_ID')
|
139
150
|
ENV.delete('AWS_SECRET_ACCESS_KEY')
|
140
|
-
|
151
|
+
retries = 0
|
152
|
+
begin
|
153
|
+
cred_obj = Aws::InstanceProfileCredentials.new
|
154
|
+
if cred_obj.nil?
|
155
|
+
retries += 1
|
156
|
+
MU.log "Failed to fetch AWS instance profile credentials, attempt #{retries.to_s}/10", MU::WARN
|
157
|
+
sleep 3
|
158
|
+
end
|
159
|
+
end while cred_obj.nil? and retries < 10
|
141
160
|
# if name.nil?
|
142
161
|
# Aws.config = {region: ENV['EC2_REGION']}
|
143
162
|
# end
|
144
163
|
end
|
164
|
+
if cred_obj.nil?
|
165
|
+
MU.log "cred_obj is nil and hosted? says #{hosted?.to_s}", MU::WARN, details: name
|
166
|
+
end
|
145
167
|
|
146
168
|
if name.nil?
|
147
169
|
@@creds_loaded["#default"] = cred_obj
|
@@ -172,18 +194,34 @@ module MU
|
|
172
194
|
end
|
173
195
|
end
|
174
196
|
|
175
|
-
# Tag
|
197
|
+
# Tag an EC2 resource
|
176
198
|
#
|
177
199
|
# @param resource [String]: The cloud provider identifier of the resource to tag
|
178
200
|
# @param region [String]: The cloud provider region
|
201
|
+
# @param credentials [String]: Credentials to authorize API requests
|
202
|
+
# @param optional [Boolean]: Whether to apply our optional generic tags
|
203
|
+
# @param nametag [String]: A +Name+ tag to apply
|
204
|
+
# @param othertags [Array<Hash>]: Miscellaneous custom tags, in Basket of Kittens style
|
179
205
|
# @return [void]
|
180
|
-
def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil)
|
206
|
+
def self.createStandardTags(resource = nil, region: MU.curRegion, credentials: nil, optional: true, nametag: nil, othertags: nil)
|
181
207
|
tags = []
|
182
208
|
MU::MommaCat.listStandardTags.each_pair { |name, value|
|
183
|
-
if !value.nil?
|
184
|
-
tags << {key: name, value: value}
|
185
|
-
end
|
209
|
+
tags << {key: name, value: value} if !value.nil?
|
186
210
|
}
|
211
|
+
if optional
|
212
|
+
MU::MommaCat.listOptionalTags.each { |key, value|
|
213
|
+
tags << {key: name, value: value} if !value.nil?
|
214
|
+
}
|
215
|
+
end
|
216
|
+
if nametag
|
217
|
+
tags << { key: "Name", value: nametag }
|
218
|
+
end
|
219
|
+
if othertags
|
220
|
+
othertags.each { |tag|
|
221
|
+
tags << { key: tag['key'], value: tag['value'] }
|
222
|
+
}
|
223
|
+
end
|
224
|
+
|
187
225
|
if MU::Cloud::CloudFormation.emitCloudFormation
|
188
226
|
return tags
|
189
227
|
end
|
@@ -205,6 +243,7 @@ module MU
|
|
205
243
|
end
|
206
244
|
end
|
207
245
|
MU.log "Created standard tags for resource #{resource}", MU::DEBUG, details: caller
|
246
|
+
|
208
247
|
end
|
209
248
|
|
210
249
|
@@myVPCObj = nil
|
@@ -283,52 +322,13 @@ module MU
|
|
283
322
|
)
|
284
323
|
end
|
285
324
|
|
286
|
-
# Tag EC2 resources.
|
287
|
-
#
|
288
|
-
# @param resources [Array<String>]: The cloud provider identifier of the resource to tag
|
289
|
-
# @param key [String]: The name of the tag to create
|
290
|
-
# @param value [String]: The value of the tag
|
291
|
-
# @param region [String]: The cloud provider region
|
292
|
-
# @return [void,<Hash>]
|
293
|
-
def self.createTag(key, value, resources = [], region: myRegion, credentials: nil)
|
294
|
-
|
295
|
-
if !MU::Cloud::CloudFormation.emitCloudFormation
|
296
|
-
begin
|
297
|
-
MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_tags(
|
298
|
-
resources: resources,
|
299
|
-
tags: [
|
300
|
-
{
|
301
|
-
key: key,
|
302
|
-
value: value
|
303
|
-
}
|
304
|
-
]
|
305
|
-
)
|
306
|
-
rescue Aws::EC2::Errors::ServiceError => e
|
307
|
-
MU.log "Got #{e.inspect} tagging #{resources.size.to_s} resources with #{key}=#{value}", MU::WARN, details: resources if attempts > 1
|
308
|
-
if attempts < 5
|
309
|
-
attempts = attempts + 1
|
310
|
-
sleep 15
|
311
|
-
retry
|
312
|
-
else
|
313
|
-
raise e
|
314
|
-
end
|
315
|
-
end
|
316
|
-
MU.log "Created tag #{key} with value #{value}", MU::DEBUG, details: resources
|
317
|
-
else
|
318
|
-
return {
|
319
|
-
"Key" => key,
|
320
|
-
"Value" => value
|
321
|
-
}
|
322
|
-
end
|
323
|
-
end
|
324
|
-
|
325
325
|
@@azs = {}
|
326
326
|
# List the Availability Zones associated with a given Amazon Web Services
|
327
327
|
# region. If no region is given, search the one in which this MU master
|
328
328
|
# server resides.
|
329
329
|
# @param region [String]: The region to search.
|
330
330
|
# @return [Array<String>]: The Availability Zones in this region.
|
331
|
-
def self.listAZs(region: MU.curRegion,
|
331
|
+
def self.listAZs(region: MU.curRegion, credentials: nil)
|
332
332
|
cfg = credConfig(credentials)
|
333
333
|
return [] if !cfg
|
334
334
|
if !region.nil? and @@azs[region]
|
@@ -356,6 +356,27 @@ module MU
|
|
356
356
|
# etc)
|
357
357
|
# @param deploy_id [MU::MommaCat]
|
358
358
|
def self.cleanDeploy(deploy_id, credentials: nil, noop: false)
|
359
|
+
|
360
|
+
if !noop
|
361
|
+
MU.log "Deleting s3://#{adminBucketName(credentials)}/#{deploy_id}-secret"
|
362
|
+
MU::Cloud::AWS.s3(credentials: credentials).delete_object(
|
363
|
+
bucket: adminBucketName(credentials),
|
364
|
+
key: "#{deploy_id}-secret"
|
365
|
+
)
|
366
|
+
listRegions(credentials: credentials).each { |r|
|
367
|
+
resp = MU::Cloud::AWS.ec2(region: r, credentials: credentials).describe_key_pairs(
|
368
|
+
filters: [{name: "key-name", values: ["deploy-#{MU.deploy_id}"]}]
|
369
|
+
)
|
370
|
+
resp.data.key_pairs.each { |keypair|
|
371
|
+
MU.log "Deleting key pair #{keypair.key_name} from #{r}"
|
372
|
+
MU::Cloud::AWS.ec2(region: r, credentials: credentials).delete_key_pair(key_name: keypair.key_name) if !noop
|
373
|
+
}
|
374
|
+
}
|
375
|
+
|
376
|
+
end
|
377
|
+
if hosted?
|
378
|
+
MU::Cloud::AWS.openFirewallForClients
|
379
|
+
end
|
359
380
|
end
|
360
381
|
|
361
382
|
# Plant a Mu deploy secret into a storage bucket somewhere for so our kittens can consume it
|
@@ -435,7 +456,7 @@ module MU
|
|
435
456
|
end
|
436
457
|
|
437
458
|
begin
|
438
|
-
Timeout.timeout(
|
459
|
+
Timeout.timeout(4) do
|
439
460
|
instance_id = open("http://169.254.169.254/latest/meta-data/instance-id").read
|
440
461
|
if !instance_id.nil? and instance_id.size > 0
|
441
462
|
@@is_in_aws = true
|
@@ -491,7 +512,32 @@ module MU
|
|
491
512
|
# @param cloudobj [MU::Cloud::AWS]: The resource from which to extract the habitat id
|
492
513
|
# @return [String,nil]
|
493
514
|
def self.habitat(cloudobj, nolookup: false, deploy: nil)
|
494
|
-
|
515
|
+
@@habmap ||= {}
|
516
|
+
# XXX whaddabout config['habitat'] HNNNGH
|
517
|
+
|
518
|
+
if cloudobj.respond_to?(:account_number) and cloudobj.account_number and
|
519
|
+
!cloudobj.account_number.empty?
|
520
|
+
return cloudobj.account_number
|
521
|
+
elsif cloudobj.config and cloudobj.config['account']
|
522
|
+
if nolookup
|
523
|
+
return cloudobj.config['account']
|
524
|
+
end
|
525
|
+
if @@habmap[cloudobj.config['account']]
|
526
|
+
return @@habmap[cloudobj.config['account']]
|
527
|
+
end
|
528
|
+
deploy ||= cloudobj.deploy if cloudobj.respond_to?(:deploy)
|
529
|
+
|
530
|
+
MU.log "Incomplete implementation: MU::Cloud::AWS.habitat", MU::DEBUG, details: deploy
|
531
|
+
|
532
|
+
# accountobj = accountLookup(cloudobj.config['account'], deploy, raise_on_fail: false)
|
533
|
+
|
534
|
+
# if accountobj
|
535
|
+
# @@habmap[cloudobj.config['account']] = accountobj.cloud_id
|
536
|
+
# return accountobj.cloud_id
|
537
|
+
# end
|
538
|
+
end
|
539
|
+
|
540
|
+
nil
|
495
541
|
end
|
496
542
|
|
497
543
|
|
@@ -506,7 +552,6 @@ module MU
|
|
506
552
|
|
507
553
|
return creds['account_number'] if creds['account_number']
|
508
554
|
|
509
|
-
user_list = MU::Cloud::AWS.iam(credentials: name).list_users.users
|
510
555
|
acct_num = MU::Cloud::AWS.iam(credentials: name).list_users.users.first.arn.split(/:/)[4]
|
511
556
|
acct_num.to_s
|
512
557
|
end
|
@@ -543,8 +588,8 @@ module MU
|
|
543
588
|
if !found
|
544
589
|
MU.log "Attempting to create log bucket #{cfg['log_bucket_name']} for credentials #{credentials}", MU::WARN
|
545
590
|
begin
|
546
|
-
|
547
|
-
rescue Aws::S3::Errors::BucketAlreadyExists
|
591
|
+
MU::Cloud::AWS.s3(credentials: credentials).create_bucket(bucket: cfg['log_bucket_name'], acl: "private")
|
592
|
+
rescue Aws::S3::Errors::BucketAlreadyExists
|
548
593
|
raise MuError, "AWS credentials #{credentials} need a log bucket, and the name #{cfg['log_bucket_name']} is unavailable. Use mu-configure to edit credentials '#{credentials}' or 'hostname'"
|
549
594
|
end
|
550
595
|
end
|
@@ -620,9 +665,9 @@ module MU
|
|
620
665
|
# Check each credential sets' resident account, then
|
621
666
|
$MU_CFG['aws'].each_pair { |acctname, cfg|
|
622
667
|
begin
|
623
|
-
|
668
|
+
MU::Cloud::AWS.iam(credentials: acctname).list_users.users
|
624
669
|
# rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
|
625
|
-
rescue
|
670
|
+
rescue StandardError => e
|
626
671
|
MU.log e.inspect, MU::WARN, details: cfg
|
627
672
|
next
|
628
673
|
end
|
@@ -653,7 +698,7 @@ module MU
|
|
653
698
|
# begin
|
654
699
|
# user_list = MU::Cloud::AWS.iam(region: credConfig['region']).list_users.users
|
655
700
|
## rescue ::Aws::IAM::Errors => e # XXX why does this NameError here?
|
656
|
-
# rescue
|
701
|
+
# rescue StandardError => e
|
657
702
|
# MU.log "Got #{e.inspect} while trying to figure out our account number", MU::WARN, details: caller
|
658
703
|
# end
|
659
704
|
# if user_list.nil? or user_list.size == 0
|
@@ -682,7 +727,6 @@ module MU
|
|
682
727
|
if @@regions.size == 0
|
683
728
|
return [] if credConfig.nil?
|
684
729
|
result = MU::Cloud::AWS.ec2(region: myRegion, credentials: credentials).describe_regions.regions
|
685
|
-
regions = []
|
686
730
|
@@regions_semaphore.synchronize {
|
687
731
|
begin
|
688
732
|
result.each { |r|
|
@@ -761,46 +805,41 @@ module MU
|
|
761
805
|
|
762
806
|
@@instance_types ||= {}
|
763
807
|
@@instance_types[region] ||= {}
|
764
|
-
next_token = nil
|
765
808
|
|
766
|
-
|
767
|
-
|
768
|
-
|
769
|
-
|
770
|
-
|
771
|
-
|
772
|
-
|
773
|
-
|
774
|
-
|
775
|
-
|
776
|
-
|
777
|
-
|
778
|
-
|
779
|
-
|
780
|
-
|
781
|
-
|
782
|
-
|
783
|
-
|
784
|
-
|
785
|
-
type: "TERM_MATCH"
|
786
|
-
}
|
787
|
-
],
|
788
|
-
next_token: next_token
|
789
|
-
)
|
790
|
-
resp.price_list.each { |pricing|
|
791
|
-
data = JSON.parse(pricing)
|
792
|
-
type = data["product"]["attributes"]["instanceType"]
|
793
|
-
next if @@instance_types[region].has_key?(type)
|
794
|
-
@@instance_types[region][type] = {}
|
795
|
-
["ecu", "vcpu", "memory", "storage"].each { |a|
|
796
|
-
@@instance_types[region][type][a] = data["product"]["attributes"][a]
|
809
|
+
# Pricing API isn't widely available, so ask a region we know supports
|
810
|
+
# it
|
811
|
+
resp = MU::Cloud::AWS.pricing(region: "us-east-1").get_products(
|
812
|
+
service_code: "AmazonEC2",
|
813
|
+
filters: [
|
814
|
+
{
|
815
|
+
field: "productFamily",
|
816
|
+
value: "Compute Instance",
|
817
|
+
type: "TERM_MATCH"
|
818
|
+
},
|
819
|
+
{
|
820
|
+
field: "tenancy",
|
821
|
+
value: "Shared",
|
822
|
+
type: "TERM_MATCH"
|
823
|
+
},
|
824
|
+
{
|
825
|
+
field: "location",
|
826
|
+
value: human_region,
|
827
|
+
type: "TERM_MATCH"
|
797
828
|
}
|
798
|
-
|
799
|
-
|
800
|
-
|
829
|
+
]
|
830
|
+
)
|
831
|
+
resp.price_list.each { |pricing|
|
832
|
+
data = JSON.parse(pricing)
|
833
|
+
type = data["product"]["attributes"]["instanceType"]
|
834
|
+
next if @@instance_types[region].has_key?(type)
|
835
|
+
@@instance_types[region][type] = {}
|
836
|
+
["ecu", "vcpu", "memory", "storage"].each { |a|
|
837
|
+
@@instance_types[region][type][a] = data["product"]["attributes"][a]
|
801
838
|
}
|
802
|
-
|
803
|
-
|
839
|
+
@@instance_types[region][type]["memory"].sub!(/ GiB/, "")
|
840
|
+
@@instance_types[region][type]["memory"] = @@instance_types[region][type]["memory"].to_f
|
841
|
+
@@instance_types[region][type]["vcpu"] = @@instance_types[region][type]["vcpu"].to_f
|
842
|
+
}
|
804
843
|
|
805
844
|
@@instance_types
|
806
845
|
end
|
@@ -1154,12 +1193,60 @@ module MU
|
|
1154
1193
|
end
|
1155
1194
|
end
|
1156
1195
|
|
1196
|
+
# Tag a resource. Defaults to applying our MU deployment identifier, if no
|
1197
|
+
# arguments other than the resource identifier are given.
|
1198
|
+
#
|
1199
|
+
# @param resource [String]: The cloud provider identifier of the resource to tag
|
1200
|
+
# @param tag_name [String]: The name of the tag to create
|
1201
|
+
# @param tag_value [String]: The value of the tag
|
1202
|
+
# @param region [String]: The cloud provider region
|
1203
|
+
# @return [void]
|
1204
|
+
def self.createTag(resource = nil,
|
1205
|
+
tag_name="MU-ID",
|
1206
|
+
tag_value=MU.deploy_id,
|
1207
|
+
region: MU.curRegion,
|
1208
|
+
credentials: nil)
|
1209
|
+
attempts = 0
|
1210
|
+
|
1211
|
+
return nil if resource.nil?
|
1212
|
+
resource = [resource] if resource.is_a?(String)
|
1213
|
+
|
1214
|
+
if !MU::Cloud::CloudFormation.emitCloudFormation
|
1215
|
+
begin
|
1216
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags(
|
1217
|
+
resources: resource,
|
1218
|
+
tags: [
|
1219
|
+
{
|
1220
|
+
key: tag_name,
|
1221
|
+
value: tag_value
|
1222
|
+
}
|
1223
|
+
]
|
1224
|
+
)
|
1225
|
+
rescue Aws::EC2::Errors::ServiceError => e
|
1226
|
+
MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1
|
1227
|
+
if attempts < 5
|
1228
|
+
attempts = attempts + 1
|
1229
|
+
sleep 15
|
1230
|
+
retry
|
1231
|
+
else
|
1232
|
+
raise e
|
1233
|
+
end
|
1234
|
+
end
|
1235
|
+
MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG
|
1236
|
+
else
|
1237
|
+
return {
|
1238
|
+
"Key" => tag_name,
|
1239
|
+
"Value" => tag_value
|
1240
|
+
}
|
1241
|
+
end
|
1242
|
+
end
|
1243
|
+
|
1157
1244
|
@syslog_port_semaphore = Mutex.new
|
1158
1245
|
# Punch AWS security group holes for client nodes to talk back to us, the
|
1159
1246
|
# Mu Master, if we're in AWS.
|
1160
1247
|
# @return [void]
|
1161
1248
|
def self.openFirewallForClients
|
1162
|
-
MU::Cloud.
|
1249
|
+
MU::Cloud.resourceClass("AWS", :FirewallRule)
|
1163
1250
|
begin
|
1164
1251
|
if File.exist?(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|
1165
1252
|
::Chef::Config.from_file(Etc.getpwuid(Process.uid).dir+"/.chef/knife.rb")
|
@@ -1205,8 +1292,8 @@ module MU
|
|
1205
1292
|
)
|
1206
1293
|
sg_id = group.group_id
|
1207
1294
|
my_sgs << sg_id
|
1208
|
-
MU::
|
1209
|
-
MU::
|
1295
|
+
MU::Cloud::AWS.createTag sg_id, "Name", my_client_sg_name
|
1296
|
+
MU::Cloud::AWS.createTag sg_id, "MU-MASTER-IP", MU.mu_public_ip
|
1210
1297
|
MU::Cloud::AWS.ec2.modify_instance_attribute(
|
1211
1298
|
instance_id: my_instance_id,
|
1212
1299
|
groups: my_sgs
|
@@ -1237,7 +1324,7 @@ module MU
|
|
1237
1324
|
end
|
1238
1325
|
|
1239
1326
|
allow_ips = ["10.0.0.0/8", "172.16.0.0/12", "192.168.0.0/16"]
|
1240
|
-
MU::MommaCat.listAllNodes.
|
1327
|
+
MU::MommaCat.listAllNodes.values.each { |data|
|
1241
1328
|
next if data.nil? or !data.is_a?(Hash)
|
1242
1329
|
["public_ip_address"].each { |key|
|
1243
1330
|
if data.has_key?(key) and !data[key].nil? and !data[key].empty?
|
@@ -1266,7 +1353,7 @@ module MU
|
|
1266
1353
|
}
|
1267
1354
|
]
|
1268
1355
|
)
|
1269
|
-
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
1356
|
+
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
1270
1357
|
MU.log "Permission disappeared from #{sg_id} (port #{port.to_s}) before I could remove it", MU::WARN, details: MU.structToHash(rule.ip_ranges)
|
1271
1358
|
end
|
1272
1359
|
end
|
@@ -1300,8 +1387,6 @@ module MU
|
|
1300
1387
|
}
|
1301
1388
|
end
|
1302
1389
|
|
1303
|
-
private
|
1304
|
-
|
1305
1390
|
# XXX we shouldn't have to do this, but AWS does not provide a way to look
|
1306
1391
|
# it up, and the pricing API only returns the human-readable strings.
|
1307
1392
|
@@regionLookup = {
|
@@ -1337,7 +1422,7 @@ module MU
|
|
1337
1422
|
# Create an AWS API client
|
1338
1423
|
# @param region [String]: Amazon region so we know what endpoint to use
|
1339
1424
|
# @param api [String]: Which API are we wrapping?
|
1340
|
-
def initialize(region:
|
1425
|
+
def initialize(region: nil, api: "EC2", credentials: nil)
|
1341
1426
|
@cred_obj = MU::Cloud::AWS.loadCredentials(credentials)
|
1342
1427
|
@credentials = MU::Cloud::AWS.credConfig(credentials, name_only: true)
|
1343
1428
|
|
@@ -1346,6 +1431,8 @@ module MU
|
|
1346
1431
|
end
|
1347
1432
|
|
1348
1433
|
params = {}
|
1434
|
+
region ||= MU::Cloud::AWS.credConfig(credentials)['region']
|
1435
|
+
region ||= MU.myRegion
|
1349
1436
|
|
1350
1437
|
if region
|
1351
1438
|
@region = region
|
@@ -1362,21 +1449,93 @@ module MU
|
|
1362
1449
|
# Catch-all for AWS client methods. Essentially a pass-through with some
|
1363
1450
|
# rescues for known silly endpoint behavior.
|
1364
1451
|
def method_missing(method_sym, *arguments)
|
1452
|
+
# make sure error symbols are loaded for our exception handling later
|
1365
1453
|
require "aws-sdk-core"
|
1454
|
+
require "aws-sdk-core/rds"
|
1455
|
+
require "aws-sdk-core/ec2"
|
1456
|
+
require "aws-sdk-core/route53"
|
1457
|
+
require "aws-sdk-core/iam"
|
1458
|
+
require "aws-sdk-core/efs"
|
1459
|
+
require "aws-sdk-core/pricing"
|
1460
|
+
require "aws-sdk-core/apigateway"
|
1461
|
+
require "aws-sdk-core/ecs"
|
1462
|
+
require "aws-sdk-core/eks"
|
1463
|
+
require "aws-sdk-core/cloudwatchlogs"
|
1464
|
+
require "aws-sdk-core/elasticloadbalancing"
|
1465
|
+
require "aws-sdk-core/elasticloadbalancingv2"
|
1466
|
+
require "aws-sdk-core/autoscaling"
|
1467
|
+
require "aws-sdk-core/client_waiters"
|
1468
|
+
require "aws-sdk-core/waiters/errors"
|
1366
1469
|
|
1367
1470
|
retries = 0
|
1368
1471
|
begin
|
1369
1472
|
MU.log "Calling #{method_sym} in #{@region}", MU::DEBUG, details: arguments
|
1370
|
-
|
1371
|
-
|
1372
|
-
|
1373
|
-
|
1374
|
-
|
1375
|
-
|
1376
|
-
|
1473
|
+
|
1474
|
+
retval = if !arguments.nil? and arguments.size == 1
|
1475
|
+
@api.method(method_sym).call(arguments[0])
|
1476
|
+
elsif !arguments.nil? and arguments.size > 0
|
1477
|
+
@api.method(method_sym).call(*arguments)
|
1478
|
+
else
|
1479
|
+
@api.method(method_sym).call
|
1480
|
+
end
|
1481
|
+
|
1482
|
+
if !retval.nil?
|
1483
|
+
begin
|
1484
|
+
page_markers = [:marker, :next_token]
|
1485
|
+
paginator = nil
|
1486
|
+
new_page = nil
|
1487
|
+
[:next_token, :marker].each { |m|
|
1488
|
+
if !retval.nil? and retval.respond_to?(m)
|
1489
|
+
paginator = m
|
1490
|
+
new_page = retval.send(paginator)
|
1491
|
+
break
|
1492
|
+
end
|
1493
|
+
}
|
1494
|
+
|
1495
|
+
if paginator and new_page and !new_page.empty?
|
1496
|
+
resp = retval.respond_to?(:__getobj__) ? retval.__getobj__ : retval
|
1497
|
+
concat_to = resp.class.instance_methods(false).reject { |m|
|
1498
|
+
m.to_s.match(/=$/) or m == paginator or resp.send(m).nil? or !resp.send(m).is_a?(Array)
|
1499
|
+
}
|
1500
|
+
if concat_to.size != 1
|
1501
|
+
MU.log "Tried to figure out where I might append paginated results for a #{resp.class.name}, but failed", MU::DEBUG, details: concat_to
|
1502
|
+
else
|
1503
|
+
concat_to = concat_to.first
|
1504
|
+
new_args = arguments ? arguments.dup : [{}]
|
1505
|
+
begin
|
1506
|
+
if new_args.is_a?(Array)
|
1507
|
+
new_args << {} if new_args.empty?
|
1508
|
+
if new_args.size == 1 and new_args.first.is_a?(Hash)
|
1509
|
+
new_args[0][paginator] = new_page
|
1510
|
+
else
|
1511
|
+
MU.log "I don't know how to insert a #{paginator} into these arguments for #{method_sym}", MU::WARN, details: new_args
|
1512
|
+
end
|
1513
|
+
elsif new_args.is_a?(Hash)
|
1514
|
+
new_args[paginator] = new_page
|
1515
|
+
end
|
1516
|
+
|
1517
|
+
MU.log "Attempting magic pagination for #{method_sym}", MU::DEBUG, details: new_args
|
1518
|
+
|
1519
|
+
# resp = if !arguments.nil? and arguments.size == 1
|
1520
|
+
# @api.method(method_sym).call(new_args[0])
|
1521
|
+
# elsif !arguments.nil? and arguments.size > 0
|
1522
|
+
resp = @api.method(method_sym).call(*new_args)
|
1523
|
+
# end
|
1524
|
+
break if resp.nil?
|
1525
|
+
resp = resp.__getobj__ if resp.respond_to?(:__getobj__)
|
1526
|
+
retval.send(concat_to).concat(resp.send(concat_to))
|
1527
|
+
new_page = resp.send(paginator) if !resp.nil?
|
1528
|
+
end while !resp.nil? and !new_page.nil? and !new_page.empty?
|
1529
|
+
end
|
1530
|
+
end
|
1531
|
+
rescue StandardError => e
|
1532
|
+
MU.log "Made a good-faith effort to auto-paginate API call to #{method_sym} and failed with #{e.message}", MU::DEBUG, details: arguments
|
1533
|
+
raise e
|
1534
|
+
end
|
1377
1535
|
end
|
1536
|
+
|
1378
1537
|
return retval
|
1379
|
-
rescue Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
1538
|
+
rescue Aws::RDS::Errors::Throttling, Aws::EC2::Errors::InternalError, Aws::EC2::Errors::RequestLimitExceeded, Aws::EC2::Errors::Unavailable, Aws::Route53::Errors::Throttling, Aws::ElasticLoadBalancing::Errors::HttpFailureException, Aws::EC2::Errors::Http503Error, Aws::AutoScaling::Errors::Http503Error, Aws::AutoScaling::Errors::InternalFailure, Aws::AutoScaling::Errors::ServiceUnavailable, Aws::Route53::Errors::ServiceUnavailable, Aws::ElasticLoadBalancing::Errors::Throttling, Aws::RDS::Errors::ClientUnavailable, Aws::Waiters::Errors::UnexpectedError, Aws::ElasticLoadBalancing::Errors::ServiceUnavailable, Aws::ElasticLoadBalancingV2::Errors::Throttling, Seahorse::Client::NetworkingError, Aws::IAM::Errors::Throttling, Aws::EFS::Errors::ThrottlingException, Aws::Pricing::Errors::ThrottlingException, Aws::APIGateway::Errors::TooManyRequestsException, Aws::ECS::Errors::ThrottlingException, Net::ReadTimeout, Faraday::TimeoutError, Aws::CloudWatchLogs::Errors::ThrottlingException => e
|
1380
1539
|
if e.class.name == "Seahorse::Client::NetworkingError" and e.message.match(/Name or service not known/)
|
1381
1540
|
MU.log e.inspect, MU::ERR
|
1382
1541
|
raise e
|
@@ -1397,7 +1556,7 @@ module MU
|
|
1397
1556
|
MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}, waiting #{interval.to_s}s and retrying. Args were: #{arguments}", debuglevel, details: caller
|
1398
1557
|
sleep interval
|
1399
1558
|
retry
|
1400
|
-
rescue
|
1559
|
+
rescue StandardError => e
|
1401
1560
|
MU.log "Got #{e.inspect} calling EC2's #{method_sym} in #{@region} with credentials #{@credentials}", MU::DEBUG, details: arguments
|
1402
1561
|
raise e
|
1403
1562
|
end
|