cloud-mu 3.1.3 → 3.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +15 -3
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +21 -13
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +4 -4
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +147 -37
- data/cloud-mu.gemspec +22 -20
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +3 -2
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +6 -16
- data/modules/mu.rb +158 -111
- data/modules/mu/adoption.rb +404 -71
- data/modules/mu/cleanup.rb +221 -306
- data/modules/mu/cloud.rb +129 -1633
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +171 -1767
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +4 -4
- data/modules/mu/config/container_cluster.rb +9 -4
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +10 -9
- data/modules/mu/config/doc_helpers.rb +516 -0
- data/modules/mu/config/endpoint.rb +5 -4
- data/modules/mu/config/firewall_rule.rb +103 -4
- data/modules/mu/config/folder.rb +4 -4
- data/modules/mu/config/function.rb +19 -10
- data/modules/mu/config/group.rb +4 -4
- data/modules/mu/config/habitat.rb +4 -4
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +60 -14
- data/modules/mu/config/log.rb +4 -4
- data/modules/mu/config/msg_queue.rb +4 -4
- data/modules/mu/config/nosqldb.rb +4 -4
- data/modules/mu/config/notifier.rb +10 -21
- data/modules/mu/config/ref.rb +411 -0
- data/modules/mu/config/role.rb +4 -4
- data/modules/mu/config/schema_helpers.rb +509 -0
- data/modules/mu/config/search_domain.rb +4 -4
- data/modules/mu/config/server.rb +98 -71
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +5 -9
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +200 -0
- data/modules/mu/config/user.rb +4 -4
- data/modules/mu/config/vpc.rb +71 -27
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +3 -2
- data/modules/mu/deploy.rb +43 -26
- data/modules/mu/groomer.rb +17 -2
- data/modules/mu/groomers/ansible.rb +188 -41
- data/modules/mu/groomers/chef.rb +116 -55
- data/modules/mu/logger.rb +127 -148
- data/modules/mu/master.rb +410 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -3
- data/modules/mu/mommacat.rb +218 -2612
- data/modules/mu/mommacat/daemon.rb +403 -0
- data/modules/mu/mommacat/naming.rb +473 -0
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +722 -0
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +380 -122
- data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
- data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
- data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
- data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
- data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
- data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
- data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
- data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
- data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
- data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
- data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +29 -9
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
- data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
- data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
- data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
- data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +68 -30
- data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
- data/modules/mu/{clouds → providers}/google/database.rb +11 -21
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
- data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
- data/modules/mu/{clouds → providers}/google/function.rb +140 -168
- data/modules/mu/{clouds → providers}/google/group.rb +29 -34
- data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
- data/modules/mu/{clouds → providers}/google/role.rb +94 -58
- data/modules/mu/{clouds → providers}/google/server.rb +243 -156
- data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
- data/modules/mu/{clouds → providers}/google/user.rb +95 -31
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +17 -5
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +240 -154
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1985
- data/modules/mu/clouds/aws/endpoint.rb +0 -592
|
@@ -18,7 +18,7 @@ module MU
|
|
|
18
18
|
class AWS
|
|
19
19
|
# A firewall ruleset as configured in {MU::Config::BasketofKittens::firewall_rules}
|
|
20
20
|
class FirewallRule < MU::Cloud::FirewallRule
|
|
21
|
-
require "mu/
|
|
21
|
+
require "mu/providers/aws/vpc"
|
|
22
22
|
|
|
23
23
|
@admin_sgs = Hash.new
|
|
24
24
|
@admin_sg_semaphore = Mutex.new
|
|
@@ -54,13 +54,12 @@ module MU
|
|
|
54
54
|
|
|
55
55
|
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
|
|
56
56
|
@cloud_id = secgroup.group_id
|
|
57
|
-
rescue Aws::EC2::Errors::InvalidGroupDuplicate
|
|
57
|
+
rescue Aws::EC2::Errors::InvalidGroupDuplicate
|
|
58
58
|
MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
|
|
59
59
|
filters = [{name: "group-name", values: [groupname]}]
|
|
60
60
|
filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
|
|
61
61
|
|
|
62
62
|
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
|
|
63
|
-
deploy_id = @deploy.deploy_id if !@deploy_id.nil?
|
|
64
63
|
if secgroup.nil?
|
|
65
64
|
raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
|
|
66
65
|
end
|
|
@@ -69,24 +68,24 @@ module MU
|
|
|
69
68
|
|
|
70
69
|
begin
|
|
71
70
|
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
|
|
72
|
-
rescue Aws::EC2::Errors::InvalidGroupNotFound
|
|
71
|
+
rescue Aws::EC2::Errors::InvalidGroupNotFound
|
|
73
72
|
MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
|
|
74
73
|
sleep 10
|
|
75
74
|
retry
|
|
76
75
|
end
|
|
77
76
|
|
|
78
77
|
MU::Cloud::AWS.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
|
|
79
|
-
MU::
|
|
78
|
+
MU::Cloud::AWS.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
|
|
80
79
|
|
|
81
80
|
if @config['optional_tags']
|
|
82
81
|
MU::MommaCat.listOptionalTags.each { |key, value|
|
|
83
|
-
MU::
|
|
82
|
+
MU::Cloud::AWS.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
|
84
83
|
}
|
|
85
84
|
end
|
|
86
85
|
|
|
87
86
|
if @config['tags']
|
|
88
87
|
@config['tags'].each { |tag|
|
|
89
|
-
MU::
|
|
88
|
+
MU::Cloud::AWS.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
|
90
89
|
}
|
|
91
90
|
end
|
|
92
91
|
|
|
@@ -180,7 +179,7 @@ module MU
|
|
|
180
179
|
ip_permissions: ec2_rule
|
|
181
180
|
)
|
|
182
181
|
end
|
|
183
|
-
rescue Aws::EC2::Errors::InvalidPermissionDuplicate
|
|
182
|
+
rescue Aws::EC2::Errors::InvalidPermissionDuplicate
|
|
184
183
|
MU.log "Attempt to add duplicate rule to #{@cloud_id}", MU::DEBUG, details: ec2_rule
|
|
185
184
|
# Ensure that, at least, the description field gets updated on
|
|
186
185
|
# existing rules
|
|
@@ -246,7 +245,7 @@ module MU
|
|
|
246
245
|
# Reverse-map our cloud description into a runnable config hash.
|
|
247
246
|
# We assume that any values we have in +@config+ are placeholders, and
|
|
248
247
|
# calculate our own accordingly based on what's live in the cloud.
|
|
249
|
-
def toKitten(
|
|
248
|
+
def toKitten(**_args)
|
|
250
249
|
bok = {
|
|
251
250
|
"cloud" => "AWS",
|
|
252
251
|
"credentials" => @config['credentials'],
|
|
@@ -382,24 +381,24 @@ module MU
|
|
|
382
381
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
|
383
382
|
# @param region [String]: The cloud provider region
|
|
384
383
|
# @return [void]
|
|
385
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
386
|
-
filters =
|
|
387
|
-
|
|
388
|
-
filters = [
|
|
384
|
+
def self.cleanup(noop: false, deploy_id: MU.deploy_id, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
|
385
|
+
filters = if flags and flags["vpc_id"]
|
|
386
|
+
[
|
|
389
387
|
{name: "vpc-id", values: [flags["vpc_id"]]}
|
|
390
388
|
]
|
|
391
389
|
else
|
|
392
390
|
filters = [
|
|
393
|
-
{name: "tag:MU-ID", values: [
|
|
391
|
+
{name: "tag:MU-ID", values: [deploy_id]}
|
|
394
392
|
]
|
|
395
393
|
if !ignoremaster
|
|
396
394
|
filters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
|
397
395
|
end
|
|
396
|
+
filters
|
|
398
397
|
end
|
|
399
398
|
|
|
400
399
|
# Some services create sneaky rogue ENIs which then block removal of
|
|
401
400
|
# associated security groups. Find them and fry them.
|
|
402
|
-
MU::Cloud
|
|
401
|
+
MU::Cloud.resourceClass("AWS", "VPC").purge_interfaces(noop, filters, region: region, credentials: credentials)
|
|
403
402
|
|
|
404
403
|
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
|
|
405
404
|
filters: filters
|
|
@@ -409,75 +408,8 @@ module MU
|
|
|
409
408
|
MU.log "Revoking rules in EC2 Security Group #{sg.group_name} (#{sg.group_id})"
|
|
410
409
|
|
|
411
410
|
if !noop
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
sg.ip_permissions.each { |hole|
|
|
415
|
-
ingress_to_revoke << MU.structToHash(hole)
|
|
416
|
-
ingress_to_revoke.each { |rule|
|
|
417
|
-
if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs] .size == 0
|
|
418
|
-
rule.delete(:user_id_group_pairs)
|
|
419
|
-
elsif !rule[:user_id_group_pairs].nil?
|
|
420
|
-
rule[:user_id_group_pairs].each { |group_ref|
|
|
421
|
-
group_ref = MU.structToHash(group_ref)
|
|
422
|
-
group_ref.delete(:group_name) if group_ref[:group_id]
|
|
423
|
-
}
|
|
424
|
-
end
|
|
425
|
-
|
|
426
|
-
if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
|
|
427
|
-
rule.delete(:ip_ranges)
|
|
428
|
-
end
|
|
429
|
-
|
|
430
|
-
if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
|
|
431
|
-
rule.delete(:prefix_list_ids)
|
|
432
|
-
end
|
|
433
|
-
|
|
434
|
-
if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
|
|
435
|
-
rule.delete(:ipv_6_ranges)
|
|
436
|
-
end
|
|
437
|
-
}
|
|
438
|
-
}
|
|
439
|
-
sg.ip_permissions_egress.each { |hole|
|
|
440
|
-
egress_to_revoke << MU.structToHash(hole)
|
|
441
|
-
egress_to_revoke.each { |rule|
|
|
442
|
-
if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
|
|
443
|
-
rule.delete(:user_id_group_pairs)
|
|
444
|
-
elsif !rule[:user_id_group_pairs].nil?
|
|
445
|
-
rule[:user_id_group_pairs].each { |group_ref|
|
|
446
|
-
group_ref = MU.structToHash(group_ref)
|
|
447
|
-
group_ref.delete(:group_name) if group_ref[:group_id]
|
|
448
|
-
}
|
|
449
|
-
end
|
|
450
|
-
|
|
451
|
-
if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
|
|
452
|
-
rule.delete(:ip_ranges)
|
|
453
|
-
end
|
|
454
|
-
|
|
455
|
-
if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
|
|
456
|
-
rule.delete(:prefix_list_ids)
|
|
457
|
-
end
|
|
458
|
-
|
|
459
|
-
if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
|
|
460
|
-
rule.delete(:ipv_6_ranges)
|
|
461
|
-
end
|
|
462
|
-
}
|
|
463
|
-
}
|
|
464
|
-
begin
|
|
465
|
-
|
|
466
|
-
if ingress_to_revoke.size > 0
|
|
467
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
|
|
468
|
-
group_id: sg.group_id,
|
|
469
|
-
ip_permissions: ingress_to_revoke
|
|
470
|
-
)
|
|
471
|
-
end
|
|
472
|
-
if egress_to_revoke.size > 0
|
|
473
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
|
|
474
|
-
group_id: sg.group_id,
|
|
475
|
-
ip_permissions: egress_to_revoke
|
|
476
|
-
)
|
|
477
|
-
end
|
|
478
|
-
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
|
479
|
-
MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
|
|
480
|
-
end
|
|
411
|
+
revoke_rules(sg, region: region, credentials: credentials)
|
|
412
|
+
revoke_rules(sg, egress: true, region: region, credentials: credentials)
|
|
481
413
|
end
|
|
482
414
|
}
|
|
483
415
|
|
|
@@ -485,61 +417,132 @@ module MU
|
|
|
485
417
|
next if sg.group_name == "default"
|
|
486
418
|
MU.log "Removing EC2 Security Group #{sg.group_name}"
|
|
487
419
|
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
{ name: "vpc-id", values: [sg.vpc_id] }
|
|
506
|
-
]
|
|
507
|
-
).security_groups
|
|
508
|
-
if default_resp and default_resp.size == 1
|
|
509
|
-
default_sg = default_resp.first.group_id
|
|
510
|
-
eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
|
|
511
|
-
filters: [ {name: "group-id", values: [sg.group_id]} ]
|
|
512
|
-
)
|
|
513
|
-
if eni_resp and eni_resp.data and
|
|
514
|
-
eni_resp.data.network_interfaces
|
|
515
|
-
eni_resp.data.network_interfaces.each { |iface|
|
|
516
|
-
iface_groups = iface.groups.map { |sg| sg.group_id }
|
|
517
|
-
iface_groups.delete(sg.group_id)
|
|
518
|
-
iface_groups << default_sg if iface_groups.empty?
|
|
519
|
-
MU.log "Attempting to remove #{sg.group_id} from ENI #{iface.network_interface_id}"
|
|
420
|
+
on_retry = Proc.new {
|
|
421
|
+
# try to get out from under loose network interfaces with which
|
|
422
|
+
# we're associated
|
|
423
|
+
if sg.vpc_id
|
|
424
|
+
default_sg = MU::Cloud.resourceClass("AWS", "VPC").getDefaultSg(sg.vpc_id, region: region, credentials: credentials)
|
|
425
|
+
if default_sg
|
|
426
|
+
eni_resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_network_interfaces(
|
|
427
|
+
filters: [ {name: "group-id", values: [sg.group_id]} ]
|
|
428
|
+
)
|
|
429
|
+
if eni_resp and eni_resp.data and
|
|
430
|
+
eni_resp.data.network_interfaces
|
|
431
|
+
eni_resp.data.network_interfaces.each { |iface|
|
|
432
|
+
iface_groups = iface.groups.map { |if_sg| if_sg.group_id }
|
|
433
|
+
iface_groups.delete(sg.group_id)
|
|
434
|
+
iface_groups << default_sg if iface_groups.empty?
|
|
435
|
+
MU.log "Attempting to remove #{sg.group_id} (#{sg.group_name}) from ENI #{iface.network_interface_id}"
|
|
436
|
+
begin
|
|
520
437
|
MU::Cloud::AWS.ec2(credentials: credentials, region: region).modify_network_interface_attribute(
|
|
521
438
|
network_interface_id: iface.network_interface_id,
|
|
522
439
|
groups: iface_groups
|
|
523
440
|
)
|
|
524
|
-
|
|
525
|
-
|
|
441
|
+
rescue ::Aws::EC2::Errors::InvalidNetworkInterfaceIDNotFound
|
|
442
|
+
# fine by me
|
|
443
|
+
rescue ::Aws::EC2::Errors::AuthFailure
|
|
444
|
+
MU.log "Permission denied attempting to trim Security Group list for #{iface.network_interface_id}", MU::WARN, details: iface.groups.map { |g| g.group_name }.join(",")+" => default"
|
|
445
|
+
end
|
|
446
|
+
}
|
|
526
447
|
end
|
|
527
448
|
end
|
|
449
|
+
end
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
if !noop
|
|
453
|
+
MU.retrier([Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse], ignoreme: [Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, on_retry: on_retry) {
|
|
454
|
+
begin
|
|
455
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id)
|
|
456
|
+
rescue Aws::EC2::Errors::CannotDelete => e
|
|
457
|
+
MU.log e.message, MU::WARN
|
|
458
|
+
end
|
|
459
|
+
}
|
|
460
|
+
end
|
|
461
|
+
|
|
462
|
+
}
|
|
463
|
+
end
|
|
464
|
+
|
|
465
|
+
def self.revoke_rules(sg, egress: false, region: MU.myregion, credentials: nil)
|
|
466
|
+
holes = sg.send(egress ? :ip_permissions_egress : :ip_permissions)
|
|
467
|
+
|
|
468
|
+
to_revoke = []
|
|
469
|
+
|
|
470
|
+
holes.each { |hole|
|
|
471
|
+
to_revoke << MU.structToHash(hole)
|
|
472
|
+
to_revoke.each { |rule|
|
|
473
|
+
if !rule[:user_id_group_pairs].nil? and rule[:user_id_group_pairs].size == 0
|
|
474
|
+
rule.delete(:user_id_group_pairs)
|
|
475
|
+
elsif !rule[:user_id_group_pairs].nil?
|
|
476
|
+
rule[:user_id_group_pairs].each { |group_ref|
|
|
477
|
+
group_ref = MU.structToHash(group_ref)
|
|
478
|
+
group_ref.delete(:group_name) if group_ref[:group_id]
|
|
479
|
+
}
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
if !rule[:ip_ranges].nil? and rule[:ip_ranges].size == 0
|
|
483
|
+
rule.delete(:ip_ranges)
|
|
484
|
+
end
|
|
485
|
+
|
|
486
|
+
if !rule[:prefix_list_ids].nil? and rule[:prefix_list_ids].size == 0
|
|
487
|
+
rule.delete(:prefix_list_ids)
|
|
488
|
+
end
|
|
489
|
+
|
|
490
|
+
if !rule[:ipv_6_ranges].nil? and rule[:ipv_6_ranges].size == 0
|
|
491
|
+
rule.delete(:ipv_6_ranges)
|
|
492
|
+
end
|
|
493
|
+
}
|
|
494
|
+
}
|
|
528
495
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
496
|
+
if to_revoke.size > 0
|
|
497
|
+
begin
|
|
498
|
+
if egress
|
|
499
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
|
|
500
|
+
group_id: sg.group_id,
|
|
501
|
+
ip_permissions: to_revoke
|
|
502
|
+
)
|
|
532
503
|
else
|
|
533
|
-
MU.
|
|
504
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
|
|
505
|
+
group_id: sg.group_id,
|
|
506
|
+
ip_permissions: to_revoke
|
|
507
|
+
)
|
|
534
508
|
end
|
|
509
|
+
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
|
510
|
+
MU.log "Rule in #{sg.group_id} disappeared before I could remove it", MU::WARN
|
|
535
511
|
end
|
|
512
|
+
end
|
|
513
|
+
|
|
514
|
+
end
|
|
515
|
+
private_class_method :revoke_rules
|
|
516
|
+
|
|
517
|
+
# Return an AWS-specific chunk of schema commonly used in the +ingress_rules+ parameter of other resource types.
|
|
518
|
+
# @return [Hash]
|
|
519
|
+
def self.ingressRuleAddtlSchema
|
|
520
|
+
{
|
|
521
|
+
"items" => {
|
|
522
|
+
"properties" => {
|
|
523
|
+
"sgs" => {
|
|
524
|
+
"type" => "array",
|
|
525
|
+
"items" => {
|
|
526
|
+
"description" => "Other AWS Security Groups; resources that are associated with this group will have this rule applied to their traffic",
|
|
527
|
+
"type" => "string"
|
|
528
|
+
}
|
|
529
|
+
},
|
|
530
|
+
"lbs" => {
|
|
531
|
+
"type" => "array",
|
|
532
|
+
"items" => {
|
|
533
|
+
"description" => "AWS Load Balancers which will have this rule applied to their traffic",
|
|
534
|
+
"type" => "string"
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
}
|
|
536
539
|
}
|
|
537
540
|
end
|
|
538
541
|
|
|
539
542
|
# Cloud-specific configuration properties.
|
|
540
|
-
# @param
|
|
543
|
+
# @param _config [MU::Config]: The calling MU::Config object
|
|
541
544
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
|
542
|
-
def self.schema(
|
|
545
|
+
def self.schema(_config)
|
|
543
546
|
toplevel_required = []
|
|
544
547
|
schema = {
|
|
545
548
|
"rules" => {
|
|
@@ -645,36 +648,16 @@ module MU
|
|
|
645
648
|
|
|
646
649
|
if rule['firewall_rules']
|
|
647
650
|
rule['firewall_rules'].each { |sg|
|
|
648
|
-
if sg
|
|
649
|
-
|
|
650
|
-
"type" => "firewall_rule",
|
|
651
|
-
"name" => sg.name,
|
|
652
|
-
"no_create_wait" => true
|
|
653
|
-
}
|
|
654
|
-
elsif sg['name'] and !sg['deploy_id']
|
|
655
|
-
acl["dependencies"] << {
|
|
656
|
-
"type" => "firewall_rule",
|
|
657
|
-
"name" => sg['name'],
|
|
658
|
-
"no_create_wait" => true
|
|
659
|
-
}
|
|
651
|
+
if sg['name'] and !sg['deploy_id']
|
|
652
|
+
MU::Config.addDependency(acl, sg['name'], "firewall_rule", no_create_wait: true)
|
|
660
653
|
end
|
|
661
654
|
}
|
|
662
655
|
end
|
|
663
656
|
|
|
664
657
|
if rule['loadbalancers']
|
|
665
658
|
rule['loadbalancers'].each { |lb|
|
|
666
|
-
if lb
|
|
667
|
-
|
|
668
|
-
"type" => "loadbalancer",
|
|
669
|
-
"name" => lb.name,
|
|
670
|
-
"phase" => "groom"
|
|
671
|
-
}
|
|
672
|
-
elsif lb['name'] and !lb['deploy_id']
|
|
673
|
-
acl["dependencies"] << {
|
|
674
|
-
"type" => "loadbalancer",
|
|
675
|
-
"name" => lb['name'],
|
|
676
|
-
"phase" => "groom"
|
|
677
|
-
}
|
|
659
|
+
if lb['name'] and !lb['deploy_id']
|
|
660
|
+
MU::Config.addDependency(acl, lb['name'], "loadbalancer", phase: "groom")
|
|
678
661
|
end
|
|
679
662
|
}
|
|
680
663
|
end
|
|
@@ -716,32 +699,7 @@ module MU
|
|
|
716
699
|
|
|
717
700
|
private
|
|
718
701
|
|
|
719
|
-
|
|
720
|
-
# Manufacture an EC2 security group. The second parameter, rules, is an
|
|
721
|
-
# "ingress_rules" structure parsed and validated by MU::Config.
|
|
722
|
-
#########################################################################
|
|
723
|
-
def setRules(rules, add_to_self: false, ingress: true, egress: false)
|
|
724
|
-
describe
|
|
725
|
-
# XXX warn about attempt to set rules before we exist
|
|
726
|
-
return if rules.nil? or rules.size == 0 or !@cloud_id
|
|
727
|
-
|
|
728
|
-
# add_to_self means that this security is a "member" of its own rules
|
|
729
|
-
# (which is to say, objects that have this SG are allowed in my these
|
|
730
|
-
# rules)
|
|
731
|
-
if add_to_self
|
|
732
|
-
rules.each { |rule|
|
|
733
|
-
if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
|
|
734
|
-
new_rule = rule.clone
|
|
735
|
-
new_rule.delete('hosts')
|
|
736
|
-
rule['sgs'] = Array.new if rule['sgs'].nil?
|
|
737
|
-
rule['sgs'] << @cloud_id
|
|
738
|
-
end
|
|
739
|
-
}
|
|
740
|
-
end
|
|
741
|
-
|
|
742
|
-
ec2_rules = convertToEc2(rules)
|
|
743
|
-
ext_permissions = MU.structToHash(cloud_desc.ip_permissions)
|
|
744
|
-
|
|
702
|
+
def purge_extraneous_rules(ec2_rules, ext_permissions)
|
|
745
703
|
# Purge any old rules that we're sure we created (check the comment)
|
|
746
704
|
# but which are no longer configured.
|
|
747
705
|
ext_permissions.each { |ext_rule|
|
|
@@ -778,97 +736,109 @@ module MU
|
|
|
778
736
|
ip_permissions: [ext_rule]
|
|
779
737
|
)
|
|
780
738
|
end
|
|
781
|
-
|
|
782
739
|
}
|
|
740
|
+
end
|
|
783
741
|
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
}
|
|
803
|
-
break
|
|
804
|
-
end
|
|
805
|
-
}
|
|
806
|
-
if haverule and !different
|
|
807
|
-
MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
|
|
808
|
-
next
|
|
742
|
+
#########################################################################
|
|
743
|
+
# Manufacture an EC2 security group. The second parameter, rules, is an
|
|
744
|
+
# "ingress_rules" structure parsed and validated by MU::Config.
|
|
745
|
+
#########################################################################
|
|
746
|
+
def setRules(rules, add_to_self: false, ingress: true, egress: false)
|
|
747
|
+
# XXX warn about attempt to set rules before we exist
|
|
748
|
+
return if rules.nil? or rules.size == 0 or !@cloud_id
|
|
749
|
+
|
|
750
|
+
# add_to_self means that this security is a "member" of its own rules
|
|
751
|
+
# (which is to say, objects that have this SG are allowed in my these
|
|
752
|
+
# rules)
|
|
753
|
+
if add_to_self
|
|
754
|
+
rules.each { |rule|
|
|
755
|
+
if rule['sgs'].nil? or !rule['sgs'].include?(@cloud_id)
|
|
756
|
+
new_rule = rule.clone
|
|
757
|
+
new_rule.delete('hosts')
|
|
758
|
+
rule['sgs'] = Array.new if rule['sgs'].nil?
|
|
759
|
+
rule['sgs'] << @cloud_id
|
|
809
760
|
end
|
|
761
|
+
}
|
|
762
|
+
end
|
|
810
763
|
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
764
|
+
ec2_rules = convertToEc2(rules)
|
|
765
|
+
return if ec2_rules.nil?
|
|
766
|
+
|
|
767
|
+
ext_permissions = MU.structToHash(cloud_desc(use_cache: false).ip_permissions)
|
|
768
|
+
|
|
769
|
+
purge_extraneous_rules(ec2_rules, ext_permissions)
|
|
770
|
+
|
|
771
|
+
ec2_rules.uniq!
|
|
772
|
+
ec2_rules.each { |rule|
|
|
773
|
+
haverule = nil
|
|
774
|
+
different = false
|
|
775
|
+
ext_permissions.each { |ext_rule|
|
|
776
|
+
if rule[:from_port] == ext_rule[:from_port] and
|
|
777
|
+
rule[:to_port] == ext_rule[:to_port] and
|
|
778
|
+
rule[:ip_protocol] == ext_rule[:ip_protocol]
|
|
779
|
+
haverule = ext_rule
|
|
780
|
+
ext_rule.keys.each { |k|
|
|
781
|
+
if ext_rule[k].nil? or ext_rule[k] == []
|
|
782
|
+
haverule.delete(k)
|
|
823
783
|
end
|
|
784
|
+
different = true if rule[k] != ext_rule[k]
|
|
785
|
+
}
|
|
786
|
+
break
|
|
787
|
+
end
|
|
788
|
+
}
|
|
789
|
+
if haverule and !different
|
|
790
|
+
MU.log "Security Group rule already up-to-date in #{@mu_name}", MU::DEBUG, details: rule
|
|
791
|
+
next
|
|
792
|
+
end
|
|
793
|
+
|
|
794
|
+
MU.log "Setting #{ingress ? "ingress" : "egress"} rule in Security Group #{@mu_name} (#{@cloud_id})", MU::NOTICE, details: rule
|
|
795
|
+
|
|
796
|
+
MU.retrier([Aws::EC2::Errors::InvalidGroupNotFound], max: 10, wait: 10, ignoreme: [Aws::EC2::Errors::InvalidPermissionDuplicate]) {
|
|
797
|
+
if ingress
|
|
798
|
+
if haverule
|
|
824
799
|
begin
|
|
825
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).
|
|
800
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_ingress(
|
|
826
801
|
group_id: @cloud_id,
|
|
827
|
-
ip_permissions: [
|
|
802
|
+
ip_permissions: [haverule]
|
|
828
803
|
)
|
|
829
|
-
rescue Aws::EC2::Errors::
|
|
830
|
-
MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
|
|
831
|
-
raise e
|
|
804
|
+
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
|
832
805
|
end
|
|
833
806
|
end
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
if haverule
|
|
837
|
-
begin
|
|
838
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
|
|
839
|
-
group_id: @cloud_id,
|
|
840
|
-
ip_permissions: [haverule]
|
|
841
|
-
)
|
|
842
|
-
rescue Aws::EC2::Errors::InvalidPermissionNotFound => e
|
|
843
|
-
end
|
|
844
|
-
end
|
|
845
|
-
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
|
807
|
+
begin
|
|
808
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
|
|
846
809
|
group_id: @cloud_id,
|
|
847
810
|
ip_permissions: [rule]
|
|
848
811
|
)
|
|
812
|
+
rescue Aws::EC2::Errors::InvalidParameterCombination => e
|
|
813
|
+
MU.log "FirewallRule #{@mu_name} had a bogus rule: #{e.message}", MU::ERR, details: rule
|
|
814
|
+
raise e
|
|
849
815
|
end
|
|
816
|
+
end
|
|
850
817
|
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
818
|
+
if egress
|
|
819
|
+
if haverule
|
|
820
|
+
begin
|
|
821
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).revoke_security_group_egress(
|
|
822
|
+
group_id: @cloud_id,
|
|
823
|
+
ip_permissions: [haverule]
|
|
824
|
+
)
|
|
825
|
+
rescue Aws::EC2::Errors::InvalidPermissionNotFound
|
|
826
|
+
end
|
|
859
827
|
end
|
|
860
|
-
|
|
861
|
-
|
|
828
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
|
829
|
+
group_id: @cloud_id,
|
|
830
|
+
ip_permissions: [rule]
|
|
831
|
+
)
|
|
862
832
|
end
|
|
863
833
|
}
|
|
864
|
-
|
|
834
|
+
}
|
|
865
835
|
|
|
866
836
|
end
|
|
867
837
|
|
|
868
|
-
|
|
869
|
-
# Convert our config languages description of firewall rules into
|
|
870
|
-
#
|
|
871
|
-
|
|
838
|
+
#######################################################################
|
|
839
|
+
# Convert our config languages description of firewall rules into
|
|
840
|
+
# Amazon's. Our rule structure is as defined in MU::Config.
|
|
841
|
+
#######################################################################
|
|
872
842
|
def convertToEc2(rules)
|
|
873
843
|
ec2_rules = []
|
|
874
844
|
if rules != nil
|
|
@@ -890,8 +860,11 @@ module MU
|
|
|
890
860
|
p_start = rule['port'].to_i
|
|
891
861
|
p_end = rule['port'].to_i
|
|
892
862
|
elsif rule['proto'] != "icmp"
|
|
893
|
-
|
|
863
|
+
MU.log "Can't create a TCP or UDP security group rule without specifying ports, assuming 'all'", MU::WARN, details: rule
|
|
864
|
+
p_start = "0"
|
|
865
|
+
p_end = "65535"
|
|
894
866
|
end
|
|
867
|
+
|
|
895
868
|
if rule['proto'] != "icmp"
|
|
896
869
|
if p_start.nil? or p_end.nil?
|
|
897
870
|
raise MuError, "Got nil ports out of rule #{rule}"
|
|
@@ -992,8 +965,8 @@ module MU
|
|
|
992
965
|
ec2_rules << ec2_rule
|
|
993
966
|
}
|
|
994
967
|
end
|
|
995
|
-
|
|
996
|
-
|
|
968
|
+
|
|
969
|
+
ec2_rules.uniq
|
|
997
970
|
end
|
|
998
971
|
|
|
999
972
|
end #class
|