cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Berksfile +16 -54
- data/Berksfile.lock +14 -62
- data/bin/mu-aws-setup +131 -108
- data/bin/mu-configure +311 -74
- data/bin/mu-gcp-setup +84 -62
- data/bin/mu-load-config.rb +46 -2
- data/bin/mu-self-update +11 -9
- data/bin/mu-upload-chef-artifacts +4 -4
- data/{mu.gemspec → cloud-mu.gemspec} +2 -2
- data/cookbooks/awscli/Berksfile +8 -0
- data/cookbooks/mu-activedirectory/Berksfile +11 -0
- data/cookbooks/mu-firewall/Berksfile +9 -0
- data/cookbooks/mu-firewall/metadata.rb +1 -1
- data/cookbooks/mu-glusterfs/Berksfile +10 -0
- data/cookbooks/mu-jenkins/Berksfile +14 -0
- data/cookbooks/mu-master/Berksfile +23 -0
- data/cookbooks/mu-master/attributes/default.rb +1 -1
- data/cookbooks/mu-master/metadata.rb +2 -2
- data/cookbooks/mu-master/recipes/default.rb +1 -1
- data/cookbooks/mu-master/recipes/init.rb +7 -3
- data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
- data/cookbooks/mu-mongo/Berksfile +10 -0
- data/cookbooks/mu-openvpn/Berksfile +11 -0
- data/cookbooks/mu-php54/Berksfile +13 -0
- data/cookbooks/mu-splunk/Berksfile +10 -0
- data/cookbooks/mu-tools/Berksfile +21 -0
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
- data/cookbooks/mu-utility/Berksfile +9 -0
- data/cookbooks/mu-utility/metadata.rb +2 -1
- data/cookbooks/nagios/Berksfile +7 -4
- data/cookbooks/s3fs/Berksfile +9 -0
- data/environments/dev.json +6 -6
- data/environments/prod.json +6 -6
- data/modules/mu.rb +20 -42
- data/modules/mu/cleanup.rb +102 -100
- data/modules/mu/cloud.rb +90 -28
- data/modules/mu/clouds/aws.rb +449 -218
- data/modules/mu/clouds/aws/alarm.rb +29 -17
- data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
- data/modules/mu/clouds/aws/collection.rb +25 -18
- data/modules/mu/clouds/aws/container_cluster.rb +73 -66
- data/modules/mu/clouds/aws/database.rb +124 -116
- data/modules/mu/clouds/aws/dnszone.rb +27 -20
- data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
- data/modules/mu/clouds/aws/folder.rb +18 -3
- data/modules/mu/clouds/aws/function.rb +77 -23
- data/modules/mu/clouds/aws/group.rb +19 -12
- data/modules/mu/clouds/aws/habitat.rb +153 -0
- data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
- data/modules/mu/clouds/aws/log.rb +30 -23
- data/modules/mu/clouds/aws/msg_queue.rb +29 -20
- data/modules/mu/clouds/aws/notifier.rb +222 -0
- data/modules/mu/clouds/aws/role.rb +178 -90
- data/modules/mu/clouds/aws/search_domain.rb +40 -24
- data/modules/mu/clouds/aws/server.rb +169 -137
- data/modules/mu/clouds/aws/server_pool.rb +60 -83
- data/modules/mu/clouds/aws/storage_pool.rb +59 -31
- data/modules/mu/clouds/aws/user.rb +36 -27
- data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
- data/modules/mu/clouds/aws/vpc.rb +250 -189
- data/modules/mu/clouds/azure.rb +132 -0
- data/modules/mu/clouds/cloudformation.rb +65 -1
- data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
- data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
- data/modules/mu/clouds/cloudformation/collection.rb +7 -0
- data/modules/mu/clouds/cloudformation/database.rb +7 -0
- data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
- data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
- data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
- data/modules/mu/clouds/cloudformation/log.rb +7 -0
- data/modules/mu/clouds/cloudformation/server.rb +7 -0
- data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
- data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
- data/modules/mu/clouds/google.rb +214 -110
- data/modules/mu/clouds/google/container_cluster.rb +42 -24
- data/modules/mu/clouds/google/database.rb +15 -6
- data/modules/mu/clouds/google/firewall_rule.rb +17 -25
- data/modules/mu/clouds/google/group.rb +13 -5
- data/modules/mu/clouds/google/habitat.rb +105 -0
- data/modules/mu/clouds/google/loadbalancer.rb +28 -20
- data/modules/mu/clouds/google/server.rb +93 -354
- data/modules/mu/clouds/google/server_pool.rb +18 -10
- data/modules/mu/clouds/google/user.rb +22 -14
- data/modules/mu/clouds/google/vpc.rb +97 -69
- data/modules/mu/config.rb +133 -38
- data/modules/mu/config/alarm.rb +25 -0
- data/modules/mu/config/cache_cluster.rb +5 -3
- data/modules/mu/config/cache_cluster.yml +23 -0
- data/modules/mu/config/database.rb +25 -16
- data/modules/mu/config/database.yml +3 -3
- data/modules/mu/config/function.rb +1 -2
- data/modules/mu/config/{project.rb → habitat.rb} +10 -10
- data/modules/mu/config/notifier.rb +85 -0
- data/modules/mu/config/notifier.yml +9 -0
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/search_domain.yml +2 -2
- data/modules/mu/config/server.rb +13 -1
- data/modules/mu/config/server.yml +3 -3
- data/modules/mu/config/server_pool.rb +3 -1
- data/modules/mu/config/storage_pool.rb +3 -1
- data/modules/mu/config/storage_pool.yml +19 -0
- data/modules/mu/config/vpc.rb +70 -8
- data/modules/mu/groomers/chef.rb +2 -3
- data/modules/mu/kittens.rb +500 -122
- data/modules/mu/master.rb +5 -5
- data/modules/mu/mommacat.rb +151 -91
- data/modules/tests/super_complex_bok.yml +12 -0
- data/modules/tests/super_simple_bok.yml +12 -0
- data/spec/mu/clouds/azure_spec.rb +82 -0
- data/spec/spec_helper.rb +105 -0
- metadata +26 -5
- data/modules/mu/clouds/aws/notification.rb +0 -139
- data/modules/mu/config/notification.rb +0 -44
@@ -64,7 +64,7 @@ module MU
|
|
64
64
|
if @config['all_account_vpcs']
|
65
65
|
# If we've been told to make this domain available account-wide, do so
|
66
66
|
MU::Cloud::AWS.listRegions(@config['us_only']).each { |region|
|
67
|
-
known_vpcs = MU::Cloud::AWS.ec2(region).describe_vpcs.vpcs
|
67
|
+
known_vpcs = MU::Cloud::AWS.ec2(region: region).describe_vpcs.vpcs
|
68
68
|
|
69
69
|
MU.log "Enumerating VPCs in #{region}", MU::DEBUG, details: known_vpcs
|
70
70
|
|
@@ -330,11 +330,11 @@ module MU
|
|
330
330
|
# @param vpc_id [String]: The cloud identifier of the VPC
|
331
331
|
# @param region [String]: The cloud provider's region
|
332
332
|
# @param remove [Boolean]: Whether to remove access (default: grant access)
|
333
|
-
def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false)
|
333
|
+
def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false, credentials: nil)
|
334
334
|
|
335
335
|
if !remove
|
336
336
|
MU.log "Granting VPC #{vpc_id} access to zone #{id}"
|
337
|
-
MU::Cloud::AWS.route53(
|
337
|
+
MU::Cloud::AWS.route53(credentials: credentials).associate_vpc_with_hosted_zone(
|
338
338
|
hosted_zone_id: id,
|
339
339
|
vpc: {
|
340
340
|
:vpc_id => vpc_id,
|
@@ -345,7 +345,7 @@ module MU
|
|
345
345
|
else
|
346
346
|
MU.log "Revoking VPC #{vpc_id} access to zone #{id}"
|
347
347
|
begin
|
348
|
-
MU::Cloud::AWS.route53(
|
348
|
+
MU::Cloud::AWS.route53(credentials: credentials).disassociate_vpc_from_hosted_zone(
|
349
349
|
hosted_zone_id: id,
|
350
350
|
vpc: {
|
351
351
|
:vpc_id => vpc_id,
|
@@ -400,7 +400,7 @@ module MU
|
|
400
400
|
target_zone = "/hostedzone/"+alias_zone if !alias_zone.match(/^\/hostedzone\//)
|
401
401
|
else
|
402
402
|
MU::Cloud::AWS.listRegions.each { |region|
|
403
|
-
MU::Cloud::AWS.elb(region).describe_load_balancers.load_balancer_descriptions.each { |elb|
|
403
|
+
MU::Cloud::AWS.elb(region: region).describe_load_balancers.load_balancer_descriptions.each { |elb|
|
404
404
|
elb_dns = elb.dns_name.downcase
|
405
405
|
elb_dns.chomp!(".")
|
406
406
|
if target_name == elb_dns
|
@@ -657,14 +657,21 @@ module MU
|
|
657
657
|
end
|
658
658
|
end
|
659
659
|
|
660
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
661
|
+
# is it localized to a region/zone?
|
662
|
+
# @return [Boolean]
|
663
|
+
def self.isGlobal?
|
664
|
+
true
|
665
|
+
end
|
666
|
+
|
660
667
|
# Called by {MU::Cleanup}. Locates resources that were created by the
|
661
668
|
# currently-loaded deployment, and purges them.
|
662
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
669
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
663
670
|
checks_to_clean = []
|
664
671
|
threads = []
|
665
|
-
MU::Cloud::AWS.route53(
|
672
|
+
MU::Cloud::AWS.route53(credentials: credentials).list_health_checks.health_checks.each { |check|
|
666
673
|
begin
|
667
|
-
tags = MU::Cloud::AWS.route53(
|
674
|
+
tags = MU::Cloud::AWS.route53(credentials: credentials).list_tags_for_resource(
|
668
675
|
resource_type: "healthcheck",
|
669
676
|
resource_id: check.id
|
670
677
|
).resource_tag_set.tags
|
@@ -692,7 +699,7 @@ module MU
|
|
692
699
|
MU.log "Removing health check #{check.id}"
|
693
700
|
retries = 5
|
694
701
|
begin
|
695
|
-
MU::Cloud::AWS.route53(
|
702
|
+
MU::Cloud::AWS.route53(credentials: credentials).delete_health_check(health_check_id: check.id) if !noop
|
696
703
|
rescue Aws::Route53::Errors::NoSuchHealthCheck => e
|
697
704
|
MU.log "Health Check '#{check.id}' disappeared before I could remove it", MU::WARN, details: e.inspect
|
698
705
|
rescue Aws::Route53::Errors::InvalidInput => e
|
@@ -721,11 +728,11 @@ module MU
|
|
721
728
|
if !noop
|
722
729
|
begin
|
723
730
|
# Clean up resource records first
|
724
|
-
rrsets = MU::Cloud::AWS.route53(
|
731
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
|
725
732
|
rrsets.resource_record_sets.each { |rrset|
|
726
733
|
next if zone.name == rrset.name and (rrset.type == "NS" or rrset.type == "SOA")
|
727
734
|
records = []
|
728
|
-
MU::Cloud::AWS.route53(
|
735
|
+
MU::Cloud::AWS.route53(credentials: credentials).change_resource_record_sets(
|
729
736
|
hosted_zone_id: zone.id,
|
730
737
|
change_batch: {
|
731
738
|
changes: [
|
@@ -738,7 +745,7 @@ module MU
|
|
738
745
|
)
|
739
746
|
}
|
740
747
|
|
741
|
-
MU::Cloud::AWS.route53(
|
748
|
+
MU::Cloud::AWS.route53(credentials: credentials).delete_hosted_zone(id: zone.id)
|
742
749
|
rescue Aws::Route53::Errors::PriorRequestNotComplete
|
743
750
|
MU.log "Still waiting for all records in DNS Zone '#{zone.name}' (#{zone.id}) to delete", MU::WARN
|
744
751
|
sleep 20
|
@@ -754,17 +761,17 @@ module MU
|
|
754
761
|
}
|
755
762
|
|
756
763
|
# Lets try cleaning MU DNS records in all zones.
|
757
|
-
MU::Cloud::AWS.route53(
|
764
|
+
MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones.hosted_zones.each { |zone|
|
758
765
|
begin
|
759
766
|
zone_rrsets = []
|
760
|
-
rrsets = MU::Cloud::AWS.route53(
|
767
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
|
761
768
|
rrsets.resource_record_sets.each { |record|
|
762
769
|
zone_rrsets << record
|
763
770
|
}
|
764
771
|
|
765
772
|
# AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
|
766
773
|
while rrsets.next_record_name && rrsets.next_record_type
|
767
|
-
rrsets = MU::Cloud::AWS.route53(
|
774
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
|
768
775
|
rrsets.resource_record_sets.each { |record|
|
769
776
|
zone_rrsets << record
|
770
777
|
}
|
@@ -871,10 +878,10 @@ module MU
|
|
871
878
|
# @param region [String]: The cloud provider region
|
872
879
|
# @param flags [Hash]: Optional flags
|
873
880
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching DNSZones
|
874
|
-
def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, flags: {})
|
881
|
+
def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, credentials: nil, flags: {})
|
875
882
|
matches = {}
|
876
883
|
|
877
|
-
resp = MU::Cloud::AWS.route53(
|
884
|
+
resp = MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones(
|
878
885
|
max_items: 100
|
879
886
|
)
|
880
887
|
|
@@ -882,13 +889,13 @@ module MU
|
|
882
889
|
if !cloud_id.nil? and !cloud_id.empty?
|
883
890
|
if zone.id == cloud_id
|
884
891
|
begin
|
885
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
892
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
886
893
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
887
894
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
888
895
|
end
|
889
896
|
elsif zone.name == cloud_id or zone.name == cloud_id+"."
|
890
897
|
begin
|
891
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
898
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
892
899
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
893
900
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
894
901
|
end
|
@@ -896,7 +903,7 @@ module MU
|
|
896
903
|
end
|
897
904
|
if !deploy_id.nil? and !deploy_id.empty? and zone.config.comment == deploy_id
|
898
905
|
begin
|
899
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
906
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
900
907
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
901
908
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
902
909
|
end
|
@@ -51,7 +51,6 @@ module MU
|
|
51
51
|
vpc_id = @vpc.cloud_id if !@vpc.nil?
|
52
52
|
groupname = @mu_name
|
53
53
|
description = groupname
|
54
|
-
MU.log "Creating EC2 Security Group #{groupname}"
|
55
54
|
|
56
55
|
sg_struct = {
|
57
56
|
:group_name => groupname,
|
@@ -62,14 +61,16 @@ module MU
|
|
62
61
|
end
|
63
62
|
|
64
63
|
begin
|
65
|
-
|
64
|
+
MU.log "Creating EC2 Security Group #{groupname}", details: sg_struct
|
65
|
+
|
66
|
+
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
|
66
67
|
@cloud_id = secgroup.group_id
|
67
68
|
rescue Aws::EC2::Errors::InvalidGroupDuplicate => e
|
68
69
|
MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
|
69
70
|
filters = [{name: "group-name", values: [groupname]}]
|
70
71
|
filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
|
71
72
|
|
72
|
-
secgroup = MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(filters: filters).security_groups.first
|
73
|
+
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
|
73
74
|
deploy_id = @deploy.deploy_id if !@deploy_id.nil?
|
74
75
|
if secgroup.nil?
|
75
76
|
raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
|
@@ -78,25 +79,25 @@ module MU
|
|
78
79
|
end
|
79
80
|
|
80
81
|
begin
|
81
|
-
MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(group_ids: [secgroup.group_id])
|
82
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
|
82
83
|
rescue Aws::EC2::Errors::InvalidGroupNotFound => e
|
83
84
|
MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
|
84
85
|
sleep 10
|
85
86
|
retry
|
86
87
|
end
|
87
88
|
|
88
|
-
MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'])
|
89
|
-
MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'])
|
89
|
+
MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
|
90
|
+
MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
|
90
91
|
|
91
92
|
if @config['optional_tags']
|
92
93
|
MU::MommaCat.listOptionalTags.each { |key, value|
|
93
|
-
MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'])
|
94
|
+
MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
94
95
|
}
|
95
96
|
end
|
96
97
|
|
97
98
|
if @config['tags']
|
98
99
|
@config['tags'].each { |tag|
|
99
|
-
MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'])
|
100
|
+
MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
100
101
|
}
|
101
102
|
end
|
102
103
|
|
@@ -167,12 +168,12 @@ module MU
|
|
167
168
|
|
168
169
|
begin
|
169
170
|
if egress
|
170
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
|
171
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
171
172
|
group_id: @cloud_id,
|
172
173
|
ip_permissions: ec2_rule
|
173
174
|
)
|
174
175
|
else
|
175
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
|
176
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
|
176
177
|
group_id: @cloud_id,
|
177
178
|
ip_permissions: ec2_rule
|
178
179
|
)
|
@@ -185,7 +186,7 @@ module MU
|
|
185
186
|
# Canonical Amazon Resource Number for this resource
|
186
187
|
# @return [String]
|
187
188
|
def arn
|
188
|
-
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.
|
189
|
+
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":security-group/"+@cloud_id
|
189
190
|
end
|
190
191
|
|
191
192
|
# Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
|
@@ -195,11 +196,11 @@ module MU
|
|
195
196
|
# @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
|
196
197
|
# @param flags [Hash]: Optional flags
|
197
198
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching FirewallRules
|
198
|
-
def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
|
199
|
+
def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
|
199
200
|
|
200
201
|
if !cloud_id.nil? and !cloud_id.empty?
|
201
202
|
begin
|
202
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(group_ids: [cloud_id])
|
203
|
+
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(group_ids: [cloud_id])
|
203
204
|
return {cloud_id => resp.data.security_groups.first}
|
204
205
|
rescue ArgumentError => e
|
205
206
|
MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::WARN, details: caller
|
@@ -212,7 +213,7 @@ module MU
|
|
212
213
|
|
213
214
|
map = {}
|
214
215
|
if !tag_key.nil? and !tag_value.nil?
|
215
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
|
216
|
+
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
|
216
217
|
filters: [
|
217
218
|
{name: "tag:#{tag_key}", values: [tag_value]}
|
218
219
|
]
|
@@ -227,12 +228,19 @@ module MU
|
|
227
228
|
map
|
228
229
|
end
|
229
230
|
|
231
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
232
|
+
# is it localized to a region/zone?
|
233
|
+
# @return [Boolean]
|
234
|
+
def self.isGlobal?
|
235
|
+
false
|
236
|
+
end
|
237
|
+
|
230
238
|
# Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
|
231
239
|
# @param noop [Boolean]: If true, will only print what would be done
|
232
240
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
233
241
|
# @param region [String]: The cloud provider region
|
234
242
|
# @return [void]
|
235
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
243
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
236
244
|
tagfilters = [
|
237
245
|
{name: "tag:MU-ID", values: [MU.deploy_id]}
|
238
246
|
]
|
@@ -240,7 +248,7 @@ module MU
|
|
240
248
|
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
241
249
|
end
|
242
250
|
|
243
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
|
251
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
|
244
252
|
filters: tagfilters
|
245
253
|
)
|
246
254
|
|
@@ -303,13 +311,13 @@ module MU
|
|
303
311
|
begin
|
304
312
|
|
305
313
|
if ingress_to_revoke.size > 0
|
306
|
-
MU::Cloud::AWS.ec2(region).revoke_security_group_ingress(
|
314
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
|
307
315
|
group_id: sg.group_id,
|
308
316
|
ip_permissions: ingress_to_revoke
|
309
317
|
)
|
310
318
|
end
|
311
319
|
if egress_to_revoke.size > 0
|
312
|
-
MU::Cloud::AWS.ec2(region).revoke_security_group_egress(
|
320
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
|
313
321
|
group_id: sg.group_id,
|
314
322
|
ip_permissions: egress_to_revoke
|
315
323
|
)
|
@@ -325,7 +333,7 @@ module MU
|
|
325
333
|
|
326
334
|
retries = 0
|
327
335
|
begin
|
328
|
-
MU::Cloud::AWS.ec2(region).delete_security_group(group_id: sg.group_id) if !noop
|
336
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
|
329
337
|
rescue Aws::EC2::Errors::InvalidGroupNotFound
|
330
338
|
MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
|
331
339
|
rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
|
@@ -459,13 +467,13 @@ module MU
|
|
459
467
|
MU.log "Rules for EC2 Security Group #{@mu_name} (#{@cloud_id}): #{ec2_rules}", MU::DEBUG
|
460
468
|
begin
|
461
469
|
if ingress
|
462
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
|
470
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
|
463
471
|
group_id: @cloud_id,
|
464
472
|
ip_permissions: ec2_rules
|
465
473
|
)
|
466
474
|
end
|
467
475
|
if egress
|
468
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
|
476
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
469
477
|
group_id: @cloud_id,
|
470
478
|
ip_permissions: ec2_rules
|
471
479
|
)
|
@@ -571,7 +579,7 @@ module MU
|
|
571
579
|
if !lb.nil? and !lb.cloud_desc.nil?
|
572
580
|
lb.cloud_desc.security_groups.each { |lb_sg|
|
573
581
|
ec2_rule[:user_id_group_pairs] << {
|
574
|
-
user_id: MU.
|
582
|
+
user_id: MU::Cloud::AWS.credToAcct(@config['credentials']),
|
575
583
|
group_id: lb_sg
|
576
584
|
}
|
577
585
|
}
|
@@ -48,21 +48,36 @@ module MU
|
|
48
48
|
}
|
49
49
|
end
|
50
50
|
|
51
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
52
|
+
# is it localized to a region/zone?
|
53
|
+
# @return [Boolean]
|
54
|
+
def self.isGlobal?
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
51
58
|
# Remove all logs associated with the currently loaded deployment.
|
52
59
|
# @param noop [Boolean]: If true, will only print what would be done
|
53
60
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
54
61
|
# @param region [String]: The cloud provider region
|
55
62
|
# @return [void]
|
56
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
63
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
57
64
|
end
|
58
65
|
|
59
|
-
# Locate an existing
|
66
|
+
# Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
|
60
67
|
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
61
68
|
# @param region [String]: The cloud provider region.
|
62
69
|
# @param flags [Hash]: Optional flags
|
63
70
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
|
64
|
-
def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
|
71
|
+
def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
|
65
72
|
found = nil
|
73
|
+
|
74
|
+
if cloud_id
|
75
|
+
else
|
76
|
+
resp = MU::Cloud::AWS.orgs(credentials: credentials).describe_organization
|
77
|
+
found ||= {}
|
78
|
+
found[resp.organization.id] = resp.organization
|
79
|
+
end
|
80
|
+
|
66
81
|
found
|
67
82
|
end
|
68
83
|
|
@@ -35,16 +35,23 @@ module MU
|
|
35
35
|
@mu_name ||= @deploy.getResourceName(@config["name"])
|
36
36
|
end
|
37
37
|
|
38
|
-
|
38
|
+
# Given an IAM role name, resolve to ARN. Will attempt to identify any
|
39
|
+
# sibling Mu role resources by this name first, and failing that, will
|
40
|
+
# do a plain get_role() to the IAM API for the provided name.
|
41
|
+
# @param name [String]
|
39
42
|
def get_role_arn(name)
|
43
|
+
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
44
|
+
return sib_role.cloudobj.arn if sib_role
|
45
|
+
|
40
46
|
begin
|
41
|
-
role = MU::Cloud::AWS.iam(@config['
|
47
|
+
role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
|
42
48
|
role_name: name.to_s
|
43
49
|
})
|
44
50
|
return role['role']['arn']
|
45
51
|
rescue Exception => e
|
46
|
-
|
52
|
+
MU.log "#{e}", MU::ERR
|
47
53
|
end
|
54
|
+
nil
|
48
55
|
end
|
49
56
|
|
50
57
|
def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
|
@@ -52,7 +59,7 @@ module MU
|
|
52
59
|
## get vpc_id
|
53
60
|
## get sub_id and verify its in the same vpc
|
54
61
|
## get sg_id and verify its in the same vpc
|
55
|
-
ec2_client = MU::Cloud::AWS.ec2(region)
|
62
|
+
ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
|
56
63
|
|
57
64
|
vpc_filter = ec2_client.describe_vpcs({
|
58
65
|
filters: [{ name: 'tag-value', values: [vpc_name] }]
|
@@ -94,7 +101,7 @@ module MU
|
|
94
101
|
def assign_tag(resource_arn, tag_list, region=@config['region'])
|
95
102
|
begin
|
96
103
|
tag_list.each do |each_pair|
|
97
|
-
tag_resp = MU::Cloud::AWS.lambda(region).tag_resource({
|
104
|
+
tag_resp = MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
|
98
105
|
resource: resource_arn,
|
99
106
|
tags: each_pair
|
100
107
|
})
|
@@ -164,11 +171,22 @@ module MU
|
|
164
171
|
lambda_properties[:vpc_config] = vpc_conf
|
165
172
|
end
|
166
173
|
|
167
|
-
|
174
|
+
retries = 0
|
175
|
+
begin
|
176
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
177
|
+
rescue Aws::Lambda::Errors::InvalidParameterValueException => e
|
178
|
+
# Freshly-made IAM roles sometimes aren't really ready
|
179
|
+
if retries < 5
|
180
|
+
sleep 10
|
181
|
+
retries += 1
|
182
|
+
retry
|
183
|
+
end
|
184
|
+
raise e
|
185
|
+
end
|
168
186
|
end
|
169
187
|
|
170
188
|
def groom
|
171
|
-
desc = MU::Cloud::AWS.lambda(@config['region']).get_function(
|
189
|
+
desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
|
172
190
|
function_name: @mu_name
|
173
191
|
)
|
174
192
|
func_arn = desc.configuration.function_arn if !desc.empty?
|
@@ -199,7 +217,7 @@ module MU
|
|
199
217
|
|
200
218
|
MU.log trigger_properties, MU::DEBUG
|
201
219
|
begin
|
202
|
-
add_trigger = MU::Cloud::AWS.lambda(@config['region']).add_permission(trigger_properties)
|
220
|
+
add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
|
203
221
|
rescue Aws::Lambda::Errors::ResourceConflictException
|
204
222
|
# XXX check properly for existence
|
205
223
|
end
|
@@ -216,11 +234,11 @@ module MU
|
|
216
234
|
arn = nil
|
217
235
|
case svc.downcase
|
218
236
|
when 'sns'
|
219
|
-
arn = "arn:aws:sns:#{@config['region']}:#{MU.
|
237
|
+
arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
220
238
|
when 'alarm','events', 'event', 'cloudwatch_event'
|
221
|
-
arn = "arn:aws:events:#{@config['region']}:#{MU.
|
239
|
+
arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
|
222
240
|
when 'apigateway'
|
223
|
-
arn = "arn:aws:apigateway:#{@config['region']}:#{MU.
|
241
|
+
arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
224
242
|
when 's3'
|
225
243
|
arn = ''
|
226
244
|
end
|
@@ -237,15 +255,16 @@ module MU
|
|
237
255
|
case trig_type
|
238
256
|
|
239
257
|
when 'sns'
|
240
|
-
|
241
|
-
sns_client = MU::Cloud::AWS.sns(@config['region'])
|
258
|
+
# XXX don't do this, use MU::Cloud::AWS::Notification
|
259
|
+
sns_client = MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials'])
|
242
260
|
sub_to_what = sns_client.subscribe({
|
243
261
|
topic_arn: trig_arn,
|
244
262
|
protocol: protocol,
|
245
263
|
endpoint: func_arn
|
246
264
|
})
|
247
265
|
when 'event','cloudwatch_event', 'events'
|
248
|
-
|
266
|
+
# XXX don't do this, use MU::Cloud::AWS::Log
|
267
|
+
client = MU::Cloud::AWS.cloudwatch_events(region: @config['region'], credentials: @config['credentials']).put_targets({
|
249
268
|
rule: @config['trigger']['name'],
|
250
269
|
targets: [
|
251
270
|
{
|
@@ -268,23 +287,27 @@ module MU
|
|
268
287
|
return deploy_struct
|
269
288
|
end
|
270
289
|
|
271
|
-
|
272
|
-
|
290
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
291
|
+
# is it localized to a region/zone?
|
292
|
+
# @return [Boolean]
|
293
|
+
def self.isGlobal?
|
294
|
+
false
|
295
|
+
end
|
273
296
|
|
274
297
|
# Remove all functions associated with the currently loaded deployment.
|
275
298
|
# @param noop [Boolean]: If true, will only print what would be done
|
276
299
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
277
300
|
# @param region [String]: The cloud provider region
|
278
301
|
# @return [void]
|
279
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
280
|
-
MU::Cloud::AWS.lambda(region).list_functions.functions.each { |f|
|
281
|
-
desc = MU::Cloud::AWS.lambda(region).get_function(
|
302
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
303
|
+
MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
|
304
|
+
desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
|
282
305
|
function_name: f.function_name
|
283
306
|
)
|
284
307
|
if desc.tags and desc.tags["MU-ID"] == MU.deploy_id
|
285
308
|
MU.log "Deleting Lambda function #{f.function_name}"
|
286
309
|
if !noop
|
287
|
-
MU::Cloud::AWS.lambda(region).delete_function(
|
310
|
+
MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
|
288
311
|
function_name: f.function_name
|
289
312
|
)
|
290
313
|
end
|
@@ -304,10 +327,10 @@ module MU
|
|
304
327
|
# @param region [String]: The cloud provider region.
|
305
328
|
# @param flags [Hash]: Optional flags
|
306
329
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
|
307
|
-
def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, flags: {})
|
330
|
+
def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, credentials: nil, flags: {})
|
308
331
|
func = nil
|
309
332
|
if !func_name.nil?
|
310
|
-
all_functions = MU::Cloud::AWS.lambda(region).list_functions
|
333
|
+
all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
|
311
334
|
if all_functions.include?(func_name)
|
312
335
|
all_functions.functions.each do |x|
|
313
336
|
if x.function_name == func_name
|
@@ -329,7 +352,13 @@ module MU
|
|
329
352
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
330
353
|
def self.schema(config)
|
331
354
|
toplevel_required = []
|
332
|
-
schema = {
|
355
|
+
schema = {
|
356
|
+
"iam_role" => {
|
357
|
+
"type" => "string",
|
358
|
+
"description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with the AWSLambdaBasicExecutionRole policy attached. To grant other permissions for your function, create a Mu 'role' resource and use the 'import' and 'policies' parameters to add permissions. See also: https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html"
|
359
|
+
}
|
360
|
+
# XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
|
361
|
+
}
|
333
362
|
[toplevel_required, schema]
|
334
363
|
end
|
335
364
|
|
@@ -340,6 +369,31 @@ module MU
|
|
340
369
|
def self.validateConfig(function, configurator)
|
341
370
|
ok = true
|
342
371
|
|
372
|
+
if !function['iam_role']
|
373
|
+
roledesc = {
|
374
|
+
"name" => function['name']+"execrole",
|
375
|
+
"credentials" => function['credentials'],
|
376
|
+
"can_assume" => [
|
377
|
+
{
|
378
|
+
"entity_id" => "lambda.amazonaws.com",
|
379
|
+
"entity_type" => "service"
|
380
|
+
}
|
381
|
+
],
|
382
|
+
"import" => [
|
383
|
+
"AWSLambdaBasicExecutionRole"
|
384
|
+
]
|
385
|
+
}
|
386
|
+
configurator.insertKitten(roledesc, "roles")
|
387
|
+
|
388
|
+
function['dependencies'] ||= []
|
389
|
+
function['iam_role'] = function['name']+"execrole"
|
390
|
+
|
391
|
+
function['dependencies'] << {
|
392
|
+
"type" => "role",
|
393
|
+
"name" => function['name']+"execrole"
|
394
|
+
}
|
395
|
+
end
|
396
|
+
|
343
397
|
ok
|
344
398
|
end
|
345
399
|
|