cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Berksfile +16 -54
- data/Berksfile.lock +14 -62
- data/bin/mu-aws-setup +131 -108
- data/bin/mu-configure +311 -74
- data/bin/mu-gcp-setup +84 -62
- data/bin/mu-load-config.rb +46 -2
- data/bin/mu-self-update +11 -9
- data/bin/mu-upload-chef-artifacts +4 -4
- data/{mu.gemspec → cloud-mu.gemspec} +2 -2
- data/cookbooks/awscli/Berksfile +8 -0
- data/cookbooks/mu-activedirectory/Berksfile +11 -0
- data/cookbooks/mu-firewall/Berksfile +9 -0
- data/cookbooks/mu-firewall/metadata.rb +1 -1
- data/cookbooks/mu-glusterfs/Berksfile +10 -0
- data/cookbooks/mu-jenkins/Berksfile +14 -0
- data/cookbooks/mu-master/Berksfile +23 -0
- data/cookbooks/mu-master/attributes/default.rb +1 -1
- data/cookbooks/mu-master/metadata.rb +2 -2
- data/cookbooks/mu-master/recipes/default.rb +1 -1
- data/cookbooks/mu-master/recipes/init.rb +7 -3
- data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
- data/cookbooks/mu-mongo/Berksfile +10 -0
- data/cookbooks/mu-openvpn/Berksfile +11 -0
- data/cookbooks/mu-php54/Berksfile +13 -0
- data/cookbooks/mu-splunk/Berksfile +10 -0
- data/cookbooks/mu-tools/Berksfile +21 -0
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
- data/cookbooks/mu-utility/Berksfile +9 -0
- data/cookbooks/mu-utility/metadata.rb +2 -1
- data/cookbooks/nagios/Berksfile +7 -4
- data/cookbooks/s3fs/Berksfile +9 -0
- data/environments/dev.json +6 -6
- data/environments/prod.json +6 -6
- data/modules/mu.rb +20 -42
- data/modules/mu/cleanup.rb +102 -100
- data/modules/mu/cloud.rb +90 -28
- data/modules/mu/clouds/aws.rb +449 -218
- data/modules/mu/clouds/aws/alarm.rb +29 -17
- data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
- data/modules/mu/clouds/aws/collection.rb +25 -18
- data/modules/mu/clouds/aws/container_cluster.rb +73 -66
- data/modules/mu/clouds/aws/database.rb +124 -116
- data/modules/mu/clouds/aws/dnszone.rb +27 -20
- data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
- data/modules/mu/clouds/aws/folder.rb +18 -3
- data/modules/mu/clouds/aws/function.rb +77 -23
- data/modules/mu/clouds/aws/group.rb +19 -12
- data/modules/mu/clouds/aws/habitat.rb +153 -0
- data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
- data/modules/mu/clouds/aws/log.rb +30 -23
- data/modules/mu/clouds/aws/msg_queue.rb +29 -20
- data/modules/mu/clouds/aws/notifier.rb +222 -0
- data/modules/mu/clouds/aws/role.rb +178 -90
- data/modules/mu/clouds/aws/search_domain.rb +40 -24
- data/modules/mu/clouds/aws/server.rb +169 -137
- data/modules/mu/clouds/aws/server_pool.rb +60 -83
- data/modules/mu/clouds/aws/storage_pool.rb +59 -31
- data/modules/mu/clouds/aws/user.rb +36 -27
- data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
- data/modules/mu/clouds/aws/vpc.rb +250 -189
- data/modules/mu/clouds/azure.rb +132 -0
- data/modules/mu/clouds/cloudformation.rb +65 -1
- data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
- data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
- data/modules/mu/clouds/cloudformation/collection.rb +7 -0
- data/modules/mu/clouds/cloudformation/database.rb +7 -0
- data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
- data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
- data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
- data/modules/mu/clouds/cloudformation/log.rb +7 -0
- data/modules/mu/clouds/cloudformation/server.rb +7 -0
- data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
- data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
- data/modules/mu/clouds/google.rb +214 -110
- data/modules/mu/clouds/google/container_cluster.rb +42 -24
- data/modules/mu/clouds/google/database.rb +15 -6
- data/modules/mu/clouds/google/firewall_rule.rb +17 -25
- data/modules/mu/clouds/google/group.rb +13 -5
- data/modules/mu/clouds/google/habitat.rb +105 -0
- data/modules/mu/clouds/google/loadbalancer.rb +28 -20
- data/modules/mu/clouds/google/server.rb +93 -354
- data/modules/mu/clouds/google/server_pool.rb +18 -10
- data/modules/mu/clouds/google/user.rb +22 -14
- data/modules/mu/clouds/google/vpc.rb +97 -69
- data/modules/mu/config.rb +133 -38
- data/modules/mu/config/alarm.rb +25 -0
- data/modules/mu/config/cache_cluster.rb +5 -3
- data/modules/mu/config/cache_cluster.yml +23 -0
- data/modules/mu/config/database.rb +25 -16
- data/modules/mu/config/database.yml +3 -3
- data/modules/mu/config/function.rb +1 -2
- data/modules/mu/config/{project.rb → habitat.rb} +10 -10
- data/modules/mu/config/notifier.rb +85 -0
- data/modules/mu/config/notifier.yml +9 -0
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/search_domain.yml +2 -2
- data/modules/mu/config/server.rb +13 -1
- data/modules/mu/config/server.yml +3 -3
- data/modules/mu/config/server_pool.rb +3 -1
- data/modules/mu/config/storage_pool.rb +3 -1
- data/modules/mu/config/storage_pool.yml +19 -0
- data/modules/mu/config/vpc.rb +70 -8
- data/modules/mu/groomers/chef.rb +2 -3
- data/modules/mu/kittens.rb +500 -122
- data/modules/mu/master.rb +5 -5
- data/modules/mu/mommacat.rb +151 -91
- data/modules/tests/super_complex_bok.yml +12 -0
- data/modules/tests/super_simple_bok.yml +12 -0
- data/spec/mu/clouds/azure_spec.rb +82 -0
- data/spec/spec_helper.rb +105 -0
- metadata +26 -5
- data/modules/mu/clouds/aws/notification.rb +0 -139
- data/modules/mu/config/notification.rb +0 -44
@@ -64,7 +64,7 @@ module MU
|
|
64
64
|
if @config['all_account_vpcs']
|
65
65
|
# If we've been told to make this domain available account-wide, do so
|
66
66
|
MU::Cloud::AWS.listRegions(@config['us_only']).each { |region|
|
67
|
-
known_vpcs = MU::Cloud::AWS.ec2(region).describe_vpcs.vpcs
|
67
|
+
known_vpcs = MU::Cloud::AWS.ec2(region: region).describe_vpcs.vpcs
|
68
68
|
|
69
69
|
MU.log "Enumerating VPCs in #{region}", MU::DEBUG, details: known_vpcs
|
70
70
|
|
@@ -330,11 +330,11 @@ module MU
|
|
330
330
|
# @param vpc_id [String]: The cloud identifier of the VPC
|
331
331
|
# @param region [String]: The cloud provider's region
|
332
332
|
# @param remove [Boolean]: Whether to remove access (default: grant access)
|
333
|
-
def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false)
|
333
|
+
def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false, credentials: nil)
|
334
334
|
|
335
335
|
if !remove
|
336
336
|
MU.log "Granting VPC #{vpc_id} access to zone #{id}"
|
337
|
-
MU::Cloud::AWS.route53(
|
337
|
+
MU::Cloud::AWS.route53(credentials: credentials).associate_vpc_with_hosted_zone(
|
338
338
|
hosted_zone_id: id,
|
339
339
|
vpc: {
|
340
340
|
:vpc_id => vpc_id,
|
@@ -345,7 +345,7 @@ module MU
|
|
345
345
|
else
|
346
346
|
MU.log "Revoking VPC #{vpc_id} access to zone #{id}"
|
347
347
|
begin
|
348
|
-
MU::Cloud::AWS.route53(
|
348
|
+
MU::Cloud::AWS.route53(credentials: credentials).disassociate_vpc_from_hosted_zone(
|
349
349
|
hosted_zone_id: id,
|
350
350
|
vpc: {
|
351
351
|
:vpc_id => vpc_id,
|
@@ -400,7 +400,7 @@ module MU
|
|
400
400
|
target_zone = "/hostedzone/"+alias_zone if !alias_zone.match(/^\/hostedzone\//)
|
401
401
|
else
|
402
402
|
MU::Cloud::AWS.listRegions.each { |region|
|
403
|
-
MU::Cloud::AWS.elb(region).describe_load_balancers.load_balancer_descriptions.each { |elb|
|
403
|
+
MU::Cloud::AWS.elb(region: region).describe_load_balancers.load_balancer_descriptions.each { |elb|
|
404
404
|
elb_dns = elb.dns_name.downcase
|
405
405
|
elb_dns.chomp!(".")
|
406
406
|
if target_name == elb_dns
|
@@ -657,14 +657,21 @@ module MU
|
|
657
657
|
end
|
658
658
|
end
|
659
659
|
|
660
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
661
|
+
# is it localized to a region/zone?
|
662
|
+
# @return [Boolean]
|
663
|
+
def self.isGlobal?
|
664
|
+
true
|
665
|
+
end
|
666
|
+
|
660
667
|
# Called by {MU::Cleanup}. Locates resources that were created by the
|
661
668
|
# currently-loaded deployment, and purges them.
|
662
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
669
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
663
670
|
checks_to_clean = []
|
664
671
|
threads = []
|
665
|
-
MU::Cloud::AWS.route53(
|
672
|
+
MU::Cloud::AWS.route53(credentials: credentials).list_health_checks.health_checks.each { |check|
|
666
673
|
begin
|
667
|
-
tags = MU::Cloud::AWS.route53(
|
674
|
+
tags = MU::Cloud::AWS.route53(credentials: credentials).list_tags_for_resource(
|
668
675
|
resource_type: "healthcheck",
|
669
676
|
resource_id: check.id
|
670
677
|
).resource_tag_set.tags
|
@@ -692,7 +699,7 @@ module MU
|
|
692
699
|
MU.log "Removing health check #{check.id}"
|
693
700
|
retries = 5
|
694
701
|
begin
|
695
|
-
MU::Cloud::AWS.route53(
|
702
|
+
MU::Cloud::AWS.route53(credentials: credentials).delete_health_check(health_check_id: check.id) if !noop
|
696
703
|
rescue Aws::Route53::Errors::NoSuchHealthCheck => e
|
697
704
|
MU.log "Health Check '#{check.id}' disappeared before I could remove it", MU::WARN, details: e.inspect
|
698
705
|
rescue Aws::Route53::Errors::InvalidInput => e
|
@@ -721,11 +728,11 @@ module MU
|
|
721
728
|
if !noop
|
722
729
|
begin
|
723
730
|
# Clean up resource records first
|
724
|
-
rrsets = MU::Cloud::AWS.route53(
|
731
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
|
725
732
|
rrsets.resource_record_sets.each { |rrset|
|
726
733
|
next if zone.name == rrset.name and (rrset.type == "NS" or rrset.type == "SOA")
|
727
734
|
records = []
|
728
|
-
MU::Cloud::AWS.route53(
|
735
|
+
MU::Cloud::AWS.route53(credentials: credentials).change_resource_record_sets(
|
729
736
|
hosted_zone_id: zone.id,
|
730
737
|
change_batch: {
|
731
738
|
changes: [
|
@@ -738,7 +745,7 @@ module MU
|
|
738
745
|
)
|
739
746
|
}
|
740
747
|
|
741
|
-
MU::Cloud::AWS.route53(
|
748
|
+
MU::Cloud::AWS.route53(credentials: credentials).delete_hosted_zone(id: zone.id)
|
742
749
|
rescue Aws::Route53::Errors::PriorRequestNotComplete
|
743
750
|
MU.log "Still waiting for all records in DNS Zone '#{zone.name}' (#{zone.id}) to delete", MU::WARN
|
744
751
|
sleep 20
|
@@ -754,17 +761,17 @@ module MU
|
|
754
761
|
}
|
755
762
|
|
756
763
|
# Lets try cleaning MU DNS records in all zones.
|
757
|
-
MU::Cloud::AWS.route53(
|
764
|
+
MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones.hosted_zones.each { |zone|
|
758
765
|
begin
|
759
766
|
zone_rrsets = []
|
760
|
-
rrsets = MU::Cloud::AWS.route53(
|
767
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
|
761
768
|
rrsets.resource_record_sets.each { |record|
|
762
769
|
zone_rrsets << record
|
763
770
|
}
|
764
771
|
|
765
772
|
# AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
|
766
773
|
while rrsets.next_record_name && rrsets.next_record_type
|
767
|
-
rrsets = MU::Cloud::AWS.route53(
|
774
|
+
rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
|
768
775
|
rrsets.resource_record_sets.each { |record|
|
769
776
|
zone_rrsets << record
|
770
777
|
}
|
@@ -871,10 +878,10 @@ module MU
|
|
871
878
|
# @param region [String]: The cloud provider region
|
872
879
|
# @param flags [Hash]: Optional flags
|
873
880
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching DNSZones
|
874
|
-
def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, flags: {})
|
881
|
+
def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, credentials: nil, flags: {})
|
875
882
|
matches = {}
|
876
883
|
|
877
|
-
resp = MU::Cloud::AWS.route53(
|
884
|
+
resp = MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones(
|
878
885
|
max_items: 100
|
879
886
|
)
|
880
887
|
|
@@ -882,13 +889,13 @@ module MU
|
|
882
889
|
if !cloud_id.nil? and !cloud_id.empty?
|
883
890
|
if zone.id == cloud_id
|
884
891
|
begin
|
885
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
892
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
886
893
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
887
894
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
888
895
|
end
|
889
896
|
elsif zone.name == cloud_id or zone.name == cloud_id+"."
|
890
897
|
begin
|
891
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
898
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
892
899
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
893
900
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
894
901
|
end
|
@@ -896,7 +903,7 @@ module MU
|
|
896
903
|
end
|
897
904
|
if !deploy_id.nil? and !deploy_id.empty? and zone.config.comment == deploy_id
|
898
905
|
begin
|
899
|
-
matches[zone.id] = MU::Cloud::AWS.route53(
|
906
|
+
matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
|
900
907
|
rescue Aws::Route53::Errors::NoSuchHostedZone
|
901
908
|
MU.log "Hosted zone #{zone.id} doesn't exist"
|
902
909
|
end
|
@@ -51,7 +51,6 @@ module MU
|
|
51
51
|
vpc_id = @vpc.cloud_id if !@vpc.nil?
|
52
52
|
groupname = @mu_name
|
53
53
|
description = groupname
|
54
|
-
MU.log "Creating EC2 Security Group #{groupname}"
|
55
54
|
|
56
55
|
sg_struct = {
|
57
56
|
:group_name => groupname,
|
@@ -62,14 +61,16 @@ module MU
|
|
62
61
|
end
|
63
62
|
|
64
63
|
begin
|
65
|
-
|
64
|
+
MU.log "Creating EC2 Security Group #{groupname}", details: sg_struct
|
65
|
+
|
66
|
+
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
|
66
67
|
@cloud_id = secgroup.group_id
|
67
68
|
rescue Aws::EC2::Errors::InvalidGroupDuplicate => e
|
68
69
|
MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
|
69
70
|
filters = [{name: "group-name", values: [groupname]}]
|
70
71
|
filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
|
71
72
|
|
72
|
-
secgroup = MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(filters: filters).security_groups.first
|
73
|
+
secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
|
73
74
|
deploy_id = @deploy.deploy_id if !@deploy_id.nil?
|
74
75
|
if secgroup.nil?
|
75
76
|
raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
|
@@ -78,25 +79,25 @@ module MU
|
|
78
79
|
end
|
79
80
|
|
80
81
|
begin
|
81
|
-
MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(group_ids: [secgroup.group_id])
|
82
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
|
82
83
|
rescue Aws::EC2::Errors::InvalidGroupNotFound => e
|
83
84
|
MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
|
84
85
|
sleep 10
|
85
86
|
retry
|
86
87
|
end
|
87
88
|
|
88
|
-
MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'])
|
89
|
-
MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'])
|
89
|
+
MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
|
90
|
+
MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
|
90
91
|
|
91
92
|
if @config['optional_tags']
|
92
93
|
MU::MommaCat.listOptionalTags.each { |key, value|
|
93
|
-
MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'])
|
94
|
+
MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
|
94
95
|
}
|
95
96
|
end
|
96
97
|
|
97
98
|
if @config['tags']
|
98
99
|
@config['tags'].each { |tag|
|
99
|
-
MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'])
|
100
|
+
MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
|
100
101
|
}
|
101
102
|
end
|
102
103
|
|
@@ -167,12 +168,12 @@ module MU
|
|
167
168
|
|
168
169
|
begin
|
169
170
|
if egress
|
170
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
|
171
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
171
172
|
group_id: @cloud_id,
|
172
173
|
ip_permissions: ec2_rule
|
173
174
|
)
|
174
175
|
else
|
175
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
|
176
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
|
176
177
|
group_id: @cloud_id,
|
177
178
|
ip_permissions: ec2_rule
|
178
179
|
)
|
@@ -185,7 +186,7 @@ module MU
|
|
185
186
|
# Canonical Amazon Resource Number for this resource
|
186
187
|
# @return [String]
|
187
188
|
def arn
|
188
|
-
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.
|
189
|
+
"arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":security-group/"+@cloud_id
|
189
190
|
end
|
190
191
|
|
191
192
|
# Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
|
@@ -195,11 +196,11 @@ module MU
|
|
195
196
|
# @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
|
196
197
|
# @param flags [Hash]: Optional flags
|
197
198
|
# @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching FirewallRules
|
198
|
-
def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
|
199
|
+
def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
|
199
200
|
|
200
201
|
if !cloud_id.nil? and !cloud_id.empty?
|
201
202
|
begin
|
202
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(group_ids: [cloud_id])
|
203
|
+
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(group_ids: [cloud_id])
|
203
204
|
return {cloud_id => resp.data.security_groups.first}
|
204
205
|
rescue ArgumentError => e
|
205
206
|
MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::WARN, details: caller
|
@@ -212,7 +213,7 @@ module MU
|
|
212
213
|
|
213
214
|
map = {}
|
214
215
|
if !tag_key.nil? and !tag_value.nil?
|
215
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
|
216
|
+
resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
|
216
217
|
filters: [
|
217
218
|
{name: "tag:#{tag_key}", values: [tag_value]}
|
218
219
|
]
|
@@ -227,12 +228,19 @@ module MU
|
|
227
228
|
map
|
228
229
|
end
|
229
230
|
|
231
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
232
|
+
# is it localized to a region/zone?
|
233
|
+
# @return [Boolean]
|
234
|
+
def self.isGlobal?
|
235
|
+
false
|
236
|
+
end
|
237
|
+
|
230
238
|
# Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
|
231
239
|
# @param noop [Boolean]: If true, will only print what would be done
|
232
240
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
233
241
|
# @param region [String]: The cloud provider region
|
234
242
|
# @return [void]
|
235
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
243
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
236
244
|
tagfilters = [
|
237
245
|
{name: "tag:MU-ID", values: [MU.deploy_id]}
|
238
246
|
]
|
@@ -240,7 +248,7 @@ module MU
|
|
240
248
|
tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
|
241
249
|
end
|
242
250
|
|
243
|
-
resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
|
251
|
+
resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
|
244
252
|
filters: tagfilters
|
245
253
|
)
|
246
254
|
|
@@ -303,13 +311,13 @@ module MU
|
|
303
311
|
begin
|
304
312
|
|
305
313
|
if ingress_to_revoke.size > 0
|
306
|
-
MU::Cloud::AWS.ec2(region).revoke_security_group_ingress(
|
314
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
|
307
315
|
group_id: sg.group_id,
|
308
316
|
ip_permissions: ingress_to_revoke
|
309
317
|
)
|
310
318
|
end
|
311
319
|
if egress_to_revoke.size > 0
|
312
|
-
MU::Cloud::AWS.ec2(region).revoke_security_group_egress(
|
320
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
|
313
321
|
group_id: sg.group_id,
|
314
322
|
ip_permissions: egress_to_revoke
|
315
323
|
)
|
@@ -325,7 +333,7 @@ module MU
|
|
325
333
|
|
326
334
|
retries = 0
|
327
335
|
begin
|
328
|
-
MU::Cloud::AWS.ec2(region).delete_security_group(group_id: sg.group_id) if !noop
|
336
|
+
MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
|
329
337
|
rescue Aws::EC2::Errors::InvalidGroupNotFound
|
330
338
|
MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
|
331
339
|
rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
|
@@ -459,13 +467,13 @@ module MU
|
|
459
467
|
MU.log "Rules for EC2 Security Group #{@mu_name} (#{@cloud_id}): #{ec2_rules}", MU::DEBUG
|
460
468
|
begin
|
461
469
|
if ingress
|
462
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
|
470
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
|
463
471
|
group_id: @cloud_id,
|
464
472
|
ip_permissions: ec2_rules
|
465
473
|
)
|
466
474
|
end
|
467
475
|
if egress
|
468
|
-
MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
|
476
|
+
MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
|
469
477
|
group_id: @cloud_id,
|
470
478
|
ip_permissions: ec2_rules
|
471
479
|
)
|
@@ -571,7 +579,7 @@ module MU
|
|
571
579
|
if !lb.nil? and !lb.cloud_desc.nil?
|
572
580
|
lb.cloud_desc.security_groups.each { |lb_sg|
|
573
581
|
ec2_rule[:user_id_group_pairs] << {
|
574
|
-
user_id: MU.
|
582
|
+
user_id: MU::Cloud::AWS.credToAcct(@config['credentials']),
|
575
583
|
group_id: lb_sg
|
576
584
|
}
|
577
585
|
}
|
@@ -48,21 +48,36 @@ module MU
|
|
48
48
|
}
|
49
49
|
end
|
50
50
|
|
51
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
52
|
+
# is it localized to a region/zone?
|
53
|
+
# @return [Boolean]
|
54
|
+
def self.isGlobal?
|
55
|
+
true
|
56
|
+
end
|
57
|
+
|
51
58
|
# Remove all logs associated with the currently loaded deployment.
|
52
59
|
# @param noop [Boolean]: If true, will only print what would be done
|
53
60
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
54
61
|
# @param region [String]: The cloud provider region
|
55
62
|
# @return [void]
|
56
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
63
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
57
64
|
end
|
58
65
|
|
59
|
-
# Locate an existing
|
66
|
+
# Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
|
60
67
|
# @param cloud_id [String]: The cloud provider's identifier for this resource.
|
61
68
|
# @param region [String]: The cloud provider region.
|
62
69
|
# @param flags [Hash]: Optional flags
|
63
70
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
|
64
|
-
def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
|
71
|
+
def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
|
65
72
|
found = nil
|
73
|
+
|
74
|
+
if cloud_id
|
75
|
+
else
|
76
|
+
resp = MU::Cloud::AWS.orgs(credentials: credentials).describe_organization
|
77
|
+
found ||= {}
|
78
|
+
found[resp.organization.id] = resp.organization
|
79
|
+
end
|
80
|
+
|
66
81
|
found
|
67
82
|
end
|
68
83
|
|
@@ -35,16 +35,23 @@ module MU
|
|
35
35
|
@mu_name ||= @deploy.getResourceName(@config["name"])
|
36
36
|
end
|
37
37
|
|
38
|
-
|
38
|
+
# Given an IAM role name, resolve to ARN. Will attempt to identify any
|
39
|
+
# sibling Mu role resources by this name first, and failing that, will
|
40
|
+
# do a plain get_role() to the IAM API for the provided name.
|
41
|
+
# @param name [String]
|
39
42
|
def get_role_arn(name)
|
43
|
+
sib_role = @deploy.findLitterMate(name: name, type: "roles")
|
44
|
+
return sib_role.cloudobj.arn if sib_role
|
45
|
+
|
40
46
|
begin
|
41
|
-
role = MU::Cloud::AWS.iam(@config['
|
47
|
+
role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
|
42
48
|
role_name: name.to_s
|
43
49
|
})
|
44
50
|
return role['role']['arn']
|
45
51
|
rescue Exception => e
|
46
|
-
|
52
|
+
MU.log "#{e}", MU::ERR
|
47
53
|
end
|
54
|
+
nil
|
48
55
|
end
|
49
56
|
|
50
57
|
def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
|
@@ -52,7 +59,7 @@ module MU
|
|
52
59
|
## get vpc_id
|
53
60
|
## get sub_id and verify its in the same vpc
|
54
61
|
## get sg_id and verify its in the same vpc
|
55
|
-
ec2_client = MU::Cloud::AWS.ec2(region)
|
62
|
+
ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
|
56
63
|
|
57
64
|
vpc_filter = ec2_client.describe_vpcs({
|
58
65
|
filters: [{ name: 'tag-value', values: [vpc_name] }]
|
@@ -94,7 +101,7 @@ module MU
|
|
94
101
|
def assign_tag(resource_arn, tag_list, region=@config['region'])
|
95
102
|
begin
|
96
103
|
tag_list.each do |each_pair|
|
97
|
-
tag_resp = MU::Cloud::AWS.lambda(region).tag_resource({
|
104
|
+
tag_resp = MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
|
98
105
|
resource: resource_arn,
|
99
106
|
tags: each_pair
|
100
107
|
})
|
@@ -164,11 +171,22 @@ module MU
|
|
164
171
|
lambda_properties[:vpc_config] = vpc_conf
|
165
172
|
end
|
166
173
|
|
167
|
-
|
174
|
+
retries = 0
|
175
|
+
begin
|
176
|
+
MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
|
177
|
+
rescue Aws::Lambda::Errors::InvalidParameterValueException => e
|
178
|
+
# Freshly-made IAM roles sometimes aren't really ready
|
179
|
+
if retries < 5
|
180
|
+
sleep 10
|
181
|
+
retries += 1
|
182
|
+
retry
|
183
|
+
end
|
184
|
+
raise e
|
185
|
+
end
|
168
186
|
end
|
169
187
|
|
170
188
|
def groom
|
171
|
-
desc = MU::Cloud::AWS.lambda(@config['region']).get_function(
|
189
|
+
desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
|
172
190
|
function_name: @mu_name
|
173
191
|
)
|
174
192
|
func_arn = desc.configuration.function_arn if !desc.empty?
|
@@ -199,7 +217,7 @@ module MU
|
|
199
217
|
|
200
218
|
MU.log trigger_properties, MU::DEBUG
|
201
219
|
begin
|
202
|
-
add_trigger = MU::Cloud::AWS.lambda(@config['region']).add_permission(trigger_properties)
|
220
|
+
add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
|
203
221
|
rescue Aws::Lambda::Errors::ResourceConflictException
|
204
222
|
# XXX check properly for existence
|
205
223
|
end
|
@@ -216,11 +234,11 @@ module MU
|
|
216
234
|
arn = nil
|
217
235
|
case svc.downcase
|
218
236
|
when 'sns'
|
219
|
-
arn = "arn:aws:sns:#{@config['region']}:#{MU.
|
237
|
+
arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
220
238
|
when 'alarm','events', 'event', 'cloudwatch_event'
|
221
|
-
arn = "arn:aws:events:#{@config['region']}:#{MU.
|
239
|
+
arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
|
222
240
|
when 'apigateway'
|
223
|
-
arn = "arn:aws:apigateway:#{@config['region']}:#{MU.
|
241
|
+
arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
|
224
242
|
when 's3'
|
225
243
|
arn = ''
|
226
244
|
end
|
@@ -237,15 +255,16 @@ module MU
|
|
237
255
|
case trig_type
|
238
256
|
|
239
257
|
when 'sns'
|
240
|
-
|
241
|
-
sns_client = MU::Cloud::AWS.sns(@config['region'])
|
258
|
+
# XXX don't do this, use MU::Cloud::AWS::Notification
|
259
|
+
sns_client = MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials'])
|
242
260
|
sub_to_what = sns_client.subscribe({
|
243
261
|
topic_arn: trig_arn,
|
244
262
|
protocol: protocol,
|
245
263
|
endpoint: func_arn
|
246
264
|
})
|
247
265
|
when 'event','cloudwatch_event', 'events'
|
248
|
-
|
266
|
+
# XXX don't do this, use MU::Cloud::AWS::Log
|
267
|
+
client = MU::Cloud::AWS.cloudwatch_events(region: @config['region'], credentials: @config['credentials']).put_targets({
|
249
268
|
rule: @config['trigger']['name'],
|
250
269
|
targets: [
|
251
270
|
{
|
@@ -268,23 +287,27 @@ module MU
|
|
268
287
|
return deploy_struct
|
269
288
|
end
|
270
289
|
|
271
|
-
|
272
|
-
|
290
|
+
# Does this resource type exist as a global (cloud-wide) artifact, or
|
291
|
+
# is it localized to a region/zone?
|
292
|
+
# @return [Boolean]
|
293
|
+
def self.isGlobal?
|
294
|
+
false
|
295
|
+
end
|
273
296
|
|
274
297
|
# Remove all functions associated with the currently loaded deployment.
|
275
298
|
# @param noop [Boolean]: If true, will only print what would be done
|
276
299
|
# @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
|
277
300
|
# @param region [String]: The cloud provider region
|
278
301
|
# @return [void]
|
279
|
-
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
|
280
|
-
MU::Cloud::AWS.lambda(region).list_functions.functions.each { |f|
|
281
|
-
desc = MU::Cloud::AWS.lambda(region).get_function(
|
302
|
+
def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
|
303
|
+
MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
|
304
|
+
desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
|
282
305
|
function_name: f.function_name
|
283
306
|
)
|
284
307
|
if desc.tags and desc.tags["MU-ID"] == MU.deploy_id
|
285
308
|
MU.log "Deleting Lambda function #{f.function_name}"
|
286
309
|
if !noop
|
287
|
-
MU::Cloud::AWS.lambda(region).delete_function(
|
310
|
+
MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
|
288
311
|
function_name: f.function_name
|
289
312
|
)
|
290
313
|
end
|
@@ -304,10 +327,10 @@ module MU
|
|
304
327
|
# @param region [String]: The cloud provider region.
|
305
328
|
# @param flags [Hash]: Optional flags
|
306
329
|
# @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
|
307
|
-
def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, flags: {})
|
330
|
+
def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, credentials: nil, flags: {})
|
308
331
|
func = nil
|
309
332
|
if !func_name.nil?
|
310
|
-
all_functions = MU::Cloud::AWS.lambda(region).list_functions
|
333
|
+
all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
|
311
334
|
if all_functions.include?(func_name)
|
312
335
|
all_functions.functions.each do |x|
|
313
336
|
if x.function_name == func_name
|
@@ -329,7 +352,13 @@ module MU
|
|
329
352
|
# @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
|
330
353
|
def self.schema(config)
|
331
354
|
toplevel_required = []
|
332
|
-
schema = {
|
355
|
+
schema = {
|
356
|
+
"iam_role" => {
|
357
|
+
"type" => "string",
|
358
|
+
"description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with the AWSLambdaBasicExecutionRole policy attached. To grant other permissions for your function, create a Mu 'role' resource and use the 'import' and 'policies' parameters to add permissions. See also: https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html"
|
359
|
+
}
|
360
|
+
# XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
|
361
|
+
}
|
333
362
|
[toplevel_required, schema]
|
334
363
|
end
|
335
364
|
|
@@ -340,6 +369,31 @@ module MU
|
|
340
369
|
def self.validateConfig(function, configurator)
|
341
370
|
ok = true
|
342
371
|
|
372
|
+
if !function['iam_role']
|
373
|
+
roledesc = {
|
374
|
+
"name" => function['name']+"execrole",
|
375
|
+
"credentials" => function['credentials'],
|
376
|
+
"can_assume" => [
|
377
|
+
{
|
378
|
+
"entity_id" => "lambda.amazonaws.com",
|
379
|
+
"entity_type" => "service"
|
380
|
+
}
|
381
|
+
],
|
382
|
+
"import" => [
|
383
|
+
"AWSLambdaBasicExecutionRole"
|
384
|
+
]
|
385
|
+
}
|
386
|
+
configurator.insertKitten(roledesc, "roles")
|
387
|
+
|
388
|
+
function['dependencies'] ||= []
|
389
|
+
function['iam_role'] = function['name']+"execrole"
|
390
|
+
|
391
|
+
function['dependencies'] << {
|
392
|
+
"type" => "role",
|
393
|
+
"name" => function['name']+"execrole"
|
394
|
+
}
|
395
|
+
end
|
396
|
+
|
343
397
|
ok
|
344
398
|
end
|
345
399
|
|