cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +16 -54
  3. data/Berksfile.lock +14 -62
  4. data/bin/mu-aws-setup +131 -108
  5. data/bin/mu-configure +311 -74
  6. data/bin/mu-gcp-setup +84 -62
  7. data/bin/mu-load-config.rb +46 -2
  8. data/bin/mu-self-update +11 -9
  9. data/bin/mu-upload-chef-artifacts +4 -4
  10. data/{mu.gemspec → cloud-mu.gemspec} +2 -2
  11. data/cookbooks/awscli/Berksfile +8 -0
  12. data/cookbooks/mu-activedirectory/Berksfile +11 -0
  13. data/cookbooks/mu-firewall/Berksfile +9 -0
  14. data/cookbooks/mu-firewall/metadata.rb +1 -1
  15. data/cookbooks/mu-glusterfs/Berksfile +10 -0
  16. data/cookbooks/mu-jenkins/Berksfile +14 -0
  17. data/cookbooks/mu-master/Berksfile +23 -0
  18. data/cookbooks/mu-master/attributes/default.rb +1 -1
  19. data/cookbooks/mu-master/metadata.rb +2 -2
  20. data/cookbooks/mu-master/recipes/default.rb +1 -1
  21. data/cookbooks/mu-master/recipes/init.rb +7 -3
  22. data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
  23. data/cookbooks/mu-mongo/Berksfile +10 -0
  24. data/cookbooks/mu-openvpn/Berksfile +11 -0
  25. data/cookbooks/mu-php54/Berksfile +13 -0
  26. data/cookbooks/mu-splunk/Berksfile +10 -0
  27. data/cookbooks/mu-tools/Berksfile +21 -0
  28. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  29. data/cookbooks/mu-utility/Berksfile +9 -0
  30. data/cookbooks/mu-utility/metadata.rb +2 -1
  31. data/cookbooks/nagios/Berksfile +7 -4
  32. data/cookbooks/s3fs/Berksfile +9 -0
  33. data/environments/dev.json +6 -6
  34. data/environments/prod.json +6 -6
  35. data/modules/mu.rb +20 -42
  36. data/modules/mu/cleanup.rb +102 -100
  37. data/modules/mu/cloud.rb +90 -28
  38. data/modules/mu/clouds/aws.rb +449 -218
  39. data/modules/mu/clouds/aws/alarm.rb +29 -17
  40. data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
  41. data/modules/mu/clouds/aws/collection.rb +25 -18
  42. data/modules/mu/clouds/aws/container_cluster.rb +73 -66
  43. data/modules/mu/clouds/aws/database.rb +124 -116
  44. data/modules/mu/clouds/aws/dnszone.rb +27 -20
  45. data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
  46. data/modules/mu/clouds/aws/folder.rb +18 -3
  47. data/modules/mu/clouds/aws/function.rb +77 -23
  48. data/modules/mu/clouds/aws/group.rb +19 -12
  49. data/modules/mu/clouds/aws/habitat.rb +153 -0
  50. data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
  51. data/modules/mu/clouds/aws/log.rb +30 -23
  52. data/modules/mu/clouds/aws/msg_queue.rb +29 -20
  53. data/modules/mu/clouds/aws/notifier.rb +222 -0
  54. data/modules/mu/clouds/aws/role.rb +178 -90
  55. data/modules/mu/clouds/aws/search_domain.rb +40 -24
  56. data/modules/mu/clouds/aws/server.rb +169 -137
  57. data/modules/mu/clouds/aws/server_pool.rb +60 -83
  58. data/modules/mu/clouds/aws/storage_pool.rb +59 -31
  59. data/modules/mu/clouds/aws/user.rb +36 -27
  60. data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
  61. data/modules/mu/clouds/aws/vpc.rb +250 -189
  62. data/modules/mu/clouds/azure.rb +132 -0
  63. data/modules/mu/clouds/cloudformation.rb +65 -1
  64. data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
  65. data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
  66. data/modules/mu/clouds/cloudformation/collection.rb +7 -0
  67. data/modules/mu/clouds/cloudformation/database.rb +7 -0
  68. data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
  69. data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
  70. data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
  71. data/modules/mu/clouds/cloudformation/log.rb +7 -0
  72. data/modules/mu/clouds/cloudformation/server.rb +7 -0
  73. data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
  74. data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
  75. data/modules/mu/clouds/google.rb +214 -110
  76. data/modules/mu/clouds/google/container_cluster.rb +42 -24
  77. data/modules/mu/clouds/google/database.rb +15 -6
  78. data/modules/mu/clouds/google/firewall_rule.rb +17 -25
  79. data/modules/mu/clouds/google/group.rb +13 -5
  80. data/modules/mu/clouds/google/habitat.rb +105 -0
  81. data/modules/mu/clouds/google/loadbalancer.rb +28 -20
  82. data/modules/mu/clouds/google/server.rb +93 -354
  83. data/modules/mu/clouds/google/server_pool.rb +18 -10
  84. data/modules/mu/clouds/google/user.rb +22 -14
  85. data/modules/mu/clouds/google/vpc.rb +97 -69
  86. data/modules/mu/config.rb +133 -38
  87. data/modules/mu/config/alarm.rb +25 -0
  88. data/modules/mu/config/cache_cluster.rb +5 -3
  89. data/modules/mu/config/cache_cluster.yml +23 -0
  90. data/modules/mu/config/database.rb +25 -16
  91. data/modules/mu/config/database.yml +3 -3
  92. data/modules/mu/config/function.rb +1 -2
  93. data/modules/mu/config/{project.rb → habitat.rb} +10 -10
  94. data/modules/mu/config/notifier.rb +85 -0
  95. data/modules/mu/config/notifier.yml +9 -0
  96. data/modules/mu/config/role.rb +1 -1
  97. data/modules/mu/config/search_domain.yml +2 -2
  98. data/modules/mu/config/server.rb +13 -1
  99. data/modules/mu/config/server.yml +3 -3
  100. data/modules/mu/config/server_pool.rb +3 -1
  101. data/modules/mu/config/storage_pool.rb +3 -1
  102. data/modules/mu/config/storage_pool.yml +19 -0
  103. data/modules/mu/config/vpc.rb +70 -8
  104. data/modules/mu/groomers/chef.rb +2 -3
  105. data/modules/mu/kittens.rb +500 -122
  106. data/modules/mu/master.rb +5 -5
  107. data/modules/mu/mommacat.rb +151 -91
  108. data/modules/tests/super_complex_bok.yml +12 -0
  109. data/modules/tests/super_simple_bok.yml +12 -0
  110. data/spec/mu/clouds/azure_spec.rb +82 -0
  111. data/spec/spec_helper.rb +105 -0
  112. metadata +26 -5
  113. data/modules/mu/clouds/aws/notification.rb +0 -139
  114. data/modules/mu/config/notification.rb +0 -44
@@ -64,7 +64,7 @@ module MU
64
64
  if @config['all_account_vpcs']
65
65
  # If we've been told to make this domain available account-wide, do so
66
66
  MU::Cloud::AWS.listRegions(@config['us_only']).each { |region|
67
- known_vpcs = MU::Cloud::AWS.ec2(region).describe_vpcs.vpcs
67
+ known_vpcs = MU::Cloud::AWS.ec2(region: region).describe_vpcs.vpcs
68
68
 
69
69
  MU.log "Enumerating VPCs in #{region}", MU::DEBUG, details: known_vpcs
70
70
 
@@ -330,11 +330,11 @@ module MU
330
330
  # @param vpc_id [String]: The cloud identifier of the VPC
331
331
  # @param region [String]: The cloud provider's region
332
332
  # @param remove [Boolean]: Whether to remove access (default: grant access)
333
- def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false)
333
+ def self.toggleVPCAccess(id: nil, vpc_id: nil, region: MU.curRegion, remove: false, credentials: nil)
334
334
 
335
335
  if !remove
336
336
  MU.log "Granting VPC #{vpc_id} access to zone #{id}"
337
- MU::Cloud::AWS.route53(region).associate_vpc_with_hosted_zone(
337
+ MU::Cloud::AWS.route53(credentials: credentials).associate_vpc_with_hosted_zone(
338
338
  hosted_zone_id: id,
339
339
  vpc: {
340
340
  :vpc_id => vpc_id,
@@ -345,7 +345,7 @@ module MU
345
345
  else
346
346
  MU.log "Revoking VPC #{vpc_id} access to zone #{id}"
347
347
  begin
348
- MU::Cloud::AWS.route53(region).disassociate_vpc_from_hosted_zone(
348
+ MU::Cloud::AWS.route53(credentials: credentials).disassociate_vpc_from_hosted_zone(
349
349
  hosted_zone_id: id,
350
350
  vpc: {
351
351
  :vpc_id => vpc_id,
@@ -400,7 +400,7 @@ module MU
400
400
  target_zone = "/hostedzone/"+alias_zone if !alias_zone.match(/^\/hostedzone\//)
401
401
  else
402
402
  MU::Cloud::AWS.listRegions.each { |region|
403
- MU::Cloud::AWS.elb(region).describe_load_balancers.load_balancer_descriptions.each { |elb|
403
+ MU::Cloud::AWS.elb(region: region).describe_load_balancers.load_balancer_descriptions.each { |elb|
404
404
  elb_dns = elb.dns_name.downcase
405
405
  elb_dns.chomp!(".")
406
406
  if target_name == elb_dns
@@ -657,14 +657,21 @@ module MU
657
657
  end
658
658
  end
659
659
 
660
+ # Does this resource type exist as a global (cloud-wide) artifact, or
661
+ # is it localized to a region/zone?
662
+ # @return [Boolean]
663
+ def self.isGlobal?
664
+ true
665
+ end
666
+
660
667
  # Called by {MU::Cleanup}. Locates resources that were created by the
661
668
  # currently-loaded deployment, and purges them.
662
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
669
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
663
670
  checks_to_clean = []
664
671
  threads = []
665
- MU::Cloud::AWS.route53(region).list_health_checks.health_checks.each { |check|
672
+ MU::Cloud::AWS.route53(credentials: credentials).list_health_checks.health_checks.each { |check|
666
673
  begin
667
- tags = MU::Cloud::AWS.route53(region).list_tags_for_resource(
674
+ tags = MU::Cloud::AWS.route53(credentials: credentials).list_tags_for_resource(
668
675
  resource_type: "healthcheck",
669
676
  resource_id: check.id
670
677
  ).resource_tag_set.tags
@@ -692,7 +699,7 @@ module MU
692
699
  MU.log "Removing health check #{check.id}"
693
700
  retries = 5
694
701
  begin
695
- MU::Cloud::AWS.route53(region).delete_health_check(health_check_id: check.id) if !noop
702
+ MU::Cloud::AWS.route53(credentials: credentials).delete_health_check(health_check_id: check.id) if !noop
696
703
  rescue Aws::Route53::Errors::NoSuchHealthCheck => e
697
704
  MU.log "Health Check '#{check.id}' disappeared before I could remove it", MU::WARN, details: e.inspect
698
705
  rescue Aws::Route53::Errors::InvalidInput => e
@@ -721,11 +728,11 @@ module MU
721
728
  if !noop
722
729
  begin
723
730
  # Clean up resource records first
724
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id)
731
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
725
732
  rrsets.resource_record_sets.each { |rrset|
726
733
  next if zone.name == rrset.name and (rrset.type == "NS" or rrset.type == "SOA")
727
734
  records = []
728
- MU::Cloud::AWS.route53(region).change_resource_record_sets(
735
+ MU::Cloud::AWS.route53(credentials: credentials).change_resource_record_sets(
729
736
  hosted_zone_id: zone.id,
730
737
  change_batch: {
731
738
  changes: [
@@ -738,7 +745,7 @@ module MU
738
745
  )
739
746
  }
740
747
 
741
- MU::Cloud::AWS.route53(region).delete_hosted_zone(id: zone.id)
748
+ MU::Cloud::AWS.route53(credentials: credentials).delete_hosted_zone(id: zone.id)
742
749
  rescue Aws::Route53::Errors::PriorRequestNotComplete
743
750
  MU.log "Still waiting for all records in DNS Zone '#{zone.name}' (#{zone.id}) to delete", MU::WARN
744
751
  sleep 20
@@ -754,17 +761,17 @@ module MU
754
761
  }
755
762
 
756
763
  # Lets try cleaning MU DNS records in all zones.
757
- MU::Cloud::AWS.route53(region).list_hosted_zones.hosted_zones.each { |zone|
764
+ MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones.hosted_zones.each { |zone|
758
765
  begin
759
766
  zone_rrsets = []
760
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id)
767
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id)
761
768
  rrsets.resource_record_sets.each { |record|
762
769
  zone_rrsets << record
763
770
  }
764
771
 
765
772
  # AWS API returns a maximum of 100 results. DNS zones are likely to have more than 100 records, lets page and make sure we grab all records in a given zone
766
773
  while rrsets.next_record_name && rrsets.next_record_type
767
- rrsets = MU::Cloud::AWS.route53(region).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
774
+ rrsets = MU::Cloud::AWS.route53(credentials: credentials).list_resource_record_sets(hosted_zone_id: zone.id, start_record_name: rrsets.next_record_name, start_record_type: rrsets.next_record_type)
768
775
  rrsets.resource_record_sets.each { |record|
769
776
  zone_rrsets << record
770
777
  }
@@ -871,10 +878,10 @@ module MU
871
878
  # @param region [String]: The cloud provider region
872
879
  # @param flags [Hash]: Optional flags
873
880
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching DNSZones
874
- def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, flags: {})
881
+ def self.find(cloud_id: nil, deploy_id: MU.deploy_id, region: MU.curRegion, credentials: nil, flags: {})
875
882
  matches = {}
876
883
 
877
- resp = MU::Cloud::AWS.route53(region).list_hosted_zones(
884
+ resp = MU::Cloud::AWS.route53(credentials: credentials).list_hosted_zones(
878
885
  max_items: 100
879
886
  )
880
887
 
@@ -882,13 +889,13 @@ module MU
882
889
  if !cloud_id.nil? and !cloud_id.empty?
883
890
  if zone.id == cloud_id
884
891
  begin
885
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
892
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
886
893
  rescue Aws::Route53::Errors::NoSuchHostedZone
887
894
  MU.log "Hosted zone #{zone.id} doesn't exist"
888
895
  end
889
896
  elsif zone.name == cloud_id or zone.name == cloud_id+"."
890
897
  begin
891
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
898
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
892
899
  rescue Aws::Route53::Errors::NoSuchHostedZone
893
900
  MU.log "Hosted zone #{zone.id} doesn't exist"
894
901
  end
@@ -896,7 +903,7 @@ module MU
896
903
  end
897
904
  if !deploy_id.nil? and !deploy_id.empty? and zone.config.comment == deploy_id
898
905
  begin
899
- matches[zone.id] = MU::Cloud::AWS.route53(region).get_hosted_zone(id: zone.id).hosted_zone
906
+ matches[zone.id] = MU::Cloud::AWS.route53(credentials: credentials).get_hosted_zone(id: zone.id).hosted_zone
900
907
  rescue Aws::Route53::Errors::NoSuchHostedZone
901
908
  MU.log "Hosted zone #{zone.id} doesn't exist"
902
909
  end
@@ -51,7 +51,6 @@ module MU
51
51
  vpc_id = @vpc.cloud_id if !@vpc.nil?
52
52
  groupname = @mu_name
53
53
  description = groupname
54
- MU.log "Creating EC2 Security Group #{groupname}"
55
54
 
56
55
  sg_struct = {
57
56
  :group_name => groupname,
@@ -62,14 +61,16 @@ module MU
62
61
  end
63
62
 
64
63
  begin
65
- secgroup = MU::Cloud::AWS.ec2(@config['region']).create_security_group(sg_struct)
64
+ MU.log "Creating EC2 Security Group #{groupname}", details: sg_struct
65
+
66
+ secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).create_security_group(sg_struct)
66
67
  @cloud_id = secgroup.group_id
67
68
  rescue Aws::EC2::Errors::InvalidGroupDuplicate => e
68
69
  MU.log "EC2 Security Group #{groupname} already exists, using it", MU::NOTICE
69
70
  filters = [{name: "group-name", values: [groupname]}]
70
71
  filters << {name: "vpc-id", values: [vpc_id]} if !vpc_id.nil?
71
72
 
72
- secgroup = MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(filters: filters).security_groups.first
73
+ secgroup = MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(filters: filters).security_groups.first
73
74
  deploy_id = @deploy.deploy_id if !@deploy_id.nil?
74
75
  if secgroup.nil?
75
76
  raise MuError, "Failed to locate security group named #{groupname}, even though EC2 says it already exists", caller
@@ -78,25 +79,25 @@ module MU
78
79
  end
79
80
 
80
81
  begin
81
- MU::Cloud::AWS.ec2(@config['region']).describe_security_groups(group_ids: [secgroup.group_id])
82
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).describe_security_groups(group_ids: [secgroup.group_id])
82
83
  rescue Aws::EC2::Errors::InvalidGroupNotFound => e
83
84
  MU.log "#{secgroup.group_id} not yet ready, waiting...", MU::NOTICE
84
85
  sleep 10
85
86
  retry
86
87
  end
87
88
 
88
- MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'])
89
- MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'])
89
+ MU::MommaCat.createStandardTags(secgroup.group_id, region: @config['region'], credentials: @config['credentials'])
90
+ MU::MommaCat.createTag(secgroup.group_id, "Name", groupname, region: @config['region'], credentials: @config['credentials'])
90
91
 
91
92
  if @config['optional_tags']
92
93
  MU::MommaCat.listOptionalTags.each { |key, value|
93
- MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'])
94
+ MU::MommaCat.createTag(secgroup.group_id, key, value, region: @config['region'], credentials: @config['credentials'])
94
95
  }
95
96
  end
96
97
 
97
98
  if @config['tags']
98
99
  @config['tags'].each { |tag|
99
- MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'])
100
+ MU::MommaCat.createTag(secgroup.group_id, tag['key'], tag['value'], region: @config['region'], credentials: @config['credentials'])
100
101
  }
101
102
  end
102
103
 
@@ -167,12 +168,12 @@ module MU
167
168
 
168
169
  begin
169
170
  if egress
170
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
171
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
171
172
  group_id: @cloud_id,
172
173
  ip_permissions: ec2_rule
173
174
  )
174
175
  else
175
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
176
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
176
177
  group_id: @cloud_id,
177
178
  ip_permissions: ec2_rule
178
179
  )
@@ -185,7 +186,7 @@ module MU
185
186
  # Canonical Amazon Resource Number for this resource
186
187
  # @return [String]
187
188
  def arn
188
- "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU.account_number+":security-group/"+@cloud_id
189
+ "arn:"+(MU::Cloud::AWS.isGovCloud?(@config["region"]) ? "aws-us-gov" : "aws")+":ec2:"+@config['region']+":"+MU::Cloud::AWS.credToAcct(@config['credentials'])+":security-group/"+@cloud_id
189
190
  end
190
191
 
191
192
  # Locate an existing security group or groups and return an array containing matching AWS resource descriptors for those that match.
@@ -195,11 +196,11 @@ module MU
195
196
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
196
197
  # @param flags [Hash]: Optional flags
197
198
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching FirewallRules
198
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
199
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, credentials: nil, flags: {})
199
200
 
200
201
  if !cloud_id.nil? and !cloud_id.empty?
201
202
  begin
202
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(group_ids: [cloud_id])
203
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(group_ids: [cloud_id])
203
204
  return {cloud_id => resp.data.security_groups.first}
204
205
  rescue ArgumentError => e
205
206
  MU.log "Attempting to load #{cloud_id}: #{e.inspect}", MU::WARN, details: caller
@@ -212,7 +213,7 @@ module MU
212
213
 
213
214
  map = {}
214
215
  if !tag_key.nil? and !tag_value.nil?
215
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
216
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).describe_security_groups(
216
217
  filters: [
217
218
  {name: "tag:#{tag_key}", values: [tag_value]}
218
219
  ]
@@ -227,12 +228,19 @@ module MU
227
228
  map
228
229
  end
229
230
 
231
+ # Does this resource type exist as a global (cloud-wide) artifact, or
232
+ # is it localized to a region/zone?
233
+ # @return [Boolean]
234
+ def self.isGlobal?
235
+ false
236
+ end
237
+
230
238
  # Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
231
239
  # @param noop [Boolean]: If true, will only print what would be done
232
240
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
233
241
  # @param region [String]: The cloud provider region
234
242
  # @return [void]
235
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
243
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
236
244
  tagfilters = [
237
245
  {name: "tag:MU-ID", values: [MU.deploy_id]}
238
246
  ]
@@ -240,7 +248,7 @@ module MU
240
248
  tagfilters << {name: "tag:MU-MASTER-IP", values: [MU.mu_public_ip]}
241
249
  end
242
250
 
243
- resp = MU::Cloud::AWS.ec2(region).describe_security_groups(
251
+ resp = MU::Cloud::AWS.ec2(credentials: credentials, region: region).describe_security_groups(
244
252
  filters: tagfilters
245
253
  )
246
254
 
@@ -303,13 +311,13 @@ module MU
303
311
  begin
304
312
 
305
313
  if ingress_to_revoke.size > 0
306
- MU::Cloud::AWS.ec2(region).revoke_security_group_ingress(
314
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_ingress(
307
315
  group_id: sg.group_id,
308
316
  ip_permissions: ingress_to_revoke
309
317
  )
310
318
  end
311
319
  if egress_to_revoke.size > 0
312
- MU::Cloud::AWS.ec2(region).revoke_security_group_egress(
320
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).revoke_security_group_egress(
313
321
  group_id: sg.group_id,
314
322
  ip_permissions: egress_to_revoke
315
323
  )
@@ -325,7 +333,7 @@ module MU
325
333
 
326
334
  retries = 0
327
335
  begin
328
- MU::Cloud::AWS.ec2(region).delete_security_group(group_id: sg.group_id) if !noop
336
+ MU::Cloud::AWS.ec2(credentials: credentials, region: region).delete_security_group(group_id: sg.group_id) if !noop
329
337
  rescue Aws::EC2::Errors::InvalidGroupNotFound
330
338
  MU.log "EC2 Security Group #{sg.group_name} disappeared before I could delete it!", MU::WARN
331
339
  rescue Aws::EC2::Errors::DependencyViolation, Aws::EC2::Errors::InvalidGroupInUse
@@ -459,13 +467,13 @@ module MU
459
467
  MU.log "Rules for EC2 Security Group #{@mu_name} (#{@cloud_id}): #{ec2_rules}", MU::DEBUG
460
468
  begin
461
469
  if ingress
462
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_ingress(
470
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_ingress(
463
471
  group_id: @cloud_id,
464
472
  ip_permissions: ec2_rules
465
473
  )
466
474
  end
467
475
  if egress
468
- MU::Cloud::AWS.ec2(@config['region']).authorize_security_group_egress(
476
+ MU::Cloud::AWS.ec2(region: @config['region'], credentials: @config['credentials']).authorize_security_group_egress(
469
477
  group_id: @cloud_id,
470
478
  ip_permissions: ec2_rules
471
479
  )
@@ -571,7 +579,7 @@ module MU
571
579
  if !lb.nil? and !lb.cloud_desc.nil?
572
580
  lb.cloud_desc.security_groups.each { |lb_sg|
573
581
  ec2_rule[:user_id_group_pairs] << {
574
- user_id: MU.account_number,
582
+ user_id: MU::Cloud::AWS.credToAcct(@config['credentials']),
575
583
  group_id: lb_sg
576
584
  }
577
585
  }
@@ -48,21 +48,36 @@ module MU
48
48
  }
49
49
  end
50
50
 
51
+ # Does this resource type exist as a global (cloud-wide) artifact, or
52
+ # is it localized to a region/zone?
53
+ # @return [Boolean]
54
+ def self.isGlobal?
55
+ true
56
+ end
57
+
51
58
  # Remove all logs associated with the currently loaded deployment.
52
59
  # @param noop [Boolean]: If true, will only print what would be done
53
60
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
54
61
  # @param region [String]: The cloud provider region
55
62
  # @return [void]
56
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
63
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
57
64
  end
58
65
 
59
- # Locate an existing log group.
66
+ # Locate an existing AWS organization. If no identifying parameters are specified, this will return a description of the Organization which owns the account for our credentials.
60
67
  # @param cloud_id [String]: The cloud provider's identifier for this resource.
61
68
  # @param region [String]: The cloud provider region.
62
69
  # @param flags [Hash]: Optional flags
63
70
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching log group.
64
- def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
71
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
65
72
  found = nil
73
+
74
+ if cloud_id
75
+ else
76
+ resp = MU::Cloud::AWS.orgs(credentials: credentials).describe_organization
77
+ found ||= {}
78
+ found[resp.organization.id] = resp.organization
79
+ end
80
+
66
81
  found
67
82
  end
68
83
 
@@ -35,16 +35,23 @@ module MU
35
35
  @mu_name ||= @deploy.getResourceName(@config["name"])
36
36
  end
37
37
 
38
-
38
+ # Given an IAM role name, resolve to ARN. Will attempt to identify any
39
+ # sibling Mu role resources by this name first, and failing that, will
40
+ # do a plain get_role() to the IAM API for the provided name.
41
+ # @param name [String]
39
42
  def get_role_arn(name)
43
+ sib_role = @deploy.findLitterMate(name: name, type: "roles")
44
+ return sib_role.cloudobj.arn if sib_role
45
+
40
46
  begin
41
- role = MU::Cloud::AWS.iam(@config['region']).get_role({
47
+ role = MU::Cloud::AWS.iam(credentials: @config['credentials']).get_role({
42
48
  role_name: name.to_s
43
49
  })
44
50
  return role['role']['arn']
45
51
  rescue Exception => e
46
- Mu.log "#{e}", MU::ERR
52
+ MU.log "#{e}", MU::ERR
47
53
  end
54
+ nil
48
55
  end
49
56
 
50
57
  def get_vpc_config(vpc_name, subnet_name, sg_name,region=@config['region'])
@@ -52,7 +59,7 @@ module MU
52
59
  ## get vpc_id
53
60
  ## get sub_id and verify its in the same vpc
54
61
  ## get sg_id and verify its in the same vpc
55
- ec2_client = MU::Cloud::AWS.ec2(region)
62
+ ec2_client = MU::Cloud::AWS.ec2(region: region, credentials: @config['credentials'])
56
63
 
57
64
  vpc_filter = ec2_client.describe_vpcs({
58
65
  filters: [{ name: 'tag-value', values: [vpc_name] }]
@@ -94,7 +101,7 @@ module MU
94
101
  def assign_tag(resource_arn, tag_list, region=@config['region'])
95
102
  begin
96
103
  tag_list.each do |each_pair|
97
- tag_resp = MU::Cloud::AWS.lambda(region).tag_resource({
104
+ tag_resp = MU::Cloud::AWS.lambda(region: region, credentials: @config['credentials']).tag_resource({
98
105
  resource: resource_arn,
99
106
  tags: each_pair
100
107
  })
@@ -164,11 +171,22 @@ module MU
164
171
  lambda_properties[:vpc_config] = vpc_conf
165
172
  end
166
173
 
167
- MU::Cloud::AWS.lambda(@config['region']).create_function(lambda_properties)
174
+ retries = 0
175
+ begin
176
+ MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).create_function(lambda_properties)
177
+ rescue Aws::Lambda::Errors::InvalidParameterValueException => e
178
+ # Freshly-made IAM roles sometimes aren't really ready
179
+ if retries < 5
180
+ sleep 10
181
+ retries += 1
182
+ retry
183
+ end
184
+ raise e
185
+ end
168
186
  end
169
187
 
170
188
  def groom
171
- desc = MU::Cloud::AWS.lambda(@config['region']).get_function(
189
+ desc = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).get_function(
172
190
  function_name: @mu_name
173
191
  )
174
192
  func_arn = desc.configuration.function_arn if !desc.empty?
@@ -199,7 +217,7 @@ module MU
199
217
 
200
218
  MU.log trigger_properties, MU::DEBUG
201
219
  begin
202
- add_trigger = MU::Cloud::AWS.lambda(@config['region']).add_permission(trigger_properties)
220
+ add_trigger = MU::Cloud::AWS.lambda(region: @config['region'], credentials: @config['credentials']).add_permission(trigger_properties)
203
221
  rescue Aws::Lambda::Errors::ResourceConflictException
204
222
  # XXX check properly for existence
205
223
  end
@@ -216,11 +234,11 @@ module MU
216
234
  arn = nil
217
235
  case svc.downcase
218
236
  when 'sns'
219
- arn = "arn:aws:sns:#{@config['region']}:#{MU.account_number}:#{name}"
237
+ arn = "arn:aws:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
220
238
  when 'alarm','events', 'event', 'cloudwatch_event'
221
- arn = "arn:aws:events:#{@config['region']}:#{MU.account_number}:rule/#{name}"
239
+ arn = "arn:aws:events:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:rule/#{name}"
222
240
  when 'apigateway'
223
- arn = "arn:aws:apigateway:#{@config['region']}:#{MU.account_number}:#{name}"
241
+ arn = "arn:aws:apigateway:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{name}"
224
242
  when 's3'
225
243
  arn = ''
226
244
  end
@@ -237,15 +255,16 @@ module MU
237
255
  case trig_type
238
256
 
239
257
  when 'sns'
240
-
241
- sns_client = MU::Cloud::AWS.sns(@config['region'])
258
+ # XXX don't do this, use MU::Cloud::AWS::Notification
259
+ sns_client = MU::Cloud::AWS.sns(region: @config['region'], credentials: @config['credentials'])
242
260
  sub_to_what = sns_client.subscribe({
243
261
  topic_arn: trig_arn,
244
262
  protocol: protocol,
245
263
  endpoint: func_arn
246
264
  })
247
265
  when 'event','cloudwatch_event', 'events'
248
- client = MU::Cloud::AWS.cloudwatch_events(@config['region']).put_targets({
266
+ # XXX don't do this, use MU::Cloud::AWS::Log
267
+ client = MU::Cloud::AWS.cloudwatch_events(region: @config['region'], credentials: @config['credentials']).put_targets({
249
268
  rule: @config['trigger']['name'],
250
269
  targets: [
251
270
  {
@@ -268,23 +287,27 @@ module MU
268
287
  return deploy_struct
269
288
  end
270
289
 
271
-
272
-
290
+ # Does this resource type exist as a global (cloud-wide) artifact, or
291
+ # is it localized to a region/zone?
292
+ # @return [Boolean]
293
+ def self.isGlobal?
294
+ false
295
+ end
273
296
 
274
297
  # Remove all functions associated with the currently loaded deployment.
275
298
  # @param noop [Boolean]: If true, will only print what would be done
276
299
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
277
300
  # @param region [String]: The cloud provider region
278
301
  # @return [void]
279
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
280
- MU::Cloud::AWS.lambda(region).list_functions.functions.each { |f|
281
- desc = MU::Cloud::AWS.lambda(region).get_function(
302
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
303
+ MU::Cloud::AWS.lambda(credentials: credentials, region: region).list_functions.functions.each { |f|
304
+ desc = MU::Cloud::AWS.lambda(credentials: credentials, region: region).get_function(
282
305
  function_name: f.function_name
283
306
  )
284
307
  if desc.tags and desc.tags["MU-ID"] == MU.deploy_id
285
308
  MU.log "Deleting Lambda function #{f.function_name}"
286
309
  if !noop
287
- MU::Cloud::AWS.lambda(region).delete_function(
310
+ MU::Cloud::AWS.lambda(credentials: credentials, region: region).delete_function(
288
311
  function_name: f.function_name
289
312
  )
290
313
  end
@@ -304,10 +327,10 @@ module MU
304
327
  # @param region [String]: The cloud provider region.
305
328
  # @param flags [Hash]: Optional flags
306
329
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching function.
307
- def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, flags: {})
330
+ def self.find(cloud_id: nil, func_name: nil, region: MU.curRegion, credentials: nil, flags: {})
308
331
  func = nil
309
332
  if !func_name.nil?
310
- all_functions = MU::Cloud::AWS.lambda(region).list_functions
333
+ all_functions = MU::Cloud::AWS.lambda(region: region, credentials: credentials).list_functions
311
334
  if all_functions.include?(func_name)
312
335
  all_functions.functions.each do |x|
313
336
  if x.function_name == func_name
@@ -329,7 +352,13 @@ module MU
329
352
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
330
353
  def self.schema(config)
331
354
  toplevel_required = []
332
- schema = {}
355
+ schema = {
356
+ "iam_role" => {
357
+ "type" => "string",
358
+ "description" => "The name of an IAM role for our Lambda function to assume. Can refer to an existing IAM role, or a sibling 'role' resource in Mu. If not specified, will create a default role with the AWSLambdaBasicExecutionRole policy attached. To grant other permissions for your function, create a Mu 'role' resource and use the 'import' and 'policies' parameters to add permissions. See also: https://docs.aws.amazon.com/lambda/latest/dg/lambda-intro-execution-role.html"
359
+ }
360
+ # XXX add some canned permission sets here, asking people to get the AWS weirdness right and then translate it into Mu-speak is just too much. Think about auto-populating when a target log group is asked for, mappings for the AWS canned policies in the URL above, writes to arbitrary S3 buckets, etc
361
+ }
333
362
  [toplevel_required, schema]
334
363
  end
335
364
 
@@ -340,6 +369,31 @@ module MU
340
369
  def self.validateConfig(function, configurator)
341
370
  ok = true
342
371
 
372
+ if !function['iam_role']
373
+ roledesc = {
374
+ "name" => function['name']+"execrole",
375
+ "credentials" => function['credentials'],
376
+ "can_assume" => [
377
+ {
378
+ "entity_id" => "lambda.amazonaws.com",
379
+ "entity_type" => "service"
380
+ }
381
+ ],
382
+ "import" => [
383
+ "AWSLambdaBasicExecutionRole"
384
+ ]
385
+ }
386
+ configurator.insertKitten(roledesc, "roles")
387
+
388
+ function['dependencies'] ||= []
389
+ function['iam_role'] = function['name']+"execrole"
390
+
391
+ function['dependencies'] << {
392
+ "type" => "role",
393
+ "name" => function['name']+"execrole"
394
+ }
395
+ end
396
+
343
397
  ok
344
398
  end
345
399