cloud-mu 2.0.0.pre.alpha9 → 2.0.0.pre.beta1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (61) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile.lock +1 -1
  3. data/README.md +2 -0
  4. data/bin/mu-configure +2 -58
  5. data/bin/mu-gen-docs +29 -4
  6. data/bin/mu-load-config.rb +0 -1
  7. data/bin/mu-user-manage +4 -0
  8. data/cloud-mu.gemspec +2 -2
  9. data/cookbooks/mu-master/recipes/default.rb +3 -4
  10. data/cookbooks/mu-master/recipes/init.rb +3 -3
  11. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  12. data/cookbooks/mu-tools/libraries/helper.rb +1 -1
  13. data/cookbooks/mu-tools/recipes/eks.rb +3 -3
  14. data/cookbooks/mu-tools/recipes/set_local_fw.rb +1 -1
  15. data/cookbooks/mu-utility/recipes/remi.rb +1 -1
  16. data/cookbooks/nagios/libraries/base.rb +4 -4
  17. data/cookbooks/nagios/libraries/contact.rb +1 -1
  18. data/cookbooks/nagios/libraries/contactgroup.rb +1 -1
  19. data/cookbooks/nagios/libraries/host.rb +2 -2
  20. data/cookbooks/nagios/libraries/hostdependency.rb +3 -3
  21. data/cookbooks/nagios/libraries/hostescalation.rb +3 -3
  22. data/cookbooks/nagios/libraries/hostgroup.rb +2 -2
  23. data/cookbooks/nagios/libraries/nagios.rb +5 -5
  24. data/cookbooks/nagios/libraries/service.rb +3 -3
  25. data/cookbooks/nagios/libraries/servicedependency.rb +2 -2
  26. data/cookbooks/nagios/libraries/serviceescalation.rb +2 -2
  27. data/cookbooks/nagios/libraries/servicegroup.rb +2 -2
  28. data/cookbooks/nagios/libraries/timeperiod.rb +1 -1
  29. data/install/installer +1 -1
  30. data/modules/mu/cleanup.rb +1 -1
  31. data/modules/mu/cloud.rb +43 -1
  32. data/modules/mu/clouds/aws.rb +55 -35
  33. data/modules/mu/clouds/aws/bucket.rb +287 -0
  34. data/modules/mu/clouds/aws/database.rb +65 -11
  35. data/modules/mu/clouds/aws/endpoint.rb +592 -0
  36. data/modules/mu/clouds/aws/firewall_rule.rb +4 -0
  37. data/modules/mu/clouds/aws/function.rb +138 -93
  38. data/modules/mu/clouds/aws/nosqldb.rb +387 -0
  39. data/modules/mu/clouds/aws/role.rb +1 -1
  40. data/modules/mu/clouds/aws/server.rb +5 -5
  41. data/modules/mu/clouds/aws/server_pool.rb +60 -3
  42. data/modules/mu/clouds/azure.rb +0 -1
  43. data/modules/mu/clouds/google.rb +34 -12
  44. data/modules/mu/clouds/google/bucket.rb +179 -0
  45. data/modules/mu/config.rb +1 -1
  46. data/modules/mu/config/bucket.rb +69 -0
  47. data/modules/mu/config/bucket.yml +10 -0
  48. data/modules/mu/config/database.rb +1 -1
  49. data/modules/mu/config/endpoint.rb +71 -0
  50. data/modules/mu/config/function.rb +6 -0
  51. data/modules/mu/config/nosqldb.rb +49 -0
  52. data/modules/mu/config/nosqldb.yml +44 -0
  53. data/modules/mu/config/notifier.yml +2 -2
  54. data/modules/mu/config/vpc.rb +0 -1
  55. data/modules/mu/defaults/amazon_images.yaml +32 -30
  56. data/modules/mu/groomers/chef.rb +1 -1
  57. data/modules/mu/kittens.rb +2430 -1511
  58. data/modules/mu/master/ldap.rb +1 -1
  59. data/modules/tests/super_complex_bok.yml +7 -0
  60. data/modules/tests/super_simple_bok.yml +7 -0
  61. metadata +11 -2
@@ -93,7 +93,7 @@ module MU
93
93
  end
94
94
 
95
95
  if @config['import']
96
- MU.log "Attaching canned #{@config['import'].size > 1 ? "policies" : "policy"} #{@config['import'].join(", ")} to role #{@mu_name}", MU::NOTICE, details: @config['credentials']
96
+ MU.log "Attaching canned #{@config['import'].size > 1 ? "policies" : "policy"} #{@config['import'].join(", ")} to role #{@mu_name}", MU::NOTICE
97
97
  configured_policies.concat(@config['import'].map { |p| p.gsub(/.*?\/([^:\/]+)$/, '\1') })
98
98
  end
99
99
 
@@ -1356,7 +1356,7 @@ module MU
1356
1356
  MU.log "Creating AMI from #{name}", details: ami_descriptor
1357
1357
  resp = nil
1358
1358
  begin
1359
- resp = MU::Cloud::AWS.ec2(region: region).create_image(ami_descriptor)
1359
+ resp = MU::Cloud::AWS.ec2(region: region, credentials: credentials).create_image(ami_descriptor)
1360
1360
  rescue Aws::EC2::Errors::InvalidAMINameDuplicate => e
1361
1361
  MU.log "AMI #{name} already exists, skipping", MU::WARN
1362
1362
  return nil
@@ -1367,7 +1367,7 @@ module MU
1367
1367
  MU.log "AMI of #{name} in region #{region}: #{ami}"
1368
1368
  if make_public
1369
1369
  MU::Cloud::AWS::Server.waitForAMI(ami, region: region, credentials: credentials)
1370
- MU::Cloud::AWS.ec2(region: region).modify_image_attribute(
1370
+ MU::Cloud::AWS.ec2(region: region, credentials: credentials).modify_image_attribute(
1371
1371
  image_id: ami,
1372
1372
  launch_permission: {add: [{group: "all"}]},
1373
1373
  attribute: "launchPermission"
@@ -1381,7 +1381,7 @@ module MU
1381
1381
  next if r == region
1382
1382
  copythreads << Thread.new {
1383
1383
  MU.dupGlobals(parent_thread_id)
1384
- copy = MU::Cloud::AWS.ec2(region: r).copy_image(
1384
+ copy = MU::Cloud::AWS.ec2(region: r, credentials: credentials).copy_image(
1385
1385
  source_region: region,
1386
1386
  source_image_id: ami,
1387
1387
  name: name,
@@ -1389,7 +1389,7 @@ module MU
1389
1389
  )
1390
1390
  MU.log "Initiated copy of #{ami} from #{region} to #{r}: #{copy.image_id}"
1391
1391
 
1392
- MU::MommaCat.createStandardTags(copy.image_id, region: r)
1392
+ MU::MommaCat.createStandardTags(copy.image_id, region: r, credentials: credentials)
1393
1393
  MU::MommaCat.createTag(copy.image_id, "Name", name, region: r, credentials: credentials)
1394
1394
  if !tags.nil?
1395
1395
  tags.each { |tag|
@@ -1398,7 +1398,7 @@ module MU
1398
1398
  end
1399
1399
  MU::Cloud::AWS::Server.waitForAMI(copy.image_id, region: r, credentials: credentials)
1400
1400
  if make_public
1401
- MU::Cloud::AWS.ec2(region: r).modify_image_attribute(
1401
+ MU::Cloud::AWS.ec2(region: r, credentials: credentials).modify_image_attribute(
1402
1402
  image_id: copy.image_id,
1403
1403
  launch_permission: {add: [{group: "all"}]},
1404
1404
  attribute: "launchPermission"
@@ -212,6 +212,29 @@ module MU
212
212
 
213
213
  # Called automatically by {MU::Deploy#createResources}
214
214
  def groom
215
+ if @config['notifications'] and @config['notifications']['topic']
216
+ # XXX expand to a full reference block for a Notification resource
217
+ arn = if @config['notifications']['topic'].match(/^arn:/)
218
+ @config['notifications']['topic']
219
+ else
220
+ "arn:#{MU::Cloud::AWS.isGovCloud?(@config['region']) ? "aws-us-gov" : "aws"}:sns:#{@config['region']}:#{MU::Cloud::AWS.credToAcct(@config['credentials'])}:#{@config['notifications']['topic']}"
221
+ end
222
+ eventmap = {
223
+ "launch" => "autoscaling:EC2_INSTANCE_LAUNCH",
224
+ "failed_launch" => "autoscaling:EC2_INSTANCE_LAUNCH_ERROR",
225
+ "terminate" => "autoscaling:EC2_INSTANCE_TERMINATE",
226
+ "failed_terminate" => "autoscaling:EC2_INSTANCE_TERMINATE_ERROR"
227
+ }
228
+ MU.log "Sending simple notifications (#{@config['notifications']['events'].join(", ")}) to #{arn}"
229
+ MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).put_notification_configuration(
230
+ auto_scaling_group_name: @mu_name,
231
+ topic_arn: arn,
232
+ notification_types: @config['notifications']['events'].map { |e|
233
+ eventmap[e]
234
+ }
235
+ )
236
+ end
237
+
215
238
  if @config['schedule']
216
239
  ext_actions = MU::Cloud::AWS.autoscale(region: @config['region'], credentials: @config['credentials']).describe_scheduled_actions(
217
240
  auto_scaling_group_name: @mu_name
@@ -357,6 +380,7 @@ module MU
357
380
  newhash
358
381
  end
359
382
  policy_params[:target_tracking_configuration] = strToSym(policy['target_tracking_configuration'])
383
+ policy_params[:target_tracking_configuration].delete(:preferred_target_group)
360
384
  if policy_params[:target_tracking_configuration][:predefined_metric_specification] and
361
385
  policy_params[:target_tracking_configuration][:predefined_metric_specification][:predefined_metric_type] == "ALBRequestCountPerTarget"
362
386
  lb_path = nil
@@ -364,10 +388,18 @@ module MU
364
388
  if @deploy.deployment["loadbalancers"].size > 1
365
389
  MU.log "Multiple load balancers attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN
366
390
  end
367
- if lb["targetgroups"].size > 1
368
- MU.log "Multiple target groups attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN
391
+ lb_path = if lb["targetgroups"].size > 1
392
+ if policy['target_tracking_configuration']["preferred_target_group"] and
393
+ lb["targetgroups"][policy['target_tracking_configuration']["preferred_target_group"]]
394
+ lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"][policy['target_tracking_configuration']["preferred_target_group"]].split(/:/)[5]
395
+ else
396
+ if policy['target_tracking_configuration']["preferred_target_group"]
397
+ MU.log "preferred_target_group was set to '#{policy["preferred_target_group"]}' but I don't see a target group by that name", MU::WARN
398
+ end
399
+ MU.log "Multiple target groups attached to Autoscale group #{@mu_name}, guessing wildly which one to use for TargetTrackingScaling policy", MU::WARN, details: lb["targetgroups"].keys
400
+ lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"].values.first.split(/:/)[5]
401
+ end
369
402
  end
370
- lb_path = lb["arn"].split(/:/)[5].sub(/^loadbalancer\//, "")+"/"+lb["targetgroups"].values.first.split(/:/)[5]
371
403
 
372
404
  policy_params[:target_tracking_configuration][:predefined_metric_specification][:resource_label] = lb_path
373
405
  end
@@ -457,6 +489,26 @@ module MU
457
489
  toplevel_required = []
458
490
 
459
491
  schema = {
492
+ "notifications" => {
493
+ "type" => "object",
494
+ "description" => "Send notifications to an SNS topic for basic AutoScaling events",
495
+ "properties" => {
496
+ "topic" => {
497
+ "type" => "string",
498
+ "description" => "The short name or ARN of an SNS topic which should receive notifications for basic Autoscaling events"
499
+ },
500
+ "events" => {
501
+ "type" => "array",
502
+ "description" => "The AutoScaling events which should generate a notification",
503
+ "items" => {
504
+ "type" => "string",
505
+ "description" => "The AutoScaling events which should generate a notification",
506
+ "enum" => ["launch", "failed_launch", "terminate", "failed_terminate"]
507
+ },
508
+ "default" => ["launch", "failed_launch", "terminate", "failed_terminate"]
509
+ }
510
+ }
511
+ },
460
512
  "generate_iam_role" => {
461
513
  "type" => "boolean",
462
514
  "default" => true,
@@ -625,6 +677,10 @@ module MU
625
677
  "type" => "float",
626
678
  "description" => "The target value for the metric."
627
679
  },
680
+ "preferred_target_group" => {
681
+ "type" => "string",
682
+ "description" => "If our load balancer has multiple target groups, prefer the one with this name instead of choosing one arbitrarily"
683
+ },
628
684
  "disable_scale_in" => {
629
685
  "type" => "boolean",
630
686
  "description" => "If set to true, new instances created by this policy will not be subject to termination by scaling in.",
@@ -665,6 +721,7 @@ module MU
665
721
  "type" => "object",
666
722
  "additionalProperties" => false,
667
723
  "required" => ["name", "value"],
724
+ "description" => "What resource to monitor with the alarm we are implicitly declaring",
668
725
  "properties" => {
669
726
  "name" => {
670
727
  "type" => "string",
@@ -110,7 +110,6 @@ module MU
110
110
 
111
111
 
112
112
  # Fetch an Azure instance metadata parameter (example: public-ipv4).
113
- # @param param [String]: The parameter name to fetch
114
113
  # @return [String, nil]
115
114
  def self.get_metadata()
116
115
  base_url = "http://169.254.169.254/metadata/instance"
@@ -71,12 +71,20 @@ module MU
71
71
  $MU_CFG['google'].keys
72
72
  end
73
73
 
74
+ # Resolve the administrative Cloud Storage bucket for a given credential
75
+ # set, or return a default.
76
+ # @param credentials [String]
77
+ # @return [String]
74
78
  def self.adminBucketName(credentials = nil)
75
79
  #XXX find a default if this particular account doesn't have a log_bucket_name configured
76
80
  cfg = credConfig(credentials)
77
81
  cfg['log_bucket_name']
78
82
  end
79
83
 
84
+ # Resolve the administrative Cloud Storage bucket for a given credential
85
+ # set, or return a default.
86
+ # @param credentials [String]
87
+ # @return [String]
80
88
  def self.adminBucketUrl(credentials = nil)
81
89
  "gs://"+adminBucketName(credentials)+"/"
82
90
  end
@@ -121,7 +129,7 @@ module MU
121
129
  return name_only ? name : @@acct_to_profile_map[name.to_s]
122
130
  end
123
131
  # XXX whatever process might lead us to populate @@acct_to_profile_map with some mappings, like projectname -> account profile, goes here
124
- raise MuError, "Google credential set #{name} was requested, but I see no such working credentials in mu.yaml"
132
+ return nil
125
133
  end
126
134
  end
127
135
 
@@ -532,7 +540,7 @@ module MU
532
540
  require 'google/apis/compute_beta'
533
541
 
534
542
  if subclass.nil?
535
- @@compute_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "ComputeBeta::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], credentials: credentials)
543
+ @@compute_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ComputeBeta::ComputeService", scopes: ['https://www.googleapis.com/auth/cloud-platform', 'https://www.googleapis.com/auth/compute.readonly'], credentials: credentials)
536
544
  return @@compute_api[credentials]
537
545
  elsif subclass.is_a?(Symbol)
538
546
  return Object.const_get("::Google").const_get("Apis").const_get("ComputeBeta").const_get(subclass)
@@ -545,7 +553,7 @@ module MU
545
553
  require 'google/apis/storage_v1'
546
554
 
547
555
  if subclass.nil?
548
- @@storage_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "StorageV1::StorageService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
556
+ @@storage_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "StorageV1::StorageService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
549
557
  return @@storage_api[credentials]
550
558
  elsif subclass.is_a?(Symbol)
551
559
  return Object.const_get("::Google").const_get("Apis").const_get("StorageV1").const_get(subclass)
@@ -558,7 +566,7 @@ module MU
558
566
  require 'google/apis/iam_v1'
559
567
 
560
568
  if subclass.nil?
561
- @@iam_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "IamV1::IamService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
569
+ @@iam_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "IamV1::IamService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
562
570
  return @@iam_api[credentials]
563
571
  elsif subclass.is_a?(Symbol)
564
572
  return Object.const_get("::Google").const_get("Apis").const_get("IamV1").const_get(subclass)
@@ -572,7 +580,7 @@ module MU
572
580
 
573
581
  if subclass.nil?
574
582
  begin
575
- @@admin_directory_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: ['https://www.googleapis.com/auth/admin.directory.group.member.readonly', 'https://www.googleapis.com/auth/admin.directory.group.readonly', 'https://www.googleapis.com/auth/admin.directory.user.readonly', 'https://www.googleapis.com/auth/admin.directory.domain.readonly', 'https://www.googleapis.com/auth/admin.directory.orgunit.readonly', 'https://www.googleapis.com/auth/admin.directory.rolemanagement.readonly', 'https://www.googleapis.com/auth/admin.directory.customer.readonly'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
583
+ @@admin_directory_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "AdminDirectoryV1::DirectoryService", scopes: ['https://www.googleapis.com/auth/admin.directory.group.member.readonly', 'https://www.googleapis.com/auth/admin.directory.group.readonly', 'https://www.googleapis.com/auth/admin.directory.user.readonly', 'https://www.googleapis.com/auth/admin.directory.domain.readonly', 'https://www.googleapis.com/auth/admin.directory.orgunit.readonly', 'https://www.googleapis.com/auth/admin.directory.rolemanagement.readonly', 'https://www.googleapis.com/auth/admin.directory.customer.readonly'], masquerade: MU::Cloud::Google.credConfig(credentials)['masquerade_as'], credentials: credentials)
576
584
  rescue Signet::AuthorizationError => e
577
585
  MU.log "Cannot masquerade as #{MU::Cloud::Google.credConfig(credentials)['masquerade_as']}", MU::ERROR, details: "You can only use masquerade_as with GSuite. For more information on delegating GSuite authority to a service account, see:\nhttps://developers.google.com/identity/protocols/OAuth2ServiceAccount#delegatingauthority"
578
586
  raise e
@@ -589,7 +597,7 @@ module MU
589
597
  require 'google/apis/cloudresourcemanager_v1'
590
598
 
591
599
  if subclass.nil?
592
- @@resource_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
600
+ @@resource_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
593
601
  return @@resource_api[credentials]
594
602
  elsif subclass.is_a?(Symbol)
595
603
  return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV1").const_get(subclass)
@@ -602,7 +610,7 @@ module MU
602
610
  require 'google/apis/cloudresourcemanager_v2beta1'
603
611
 
604
612
  if subclass.nil?
605
- @@resource2_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "CloudresourcemanagerV2beta1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
613
+ @@resource2_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "CloudresourcemanagerV2beta1::CloudResourceManagerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
606
614
  return @@resource2_api[credentials]
607
615
  elsif subclass.is_a?(Symbol)
608
616
  return Object.const_get("::Google").const_get("Apis").const_get("CloudresourcemanagerV2beta1").const_get(subclass)
@@ -615,7 +623,7 @@ module MU
615
623
  require 'google/apis/container_v1'
616
624
 
617
625
  if subclass.nil?
618
- @@container_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "ContainerV1::ContainerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
626
+ @@container_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ContainerV1::ContainerService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
619
627
  return @@container_api[credentials]
620
628
  elsif subclass.is_a?(Symbol)
621
629
  return Object.const_get("::Google").const_get("Apis").const_get("ContainerV1").const_get(subclass)
@@ -628,7 +636,7 @@ module MU
628
636
  require 'google/apis/servicemanagement_v1'
629
637
 
630
638
  if subclass.nil?
631
- @@service_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "ServicemanagementV1::ServiceManagementService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
639
+ @@service_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "ServicemanagementV1::ServiceManagementService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
632
640
  return @@service_api[credentials]
633
641
  elsif subclass.is_a?(Symbol)
634
642
  return Object.const_get("::Google").const_get("Apis").const_get("ServicemanagementV1").const_get(subclass)
@@ -641,20 +649,33 @@ module MU
641
649
  require 'google/apis/sqladmin_v1beta4'
642
650
 
643
651
  if subclass.nil?
644
- @@sql_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "SqladminV1beta4::SQLAdminService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
652
+ @@sql_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "SqladminV1beta4::SQLAdminService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
645
653
  return @@sql_api[credentials]
646
654
  elsif subclass.is_a?(Symbol)
647
655
  return Object.const_get("::Google").const_get("Apis").const_get("SqladminV1beta4").const_get(subclass)
648
656
  end
649
657
  end
650
658
 
659
+ # Google's Firestore (NoSQL) Service API
660
+ # @param subclass [<Google::Apis::FirestoreV1>]: If specified, will return the class ::Google::Apis::FirestoreV1::subclass instead of an API client instance
661
+ def self.firestore(subclass = nil, credentials: nil)
662
+ require 'google/apis/firestore_v1'
663
+
664
+ if subclass.nil?
665
+ @@firestore_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "FirestoreV1::FirestoreService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
666
+ return @@firestore_api[credentials]
667
+ elsif subclass.is_a?(Symbol)
668
+ return Object.const_get("::Google").const_get("Apis").const_get("FirestoreV1").const_get(subclass)
669
+ end
670
+ end
671
+
651
672
  # Google's StackDriver Logging Service API
652
673
  # @param subclass [<Google::Apis::LoggingV2>]: If specified, will return the class ::Google::Apis::LoggingV2::subclass instead of an API client instance
653
674
  def self.logging(subclass = nil, credentials: nil)
654
675
  require 'google/apis/logging_v2'
655
676
 
656
677
  if subclass.nil?
657
- @@logging_api[credentials] ||= MU::Cloud::Google::Endpoint.new(api: "LoggingV2::LoggingService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
678
+ @@logging_api[credentials] ||= MU::Cloud::Google::GoogleEndpoint.new(api: "LoggingV2::LoggingService", scopes: ['https://www.googleapis.com/auth/cloud-platform'], credentials: credentials)
658
679
  return @@logging_api[credentials]
659
680
  elsif subclass.is_a?(Symbol)
660
681
  return Object.const_get("::Google").const_get("Apis").const_get("LoggingV2").const_get(subclass)
@@ -667,7 +688,7 @@ module MU
667
688
  # Wrapper class for Google APIs, so that we can catch some common
668
689
  # transient endpoint errors without having to spray rescues all over the
669
690
  # codebase.
670
- class Endpoint
691
+ class GoogleEndpoint
671
692
  @api = nil
672
693
  @credentials = nil
673
694
  attr_reader :issuer
@@ -945,6 +966,7 @@ module MU
945
966
  @@resource_api = {}
946
967
  @@resource2_api = {}
947
968
  @@service_api = {}
969
+ @@firestore_api = {}
948
970
  @@admin_directory_api = {}
949
971
  end
950
972
  end
@@ -0,0 +1,179 @@
1
+ # Copyright:: Copyright (c) 2019 eGlobalTech, Inc., all rights reserved
2
+ #
3
+ # Licensed under the BSD-3 license (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License in the root of the project or at
6
+ #
7
+ # http://egt-labs.com/mu/LICENSE.html
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ module MU
16
+ class Cloud
17
+ class Google
18
+ # Support for Google Cloud Storage
19
+ class Bucket < MU::Cloud::Bucket
20
+ @deploy = nil
21
+ @config = nil
22
+
23
+ attr_reader :mu_name
24
+ attr_reader :config
25
+ attr_reader :cloud_id
26
+
27
+ # @param mommacat [MU::MommaCat]: A {MU::Mommacat} object containing the deploy of which this resource is/will be a member.
28
+ # @param kitten_cfg [Hash]: The fully parsed and resolved {MU::Config} resource descriptor as defined in {MU::Config::BasketofKittens::logs}
29
+ def initialize(mommacat: nil, kitten_cfg: nil, mu_name: nil, cloud_id: nil)
30
+ @deploy = mommacat
31
+ @config = MU::Config.manxify(kitten_cfg)
32
+ @cloud_id ||= cloud_id
33
+ @mu_name ||= @deploy.getResourceName(@config["name"])
34
+ end
35
+
36
+ # Called automatically by {MU::Deploy#createResources}
37
+ def create
38
+ MU::Cloud::Google.storage(credentials: credentials).insert_bucket(@config['project'], bucket_descriptor)
39
+ @cloud_id = @mu_name.downcase
40
+ end
41
+
42
+ # Called automatically by {MU::Deploy#createResources}
43
+ def groom
44
+ current = cloud_desc
45
+ changed = false
46
+
47
+ if !current.versioning.enabled and @config['versioning']
48
+ MU.log "Enabling versioning on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
49
+ changed = true
50
+ elsif current.versioning.enabled and !@config['versioning']
51
+ MU.log "Disabling versioning on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
52
+ changed = true
53
+ end
54
+
55
+ if current.website.nil? and @config['web']
56
+ MU.log "Enabling website service on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
57
+ changed = true
58
+ elsif !current.website.nil? and !@config['web']
59
+ MU.log "Disabling website service on Cloud Storage bucket #{@cloud_id}", MU::NOTICE
60
+ changed = true
61
+ end
62
+
63
+ if changed
64
+ MU::Cloud::Google.storage(credentials: credentials).patch_bucket(@cloud_id, bucket_descriptor)
65
+ end
66
+ end
67
+
68
+ # Does this resource type exist as a global (cloud-wide) artifact, or
69
+ # is it localized to a region/zone?
70
+ # @return [Boolean]
71
+ def self.isGlobal?
72
+ true
73
+ end
74
+
75
+ # Remove all buckets associated with the currently loaded deployment.
76
+ # @param noop [Boolean]: If true, will only print what would be done
77
+ # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
78
+ # @param region [String]: The cloud provider region
79
+ # @return [void]
80
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
81
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
82
+
83
+ resp = MU::Cloud::Google.storage(credentials: credentials).list_buckets(flags['project'])
84
+ if resp and resp.items
85
+ resp.items.each { |bucket|
86
+ if bucket.labels and bucket.labels["mu-id"] == MU.deploy_id.downcase
87
+ MU.log "Deleting Cloud Storage bucket #{bucket.name}"
88
+ if !noop
89
+ MU::Cloud::Google.storage(credentials: credentials).delete_bucket(bucket.name)
90
+ end
91
+ end
92
+ }
93
+ end
94
+ end
95
+
96
+ # Return the metadata for this user cofiguration
97
+ # @return [Hash]
98
+ def notify
99
+ MU.structToHash(cloud_desc)
100
+ end
101
+
102
+ # Locate an existing bucket.
103
+ # @param cloud_id [String]: The cloud provider's identifier for this resource.
104
+ # @param region [String]: The cloud provider region.
105
+ # @param flags [Hash]: Optional flags
106
+ # @return [OpenStruct]: The cloud provider's complete descriptions of matching bucket.
107
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
108
+ found = {}
109
+ if cloud_id
110
+ found[cloud_id] = MU::Cloud::Google.storage(credentials: credentials).get_bucket(cloud_id)
111
+ end
112
+ found
113
+ end
114
+
115
+ # Cloud-specific configuration properties.
116
+ # @param config [MU::Config]: The calling MU::Config object
117
+ # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
118
+ def self.schema(config)
119
+ toplevel_required = []
120
+ schema = {
121
+ "storage_class" => {
122
+ "type" => "string",
123
+ "enum" => ["MULTI_REGIONAL", "REGIONAL", "STANDARD", "NEARLINE", "COLDLINE", "DURABLE_REDUCED_AVAILABILITY"],
124
+ "default" => "STANDARD"
125
+ }
126
+ }
127
+ [toplevel_required, schema]
128
+ end
129
+
130
+ # Cloud-specific pre-processing of {MU::Config::BasketofKittens::bucket}, bare and unvalidated.
131
+
132
+ # @param bucket [Hash]: The resource to process and validate
133
+ # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
134
+ # @return [Boolean]: True if validation succeeded, False otherwise
135
+ def self.validateConfig(bucket, configurator)
136
+ ok = true
137
+
138
+ ok
139
+ end
140
+
141
+ private
142
+
143
+ # create and return the Google::Apis::StorageV1::Bucket object used by
144
+ # both +insert_bucket+ and +patch_bucket+
145
+ def bucket_descriptor
146
+ labels = {}
147
+ MU::MommaCat.listStandardTags.each_pair { |name, value|
148
+ if !value.nil?
149
+ labels[name.downcase] = value.downcase.gsub(/[^a-z0-9\-\_]/i, "_")
150
+ end
151
+ }
152
+ labels["name"] = @mu_name.downcase
153
+
154
+ params = {
155
+ :name => @mu_name.downcase,
156
+ :labels => labels,
157
+ :storage_class => @config['storage_class'],
158
+ }
159
+
160
+ if @config['web']
161
+ params[:website] = MU::Cloud::Google.storage(:Bucket)::Website.new(
162
+ main_page_suffix: @config['web_index_object'],
163
+ not_found_page: @config['web_error_object']
164
+ )
165
+ end
166
+
167
+ if @config['versioning']
168
+ params[:versioning] = MU::Cloud::Google.storage(:Bucket)::Versioning.new(enabled: true)
169
+ else
170
+ params[:versioning] = MU::Cloud::Google.storage(:Bucket)::Versioning.new(enabled: false)
171
+ end
172
+
173
+ MU::Cloud::Google.storage(:Bucket).new(params)
174
+ end
175
+
176
+ end
177
+ end
178
+ end
179
+ end