cloud-mu 3.1.3 → 3.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +10 -2
  3. data/bin/mu-adopt +5 -1
  4. data/bin/mu-load-config.rb +2 -3
  5. data/bin/mu-run-tests +112 -27
  6. data/cloud-mu.gemspec +20 -20
  7. data/cookbooks/mu-tools/libraries/helper.rb +2 -1
  8. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  9. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  10. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  11. data/extras/image-generators/Google/centos6.yaml +1 -0
  12. data/extras/image-generators/Google/centos7.yaml +1 -1
  13. data/modules/mommacat.ru +5 -15
  14. data/modules/mu.rb +10 -14
  15. data/modules/mu/adoption.rb +20 -14
  16. data/modules/mu/cleanup.rb +13 -9
  17. data/modules/mu/cloud.rb +26 -26
  18. data/modules/mu/clouds/aws.rb +100 -59
  19. data/modules/mu/clouds/aws/alarm.rb +4 -2
  20. data/modules/mu/clouds/aws/bucket.rb +25 -21
  21. data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
  22. data/modules/mu/clouds/aws/collection.rb +21 -20
  23. data/modules/mu/clouds/aws/container_cluster.rb +47 -26
  24. data/modules/mu/clouds/aws/database.rb +57 -68
  25. data/modules/mu/clouds/aws/dnszone.rb +14 -14
  26. data/modules/mu/clouds/aws/endpoint.rb +20 -16
  27. data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
  28. data/modules/mu/clouds/aws/folder.rb +7 -7
  29. data/modules/mu/clouds/aws/function.rb +15 -12
  30. data/modules/mu/clouds/aws/group.rb +14 -10
  31. data/modules/mu/clouds/aws/habitat.rb +16 -13
  32. data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
  33. data/modules/mu/clouds/aws/log.rb +13 -10
  34. data/modules/mu/clouds/aws/msg_queue.rb +15 -8
  35. data/modules/mu/clouds/aws/nosqldb.rb +18 -11
  36. data/modules/mu/clouds/aws/notifier.rb +11 -6
  37. data/modules/mu/clouds/aws/role.rb +87 -70
  38. data/modules/mu/clouds/aws/search_domain.rb +30 -19
  39. data/modules/mu/clouds/aws/server.rb +102 -72
  40. data/modules/mu/clouds/aws/server_pool.rb +47 -28
  41. data/modules/mu/clouds/aws/storage_pool.rb +5 -6
  42. data/modules/mu/clouds/aws/user.rb +13 -10
  43. data/modules/mu/clouds/aws/vpc.rb +135 -121
  44. data/modules/mu/clouds/azure.rb +16 -9
  45. data/modules/mu/clouds/azure/container_cluster.rb +2 -3
  46. data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
  47. data/modules/mu/clouds/azure/habitat.rb +8 -6
  48. data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
  49. data/modules/mu/clouds/azure/role.rb +8 -10
  50. data/modules/mu/clouds/azure/server.rb +65 -25
  51. data/modules/mu/clouds/azure/user.rb +5 -7
  52. data/modules/mu/clouds/azure/vpc.rb +12 -15
  53. data/modules/mu/clouds/cloudformation.rb +8 -7
  54. data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
  55. data/modules/mu/clouds/google.rb +39 -24
  56. data/modules/mu/clouds/google/bucket.rb +9 -11
  57. data/modules/mu/clouds/google/container_cluster.rb +27 -42
  58. data/modules/mu/clouds/google/database.rb +6 -9
  59. data/modules/mu/clouds/google/firewall_rule.rb +11 -10
  60. data/modules/mu/clouds/google/folder.rb +16 -9
  61. data/modules/mu/clouds/google/function.rb +127 -161
  62. data/modules/mu/clouds/google/group.rb +21 -18
  63. data/modules/mu/clouds/google/habitat.rb +18 -15
  64. data/modules/mu/clouds/google/loadbalancer.rb +14 -16
  65. data/modules/mu/clouds/google/role.rb +48 -31
  66. data/modules/mu/clouds/google/server.rb +105 -105
  67. data/modules/mu/clouds/google/server_pool.rb +12 -31
  68. data/modules/mu/clouds/google/user.rb +67 -13
  69. data/modules/mu/clouds/google/vpc.rb +58 -65
  70. data/modules/mu/config.rb +89 -1738
  71. data/modules/mu/config/bucket.rb +3 -3
  72. data/modules/mu/config/collection.rb +3 -3
  73. data/modules/mu/config/container_cluster.rb +2 -2
  74. data/modules/mu/config/dnszone.rb +5 -5
  75. data/modules/mu/config/doc_helpers.rb +517 -0
  76. data/modules/mu/config/endpoint.rb +3 -3
  77. data/modules/mu/config/firewall_rule.rb +118 -3
  78. data/modules/mu/config/folder.rb +3 -3
  79. data/modules/mu/config/function.rb +2 -2
  80. data/modules/mu/config/group.rb +3 -3
  81. data/modules/mu/config/habitat.rb +3 -3
  82. data/modules/mu/config/loadbalancer.rb +3 -3
  83. data/modules/mu/config/log.rb +3 -3
  84. data/modules/mu/config/msg_queue.rb +3 -3
  85. data/modules/mu/config/nosqldb.rb +3 -3
  86. data/modules/mu/config/notifier.rb +2 -2
  87. data/modules/mu/config/ref.rb +333 -0
  88. data/modules/mu/config/role.rb +3 -3
  89. data/modules/mu/config/schema_helpers.rb +508 -0
  90. data/modules/mu/config/search_domain.rb +3 -3
  91. data/modules/mu/config/server.rb +86 -58
  92. data/modules/mu/config/server_pool.rb +2 -2
  93. data/modules/mu/config/tail.rb +189 -0
  94. data/modules/mu/config/user.rb +3 -3
  95. data/modules/mu/config/vpc.rb +44 -4
  96. data/modules/mu/defaults/Google.yaml +2 -2
  97. data/modules/mu/deploy.rb +13 -10
  98. data/modules/mu/groomer.rb +1 -1
  99. data/modules/mu/groomers/ansible.rb +69 -24
  100. data/modules/mu/groomers/chef.rb +52 -44
  101. data/modules/mu/logger.rb +17 -14
  102. data/modules/mu/master.rb +317 -2
  103. data/modules/mu/master/chef.rb +3 -4
  104. data/modules/mu/master/ldap.rb +3 -3
  105. data/modules/mu/master/ssl.rb +12 -2
  106. data/modules/mu/mommacat.rb +85 -1766
  107. data/modules/mu/mommacat/daemon.rb +394 -0
  108. data/modules/mu/mommacat/naming.rb +366 -0
  109. data/modules/mu/mommacat/storage.rb +689 -0
  110. data/modules/tests/bucket.yml +4 -0
  111. data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
  112. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  113. data/modules/tests/regrooms/bucket.yml +19 -0
  114. metadata +112 -102
@@ -250,7 +250,7 @@ module MU
250
250
  @config['master_az'] = @config['region']
251
251
  parent_arg = "projects/"+@config['project']+"/locations/"+@config['master_az']
252
252
 
253
- cluster = MU::Cloud::Google.container(credentials: @config['credentials']).create_project_location_cluster(
253
+ MU::Cloud::Google.container(credentials: @config['credentials']).create_project_location_cluster(
254
254
  parent_arg,
255
255
  requestobj
256
256
  )
@@ -277,11 +277,9 @@ module MU
277
277
 
278
278
  me = cloud_desc
279
279
 
280
- parent_arg = "projects/"+@config['project']+"/locations/"+me.location
281
-
282
280
  # Enable/disable basic auth
283
281
  authcfg = {}
284
- action = nil
282
+
285
283
  if @config['master_user'] and (me.master_auth.username != @config['master_user'] or !me.master_auth.password)
286
284
  authcfg[:username] = @config['master_user']
287
285
  authcfg[:password] = Password.pronounceable(16..18)
@@ -368,15 +366,24 @@ module MU
368
366
  }
369
367
  end
370
368
 
369
+ # map from GKE Kuberentes addon parameter names to our BoK equivalent
370
+ # fields so we can check all these programmatically
371
+ addon_map = {
372
+ :horizontal_pod_autoscaling => 'horizontal_pod_autoscaling',
373
+ :http_load_balancing => 'http_load_balancing',
374
+ :kubernetes_dashboard => 'dashboard',
375
+ :network_policy_config => 'network_policy_addon'
376
+ }
377
+
371
378
  if @config['kubernetes']
372
- if (me.addons_config.horizontal_pod_autoscaling.disabled and @config['kubernetes']['horizontal_pod_autoscaling']) or
373
- (!me.addons_config.horizontal_pod_autoscaling and !@config['kubernetes']['horizontal_pod_autoscaling']) or
374
- (me.addons_config.http_load_balancing.disabled and @config['kubernetes']['http_load_balancing']) or
375
- (!me.addons_config.http_load_balancing and !@config['kubernetes']['http_load_balancing']) or
376
- (me.addons_config.kubernetes_dashboard.disabled and @config['kubernetes']['dashboard']) or
377
- (!me.addons_config.kubernetes_dashboard and !@config['kubernetes']['dashboard']) or
378
- (me.addons_config.network_policy_config.disabled and @config['kubernetes']['network_policy_addon']) or
379
- (!me.addons_config.network_policy_config and !@config['kubernetes']['network_policy_addon'])
379
+ have_changes = false
380
+ addon_map.each_pair { |param, bok_param|
381
+ if (me.addons_config.send(param).disabled and @config['kubernetes'][bok_param]) or
382
+ (!me.addons_config.send(param) and !@config['kubernetes'][bok_param])
383
+ have_changes = true
384
+ end
385
+ }
386
+ if have_changes
380
387
  updates << { :desired_addons_config => MU::Cloud::Google.container(:AddonsConfig).new(
381
388
  horizontal_pod_autoscaling: MU::Cloud::Google.container(:HorizontalPodAutoscaling).new(
382
389
  disabled: !@config['kubernetes']['horizontal_pod_autoscaling']
@@ -471,9 +478,7 @@ module MU
471
478
  # Locate an existing ContainerCluster or ContainerClusters and return an array containing matching GCP resource descriptors for those that match.
472
479
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ContainerClusters
473
480
  def self.find(**args)
474
- args[:project] ||= args[:habitat]
475
- args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
476
- location = args[:region] || args[:availability_zone] || "-"
481
+ args = MU::Cloud::Google.findLocationArgs(args)
477
482
 
478
483
  found = {}
479
484
 
@@ -486,7 +491,7 @@ module MU
486
491
  found[args[:cloud_id]] = resp if resp
487
492
  else
488
493
  resp = begin
489
- MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{location}")
494
+ MU::Cloud::Google.container(credentials: args[:credentials]).list_project_location_clusters("projects/#{args[:project]}/locations/#{args[:location]}")
490
495
  rescue ::Google::Apis::ClientError => e
491
496
  raise e if !e.message.match(/forbidden:/)
492
497
  end
@@ -503,7 +508,7 @@ module MU
503
508
  # Reverse-map our cloud description into a runnable config hash.
504
509
  # We assume that any values we have in +@config+ are placeholders, and
505
510
  # calculate our own accordingly based on what's live in the cloud.
506
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
511
+ def toKitten(**_args)
507
512
 
508
513
  bok = {
509
514
  "cloud" => "Google",
@@ -739,7 +744,6 @@ module MU
739
744
  # @param region [String]: The cloud provider region in which to operate
740
745
  # @return [void]
741
746
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
742
- skipsnapshots = flags["skipsnapshots"]
743
747
 
744
748
  flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
745
749
  return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
@@ -756,7 +760,9 @@ module MU
756
760
  clusters.uniq.each { |cluster|
757
761
  if !cluster.resource_labels or (
758
762
  !cluster.name.match(/^#{Regexp.quote(MU.deploy_id)}\-/i) and
759
- cluster.resource_labels['mu-id'] != MU.deploy_id.downcase
763
+ (cluster.resource_labels['mu-id'] != MU.deploy_id.downcase or
764
+ (!ignoremaster and cluster.resource_labels['mu-master-ip'] != MU.mu_public_ip.gsub(/\./, "_"))
765
+ )
760
766
  )
761
767
  next
762
768
  end
@@ -1037,29 +1043,7 @@ module MU
1037
1043
  ok = false
1038
1044
  end
1039
1045
  else
1040
- user = {
1041
- "name" => cluster['name'],
1042
- "cloud" => "Google",
1043
- "project" => cluster["project"],
1044
- "credentials" => cluster["credentials"],
1045
- "type" => "service"
1046
- }
1047
- if user["name"].length < 6
1048
- user["name"] += Password.pronounceable(6)
1049
- end
1050
- configurator.insertKitten(user, "users", true)
1051
- cluster['dependencies'] ||= []
1052
- cluster['service_account'] = MU::Config::Ref.get(
1053
- type: "users",
1054
- cloud: "Google",
1055
- name: cluster["name"],
1056
- project: cluster["project"],
1057
- credentials: cluster["credentials"]
1058
- )
1059
- cluster['dependencies'] << {
1060
- "type" => "user",
1061
- "name" => user["name"]
1062
- }
1046
+ cluster = MU::Cloud::Google::User.genericServiceAccount(cluster, configurator)
1063
1047
  end
1064
1048
 
1065
1049
  if cluster['dependencies']
@@ -1223,6 +1207,7 @@ module MU
1223
1207
  @@server_config[credentials][az] = MU::Cloud::Google.container(credentials: credentials).get_project_location_server_config(parent_arg)
1224
1208
  @@server_config[credentials][az]
1225
1209
  end
1210
+ private_class_method :defaults
1226
1211
 
1227
1212
  def writeKubeConfig
1228
1213
  kube_conf = @deploy.deploy_dir+"/kubeconfig-#{@config['name']}"
@@ -68,8 +68,7 @@ module MU
68
68
  # Locate an existing Database or Databases and return an array containing matching GCP resource descriptors for those that match.
69
69
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching Databases
70
70
  def self.find(**args)
71
- args[:project] ||= args[:habitat]
72
- args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
71
+ args = MU::Cloud::Google.findLocationArgs(args)
73
72
  end
74
73
 
75
74
  # Called automatically by {MU::Deploy#createResources}
@@ -110,7 +109,7 @@ module MU
110
109
  # @return [void]
111
110
  def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
112
111
  flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
113
- skipsnapshots||= flags["skipsnapshots"]
112
+
114
113
  # instances = MU::Cloud::Google.sql(credentials: credentials).list_instances(flags['project'], filter: %Q{userLabels.mu-id:"#{MU.deploy_id.downcase}"})
115
114
  # if instances and instances.items
116
115
  # instances.items.each { |instance|
@@ -121,9 +120,9 @@ module MU
121
120
  end
122
121
 
123
122
  # Cloud-specific configuration properties.
124
- # @param config [MU::Config]: The calling MU::Config object
123
+ # @param _config [MU::Config]: The calling MU::Config object
125
124
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
126
- def self.schema(config)
125
+ def self.schema(_config)
127
126
  toplevel_required = []
128
127
  schema = {}
129
128
  [toplevel_required, schema]
@@ -131,9 +130,9 @@ module MU
131
130
 
132
131
  # Cloud-specific pre-processing of {MU::Config::BasketofKittens::databases}, bare and unvalidated.
133
132
  # @param db [Hash]: The resource to process and validate
134
- # @param configurator [MU::Config]: The overall deployment configurator of which this resource is a member
133
+ # @param _configurator [MU::Config]: The overall deployment configurator of which this resource is a member
135
134
  # @return [Boolean]: True if validation succeeded, False otherwise
136
- def self.validateConfig(db, configurator)
135
+ def self.validateConfig(db, _configurator)
137
136
  ok = true
138
137
 
139
138
  if db["create_cluster"]
@@ -144,8 +143,6 @@ module MU
144
143
  ok
145
144
  end
146
145
 
147
- private
148
-
149
146
  end #class
150
147
  end #class
151
148
  end
@@ -172,8 +172,7 @@ end
172
172
  # @param args [Hash]: Hash of named arguments passed via Ruby's double-splat
173
173
  # @return [Hash<String,OpenStruct>]: The cloud provider's complete descriptions of matching resources
174
174
  def self.find(**args)
175
- args[:project] ||= args[:habitat]
176
- args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
175
+ args = MU::Cloud::Google.findLocationArgs(args)
177
176
 
178
177
  found = {}
179
178
  resp = begin
@@ -207,11 +206,15 @@ end
207
206
  # Remove all security groups (firewall rulesets) associated with the currently loaded deployment.
208
207
  # @param noop [Boolean]: If true, will only print what would be done
209
208
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
210
- # @param region [String]: The cloud provider region
211
209
  # @return [void]
212
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
210
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
213
211
  flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
214
212
  return if !MU::Cloud::Google::Habitat.isLive?(flags["project"], credentials)
213
+ filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
214
+ if !ignoremaster and MU.mu_public_ip
215
+ filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
216
+ end
217
+ MU.log "Placeholder: Google FirewallRule artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
215
218
 
216
219
  MU::Cloud::Google.compute(credentials: credentials).delete(
217
220
  "firewall",
@@ -224,7 +227,7 @@ end
224
227
  # Reverse-map our cloud description into a runnable config hash.
225
228
  # We assume that any values we have in +@config+ are placeholders, and
226
229
  # calculate our own accordingly based on what's live in the cloud.
227
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
230
+ def toKitten(**_args)
228
231
 
229
232
  if cloud_desc.name.match(/^[a-f0-9]+$/)
230
233
  gke_ish = true
@@ -362,9 +365,9 @@ end
362
365
  end
363
366
 
364
367
  # Cloud-specific configuration properties.
365
- # @param config [MU::Config]: The calling MU::Config object
368
+ # @param _config [MU::Config]: The calling MU::Config object
366
369
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
367
- def self.schema(config = nil)
370
+ def self.schema(_config = nil)
368
371
  toplevel_required = []
369
372
  schema = {
370
373
  "rules" => {
@@ -523,7 +526,7 @@ end
523
526
  end
524
527
  }
525
528
 
526
- rules_by_class.reject! { |k, v| v.size == 0 }
529
+ rules_by_class.reject! { |_k, v| v.size == 0 }
527
530
 
528
531
  # Generate other firewall rule objects to cover the other behaviors
529
532
  # we've requested, if indeed we've done so.
@@ -543,8 +546,6 @@ end
543
546
  ok
544
547
  end
545
548
 
546
- private
547
-
548
549
  end #class
549
550
  end #class
550
551
  end
@@ -48,7 +48,7 @@ module MU
48
48
  folder_obj = MU::Cloud::Google.folder(:Folder).new(params)
49
49
 
50
50
  MU.log "Creating folder #{name_string} under #{parent}", details: folder_obj
51
- resp = MU::Cloud::Google.folder(credentials: @config['credentials']).create_folder(folder_obj, parent: parent)
51
+ MU::Cloud::Google.folder(credentials: @config['credentials']).create_folder(folder_obj, parent: parent)
52
52
 
53
53
  # Wait for list_folders output to be consistent (for the folder we
54
54
  # just created to show up)
@@ -125,10 +125,12 @@ module MU
125
125
  nil
126
126
  end
127
127
 
128
+ @cached_cloud_desc = nil
128
129
  # Return the cloud descriptor for the Folder
129
130
  # @return [Google::Apis::Core::Hashable]
130
- def cloud_desc
131
- @cached_cloud_desc ||= MU::Cloud::Google::Folder.find(cloud_id: @cloud_id, credentials: @config['credentials']).values.first
131
+ def cloud_desc(use_cache: true)
132
+ return @cached_cloud_desc if @cached_cloud_desc and use_cache
133
+ @cached_cloud_desc = MU::Cloud::Google::Folder.find(cloud_id: @cloud_id, credentials: @config['credentials']).values.first
132
134
  @habitat_id ||= @cached_cloud_desc.parent.sub(/^(folders|organizations)\//, "")
133
135
  @cached_cloud_desc
134
136
  end
@@ -160,7 +162,12 @@ module MU
160
162
  # @param noop [Boolean]: If true, will only print what would be done
161
163
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
162
164
  # @return [void]
163
- def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {}, region: MU.myRegion)
165
+ def self.cleanup(noop: false, ignoremaster: false, credentials: nil, flags: {})
166
+ filter = %Q{(labels.mu-id = "#{MU.deploy_id.downcase}")}
167
+ if !ignoremaster and MU.mu_public_ip
168
+ filter += %Q{ AND (labels.mu-master-ip = "#{MU.mu_public_ip.gsub(/\./, "_")}")}
169
+ end
170
+ MU.log "Placeholder: Google Folder artifacts do not support labels, so ignoremaster cleanup flag has no effect", MU::DEBUG, details: filter
164
171
  # We can't label GCP folders, and their names are too short to encode
165
172
  # Mu deploy IDs, so all we can do is rely on flags['known'] passed in
166
173
  # from cleanup, which relies on our metadata to know what's ours.
@@ -293,7 +300,7 @@ module MU
293
300
  # Reverse-map our cloud description into a runnable config hash.
294
301
  # We assume that any values we have in +@config+ are placeholders, and
295
302
  # calculate our own accordingly based on what's live in the cloud.
296
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
303
+ def toKitten(**args)
297
304
  bok = {
298
305
  "cloud" => "Google",
299
306
  "credentials" => @config['credentials']
@@ -310,9 +317,9 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
310
317
  credentials: @config['credentials'],
311
318
  type: "folders"
312
319
  )
313
- elsif rootparent
320
+ elsif args[:rootparent]
314
321
  bok['parent'] = {
315
- 'id' => rootparent.is_a?(String) ? rootparent : rootparent.cloud_desc.name
322
+ 'id' => args[:rootparent].is_a?(String) ? args[:rootparent] : args[:rootparent].cloud_desc.name
316
323
  }
317
324
  else
318
325
  bok['parent'] = { 'id' => cloud_desc.parent }
@@ -322,9 +329,9 @@ MU.log bok['display_name']+" generating reference", MU::NOTICE, details: cloud_d
322
329
  end
323
330
 
324
331
  # Cloud-specific configuration properties.
325
- # @param config [MU::Config]: The calling MU::Config object
332
+ # @param _config [MU::Config]: The calling MU::Config object
326
333
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
327
- def self.schema(config)
334
+ def self.schema(_config)
328
335
  toplevel_required = []
329
336
  schema = {
330
337
  "display_name" => {
@@ -108,98 +108,9 @@ module example.com/cloudfunction
108
108
 
109
109
  # Called automatically by {MU::Deploy#createResources}
110
110
  def create
111
- labels = Hash[@tags.keys.map { |k|
112
- [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
113
- ]
114
- labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
115
111
 
116
112
  location = "projects/"+@config['project']+"/locations/"+@config['region']
117
- sa = nil
118
- retries = 0
119
- begin
120
- sa_ref = MU::Config::Ref.get(@config['service_account'])
121
- sa = @deploy.findLitterMate(name: sa_ref.name, type: "users")
122
- if !sa or !sa.cloud_desc
123
- sleep 10
124
- end
125
- rescue ::Google::Apis::ClientError => e
126
- if e.message.match(/notFound:/)
127
- sleep 10
128
- retries += 1
129
- retry
130
- end
131
- end while !sa or !sa.cloud_desc and retries < 5
132
-
133
- if !sa or !sa.cloud_desc
134
- raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
135
- end
136
-
137
- desc = {
138
- name: location+"/functions/"+@mu_name.downcase,
139
- runtime: @config['runtime'],
140
- timeout: @config['timeout'].to_s+"s",
141
- # entry_point: "hello_world",
142
- entry_point: @config['handler'],
143
- description: @deploy.deploy_id,
144
- service_account_email: sa.cloud_desc.email,
145
- labels: labels,
146
- available_memory_mb: @config['memory']
147
- }
148
-
149
- # XXX This network argument is deprecated in favor of using VPC
150
- # Connectors. Which would be fine, except there's no API support for
151
- # interacting with VPC Connectors. Can't create them, can't list them,
152
- # can't do anything except pass their ids into Cloud Functions or
153
- # AppEngine and hope for the best.
154
- if @config['vpc_connector']
155
- desc[:vpc_connector] = @config['vpc_connector']
156
- elsif @vpc
157
- desc[:network] = @vpc.url.sub(/^.*?\/projects\//, 'projects/')
158
- end
159
-
160
- if @config['triggers']
161
- desc[:event_trigger] = MU::Cloud::Google.function(:EventTrigger).new(
162
- event_type: @config['triggers'].first['event'],
163
- resource: @config['triggers'].first['resource']
164
- )
165
- else
166
- desc[:https_trigger] = MU::Cloud::Google.function(:HttpsTrigger).new
167
- end
168
-
169
-
170
- if @config['environment_variable']
171
- @config['environment_variable'].each { |var|
172
- desc[:environment_variables] ||= {}
173
- desc[:environment_variables][var["key"].to_s] = var["value"].to_s
174
- }
175
- end
176
-
177
- # hello_code = nil
178
- # HELLO_WORLDS.each_pair { |runtime, code|
179
- # if @config['runtime'].match(/^#{Regexp.quote(runtime)}/)
180
- # hello_code = code
181
- # break
182
- # end
183
- # }
184
- if @config['code']['gs_url']
185
- desc[:source_archive_url] = @config['code']['gs_url']
186
- elsif @config['code']['zip_file']
187
- desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
188
- end
189
-
190
- # Dir.mktmpdir(@mu_name) { |dir|
191
- # hello_code.each_pair { |file, contents|
192
- # f = File.open(dir+"/"+file, "w")
193
- # f.puts contents
194
- # f.close
195
- # Zip::File.open(dir+"/function.zip", Zip::File::CREATE) { |z|
196
- # z.add(file, dir+"/"+file)
197
- # }
198
- # }
199
- # desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(dir+"/function.zip", @mu_name+"-cloudfunction.zip", credentials: @credentials)
200
- # }
201
-
202
- func_obj = MU::Cloud::Google.function(:CloudFunction).new(desc)
113
+ func_obj = buildDesc
203
114
  MU.log "Creating Cloud Function #{@mu_name} in #{location}", details: func_obj
204
115
  resp = MU::Cloud::Google.function(credentials: @credentials).create_project_location_function(location, func_obj)
205
116
  @cloud_id = resp.name
@@ -214,27 +125,26 @@ module example.com/cloudfunction
214
125
  labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
215
126
 
216
127
  if cloud_desc.labels != labels
217
- desc[:labels] = labels
128
+ need_update = true
218
129
  end
219
130
 
220
131
  if cloud_desc.runtime != @config['runtime']
221
- desc[:runtime] = @config['runtime']
132
+ need_update = true
222
133
  end
223
134
  if cloud_desc.timeout != @config['timeout'].to_s+"s"
224
- desc[:timeout] = @config['timeout'].to_s+"s"
135
+ need_update = true
225
136
  end
226
137
  if cloud_desc.entry_point != @config['handler']
227
- desc[:entry_point] = @config['handler']
138
+ need_update = true
228
139
  end
229
140
  if cloud_desc.available_memory_mb != @config['memory']
230
- desc[:available_memory_mb] = @config['memory']
141
+ need_update = true
231
142
  end
232
143
  if @config['environment_variable']
233
144
  @config['environment_variable'].each { |var|
234
145
  if !cloud_desc.environment_variables or
235
146
  cloud_desc.environment_variables[var["key"].to_s] != var["value"].to_s
236
- desc[:environment_variables] ||= {}
237
- desc[:environment_variables][var["key"].to_s] = var["value"].to_s
147
+ need_update = true
238
148
  end
239
149
  }
240
150
  end
@@ -242,10 +152,7 @@ module example.com/cloudfunction
242
152
  if !cloud_desc.event_trigger or
243
153
  cloud_desc.event_trigger.event_type != @config['triggers'].first['event'] or
244
154
  cloud_desc.event_trigger.resource != @config['triggers'].first['resource']
245
- desc[:event_trigger] = MU::Cloud::Google.function(:EventTrigger).new(
246
- event_type: @config['triggers'].first['event'],
247
- resource: @config['triggers'].first['resource']
248
- )
155
+ need_update = true
249
156
  end
250
157
  end
251
158
 
@@ -254,7 +161,6 @@ module example.com/cloudfunction
254
161
  File.read("#{dir}/current.zip")
255
162
  }
256
163
 
257
- source_url = nil
258
164
  new = if @config['code']['zip_file']
259
165
  File.read(@config['code']['zip_file'])
260
166
  elsif @config['code']['gs_url']
@@ -269,37 +175,21 @@ module example.com/cloudfunction
269
175
  if @config['code']['gs_url'] and
270
176
  (@config['code']['gs_url'] != cloud_desc.source_archive_url or
271
177
  current != new)
272
- desc[:source_archive_url] = @config['code']['gs_url']
178
+ need_update = true
273
179
  elsif @config['code']['zip_file'] and current != new
180
+ need_update = true
274
181
  desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
275
182
  end
276
183
 
277
- if desc.size > 0
278
- # A trigger and some code must always be specified, apparently. The
279
- # endpoint hangs indefinitely if either is missing. Charming.
280
- if !desc[:https_trigger] and !desc[:event_trigger]
281
- if cloud_desc.https_trigger
282
- desc[:https_trigger] = MU::Cloud::Google.function(:HttpsTrigger).new
283
- else
284
- desc[:event_trigger] = cloud_desc.event_trigger
285
- end
286
- end
287
- if !desc[:source_archive_url] and !desc[:source_upload_url]
288
- if cloud_desc.source_archive_url
289
- desc[:source_archive_url] = cloud_desc.source_archive_url
290
- else
291
- desc[:source_upload_url] = cloud_desc.source_upload_url
292
- end
293
- end
294
-
295
- func_obj = MU::Cloud::Google.function(:CloudFunction).new(desc)
184
+ if need_update
185
+ func_obj = buildDesc
296
186
  MU.log "Updating Cloud Function #{@mu_name}", MU::NOTICE, details: func_obj
297
187
  begin
298
- MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
299
- @cloud_id,
300
- func_obj
301
- )
302
- rescue ::Google::Apis::ClientError => e
188
+ # MU::Cloud::Google.function(credentials: @credentials).patch_project_location_function(
189
+ # @cloud_id,
190
+ # func_obj
191
+ # )
192
+ rescue ::Google::Apis::ClientError
303
193
  MU.log "Error updating Cloud Function #{@mu_name}.", MU::ERR
304
194
  if desc[:source_archive_url]
305
195
  main_file = nil
@@ -350,7 +240,7 @@ module example.com/cloudfunction
350
240
  found = MU::Cloud::Google::Function.find(credentials: credentials, region: region, project: flags["project"])
351
241
  found.each_pair { |cloud_id, desc|
352
242
  if (desc.description and desc.description == MU.deploy_id) or
353
- (desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase) or
243
+ (desc.labels and desc.labels["mu-id"] == MU.deploy_id.downcase and (ignoremaster or desc.labels["mu-master-ip"] == MU.mu_public_ip.gsub(/\./, "_"))) or
354
244
  (flags["known"] and flags["known"].include?(cloud_id))
355
245
  MU.log "Deleting Cloud Function #{desc.name}"
356
246
  if !noop
@@ -364,9 +254,7 @@ module example.com/cloudfunction
364
254
  # Locate an existing project
365
255
  # @return [Hash<OpenStruct>]: The cloud provider's complete descriptions of matching project
366
256
  def self.find(**args)
367
- args[:project] ||= args[:habitat]
368
- args[:project] ||= MU::Cloud::Google.defaultProject(args[:credentials])
369
- location = args[:region] || args[:availability_zone] || "-"
257
+ args = MU::Cloud::Google.findLocationArgs(args)
370
258
 
371
259
  found = {}
372
260
 
@@ -379,7 +267,7 @@ module example.com/cloudfunction
379
267
  found[args[:cloud_id]] = resp if resp
380
268
  else
381
269
  resp = begin
382
- MU::Cloud::Google.function(credentials: args[:credentials]).list_project_location_functions("projects/#{args[:project]}/locations/#{location}")
270
+ MU::Cloud::Google.function(credentials: args[:credentials]).list_project_location_functions("projects/#{args[:project]}/locations/#{args[:location]}")
383
271
  rescue ::Google::Apis::ClientError => e
384
272
  raise e if !e.message.match(/forbidden:/)
385
273
  end
@@ -397,7 +285,7 @@ module example.com/cloudfunction
397
285
  # Reverse-map our cloud description into a runnable config hash.
398
286
  # We assume that any values we have in +@config+ are placeholders, and
399
287
  # calculate our own accordingly based on what's live in the cloud.
400
- def toKitten(rootparent: nil, billing: nil, habitats: nil)
288
+ def toKitten(**_args)
401
289
  bok = {
402
290
  "cloud" => "Google",
403
291
  "cloud_id" => @cloud_id,
@@ -460,7 +348,6 @@ module example.com/cloudfunction
460
348
  bok
461
349
  end
462
350
 
463
-
464
351
  # Cloud-specific configuration properties.
465
352
  # @param config [MU::Config]: The calling MU::Config object
466
353
  # @return [Array<Array,Hash>]: List of required fields, and json-schema Hash of cloud-specific configuration parameters for this resource
@@ -495,8 +382,14 @@ module example.com/cloudfunction
495
382
  "type" => "string",
496
383
  "description" => "+DEPRECATED+ VPC Connector to attach, of the form +projects/my-project/locations/some-region/connectors/my-connector+. This option will be removed once proper google-cloud-sdk support for VPC Connectors becomes available, at which point we will piggyback on the normal +vpc+ stanza and resolve connectors as needed."
497
384
  },
385
+ "vpc_connector_allow_all_egress" => {
386
+ "type" => "boolean",
387
+ "default" => false,
388
+ "description" => "+DEPRECATED+ Allow VPC connector egress traffic to any IP range, instead of just private IPs. This option will be removed once proper google-cloud-sdk support for VPC Connectors becomes available, at which point we will piggyback on the normal +vpc+ stanza and resolve connectors as needed."
389
+ },
498
390
  "code" => {
499
391
  "type" => "object",
392
+ "description" => "Zipped deployment package to upload to our function.",
500
393
  "properties" => {
501
394
  "gs_url" => {
502
395
  "type" => "string",
@@ -523,7 +416,7 @@ module example.com/cloudfunction
523
416
  cloud_desc.source_archive_url.match(/^gs:\/\/([^\/]+)\/(.*)/)
524
417
  bucket = Regexp.last_match[1]
525
418
  path = Regexp.last_match[2]
526
- current = nil
419
+
527
420
  MU::Cloud::Google.storage(credentials: credentials).get_object(bucket, path, download_dest: zipfile)
528
421
  elsif cloud_desc.source_upload_url
529
422
  resp = MU::Cloud::Google.function(credentials: credentials).generate_function_download_url(
@@ -625,32 +518,7 @@ module example.com/cloudfunction
625
518
  ok = false
626
519
  end
627
520
  else
628
- user = {
629
- "name" => function['name'],
630
- "cloud" => "Google",
631
- "project" => function["project"],
632
- "credentials" => function["credentials"],
633
- "type" => "service"
634
- }
635
- if user["name"].length < 6
636
- user["name"] += Password.pronounceable(6)
637
- end
638
- if function['roles']
639
- user['roles'] = function['roles'].dup
640
- end
641
- configurator.insertKitten(user, "users", true)
642
- function['dependencies'] ||= []
643
- function['service_account'] = MU::Config::Ref.get(
644
- type: "users",
645
- cloud: "Google",
646
- name: user["name"],
647
- project: user["project"],
648
- credentials: user["credentials"]
649
- )
650
- function['dependencies'] << {
651
- "type" => "user",
652
- "name" => user["name"]
653
- }
521
+ function = MU::Cloud::Google::User.genericServiceAccount(function, configurator)
654
522
  end
655
523
 
656
524
  # siblings = configurator.haveLitterMate?(nil, "vpcs", has_multiple: true)
@@ -673,6 +541,104 @@ module example.com/cloudfunction
673
541
  ok
674
542
  end
675
543
 
544
+ private
545
+
546
+ def buildDesc
547
+ labels = Hash[@tags.keys.map { |k|
548
+ [k.downcase, @tags[k].downcase.gsub(/[^-_a-z0-9]/, '-')] }
549
+ ]
550
+ labels["name"] = MU::Cloud::Google.nameStr(@mu_name)
551
+
552
+ location = "projects/"+@config['project']+"/locations/"+@config['region']
553
+ sa = nil
554
+ retries = 0
555
+ begin
556
+ sa_ref = MU::Config::Ref.get(@config['service_account'])
557
+ sa = @deploy.findLitterMate(name: sa_ref.name, type: "users")
558
+ if !sa or !sa.cloud_desc
559
+ sleep 10
560
+ end
561
+ rescue ::Google::Apis::ClientError => e
562
+ if e.message.match(/notFound:/)
563
+ sleep 10
564
+ retries += 1
565
+ retry
566
+ end
567
+ end while !sa or !sa.cloud_desc and retries < 5
568
+
569
+ if !sa or !sa.cloud_desc
570
+ raise MuError, "Failed to get service account cloud id from #{@config['service_account'].to_s}"
571
+ end
572
+
573
+ desc = {
574
+ name: location+"/functions/"+@mu_name.downcase,
575
+ runtime: @config['runtime'],
576
+ timeout: @config['timeout'].to_s+"s",
577
+ # entry_point: "hello_world",
578
+ entry_point: @config['handler'],
579
+ description: @deploy.deploy_id,
580
+ service_account_email: sa.cloud_desc.email,
581
+ labels: labels,
582
+ available_memory_mb: @config['memory']
583
+ }
584
+
585
+ # XXX This network argument is deprecated in favor of using VPC
586
+ # Connectors. Which would be fine, except there's no API support for
587
+ # interacting with VPC Connectors. Can't create them, can't list them,
588
+ # can't do anything except pass their ids into Cloud Functions or
589
+ # AppEngine and hope for the best.
590
+ if @config['vpc_connector']
591
+ desc[:vpc_connector] = @config['vpc_connector']
592
+ desc[:vpc_connector_egress_settings] = @config['vpc_connector_allow_all_egress'] ? "ALL_TRAFFIC" : "PRIVATE_RANGES_ONLY"
593
+ pp desc
594
+ elsif @vpc
595
+ desc[:network] = @vpc.url.sub(/^.*?\/projects\//, 'projects/')
596
+ end
597
+
598
+ if @config['triggers']
599
+ desc[:event_trigger] = MU::Cloud::Google.function(:EventTrigger).new(
600
+ event_type: @config['triggers'].first['event'],
601
+ resource: @config['triggers'].first['resource']
602
+ )
603
+ else
604
+ desc[:https_trigger] = MU::Cloud::Google.function(:HttpsTrigger).new
605
+ end
606
+
607
+
608
+ if @config['environment_variable']
609
+ @config['environment_variable'].each { |var|
610
+ desc[:environment_variables] ||= {}
611
+ desc[:environment_variables][var["key"].to_s] = var["value"].to_s
612
+ }
613
+ end
614
+
615
+ # hello_code = nil
616
+ # HELLO_WORLDS.each_pair { |runtime, code|
617
+ # if @config['runtime'].match(/^#{Regexp.quote(runtime)}/)
618
+ # hello_code = code
619
+ # break
620
+ # end
621
+ # }
622
+ if @config['code']['gs_url']
623
+ desc[:source_archive_url] = @config['code']['gs_url']
624
+ elsif @config['code']['zip_file']
625
+ desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(@config['code']['zip_file'], @mu_name+"-cloudfunction.zip", credentials: @credentials)
626
+ end
627
+
628
+ # Dir.mktmpdir(@mu_name) { |dir|
629
+ # hello_code.each_pair { |file, contents|
630
+ # f = File.open(dir+"/"+file, "w")
631
+ # f.puts contents
632
+ # f.close
633
+ # Zip::File.open(dir+"/function.zip", Zip::File::CREATE) { |z|
634
+ # z.add(file, dir+"/"+file)
635
+ # }
636
+ # }
637
+ # desc[:source_archive_url] = MU::Cloud::Google::Function.uploadPackage(dir+"/function.zip", @mu_name+"-cloudfunction.zip", credentials: @credentials)
638
+ # }
639
+ MU::Cloud::Google.function(:CloudFunction).new(desc)
640
+ end
641
+
676
642
  end
677
643
  end
678
644
  end