cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +16 -54
  3. data/Berksfile.lock +14 -62
  4. data/bin/mu-aws-setup +131 -108
  5. data/bin/mu-configure +311 -74
  6. data/bin/mu-gcp-setup +84 -62
  7. data/bin/mu-load-config.rb +46 -2
  8. data/bin/mu-self-update +11 -9
  9. data/bin/mu-upload-chef-artifacts +4 -4
  10. data/{mu.gemspec → cloud-mu.gemspec} +2 -2
  11. data/cookbooks/awscli/Berksfile +8 -0
  12. data/cookbooks/mu-activedirectory/Berksfile +11 -0
  13. data/cookbooks/mu-firewall/Berksfile +9 -0
  14. data/cookbooks/mu-firewall/metadata.rb +1 -1
  15. data/cookbooks/mu-glusterfs/Berksfile +10 -0
  16. data/cookbooks/mu-jenkins/Berksfile +14 -0
  17. data/cookbooks/mu-master/Berksfile +23 -0
  18. data/cookbooks/mu-master/attributes/default.rb +1 -1
  19. data/cookbooks/mu-master/metadata.rb +2 -2
  20. data/cookbooks/mu-master/recipes/default.rb +1 -1
  21. data/cookbooks/mu-master/recipes/init.rb +7 -3
  22. data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
  23. data/cookbooks/mu-mongo/Berksfile +10 -0
  24. data/cookbooks/mu-openvpn/Berksfile +11 -0
  25. data/cookbooks/mu-php54/Berksfile +13 -0
  26. data/cookbooks/mu-splunk/Berksfile +10 -0
  27. data/cookbooks/mu-tools/Berksfile +21 -0
  28. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  29. data/cookbooks/mu-utility/Berksfile +9 -0
  30. data/cookbooks/mu-utility/metadata.rb +2 -1
  31. data/cookbooks/nagios/Berksfile +7 -4
  32. data/cookbooks/s3fs/Berksfile +9 -0
  33. data/environments/dev.json +6 -6
  34. data/environments/prod.json +6 -6
  35. data/modules/mu.rb +20 -42
  36. data/modules/mu/cleanup.rb +102 -100
  37. data/modules/mu/cloud.rb +90 -28
  38. data/modules/mu/clouds/aws.rb +449 -218
  39. data/modules/mu/clouds/aws/alarm.rb +29 -17
  40. data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
  41. data/modules/mu/clouds/aws/collection.rb +25 -18
  42. data/modules/mu/clouds/aws/container_cluster.rb +73 -66
  43. data/modules/mu/clouds/aws/database.rb +124 -116
  44. data/modules/mu/clouds/aws/dnszone.rb +27 -20
  45. data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
  46. data/modules/mu/clouds/aws/folder.rb +18 -3
  47. data/modules/mu/clouds/aws/function.rb +77 -23
  48. data/modules/mu/clouds/aws/group.rb +19 -12
  49. data/modules/mu/clouds/aws/habitat.rb +153 -0
  50. data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
  51. data/modules/mu/clouds/aws/log.rb +30 -23
  52. data/modules/mu/clouds/aws/msg_queue.rb +29 -20
  53. data/modules/mu/clouds/aws/notifier.rb +222 -0
  54. data/modules/mu/clouds/aws/role.rb +178 -90
  55. data/modules/mu/clouds/aws/search_domain.rb +40 -24
  56. data/modules/mu/clouds/aws/server.rb +169 -137
  57. data/modules/mu/clouds/aws/server_pool.rb +60 -83
  58. data/modules/mu/clouds/aws/storage_pool.rb +59 -31
  59. data/modules/mu/clouds/aws/user.rb +36 -27
  60. data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
  61. data/modules/mu/clouds/aws/vpc.rb +250 -189
  62. data/modules/mu/clouds/azure.rb +132 -0
  63. data/modules/mu/clouds/cloudformation.rb +65 -1
  64. data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
  65. data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
  66. data/modules/mu/clouds/cloudformation/collection.rb +7 -0
  67. data/modules/mu/clouds/cloudformation/database.rb +7 -0
  68. data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
  69. data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
  70. data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
  71. data/modules/mu/clouds/cloudformation/log.rb +7 -0
  72. data/modules/mu/clouds/cloudformation/server.rb +7 -0
  73. data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
  74. data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
  75. data/modules/mu/clouds/google.rb +214 -110
  76. data/modules/mu/clouds/google/container_cluster.rb +42 -24
  77. data/modules/mu/clouds/google/database.rb +15 -6
  78. data/modules/mu/clouds/google/firewall_rule.rb +17 -25
  79. data/modules/mu/clouds/google/group.rb +13 -5
  80. data/modules/mu/clouds/google/habitat.rb +105 -0
  81. data/modules/mu/clouds/google/loadbalancer.rb +28 -20
  82. data/modules/mu/clouds/google/server.rb +93 -354
  83. data/modules/mu/clouds/google/server_pool.rb +18 -10
  84. data/modules/mu/clouds/google/user.rb +22 -14
  85. data/modules/mu/clouds/google/vpc.rb +97 -69
  86. data/modules/mu/config.rb +133 -38
  87. data/modules/mu/config/alarm.rb +25 -0
  88. data/modules/mu/config/cache_cluster.rb +5 -3
  89. data/modules/mu/config/cache_cluster.yml +23 -0
  90. data/modules/mu/config/database.rb +25 -16
  91. data/modules/mu/config/database.yml +3 -3
  92. data/modules/mu/config/function.rb +1 -2
  93. data/modules/mu/config/{project.rb → habitat.rb} +10 -10
  94. data/modules/mu/config/notifier.rb +85 -0
  95. data/modules/mu/config/notifier.yml +9 -0
  96. data/modules/mu/config/role.rb +1 -1
  97. data/modules/mu/config/search_domain.yml +2 -2
  98. data/modules/mu/config/server.rb +13 -1
  99. data/modules/mu/config/server.yml +3 -3
  100. data/modules/mu/config/server_pool.rb +3 -1
  101. data/modules/mu/config/storage_pool.rb +3 -1
  102. data/modules/mu/config/storage_pool.yml +19 -0
  103. data/modules/mu/config/vpc.rb +70 -8
  104. data/modules/mu/groomers/chef.rb +2 -3
  105. data/modules/mu/kittens.rb +500 -122
  106. data/modules/mu/master.rb +5 -5
  107. data/modules/mu/mommacat.rb +151 -91
  108. data/modules/tests/super_complex_bok.yml +12 -0
  109. data/modules/tests/super_simple_bok.yml +12 -0
  110. data/spec/mu/clouds/azure_spec.rb +82 -0
  111. data/spec/spec_helper.rb +105 -0
  112. metadata +26 -5
  113. data/modules/mu/clouds/aws/notification.rb +0 -139
  114. data/modules/mu/config/notification.rb +0 -44
@@ -76,7 +76,7 @@ module MU
76
76
  # machine_type: "zones/"+az+"/machineTypes/"+size,
77
77
  machine_type: size,
78
78
  labels: labels,
79
- disks: MU::Cloud::Google::Server.diskConfig(@config, false, false),
79
+ disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
80
80
  network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
81
81
  metadata: {
82
82
  :items => [
@@ -94,7 +94,7 @@ module MU
94
94
  )
95
95
 
96
96
  MU.log "Creating instance template #{@mu_name}", details: template_obj
97
- template = MU::Cloud::Google.compute.insert_instance_template(
97
+ template = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_instance_template(
98
98
  @config['project'],
99
99
  template_obj
100
100
  )
@@ -116,7 +116,7 @@ module MU
116
116
  )
117
117
 
118
118
  MU.log "Creating region instance group manager #{@mu_name}", details: mgr_obj
119
- mgr = MU::Cloud::Google.compute.insert_region_instance_group_manager(
119
+ mgr = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_instance_group_manager(
120
120
  @config['project'],
121
121
  @config['region'],
122
122
  mgr_obj
@@ -142,7 +142,7 @@ module MU
142
142
  )
143
143
 
144
144
  MU.log "Creating autoscaler policy #{@mu_name}", details: scaler_obj
145
- MU::Cloud::Google.compute.insert_region_autoscaler(
145
+ MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_autoscaler(
146
146
  @config['project'],
147
147
  @config['region'],
148
148
  scaler_obj
@@ -165,7 +165,8 @@ module MU
165
165
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
166
166
  # @param flags [Hash]: Optional flags
167
167
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ServerPools
168
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
168
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
169
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
169
170
  MU.log "XXX ServerPool.find not yet implemented", MU::WARN
170
171
  return {}
171
172
  end
@@ -226,7 +227,7 @@ module MU
226
227
 
227
228
  real_image = nil
228
229
  begin
229
- real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s)
230
+ real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
230
231
  rescue ::Google::Apis::ClientError => e
231
232
  MU.log e.inspect, MU::WARN
232
233
  end
@@ -242,17 +243,24 @@ module MU
242
243
  ok
243
244
  end
244
245
 
246
+ # Does this resource type exist as a global (cloud-wide) artifact, or
247
+ # is it localized to a region/zone?
248
+ # @return [Boolean]
249
+ def self.isGlobal?
250
+ false
251
+ end
252
+
245
253
  # Remove all autoscale groups associated with the currently loaded deployment.
246
254
  # @param noop [Boolean]: If true, will only print what would be done
247
255
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
248
256
  # @param region [String]: The cloud provider region
249
257
  # @return [void]
250
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
251
- flags["project"] ||= MU::Cloud::Google.defaultProject
258
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
259
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
252
260
 
253
261
  if !flags["global"]
254
262
  ["region_autoscaler", "region_instance_group_manager"].each { |type|
255
- MU::Cloud::Google.compute.delete(
263
+ MU::Cloud::Google.compute(credentials: credentials).delete(
256
264
  type,
257
265
  flags["project"],
258
266
  region,
@@ -260,7 +268,7 @@ module MU
260
268
  )
261
269
  }
262
270
  else
263
- MU::Cloud::Google.compute.delete(
271
+ MU::Cloud::Google.compute(credentials: credentials).delete(
264
272
  "instance_template",
265
273
  flags["project"],
266
274
  noop
@@ -44,7 +44,7 @@ module MU
44
44
  )
45
45
  )
46
46
  MU.log "Creating service account #{@mu_name}"
47
- MU::Cloud::Google.iam.create_service_account(
47
+ MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account(
48
48
  "projects/"+@config['project'],
49
49
  req_obj
50
50
  )
@@ -57,12 +57,12 @@ module MU
57
57
  bind_human_user
58
58
  else
59
59
  if @config['create_api_key']
60
- resp = MU::Cloud::Google.iam.list_project_service_account_keys(
60
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
61
61
  cloud_desc.name
62
62
  )
63
63
  if resp.keys.size == 0
64
64
  MU.log "Generating API keys for service account #{@mu_name}"
65
- resp = MU::Cloud::Google.iam.create_service_account_key(
65
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account_key(
66
66
  cloud_desc.name
67
67
  )
68
68
  scratchitem = MU::Master.storeScratchPadSecret("Google Cloud Service Account credentials for #{@mu_name}:\n<pre style='text-align:left;'>#{resp.private_key_data}</pre>")
@@ -77,7 +77,7 @@ module MU
77
77
  if @config['type'] == "interactive"
78
78
  return nil
79
79
  else
80
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
80
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_accounts(
81
81
  "projects/"+@config["project"]
82
82
  )
83
83
 
@@ -103,14 +103,21 @@ module MU
103
103
  }
104
104
  end
105
105
 
106
+ # Does this resource type exist as a global (cloud-wide) artifact, or
107
+ # is it localized to a region/zone?
108
+ # @return [Boolean]
109
+ def self.isGlobal?
110
+ true
111
+ end
112
+
106
113
  # Remove all users associated with the currently loaded deployment.
107
114
  # @param noop [Boolean]: If true, will only print what would be done
108
115
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
109
116
  # @param region [String]: The cloud provider region
110
117
  # @return [void]
111
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
112
- flags["project"] ||= MU::Cloud::Google.defaultProject
113
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
118
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
119
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
120
+ resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
114
121
  "projects/"+flags["project"]
115
122
  )
116
123
 
@@ -120,7 +127,7 @@ module MU
120
127
  begin
121
128
  MU.log "Deleting service account #{sa.name}", details: sa
122
129
  if !noop
123
- MU::Cloud::Google.iam.delete_project_service_account(sa.name)
130
+ MU::Cloud::Google.iam(credentials: credentials).delete_project_service_account(sa.name)
124
131
  end
125
132
  rescue ::Google::Apis::ClientError => e
126
133
  raise e if !e.message.match(/^notFound: /)
@@ -130,14 +137,15 @@ module MU
130
137
  end
131
138
  end
132
139
 
133
- # Locate an existing user group.
140
+ # Locate an existing user.
134
141
  # @param cloud_id [String]: The cloud provider's identifier for this resource.
135
142
  # @param region [String]: The cloud provider region.
136
143
  # @param flags [Hash]: Optional flags
137
144
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
138
- def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
145
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
146
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
139
147
  found = nil
140
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
148
+ resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
141
149
  "projects/"+flags["project"]
142
150
  )
143
151
 
@@ -194,7 +202,7 @@ module MU
194
202
  # admin_directory only works in a GSuite environment
195
203
  if !user['name'].match(/@/i) and $MU_CFG['google']['masquerade_as']
196
204
  # XXX flesh this check out, need to test with a GSuite site
197
- pp MU::Cloud::Google.admin_directory.get_user(user['name'])
205
+ pp MU::Cloud::Google.admin_directory(credentials: user['credentials']).get_user(user['name'])
198
206
  end
199
207
 
200
208
  if user['groups'] and user['groups'].size > 0 and
@@ -215,7 +223,7 @@ module MU
215
223
 
216
224
  def bind_human_user
217
225
  bindings = []
218
- ext_policy = MU::Cloud::Google.resource_manager.get_project_iam_policy(
226
+ ext_policy = MU::Cloud::Google.resource_manager(credentials: @config['credentials']).get_project_iam_policy(
219
227
  @config['project']
220
228
  )
221
229
 
@@ -247,7 +255,7 @@ module MU
247
255
  MU.log "Adding #{@config['name']} to Google Cloud project #{@config['project']}", details: @config['roles']
248
256
 
249
257
  begin
250
- MU::Cloud::Google.resource_manager.set_project_iam_policy(
258
+ MU::Cloud::Google.resource_manager(credentials: @config['credentials']).set_project_iam_policy(
251
259
  @config['project'],
252
260
  req_obj
253
261
  )
@@ -63,7 +63,7 @@ module MU
63
63
  # i_pv4_range: @config['ip_block']
64
64
  )
65
65
  MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@config['project']}", details: networkobj
66
- resp = MU::Cloud::Google.compute.insert_network(@config['project'], networkobj)
66
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_network(@config['project'], networkobj)
67
67
  @url = resp.self_link # XXX needs to go in notify
68
68
  @cloud_id = resp.name
69
69
 
@@ -83,7 +83,7 @@ module MU
83
83
  network: @url,
84
84
  region: subnet['availability_zone']
85
85
  )
86
- resp = MU::Cloud::Google.compute.insert_subnetwork(@config['project'], subnet['availability_zone'], subnetobj)
86
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_subnetwork(@config['project'], subnet['availability_zone'], subnetobj)
87
87
 
88
88
  }
89
89
  }
@@ -121,15 +121,20 @@ module MU
121
121
  base = MU.structToHash(cloud_desc)
122
122
  base["cloud_id"] = @cloud_id
123
123
  base.merge!(@config.to_h)
124
+ if @config['name'] == "gkeprivate"
125
+ pp base.keys
126
+ puts base['cloud_id']
127
+ end
128
+
124
129
  base
125
130
  end
126
131
 
127
132
  # Describe this VPC from the cloud platform's perspective
128
133
  # @return [Hash]
129
134
  def cloud_desc
130
- @config['project'] ||= MU::Cloud::Google.defaultProject
135
+ @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
131
136
 
132
- resp = MU::Cloud::Google.compute.get_network(@config['project'], @cloud_id)
137
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).get_network(@config['project'], @cloud_id)
133
138
  if @cloud_id.nil? or @cloud_id == ""
134
139
  MU.log "Couldn't describe #{self}, @cloud_id #{@cloud_id.nil? ? "undefined" : "empty" }", MU::ERR
135
140
  return nil
@@ -137,7 +142,7 @@ module MU
137
142
 
138
143
  resp = resp.to_h
139
144
  @url ||= resp[:self_link]
140
- routes = MU::Cloud::Google.compute.list_routes(
145
+ routes = MU::Cloud::Google.compute(credentials: @config['credentials']).list_routes(
141
146
  @config['project'],
142
147
  filter: "network eq #{@cloud_id}"
143
148
  ).items
@@ -162,38 +167,66 @@ module MU
162
167
  if !@config['peers'].nil?
163
168
  count = 0
164
169
  @config['peers'].each { |peer|
165
- tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
166
- if peer['vpc']['deploy_id'].nil? and peer['vpc']['vpc_id'].nil? and tag_key.nil?
167
- peer['vpc']['deploy_id'] = @deploy.deploy_id
168
- end
170
+ if peer['vpc']['vpc_name']
171
+ peer_obj = @deploy.findLitterMate(name: peer['vpc']['vpc_name'], type: "vpcs")
172
+ if peer_obj
173
+ if peer_obj.config['peers']
174
+ skipme = false
175
+ peer_obj.config['peers'].each { |peerpeer|
176
+ if peerpeer['vpc']['vpc_name'] == @config['name'] and
177
+ (peer['vpc']['vpc_name'] <=> @config['name']) == -1
178
+ skipme = true
179
+ MU.log "VPCs #{peer['vpc']['vpc_name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
180
+ # XXX and if deploy_id matches or is unset
181
+ end
182
+ }
183
+ next if skipme
184
+ end
185
+ end
169
186
 
170
- peer_obj = MU::MommaCat.findStray(
171
- "Google",
172
- "vpcs",
173
- deploy_id: peer['vpc']['deploy_id'],
174
- cloud_id: peer['vpc']['vpc_id'],
175
- name: peer['vpc']['vpc_name'],
176
- tag_key: tag_key,
177
- tag_value: tag_value,
178
- dummy_ok: true
179
- )
187
+ else
188
+ tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
189
+ if peer['vpc']['deploy_id'].nil? and peer['vpc']['vpc_id'].nil? and tag_key.nil?
190
+ peer['vpc']['deploy_id'] = @deploy.deploy_id
191
+ end
180
192
 
181
- raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
193
+ peer_obj = MU::MommaCat.findStray(
194
+ "Google",
195
+ "vpcs",
196
+ deploy_id: peer['vpc']['deploy_id'],
197
+ cloud_id: peer['vpc']['vpc_id'],
198
+ name: peer['vpc']['vpc_name'],
199
+ tag_key: tag_key,
200
+ tag_value: tag_value,
201
+ dummy_ok: true
202
+ ).first
203
+ end
204
+
205
+ raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil?
182
206
 
183
- url = peer_obj.first.cloudobj.url || peer_obj.first.cloudobj.deploydata['self_link']
207
+ url = if peer_obj.cloudobj.url
208
+ peer_obj.cloudobj.url
209
+ elsif peer_obj.cloudobj.deploydata
210
+ peer_obj.cloudobj.deploydata['self_link']
211
+ else
212
+ pp peer_obj.cloudobj.cloud_desc
213
+ raise MuError, "Can't find the damn URL of my damn peer VPC #{peer['vpc']}"
214
+ end
215
+ cnxn_name = MU::Cloud::Google.nameStr(@mu_name+"-peer-"+count.to_s)
184
216
  peerreq = MU::Cloud::Google.compute(:NetworksAddPeeringRequest).new(
185
- name: MU::Cloud::Google.nameStr(@mu_name+"-peer-"+count.to_s),
217
+ name: cnxn_name,
186
218
  auto_create_routes: true,
187
219
  peer_network: url
188
220
  )
189
221
 
190
- MU.log "Peering #{@mu_name} with #{url}", details: peerreq
191
- MU::Cloud::Google.compute.add_network_peering(
222
+ MU.log "Peering #{@url} with #{url}, connection name is #{cnxn_name}", details: peerreq
223
+
224
+ MU::Cloud::Google.compute(credentials: @config['credentials']).add_network_peering(
192
225
  @config['project'],
193
226
  @cloud_id,
194
227
  peerreq
195
228
  )
196
-
229
+ count += 1
197
230
  }
198
231
  end
199
232
  end
@@ -204,19 +237,19 @@ module MU
204
237
  # @param tag_key [String]: A tag key to search.
205
238
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
206
239
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching VPCs
207
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
208
- flags["project"] ||= MU::Cloud::Google.defaultProject
209
- #MU.log "CALLED MU::Cloud::Google::VPC.find(#{cloud_id}, #{region}, #{tag_key}, #{tag_value}) from #{caller[0]}", MU::NOTICE, details: flags
240
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
241
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
242
+ #MU.log "CALLED MU::Cloud::Google::VPC.find(#{cloud_id}, #{region}, #{tag_key}, #{tag_value}) with credentials #{credentials} from #{caller[0]}", MU::NOTICE, details: flags
210
243
 
211
244
  resp = {}
212
245
  if cloud_id
213
- vpc = MU::Cloud::Google.compute.get_network(
246
+ vpc = MU::Cloud::Google.compute(credentials: credentials).get_network(
214
247
  flags['project'],
215
248
  cloud_id.to_s.sub(/^.*?\/([^\/]+)$/, '\1')
216
249
  )
217
250
  resp[cloud_id] = vpc if !vpc.nil?
218
251
  else # XXX other criteria
219
- MU::Cloud::Google.compute.list_networks(
252
+ MU::Cloud::Google.compute(credentials: credentials).list_networks(
220
253
  flags["project"]
221
254
  ).items.each { |vpc|
222
255
  resp[vpc.name] = vpc
@@ -224,7 +257,7 @@ module MU
224
257
  end
225
258
  #MU.log "THINGY", MU::WARN, details: resp
226
259
  resp.each_pair { |cloud_id, vpc|
227
- routes = MU::Cloud::Google.compute.list_routes(
260
+ routes = MU::Cloud::Google.compute(credentials: credentials).list_routes(
228
261
  flags["project"],
229
262
  filter: "network eq #{vpc.self_link}"
230
263
  ).items
@@ -259,7 +292,7 @@ module MU
259
292
 
260
293
  resp = nil
261
294
  MU::Cloud::Google.listRegions(@config['us_only']).each { |r|
262
- resp = MU::Cloud::Google.compute.list_subnetworks(
295
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_subnetworks(
263
296
  @config['project'],
264
297
  r,
265
298
  filter: "network eq #{network[:self_link]}"
@@ -415,13 +448,13 @@ module MU
415
448
  # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
416
449
  # @param region [String]: The cloud provider region of the target subnet.
417
450
  # @return [Boolean]
418
- def self.haveRouteToInstance?(target_instance, region: MU.curRegion)
419
- project ||= MU::Cloud::Google.defaultProject
451
+ def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)
452
+ project ||= MU::Cloud::Google.defaultProject(credentials)
420
453
  return false if MU.myCloud != "Google"
421
454
  # XXX see if we reside in the same Network and overlap subnets
422
455
  # XXX see if we peer with the target's Network
423
456
  target_instance.network_interfaces.each { |iface|
424
- resp = MU::Cloud::Google.compute.list_routes(
457
+ resp = MU::Cloud::Google.compute(credentials: credentials).list_routes(
425
458
  project,
426
459
  filter: "network eq #{iface.network}"
427
460
  )
@@ -456,19 +489,26 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
456
489
  def self.get_route_tables(subnet_ids: [], vpc_ids: [], region: MU.curRegion)
457
490
  end
458
491
 
492
+ # Does this resource type exist as a global (cloud-wide) artifact, or
493
+ # is it localized to a region/zone?
494
+ # @return [Boolean]
495
+ def self.isGlobal?
496
+ true
497
+ end
498
+
459
499
  # Remove all VPC resources associated with the currently loaded deployment.
460
500
  # @param noop [Boolean]: If true, will only print what would be done
461
501
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
462
502
  # @param region [String]: The cloud provider region
463
503
  # @return [void]
464
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
465
- flags["project"] ||= MU::Cloud::Google.defaultProject
504
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
505
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
466
506
 
467
- purge_subnets(noop, project: flags['project'])
507
+ purge_subnets(noop, project: flags['project'], credentials: credentials)
468
508
  ["route", "network"].each { |type|
469
509
  # XXX tagged routes aren't showing up in list, and the networks that own them
470
510
  # fail to delete silently
471
- MU::Cloud::Google.compute.delete(
511
+ MU::Cloud::Google.compute(credentials: credentials).delete(
472
512
  type,
473
513
  flags["project"],
474
514
  nil,
@@ -558,6 +598,8 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
558
598
  vpc['route_tables'].each { |tbl|
559
599
  newvpc = {
560
600
  "name" => vpc['name']+"-"+tbl['name'],
601
+ "credentials" => vpc['credentials'],
602
+ "virtual_name" => vpc['name'],
561
603
  "ip_block" => blocks.shift,
562
604
  "route_tables" => [tbl],
563
605
  "parent_block" => vpc['ip_block'],
@@ -587,9 +629,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
587
629
  # XXX we need routes to peered Networks too
588
630
 
589
631
  if has_nat or has_deny
590
- ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], false)
632
+ ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], false, credentials: vpc['credentials'])
591
633
  else
592
- ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"])
634
+ ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], credentials: vpc['credentials'])
593
635
  end
594
636
  if has_nat and !has_deny
595
637
  vpc['route_tables'].first["routes"] << {
@@ -613,6 +655,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
613
655
  if route['gateway'] == "#NAT"
614
656
  nat_cfg = MU::Cloud::Google::Server.genericNAT
615
657
  nat_cfg['name'] = vpc['name']+"-natstion-"+nat_count.to_s
658
+ nat_cfg['credentials'] = vpc['credentials']
616
659
  # XXX ingress/egress rules?
617
660
  # XXX for master too if applicable
618
661
  nat_cfg["application_attributes"] = {
@@ -649,10 +692,11 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
649
692
 
650
693
  private
651
694
 
652
- def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true)
695
+ def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true, credentials: nil)
653
696
  private_acl = {
654
- "name" => vpc_name+"-routables",
697
+ "name" => vpc_name+"-rt",
655
698
  "cloud" => "Google",
699
+ "credentials" => credentials,
656
700
  "project" => project,
657
701
  "vpc" => { "vpc_name" => vpc_name },
658
702
  "dependencies" => [ { "type" => "vpc", "name" => vpc_name } ],
@@ -712,7 +756,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
712
756
  end
713
757
  # several other cases missing for various types of routers (raw IPs, instance ids, etc) XXX
714
758
  elsif route['gateway'] == "#DENY"
715
- resp = MU::Cloud::Google.compute.list_routes(
759
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_routes(
716
760
  @config['project'],
717
761
  filter: "network eq #{network}"
718
762
  )
@@ -721,7 +765,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
721
765
  resp.items.each { |r|
722
766
  next if r.next_hop_gateway.nil? or !r.next_hop_gateway.match(/\/global\/gateways\/default-internet-gateway$/)
723
767
  MU.log "Removing standard route #{r.name} per our #DENY entry"
724
- MU::Cloud::Google.compute.delete_route(@config['project'], r.name)
768
+ MU::Cloud::Google.compute(credentials: @config['credentials']).delete_route(@config['project'], r.name)
725
769
  }
726
770
  end
727
771
  elsif route['gateway'] == "#INTERNET"
@@ -748,11 +792,11 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
748
792
 
749
793
  if route['gateway'] != "#DENY"
750
794
  begin
751
- MU::Cloud::Google.compute.get_route(@config['project'], routename)
795
+ MU::Cloud::Google.compute(credentials: @config['credentials']).get_route(@config['project'], routename)
752
796
  rescue ::Google::Apis::ClientError, MU::MuError => e
753
797
  if e.message.match(/notFound/)
754
798
  MU.log "Creating route #{routename} in project #{@config['project']}", details: routeobj
755
- resp = MU::Cloud::Google.compute.insert_route(@config['project'], routeobj)
799
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@config['project'], routeobj)
756
800
  else
757
801
  # TODO can't update GCP routes, would have to delete and re-create
758
802
  end
@@ -786,7 +830,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
786
830
 
787
831
  # Remove all network interfaces associated with the currently loaded deployment.
788
832
  # @param noop [Boolean]: If true, will only print what would be done
789
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
833
+ # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
790
834
  # @param region [String]: The cloud provider region
791
835
  # @return [void]
792
836
  def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
@@ -794,16 +838,17 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
794
838
 
795
839
  # Remove all subnets associated with the currently loaded deployment.
796
840
  # @param noop [Boolean]: If true, will only print what would be done
797
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
841
+ # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
798
842
  # @param regions [Array<String>]: The cloud provider regions to check
799
843
  # @return [void]
800
- def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: MU::Cloud::Google.defaultProject)
844
+ def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
845
+ project ||= MU::Cloud::Google.defaultProject(credentials)
801
846
  parent_thread_id = Thread.current.object_id
802
847
  regionthreads = []
803
848
  regions.each { |r|
804
849
  regionthreads << Thread.new {
805
850
  MU.dupGlobals(parent_thread_id)
806
- MU::Cloud::Google.compute.delete(
851
+ MU::Cloud::Google.compute(credentials: credentials).delete(
807
852
  "subnetwork",
808
853
  project,
809
854
  r,
@@ -816,23 +861,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
816
861
  end
817
862
  end
818
863
 
819
- # Remove all DHCP options sets associated with the currently loaded
820
- # deployment.
821
- # @param noop [Boolean]: If true, will only print what would be done
822
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
823
- # @param region [String]: The cloud provider region
824
- # @return [void]
825
- def self.purge_dhcpopts(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
826
- end
827
-
828
- # Remove all VPCs associated with the currently loaded deployment.
829
- # @param noop [Boolean]: If true, will only print what would be done
830
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
831
- # @param region [String]: The cloud provider region
832
- # @return [void]
833
- def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
834
- end
835
-
836
864
  protected
837
865
 
838
866
  # Subnets are almost a first-class resource. So let's kinda sorta treat
@@ -869,7 +897,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
869
897
  # Is this subnet privately-routable only, or public?
870
898
  # @return [Boolean]
871
899
  def private?
872
- routes = MU::Cloud::Google.compute.list_routes(
900
+ routes = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).list_routes(
873
901
  @parent.config['project'],
874
902
  filter: "network eq #{@parent.url}"
875
903
  ).items