cloud-mu 1.9.0.pre.beta → 2.0.0.pre.alpha

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +4 -4
  2. data/Berksfile +16 -54
  3. data/Berksfile.lock +14 -62
  4. data/bin/mu-aws-setup +131 -108
  5. data/bin/mu-configure +311 -74
  6. data/bin/mu-gcp-setup +84 -62
  7. data/bin/mu-load-config.rb +46 -2
  8. data/bin/mu-self-update +11 -9
  9. data/bin/mu-upload-chef-artifacts +4 -4
  10. data/{mu.gemspec → cloud-mu.gemspec} +2 -2
  11. data/cookbooks/awscli/Berksfile +8 -0
  12. data/cookbooks/mu-activedirectory/Berksfile +11 -0
  13. data/cookbooks/mu-firewall/Berksfile +9 -0
  14. data/cookbooks/mu-firewall/metadata.rb +1 -1
  15. data/cookbooks/mu-glusterfs/Berksfile +10 -0
  16. data/cookbooks/mu-jenkins/Berksfile +14 -0
  17. data/cookbooks/mu-master/Berksfile +23 -0
  18. data/cookbooks/mu-master/attributes/default.rb +1 -1
  19. data/cookbooks/mu-master/metadata.rb +2 -2
  20. data/cookbooks/mu-master/recipes/default.rb +1 -1
  21. data/cookbooks/mu-master/recipes/init.rb +7 -3
  22. data/cookbooks/mu-master/recipes/ssl-certs.rb +1 -0
  23. data/cookbooks/mu-mongo/Berksfile +10 -0
  24. data/cookbooks/mu-openvpn/Berksfile +11 -0
  25. data/cookbooks/mu-php54/Berksfile +13 -0
  26. data/cookbooks/mu-splunk/Berksfile +10 -0
  27. data/cookbooks/mu-tools/Berksfile +21 -0
  28. data/cookbooks/mu-tools/files/default/Mu_CA.pem +15 -15
  29. data/cookbooks/mu-utility/Berksfile +9 -0
  30. data/cookbooks/mu-utility/metadata.rb +2 -1
  31. data/cookbooks/nagios/Berksfile +7 -4
  32. data/cookbooks/s3fs/Berksfile +9 -0
  33. data/environments/dev.json +6 -6
  34. data/environments/prod.json +6 -6
  35. data/modules/mu.rb +20 -42
  36. data/modules/mu/cleanup.rb +102 -100
  37. data/modules/mu/cloud.rb +90 -28
  38. data/modules/mu/clouds/aws.rb +449 -218
  39. data/modules/mu/clouds/aws/alarm.rb +29 -17
  40. data/modules/mu/clouds/aws/cache_cluster.rb +78 -64
  41. data/modules/mu/clouds/aws/collection.rb +25 -18
  42. data/modules/mu/clouds/aws/container_cluster.rb +73 -66
  43. data/modules/mu/clouds/aws/database.rb +124 -116
  44. data/modules/mu/clouds/aws/dnszone.rb +27 -20
  45. data/modules/mu/clouds/aws/firewall_rule.rb +30 -22
  46. data/modules/mu/clouds/aws/folder.rb +18 -3
  47. data/modules/mu/clouds/aws/function.rb +77 -23
  48. data/modules/mu/clouds/aws/group.rb +19 -12
  49. data/modules/mu/clouds/aws/habitat.rb +153 -0
  50. data/modules/mu/clouds/aws/loadbalancer.rb +59 -52
  51. data/modules/mu/clouds/aws/log.rb +30 -23
  52. data/modules/mu/clouds/aws/msg_queue.rb +29 -20
  53. data/modules/mu/clouds/aws/notifier.rb +222 -0
  54. data/modules/mu/clouds/aws/role.rb +178 -90
  55. data/modules/mu/clouds/aws/search_domain.rb +40 -24
  56. data/modules/mu/clouds/aws/server.rb +169 -137
  57. data/modules/mu/clouds/aws/server_pool.rb +60 -83
  58. data/modules/mu/clouds/aws/storage_pool.rb +59 -31
  59. data/modules/mu/clouds/aws/user.rb +36 -27
  60. data/modules/mu/clouds/aws/userdata/linux.erb +101 -93
  61. data/modules/mu/clouds/aws/vpc.rb +250 -189
  62. data/modules/mu/clouds/azure.rb +132 -0
  63. data/modules/mu/clouds/cloudformation.rb +65 -1
  64. data/modules/mu/clouds/cloudformation/alarm.rb +8 -0
  65. data/modules/mu/clouds/cloudformation/cache_cluster.rb +7 -0
  66. data/modules/mu/clouds/cloudformation/collection.rb +7 -0
  67. data/modules/mu/clouds/cloudformation/database.rb +7 -0
  68. data/modules/mu/clouds/cloudformation/dnszone.rb +7 -0
  69. data/modules/mu/clouds/cloudformation/firewall_rule.rb +9 -2
  70. data/modules/mu/clouds/cloudformation/loadbalancer.rb +7 -0
  71. data/modules/mu/clouds/cloudformation/log.rb +7 -0
  72. data/modules/mu/clouds/cloudformation/server.rb +7 -0
  73. data/modules/mu/clouds/cloudformation/server_pool.rb +7 -0
  74. data/modules/mu/clouds/cloudformation/vpc.rb +7 -0
  75. data/modules/mu/clouds/google.rb +214 -110
  76. data/modules/mu/clouds/google/container_cluster.rb +42 -24
  77. data/modules/mu/clouds/google/database.rb +15 -6
  78. data/modules/mu/clouds/google/firewall_rule.rb +17 -25
  79. data/modules/mu/clouds/google/group.rb +13 -5
  80. data/modules/mu/clouds/google/habitat.rb +105 -0
  81. data/modules/mu/clouds/google/loadbalancer.rb +28 -20
  82. data/modules/mu/clouds/google/server.rb +93 -354
  83. data/modules/mu/clouds/google/server_pool.rb +18 -10
  84. data/modules/mu/clouds/google/user.rb +22 -14
  85. data/modules/mu/clouds/google/vpc.rb +97 -69
  86. data/modules/mu/config.rb +133 -38
  87. data/modules/mu/config/alarm.rb +25 -0
  88. data/modules/mu/config/cache_cluster.rb +5 -3
  89. data/modules/mu/config/cache_cluster.yml +23 -0
  90. data/modules/mu/config/database.rb +25 -16
  91. data/modules/mu/config/database.yml +3 -3
  92. data/modules/mu/config/function.rb +1 -2
  93. data/modules/mu/config/{project.rb → habitat.rb} +10 -10
  94. data/modules/mu/config/notifier.rb +85 -0
  95. data/modules/mu/config/notifier.yml +9 -0
  96. data/modules/mu/config/role.rb +1 -1
  97. data/modules/mu/config/search_domain.yml +2 -2
  98. data/modules/mu/config/server.rb +13 -1
  99. data/modules/mu/config/server.yml +3 -3
  100. data/modules/mu/config/server_pool.rb +3 -1
  101. data/modules/mu/config/storage_pool.rb +3 -1
  102. data/modules/mu/config/storage_pool.yml +19 -0
  103. data/modules/mu/config/vpc.rb +70 -8
  104. data/modules/mu/groomers/chef.rb +2 -3
  105. data/modules/mu/kittens.rb +500 -122
  106. data/modules/mu/master.rb +5 -5
  107. data/modules/mu/mommacat.rb +151 -91
  108. data/modules/tests/super_complex_bok.yml +12 -0
  109. data/modules/tests/super_simple_bok.yml +12 -0
  110. data/spec/mu/clouds/azure_spec.rb +82 -0
  111. data/spec/spec_helper.rb +105 -0
  112. metadata +26 -5
  113. data/modules/mu/clouds/aws/notification.rb +0 -139
  114. data/modules/mu/config/notification.rb +0 -44
@@ -76,7 +76,7 @@ module MU
76
76
  # machine_type: "zones/"+az+"/machineTypes/"+size,
77
77
  machine_type: size,
78
78
  labels: labels,
79
- disks: MU::Cloud::Google::Server.diskConfig(@config, false, false),
79
+ disks: MU::Cloud::Google::Server.diskConfig(@config, false, false, credentials: @config['credentials']),
80
80
  network_interfaces: MU::Cloud::Google::Server.interfaceConfig(@config, @vpc),
81
81
  metadata: {
82
82
  :items => [
@@ -94,7 +94,7 @@ module MU
94
94
  )
95
95
 
96
96
  MU.log "Creating instance template #{@mu_name}", details: template_obj
97
- template = MU::Cloud::Google.compute.insert_instance_template(
97
+ template = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_instance_template(
98
98
  @config['project'],
99
99
  template_obj
100
100
  )
@@ -116,7 +116,7 @@ module MU
116
116
  )
117
117
 
118
118
  MU.log "Creating region instance group manager #{@mu_name}", details: mgr_obj
119
- mgr = MU::Cloud::Google.compute.insert_region_instance_group_manager(
119
+ mgr = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_instance_group_manager(
120
120
  @config['project'],
121
121
  @config['region'],
122
122
  mgr_obj
@@ -142,7 +142,7 @@ module MU
142
142
  )
143
143
 
144
144
  MU.log "Creating autoscaler policy #{@mu_name}", details: scaler_obj
145
- MU::Cloud::Google.compute.insert_region_autoscaler(
145
+ MU::Cloud::Google.compute(credentials: @config['credentials']).insert_region_autoscaler(
146
146
  @config['project'],
147
147
  @config['region'],
148
148
  scaler_obj
@@ -165,7 +165,8 @@ module MU
165
165
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
166
166
  # @param flags [Hash]: Optional flags
167
167
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching ServerPools
168
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
168
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
169
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
169
170
  MU.log "XXX ServerPool.find not yet implemented", MU::WARN
170
171
  return {}
171
172
  end
@@ -226,7 +227,7 @@ module MU
226
227
 
227
228
  real_image = nil
228
229
  begin
229
- real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s)
230
+ real_image = MU::Cloud::Google::Server.fetchImage(launch['image_id'].to_s, credentials: pool['credentials'])
230
231
  rescue ::Google::Apis::ClientError => e
231
232
  MU.log e.inspect, MU::WARN
232
233
  end
@@ -242,17 +243,24 @@ module MU
242
243
  ok
243
244
  end
244
245
 
246
+ # Does this resource type exist as a global (cloud-wide) artifact, or
247
+ # is it localized to a region/zone?
248
+ # @return [Boolean]
249
+ def self.isGlobal?
250
+ false
251
+ end
252
+
245
253
  # Remove all autoscale groups associated with the currently loaded deployment.
246
254
  # @param noop [Boolean]: If true, will only print what would be done
247
255
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
248
256
  # @param region [String]: The cloud provider region
249
257
  # @return [void]
250
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
251
- flags["project"] ||= MU::Cloud::Google.defaultProject
258
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
259
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
252
260
 
253
261
  if !flags["global"]
254
262
  ["region_autoscaler", "region_instance_group_manager"].each { |type|
255
- MU::Cloud::Google.compute.delete(
263
+ MU::Cloud::Google.compute(credentials: credentials).delete(
256
264
  type,
257
265
  flags["project"],
258
266
  region,
@@ -260,7 +268,7 @@ module MU
260
268
  )
261
269
  }
262
270
  else
263
- MU::Cloud::Google.compute.delete(
271
+ MU::Cloud::Google.compute(credentials: credentials).delete(
264
272
  "instance_template",
265
273
  flags["project"],
266
274
  noop
@@ -44,7 +44,7 @@ module MU
44
44
  )
45
45
  )
46
46
  MU.log "Creating service account #{@mu_name}"
47
- MU::Cloud::Google.iam.create_service_account(
47
+ MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account(
48
48
  "projects/"+@config['project'],
49
49
  req_obj
50
50
  )
@@ -57,12 +57,12 @@ module MU
57
57
  bind_human_user
58
58
  else
59
59
  if @config['create_api_key']
60
- resp = MU::Cloud::Google.iam.list_project_service_account_keys(
60
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_account_keys(
61
61
  cloud_desc.name
62
62
  )
63
63
  if resp.keys.size == 0
64
64
  MU.log "Generating API keys for service account #{@mu_name}"
65
- resp = MU::Cloud::Google.iam.create_service_account_key(
65
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).create_service_account_key(
66
66
  cloud_desc.name
67
67
  )
68
68
  scratchitem = MU::Master.storeScratchPadSecret("Google Cloud Service Account credentials for #{@mu_name}:\n<pre style='text-align:left;'>#{resp.private_key_data}</pre>")
@@ -77,7 +77,7 @@ module MU
77
77
  if @config['type'] == "interactive"
78
78
  return nil
79
79
  else
80
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
80
+ resp = MU::Cloud::Google.iam(credentials: @config['credentials']).list_project_service_accounts(
81
81
  "projects/"+@config["project"]
82
82
  )
83
83
 
@@ -103,14 +103,21 @@ module MU
103
103
  }
104
104
  end
105
105
 
106
+ # Does this resource type exist as a global (cloud-wide) artifact, or
107
+ # is it localized to a region/zone?
108
+ # @return [Boolean]
109
+ def self.isGlobal?
110
+ true
111
+ end
112
+
106
113
  # Remove all users associated with the currently loaded deployment.
107
114
  # @param noop [Boolean]: If true, will only print what would be done
108
115
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
109
116
  # @param region [String]: The cloud provider region
110
117
  # @return [void]
111
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
112
- flags["project"] ||= MU::Cloud::Google.defaultProject
113
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
118
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
119
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
120
+ resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
114
121
  "projects/"+flags["project"]
115
122
  )
116
123
 
@@ -120,7 +127,7 @@ module MU
120
127
  begin
121
128
  MU.log "Deleting service account #{sa.name}", details: sa
122
129
  if !noop
123
- MU::Cloud::Google.iam.delete_project_service_account(sa.name)
130
+ MU::Cloud::Google.iam(credentials: credentials).delete_project_service_account(sa.name)
124
131
  end
125
132
  rescue ::Google::Apis::ClientError => e
126
133
  raise e if !e.message.match(/^notFound: /)
@@ -130,14 +137,15 @@ module MU
130
137
  end
131
138
  end
132
139
 
133
- # Locate an existing user group.
140
+ # Locate an existing user.
134
141
  # @param cloud_id [String]: The cloud provider's identifier for this resource.
135
142
  # @param region [String]: The cloud provider region.
136
143
  # @param flags [Hash]: Optional flags
137
144
  # @return [OpenStruct]: The cloud provider's complete descriptions of matching user group.
138
- def self.find(cloud_id: nil, region: MU.curRegion, flags: {})
145
+ def self.find(cloud_id: nil, region: MU.curRegion, credentials: nil, flags: {})
146
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
139
147
  found = nil
140
- resp = MU::Cloud::Google.iam.list_project_service_accounts(
148
+ resp = MU::Cloud::Google.iam(credentials: credentials).list_project_service_accounts(
141
149
  "projects/"+flags["project"]
142
150
  )
143
151
 
@@ -194,7 +202,7 @@ module MU
194
202
  # admin_directory only works in a GSuite environment
195
203
  if !user['name'].match(/@/i) and $MU_CFG['google']['masquerade_as']
196
204
  # XXX flesh this check out, need to test with a GSuite site
197
- pp MU::Cloud::Google.admin_directory.get_user(user['name'])
205
+ pp MU::Cloud::Google.admin_directory(credentials: user['credentials']).get_user(user['name'])
198
206
  end
199
207
 
200
208
  if user['groups'] and user['groups'].size > 0 and
@@ -215,7 +223,7 @@ module MU
215
223
 
216
224
  def bind_human_user
217
225
  bindings = []
218
- ext_policy = MU::Cloud::Google.resource_manager.get_project_iam_policy(
226
+ ext_policy = MU::Cloud::Google.resource_manager(credentials: @config['credentials']).get_project_iam_policy(
219
227
  @config['project']
220
228
  )
221
229
 
@@ -247,7 +255,7 @@ module MU
247
255
  MU.log "Adding #{@config['name']} to Google Cloud project #{@config['project']}", details: @config['roles']
248
256
 
249
257
  begin
250
- MU::Cloud::Google.resource_manager.set_project_iam_policy(
258
+ MU::Cloud::Google.resource_manager(credentials: @config['credentials']).set_project_iam_policy(
251
259
  @config['project'],
252
260
  req_obj
253
261
  )
@@ -63,7 +63,7 @@ module MU
63
63
  # i_pv4_range: @config['ip_block']
64
64
  )
65
65
  MU.log "Creating network #{@mu_name} (#{@config['ip_block']}) in project #{@config['project']}", details: networkobj
66
- resp = MU::Cloud::Google.compute.insert_network(@config['project'], networkobj)
66
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_network(@config['project'], networkobj)
67
67
  @url = resp.self_link # XXX needs to go in notify
68
68
  @cloud_id = resp.name
69
69
 
@@ -83,7 +83,7 @@ module MU
83
83
  network: @url,
84
84
  region: subnet['availability_zone']
85
85
  )
86
- resp = MU::Cloud::Google.compute.insert_subnetwork(@config['project'], subnet['availability_zone'], subnetobj)
86
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_subnetwork(@config['project'], subnet['availability_zone'], subnetobj)
87
87
 
88
88
  }
89
89
  }
@@ -121,15 +121,20 @@ module MU
121
121
  base = MU.structToHash(cloud_desc)
122
122
  base["cloud_id"] = @cloud_id
123
123
  base.merge!(@config.to_h)
124
+ if @config['name'] == "gkeprivate"
125
+ pp base.keys
126
+ puts base['cloud_id']
127
+ end
128
+
124
129
  base
125
130
  end
126
131
 
127
132
  # Describe this VPC from the cloud platform's perspective
128
133
  # @return [Hash]
129
134
  def cloud_desc
130
- @config['project'] ||= MU::Cloud::Google.defaultProject
135
+ @config['project'] ||= MU::Cloud::Google.defaultProject(@config['credentials'])
131
136
 
132
- resp = MU::Cloud::Google.compute.get_network(@config['project'], @cloud_id)
137
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).get_network(@config['project'], @cloud_id)
133
138
  if @cloud_id.nil? or @cloud_id == ""
134
139
  MU.log "Couldn't describe #{self}, @cloud_id #{@cloud_id.nil? ? "undefined" : "empty" }", MU::ERR
135
140
  return nil
@@ -137,7 +142,7 @@ module MU
137
142
 
138
143
  resp = resp.to_h
139
144
  @url ||= resp[:self_link]
140
- routes = MU::Cloud::Google.compute.list_routes(
145
+ routes = MU::Cloud::Google.compute(credentials: @config['credentials']).list_routes(
141
146
  @config['project'],
142
147
  filter: "network eq #{@cloud_id}"
143
148
  ).items
@@ -162,38 +167,66 @@ module MU
162
167
  if !@config['peers'].nil?
163
168
  count = 0
164
169
  @config['peers'].each { |peer|
165
- tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
166
- if peer['vpc']['deploy_id'].nil? and peer['vpc']['vpc_id'].nil? and tag_key.nil?
167
- peer['vpc']['deploy_id'] = @deploy.deploy_id
168
- end
170
+ if peer['vpc']['vpc_name']
171
+ peer_obj = @deploy.findLitterMate(name: peer['vpc']['vpc_name'], type: "vpcs")
172
+ if peer_obj
173
+ if peer_obj.config['peers']
174
+ skipme = false
175
+ peer_obj.config['peers'].each { |peerpeer|
176
+ if peerpeer['vpc']['vpc_name'] == @config['name'] and
177
+ (peer['vpc']['vpc_name'] <=> @config['name']) == -1
178
+ skipme = true
179
+ MU.log "VPCs #{peer['vpc']['vpc_name']} and #{@config['name']} both declare mutual peering connection, ignoring #{@config['name']}'s redundant declaration", MU::DEBUG
180
+ # XXX and if deploy_id matches or is unset
181
+ end
182
+ }
183
+ next if skipme
184
+ end
185
+ end
169
186
 
170
- peer_obj = MU::MommaCat.findStray(
171
- "Google",
172
- "vpcs",
173
- deploy_id: peer['vpc']['deploy_id'],
174
- cloud_id: peer['vpc']['vpc_id'],
175
- name: peer['vpc']['vpc_name'],
176
- tag_key: tag_key,
177
- tag_value: tag_value,
178
- dummy_ok: true
179
- )
187
+ else
188
+ tag_key, tag_value = peer['vpc']['tag'].split(/=/, 2) if !peer['vpc']['tag'].nil?
189
+ if peer['vpc']['deploy_id'].nil? and peer['vpc']['vpc_id'].nil? and tag_key.nil?
190
+ peer['vpc']['deploy_id'] = @deploy.deploy_id
191
+ end
180
192
 
181
- raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil? or peer_obj.first.nil?
193
+ peer_obj = MU::MommaCat.findStray(
194
+ "Google",
195
+ "vpcs",
196
+ deploy_id: peer['vpc']['deploy_id'],
197
+ cloud_id: peer['vpc']['vpc_id'],
198
+ name: peer['vpc']['vpc_name'],
199
+ tag_key: tag_key,
200
+ tag_value: tag_value,
201
+ dummy_ok: true
202
+ ).first
203
+ end
204
+
205
+ raise MuError, "No result looking for #{@mu_name}'s peer VPCs (#{peer['vpc']})" if peer_obj.nil?
182
206
 
183
- url = peer_obj.first.cloudobj.url || peer_obj.first.cloudobj.deploydata['self_link']
207
+ url = if peer_obj.cloudobj.url
208
+ peer_obj.cloudobj.url
209
+ elsif peer_obj.cloudobj.deploydata
210
+ peer_obj.cloudobj.deploydata['self_link']
211
+ else
212
+ pp peer_obj.cloudobj.cloud_desc
213
+ raise MuError, "Can't find the damn URL of my damn peer VPC #{peer['vpc']}"
214
+ end
215
+ cnxn_name = MU::Cloud::Google.nameStr(@mu_name+"-peer-"+count.to_s)
184
216
  peerreq = MU::Cloud::Google.compute(:NetworksAddPeeringRequest).new(
185
- name: MU::Cloud::Google.nameStr(@mu_name+"-peer-"+count.to_s),
217
+ name: cnxn_name,
186
218
  auto_create_routes: true,
187
219
  peer_network: url
188
220
  )
189
221
 
190
- MU.log "Peering #{@mu_name} with #{url}", details: peerreq
191
- MU::Cloud::Google.compute.add_network_peering(
222
+ MU.log "Peering #{@url} with #{url}, connection name is #{cnxn_name}", details: peerreq
223
+
224
+ MU::Cloud::Google.compute(credentials: @config['credentials']).add_network_peering(
192
225
  @config['project'],
193
226
  @cloud_id,
194
227
  peerreq
195
228
  )
196
-
229
+ count += 1
197
230
  }
198
231
  end
199
232
  end
@@ -204,19 +237,19 @@ module MU
204
237
  # @param tag_key [String]: A tag key to search.
205
238
  # @param tag_value [String]: The value of the tag specified by tag_key to match when searching by tag.
206
239
  # @return [Array<Hash<String,OpenStruct>>]: The cloud provider's complete descriptions of matching VPCs
207
- def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {})
208
- flags["project"] ||= MU::Cloud::Google.defaultProject
209
- #MU.log "CALLED MU::Cloud::Google::VPC.find(#{cloud_id}, #{region}, #{tag_key}, #{tag_value}) from #{caller[0]}", MU::NOTICE, details: flags
240
+ def self.find(cloud_id: nil, region: MU.curRegion, tag_key: "Name", tag_value: nil, flags: {}, credentials: nil)
241
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
242
+ #MU.log "CALLED MU::Cloud::Google::VPC.find(#{cloud_id}, #{region}, #{tag_key}, #{tag_value}) with credentials #{credentials} from #{caller[0]}", MU::NOTICE, details: flags
210
243
 
211
244
  resp = {}
212
245
  if cloud_id
213
- vpc = MU::Cloud::Google.compute.get_network(
246
+ vpc = MU::Cloud::Google.compute(credentials: credentials).get_network(
214
247
  flags['project'],
215
248
  cloud_id.to_s.sub(/^.*?\/([^\/]+)$/, '\1')
216
249
  )
217
250
  resp[cloud_id] = vpc if !vpc.nil?
218
251
  else # XXX other criteria
219
- MU::Cloud::Google.compute.list_networks(
252
+ MU::Cloud::Google.compute(credentials: credentials).list_networks(
220
253
  flags["project"]
221
254
  ).items.each { |vpc|
222
255
  resp[vpc.name] = vpc
@@ -224,7 +257,7 @@ module MU
224
257
  end
225
258
  #MU.log "THINGY", MU::WARN, details: resp
226
259
  resp.each_pair { |cloud_id, vpc|
227
- routes = MU::Cloud::Google.compute.list_routes(
260
+ routes = MU::Cloud::Google.compute(credentials: credentials).list_routes(
228
261
  flags["project"],
229
262
  filter: "network eq #{vpc.self_link}"
230
263
  ).items
@@ -259,7 +292,7 @@ module MU
259
292
 
260
293
  resp = nil
261
294
  MU::Cloud::Google.listRegions(@config['us_only']).each { |r|
262
- resp = MU::Cloud::Google.compute.list_subnetworks(
295
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_subnetworks(
263
296
  @config['project'],
264
297
  r,
265
298
  filter: "network eq #{network[:self_link]}"
@@ -415,13 +448,13 @@ module MU
415
448
  # @param target_instance [OpenStruct]: The cloud descriptor of the instance to check.
416
449
  # @param region [String]: The cloud provider region of the target subnet.
417
450
  # @return [Boolean]
418
- def self.haveRouteToInstance?(target_instance, region: MU.curRegion)
419
- project ||= MU::Cloud::Google.defaultProject
451
+ def self.haveRouteToInstance?(target_instance, region: MU.curRegion, credentials: nil)
452
+ project ||= MU::Cloud::Google.defaultProject(credentials)
420
453
  return false if MU.myCloud != "Google"
421
454
  # XXX see if we reside in the same Network and overlap subnets
422
455
  # XXX see if we peer with the target's Network
423
456
  target_instance.network_interfaces.each { |iface|
424
- resp = MU::Cloud::Google.compute.list_routes(
457
+ resp = MU::Cloud::Google.compute(credentials: credentials).list_routes(
425
458
  project,
426
459
  filter: "network eq #{iface.network}"
427
460
  )
@@ -456,19 +489,26 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
456
489
  def self.get_route_tables(subnet_ids: [], vpc_ids: [], region: MU.curRegion)
457
490
  end
458
491
 
492
+ # Does this resource type exist as a global (cloud-wide) artifact, or
493
+ # is it localized to a region/zone?
494
+ # @return [Boolean]
495
+ def self.isGlobal?
496
+ true
497
+ end
498
+
459
499
  # Remove all VPC resources associated with the currently loaded deployment.
460
500
  # @param noop [Boolean]: If true, will only print what would be done
461
501
  # @param ignoremaster [Boolean]: If true, will remove resources not flagged as originating from this Mu server
462
502
  # @param region [String]: The cloud provider region
463
503
  # @return [void]
464
- def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, flags: {})
465
- flags["project"] ||= MU::Cloud::Google.defaultProject
504
+ def self.cleanup(noop: false, ignoremaster: false, region: MU.curRegion, credentials: nil, flags: {})
505
+ flags["project"] ||= MU::Cloud::Google.defaultProject(credentials)
466
506
 
467
- purge_subnets(noop, project: flags['project'])
507
+ purge_subnets(noop, project: flags['project'], credentials: credentials)
468
508
  ["route", "network"].each { |type|
469
509
  # XXX tagged routes aren't showing up in list, and the networks that own them
470
510
  # fail to delete silently
471
- MU::Cloud::Google.compute.delete(
511
+ MU::Cloud::Google.compute(credentials: credentials).delete(
472
512
  type,
473
513
  flags["project"],
474
514
  nil,
@@ -558,6 +598,8 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
558
598
  vpc['route_tables'].each { |tbl|
559
599
  newvpc = {
560
600
  "name" => vpc['name']+"-"+tbl['name'],
601
+ "credentials" => vpc['credentials'],
602
+ "virtual_name" => vpc['name'],
561
603
  "ip_block" => blocks.shift,
562
604
  "route_tables" => [tbl],
563
605
  "parent_block" => vpc['ip_block'],
@@ -587,9 +629,9 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
587
629
  # XXX we need routes to peered Networks too
588
630
 
589
631
  if has_nat or has_deny
590
- ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], false)
632
+ ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], false, credentials: vpc['credentials'])
591
633
  else
592
- ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"])
634
+ ok = false if !genStandardSubnetACLs(vpc['parent_block'] || vpc['ip_block'], vpc['name'], configurator, vpc["project"], credentials: vpc['credentials'])
593
635
  end
594
636
  if has_nat and !has_deny
595
637
  vpc['route_tables'].first["routes"] << {
@@ -613,6 +655,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
613
655
  if route['gateway'] == "#NAT"
614
656
  nat_cfg = MU::Cloud::Google::Server.genericNAT
615
657
  nat_cfg['name'] = vpc['name']+"-natstion-"+nat_count.to_s
658
+ nat_cfg['credentials'] = vpc['credentials']
616
659
  # XXX ingress/egress rules?
617
660
  # XXX for master too if applicable
618
661
  nat_cfg["application_attributes"] = {
@@ -649,10 +692,11 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
649
692
 
650
693
  private
651
694
 
652
- def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true)
695
+ def self.genStandardSubnetACLs(vpc_cidr, vpc_name, configurator, project, publicroute = true, credentials: nil)
653
696
  private_acl = {
654
- "name" => vpc_name+"-routables",
697
+ "name" => vpc_name+"-rt",
655
698
  "cloud" => "Google",
699
+ "credentials" => credentials,
656
700
  "project" => project,
657
701
  "vpc" => { "vpc_name" => vpc_name },
658
702
  "dependencies" => [ { "type" => "vpc", "name" => vpc_name } ],
@@ -712,7 +756,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
712
756
  end
713
757
  # several other cases missing for various types of routers (raw IPs, instance ids, etc) XXX
714
758
  elsif route['gateway'] == "#DENY"
715
- resp = MU::Cloud::Google.compute.list_routes(
759
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).list_routes(
716
760
  @config['project'],
717
761
  filter: "network eq #{network}"
718
762
  )
@@ -721,7 +765,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
721
765
  resp.items.each { |r|
722
766
  next if r.next_hop_gateway.nil? or !r.next_hop_gateway.match(/\/global\/gateways\/default-internet-gateway$/)
723
767
  MU.log "Removing standard route #{r.name} per our #DENY entry"
724
- MU::Cloud::Google.compute.delete_route(@config['project'], r.name)
768
+ MU::Cloud::Google.compute(credentials: @config['credentials']).delete_route(@config['project'], r.name)
725
769
  }
726
770
  end
727
771
  elsif route['gateway'] == "#INTERNET"
@@ -748,11 +792,11 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
748
792
 
749
793
  if route['gateway'] != "#DENY"
750
794
  begin
751
- MU::Cloud::Google.compute.get_route(@config['project'], routename)
795
+ MU::Cloud::Google.compute(credentials: @config['credentials']).get_route(@config['project'], routename)
752
796
  rescue ::Google::Apis::ClientError, MU::MuError => e
753
797
  if e.message.match(/notFound/)
754
798
  MU.log "Creating route #{routename} in project #{@config['project']}", details: routeobj
755
- resp = MU::Cloud::Google.compute.insert_route(@config['project'], routeobj)
799
+ resp = MU::Cloud::Google.compute(credentials: @config['credentials']).insert_route(@config['project'], routeobj)
756
800
  else
757
801
  # TODO can't update GCP routes, would have to delete and re-create
758
802
  end
@@ -786,7 +830,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
786
830
 
787
831
  # Remove all network interfaces associated with the currently loaded deployment.
788
832
  # @param noop [Boolean]: If true, will only print what would be done
789
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
833
+ # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
790
834
  # @param region [String]: The cloud provider region
791
835
  # @return [void]
792
836
  def self.purge_interfaces(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
@@ -794,16 +838,17 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
794
838
 
795
839
  # Remove all subnets associated with the currently loaded deployment.
796
840
  # @param noop [Boolean]: If true, will only print what would be done
797
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
841
+ # @param tagfilters [Array<Hash>]: Labels to filter against when search for resources to purge
798
842
  # @param regions [Array<String>]: The cloud provider regions to check
799
843
  # @return [void]
800
- def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: MU::Cloud::Google.defaultProject)
844
+ def self.purge_subnets(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], regions: MU::Cloud::Google.listRegions, project: nil, credentials: nil)
845
+ project ||= MU::Cloud::Google.defaultProject(credentials)
801
846
  parent_thread_id = Thread.current.object_id
802
847
  regionthreads = []
803
848
  regions.each { |r|
804
849
  regionthreads << Thread.new {
805
850
  MU.dupGlobals(parent_thread_id)
806
- MU::Cloud::Google.compute.delete(
851
+ MU::Cloud::Google.compute(credentials: credentials).delete(
807
852
  "subnetwork",
808
853
  project,
809
854
  r,
@@ -816,23 +861,6 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
816
861
  end
817
862
  end
818
863
 
819
- # Remove all DHCP options sets associated with the currently loaded
820
- # deployment.
821
- # @param noop [Boolean]: If true, will only print what would be done
822
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
823
- # @param region [String]: The cloud provider region
824
- # @return [void]
825
- def self.purge_dhcpopts(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
826
- end
827
-
828
- # Remove all VPCs associated with the currently loaded deployment.
829
- # @param noop [Boolean]: If true, will only print what would be done
830
- # @param tagfilters [Array<Hash>]: EC2 tags to filter against when search for resources to purge
831
- # @param region [String]: The cloud provider region
832
- # @return [void]
833
- def self.purge_vpcs(noop = false, tagfilters = [{name: "tag:MU-ID", values: [MU.deploy_id]}], region: MU.curRegion)
834
- end
835
-
836
864
  protected
837
865
 
838
866
  # Subnets are almost a first-class resource. So let's kinda sorta treat
@@ -869,7 +897,7 @@ MU.log "ROUTES TO #{target_instance.name}", MU::WARN, details: resp
869
897
  # Is this subnet privately-routable only, or public?
870
898
  # @return [Boolean]
871
899
  def private?
872
- routes = MU::Cloud::Google.compute.list_routes(
900
+ routes = MU::Cloud::Google.compute(credentials: @parent.config['credentials']).list_routes(
873
901
  @parent.config['project'],
874
902
  filter: "network eq #{@parent.url}"
875
903
  ).items