cloud-mu 3.1.4 → 3.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/README.md +33 -0
- data/ansible/roles/mu-windows/defaults/main.yml +2 -0
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/handlers/main.yml +2 -0
- data/ansible/roles/mu-windows/meta/main.yml +53 -0
- data/ansible/roles/mu-windows/tasks/main.yml +36 -0
- data/ansible/roles/mu-windows/tests/inventory +2 -0
- data/ansible/roles/mu-windows/tests/test.yml +5 -0
- data/ansible/roles/mu-windows/vars/main.yml +2 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +5 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
- data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
- data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +18 -13
- data/extras/image-generators/AWS/win2k16.yaml +18 -13
- data/extras/image-generators/AWS/win2k19.yaml +21 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +158 -107
- data/modules/mu/adoption.rb +386 -59
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1632
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +926 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +135 -82
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server.yml +1 -0
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +91 -68
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +33 -19
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +123 -21
- data/modules/mu/groomers/chef.rb +64 -11
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/master/ssl.rb +0 -1
- data/modules/mu/mommacat.rb +154 -867
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +495 -0
- data/modules/mu/mommacat/storage.rb +225 -192
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +281 -64
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +708 -749
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +75 -57
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +212 -242
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +50 -41
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +94 -57
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +173 -42
- data/modules/mu/{clouds → providers}/aws/server.rb +782 -1107
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +36 -46
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
- data/modules/mu/{clouds → providers}/aws/vpc.rb +429 -849
- data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +5 -5
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +59 -37
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +14 -8
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +142 -55
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +46 -15
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +23 -0
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/includes-and-params.yaml +2 -1
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +2 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/modules/tests/win2k12.yaml +25 -0
- data/modules/tests/win2k16.yaml +25 -0
- data/modules/tests/win2k19.yaml +25 -0
- data/requirements.txt +1 -0
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +169 -93
- data/extras/image-generators/AWS/windows.yaml +0 -18
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
- data/modules/tests/needwork/win2k12.yaml +0 -13
data/modules/mu/cleanup.rb
CHANGED
@@ -30,6 +30,10 @@ module MU
|
|
30
30
|
@onlycloud = false
|
31
31
|
@skipcloud = false
|
32
32
|
|
33
|
+
# Resource types, in the order in which we generally have to clean them up
|
34
|
+
# to disentangle them from one another.
|
35
|
+
TYPES_IN_ORDER = ["Collection", "CDN", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "Job", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
|
36
|
+
|
33
37
|
# Purge all resources associated with a deployment.
|
34
38
|
# @param deploy_id [String]: The identifier of the deployment to remove (typically seen in the MU-ID tag on a resource).
|
35
39
|
# @param noop [Boolean]: Do not delete resources, merely list what would be deleted.
|
@@ -48,20 +52,14 @@ module MU
|
|
48
52
|
@onlycloud = onlycloud
|
49
53
|
@skipcloud = skipcloud
|
50
54
|
@ignoremaster = ignoremaster
|
55
|
+
@deploy_id = deploy_id
|
51
56
|
|
52
57
|
if @skipcloud and @onlycloud # you actually mean noop
|
53
58
|
@onlycloud = @skipcloud = false
|
54
59
|
@noop = true
|
55
60
|
end
|
56
61
|
|
57
|
-
|
58
|
-
MU.setVar("dataDir", Etc.getpwnam(MU.mu_user).dir+"/.mu/var")
|
59
|
-
else
|
60
|
-
MU.setVar("dataDir", MU.mainDataDir)
|
61
|
-
end
|
62
|
-
|
63
|
-
|
64
|
-
types_in_order = ["Collection", "Endpoint", "Function", "ServerPool", "ContainerCluster", "SearchDomain", "Server", "MsgQueue", "Database", "CacheCluster", "StoragePool", "LoadBalancer", "NoSQLDB", "FirewallRule", "Alarm", "Notifier", "Log", "VPC", "Role", "Group", "User", "Bucket", "DNSZone", "Collection"]
|
62
|
+
MU.setVar("dataDir", (MU.mu_user == "mu" ? MU.mainDataDir : Etc.getpwnam(MU.mu_user).dir+"/.mu/var"))
|
65
63
|
|
66
64
|
# Load up our deployment metadata
|
67
65
|
if !mommacat.nil?
|
@@ -82,172 +80,24 @@ module MU
|
|
82
80
|
rescue StandardError => e
|
83
81
|
MU.log "Can't load a deploy record for #{deploy_id} (#{e.inspect}), cleaning up resources by guesswork", MU::WARN, details: e.backtrace
|
84
82
|
MU.setVar("deploy_id", deploy_id)
|
85
|
-
|
86
83
|
end
|
87
84
|
end
|
88
85
|
|
89
|
-
regionsused = @mommacat.regionsUsed if @mommacat
|
90
|
-
credsused = @mommacat.credsUsed if @mommacat
|
91
|
-
habitatsused = @mommacat.habitatsUsed if @mommacat
|
86
|
+
@regionsused = @mommacat.regionsUsed if @mommacat
|
87
|
+
@credsused = @mommacat.credsUsed if @mommacat
|
88
|
+
@habitatsused = @mommacat.habitatsUsed if @mommacat
|
92
89
|
|
93
90
|
if !@skipcloud
|
94
|
-
creds =
|
95
|
-
MU::Cloud.availableClouds.each { |cloud|
|
96
|
-
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
|
97
|
-
if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
|
98
|
-
creds[cloud] ||= {}
|
99
|
-
cloudclass.listCredentials.each { |credset|
|
100
|
-
next if credsets and credsets.size > 0 and !credsets.include?(credset)
|
101
|
-
next if credsused and credsused.size > 0 and !credsused.include?(credset)
|
102
|
-
MU.log "Will scan #{cloud} with credentials #{credset}"
|
103
|
-
creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
|
104
|
-
}
|
105
|
-
else
|
106
|
-
if cloudclass.hosted?
|
107
|
-
creds[cloud] ||= {}
|
108
|
-
creds[cloud]["#default"] = cloudclass.listRegions
|
109
|
-
end
|
110
|
-
end
|
111
|
-
}
|
91
|
+
creds = listUsedCredentials(credsets)
|
112
92
|
|
113
|
-
parent_thread_id = Thread.current.object_id
|
114
93
|
cloudthreads = []
|
115
|
-
|
94
|
+
|
116
95
|
had_failures = false
|
117
96
|
|
118
97
|
creds.each_pair { |provider, credsets_outer|
|
119
98
|
cloudthreads << Thread.new(provider, credsets_outer) { |cloud, credsets_inner|
|
120
|
-
MU.dupGlobals(parent_thread_id)
|
121
99
|
Thread.abort_on_exception = false
|
122
|
-
|
123
|
-
habitatclass = Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get("Habitat")
|
124
|
-
credsets_inner.each_pair { |credset, acct_regions|
|
125
|
-
next if credsused and !credsused.include?(credset)
|
126
|
-
global_vs_region_semaphore = Mutex.new
|
127
|
-
global_done = {}
|
128
|
-
regionthreads = []
|
129
|
-
acct_regions.each { |r|
|
130
|
-
if regionsused
|
131
|
-
if regionsused.size > 0
|
132
|
-
next if !regionsused.include?(r)
|
133
|
-
else
|
134
|
-
next if r != cloudclass.myRegion(credset)
|
135
|
-
end
|
136
|
-
end
|
137
|
-
if regions and !regions.empty?
|
138
|
-
next if !regions.include?(r)
|
139
|
-
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}...", MU::NOTICE
|
140
|
-
end
|
141
|
-
regionthreads << Thread.new {
|
142
|
-
MU.dupGlobals(parent_thread_id)
|
143
|
-
Thread.abort_on_exception = false
|
144
|
-
MU.setVar("curRegion", r)
|
145
|
-
projects = []
|
146
|
-
if habitats
|
147
|
-
projects = habitats
|
148
|
-
else
|
149
|
-
if $MU_CFG and $MU_CFG[cloud.downcase] and
|
150
|
-
$MU_CFG[cloud.downcase][credset] and
|
151
|
-
$MU_CFG[cloud.downcase][credset]["project"]
|
152
|
-
# XXX GCP credential schema needs an array for projects
|
153
|
-
projects << $MU_CFG[cloud.downcase][credset]["project"]
|
154
|
-
end
|
155
|
-
begin
|
156
|
-
projects.concat(cloudclass.listProjects(credset))
|
157
|
-
rescue NoMethodError
|
158
|
-
end
|
159
|
-
end
|
160
|
-
|
161
|
-
if projects == []
|
162
|
-
projects << "" # dummy
|
163
|
-
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}", MU::NOTICE
|
164
|
-
end
|
165
|
-
projects.uniq!
|
166
|
-
|
167
|
-
# We do these in an order that unrolls dependent resources
|
168
|
-
# sensibly, and we hit :Collection twice because AWS
|
169
|
-
# CloudFormation sometimes fails internally.
|
170
|
-
projectthreads = []
|
171
|
-
projects.each { |project|
|
172
|
-
if habitats and !habitats.empty? and project != ""
|
173
|
-
next if !habitats.include?(project)
|
174
|
-
end
|
175
|
-
if habitatsused and !habitatsused.empty? and project != ""
|
176
|
-
next if !habitatsused.include?(project)
|
177
|
-
end
|
178
|
-
next if !habitatclass.isLive?(project, credset)
|
179
|
-
|
180
|
-
projectthreads << Thread.new {
|
181
|
-
MU.dupGlobals(parent_thread_id)
|
182
|
-
MU.setVar("curRegion", r)
|
183
|
-
Thread.abort_on_exception = false
|
184
|
-
if project != ""
|
185
|
-
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}, project #{project}", MU::NOTICE
|
186
|
-
end
|
187
|
-
|
188
|
-
MU.dupGlobals(parent_thread_id)
|
189
|
-
flags = {
|
190
|
-
"project" => project,
|
191
|
-
"onlycloud" => @onlycloud,
|
192
|
-
"skipsnapshots" => @skipsnapshots,
|
193
|
-
}
|
194
|
-
types_in_order.each { |t|
|
195
|
-
begin
|
196
|
-
skipme = false
|
197
|
-
global_vs_region_semaphore.synchronize {
|
198
|
-
MU::Cloud.loadCloudType(cloud, t)
|
199
|
-
if Object.const_get("MU").const_get("Cloud").const_get(cloud).const_get(t).isGlobal?
|
200
|
-
global_done[project] ||= []
|
201
|
-
if !global_done[project].include?(t)
|
202
|
-
global_done[project] << t
|
203
|
-
flags['global'] = true
|
204
|
-
else
|
205
|
-
skipme = true
|
206
|
-
end
|
207
|
-
end
|
208
|
-
}
|
209
|
-
next if skipme
|
210
|
-
rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
|
211
|
-
next
|
212
|
-
rescue MU::MuError, NoMethodError => e
|
213
|
-
MU.log "While checking mu/clouds/#{cloud.downcase}/#{cloudclass.cfg_name} for global-ness in cleanup: "+e.message, MU::WARN
|
214
|
-
next
|
215
|
-
rescue ::Aws::EC2::Errors::AuthFailure, ::Google::Apis::ClientError => e
|
216
|
-
MU.log e.message+" in "+r, MU::ERR
|
217
|
-
next
|
218
|
-
end
|
219
|
-
|
220
|
-
begin
|
221
|
-
if !self.call_cleanup(t, credset, cloud, flags, r)
|
222
|
-
had_failures = true
|
223
|
-
end
|
224
|
-
rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented => e
|
225
|
-
next
|
226
|
-
end
|
227
|
-
}
|
228
|
-
} # types_in_order.each { |t|
|
229
|
-
} # projects.each { |project|
|
230
|
-
projectthreads.each do |t|
|
231
|
-
t.join
|
232
|
-
end
|
233
|
-
|
234
|
-
# XXX move to MU::AWS
|
235
|
-
if cloud == "AWS"
|
236
|
-
resp = MU::Cloud::AWS.ec2(region: r, credentials: credset).describe_key_pairs(
|
237
|
-
filters: [{name: "key-name", values: [keyname]}]
|
238
|
-
)
|
239
|
-
resp.data.key_pairs.each { |keypair|
|
240
|
-
MU.log "Deleting key pair #{keypair.key_name} from #{r}"
|
241
|
-
MU::Cloud::AWS.ec2(region: r, credentials: credset).delete_key_pair(key_name: keypair.key_name) if !@noop
|
242
|
-
}
|
243
|
-
end
|
244
|
-
} # regionthreads << Thread.new {
|
245
|
-
} # acct_regions.each { |r|
|
246
|
-
regionthreads.each do |t|
|
247
|
-
t.join
|
248
|
-
end
|
249
|
-
|
250
|
-
} # credsets.each_pair { |credset, acct_regions|
|
100
|
+
cleanCloud(cloud, habitats, regions, credsets_inner)
|
251
101
|
} # cloudthreads << Thread.new(provider, credsets) { |cloud, credsets_outer|
|
252
102
|
cloudthreads.each do |t|
|
253
103
|
t.join
|
@@ -259,24 +109,21 @@ module MU
|
|
259
109
|
# once they're all done.
|
260
110
|
creds.each_pair { |provider, credsets_inner|
|
261
111
|
credsets_inner.keys.each { |credset|
|
262
|
-
next if credsused and
|
112
|
+
next if @credsused and !@credsused.include?(credset)
|
263
113
|
["Habitat", "Folder"].each { |t|
|
264
114
|
flags = {
|
265
115
|
"onlycloud" => @onlycloud,
|
266
116
|
"skipsnapshots" => @skipsnapshots
|
267
117
|
}
|
268
|
-
if !
|
118
|
+
if !call_cleanup(t, credset, provider, flags, nil)
|
269
119
|
had_failures = true
|
270
120
|
end
|
271
121
|
}
|
272
122
|
}
|
273
123
|
}
|
274
124
|
|
275
|
-
MU::Cloud::Google.removeDeploySecretsAndRoles(MU.deploy_id)
|
276
|
-
# XXX port AWS equivalent behavior and add a MU::Cloud wrapper
|
277
|
-
|
278
125
|
creds.each_pair { |provider, credsets_inner|
|
279
|
-
cloudclass =
|
126
|
+
cloudclass = MU::Cloud.cloudClass(provider)
|
280
127
|
credsets_inner.keys.each { |c|
|
281
128
|
cloudclass.cleanDeploy(MU.deploy_id, credentials: c, noop: @noop)
|
282
129
|
}
|
@@ -284,44 +131,16 @@ module MU
|
|
284
131
|
end
|
285
132
|
|
286
133
|
# Scrub any residual Chef records with matching tags
|
287
|
-
if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0) and
|
288
|
-
|
289
|
-
MU::Groomer
|
290
|
-
|
291
|
-
|
292
|
-
end
|
293
|
-
deadnodes = []
|
294
|
-
Chef::Config[:environment] = MU.environment
|
295
|
-
q = Chef::Search::Query.new
|
296
|
-
begin
|
297
|
-
q.search("node", "tags_MU-ID:#{MU.deploy_id}").each { |item|
|
298
|
-
next if item.is_a?(Integer)
|
299
|
-
item.each { |node|
|
300
|
-
deadnodes << node.name
|
301
|
-
}
|
302
|
-
}
|
303
|
-
rescue Net::HTTPServerException
|
304
|
-
end
|
305
|
-
|
306
|
-
begin
|
307
|
-
q.search("node", "name:#{MU.deploy_id}-*").each { |item|
|
308
|
-
next if item.is_a?(Integer)
|
309
|
-
item.each { |node|
|
310
|
-
deadnodes << node.name
|
311
|
-
}
|
312
|
-
}
|
313
|
-
rescue Net::HTTPServerException
|
314
|
-
end
|
315
|
-
MU.log "Missed some Chef resources in node cleanup, purging now", MU::NOTICE if deadnodes.size > 0
|
316
|
-
deadnodes.uniq.each { |node|
|
317
|
-
MU::Groomer::Chef.cleanup(node, [], noop)
|
318
|
-
}
|
319
|
-
rescue LoadError
|
320
|
-
end
|
134
|
+
if !@onlycloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0) and !@noop
|
135
|
+
MU.supportedGroomers.each { |g|
|
136
|
+
groomer = MU::Groomer.loadGroomer(g)
|
137
|
+
groomer.cleanup(MU.deploy_id, @noop)
|
138
|
+
}
|
321
139
|
end
|
322
140
|
|
323
141
|
if had_failures
|
324
142
|
MU.log "Had cleanup failures, exiting", MU::ERR
|
143
|
+
File.unlink("#{deploy_dir}/.cleanup") if !@noop
|
325
144
|
exit 1
|
326
145
|
end
|
327
146
|
|
@@ -329,99 +148,192 @@ module MU
|
|
329
148
|
@mommacat.purge!
|
330
149
|
end
|
331
150
|
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
ssharchive = "#{sshdir}/archive"
|
336
|
-
|
337
|
-
Dir.mkdir(sshdir, 0700) if !Dir.exist?(sshdir) and !@noop
|
338
|
-
Dir.mkdir(ssharchive, 0700) if !Dir.exist?(ssharchive) and !@noop
|
151
|
+
if !@onlycloud
|
152
|
+
MU::Master.purgeDeployFromSSH(MU.deploy_id, noop: @noop)
|
153
|
+
end
|
339
154
|
|
340
|
-
|
341
|
-
|
342
|
-
MU.log "Moving #{sshdir}/#{keyname} to #{ssharchive}/#{keyname}"
|
343
|
-
if !@noop
|
344
|
-
File.rename("#{sshdir}/#{keyname}", "#{ssharchive}/#{keyname}")
|
345
|
-
end
|
155
|
+
if !@noop and !@skipcloud and (@mommacat.nil? or @mommacat.numKittens(types: ["Server", "ServerPool"]) > 0)
|
156
|
+
# MU::Master.syncMonitoringConfig
|
346
157
|
end
|
347
158
|
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
361
|
-
end
|
362
|
-
newlines << line if !delete_block
|
363
|
-
}
|
364
|
-
f.rewind
|
365
|
-
f.truncate(0)
|
366
|
-
f.puts(newlines)
|
367
|
-
f.flush
|
368
|
-
f.flock(File::LOCK_UN)
|
159
|
+
end
|
160
|
+
|
161
|
+
def self.listUsedCredentials(credsets)
|
162
|
+
creds = {}
|
163
|
+
MU::Cloud.availableClouds.each { |cloud|
|
164
|
+
cloudclass = MU::Cloud.cloudClass(cloud)
|
165
|
+
if $MU_CFG[cloud.downcase] and $MU_CFG[cloud.downcase].size > 0
|
166
|
+
creds[cloud] ||= {}
|
167
|
+
cloudclass.listCredentials.each { |credset|
|
168
|
+
next if credsets and credsets.size > 0 and !credsets.include?(credset)
|
169
|
+
next if @credsused and @credsused.size > 0 and !@credsused.include?(credset)
|
170
|
+
MU.log "Will scan #{cloud} with credentials #{credset}"
|
171
|
+
creds[cloud][credset] = cloudclass.listRegions(credentials: credset)
|
369
172
|
}
|
173
|
+
else
|
174
|
+
if cloudclass.hosted?
|
175
|
+
creds[cloud] ||= {}
|
176
|
+
creds[cloud]["#default"] = cloudclass.listRegions
|
177
|
+
end
|
370
178
|
end
|
371
|
-
|
372
|
-
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
378
|
-
|
379
|
-
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
386
|
-
|
387
|
-
|
388
|
-
|
389
|
-
|
390
|
-
f.flock(File::LOCK_UN)
|
391
|
-
}
|
179
|
+
}
|
180
|
+
creds
|
181
|
+
end
|
182
|
+
private_class_method :listUsedCredentials
|
183
|
+
|
184
|
+
def self.cleanCloud(cloud, habitats, regions, credsets)
|
185
|
+
cloudclass = MU::Cloud.cloudClass(cloud)
|
186
|
+
credsets.each_pair { |credset, acct_regions|
|
187
|
+
next if @credsused and !@credsused.include?(credset)
|
188
|
+
global_vs_region_semaphore = Mutex.new
|
189
|
+
global_done = {}
|
190
|
+
regionthreads = []
|
191
|
+
acct_regions.each { |r|
|
192
|
+
if @regionsused
|
193
|
+
if @regionsused.size > 0
|
194
|
+
next if !@regionsused.include?(r)
|
195
|
+
else
|
196
|
+
next if r != cloudclass.myRegion(credset)
|
197
|
+
end
|
392
198
|
end
|
393
|
-
|
394
|
-
|
199
|
+
if regions and !regions.empty?
|
200
|
+
next if !regions.include?(r)
|
201
|
+
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{r}...", MU::NOTICE
|
202
|
+
end
|
203
|
+
regionthreads << Thread.new {
|
204
|
+
Thread.abort_on_exception = false
|
205
|
+
MU.setVar("curRegion", r)
|
206
|
+
cleanRegion(cloud, credset, r, global_vs_region_semaphore, global_done, habitats)
|
207
|
+
} # regionthreads << Thread.new {
|
208
|
+
} # acct_regions.each { |r|
|
209
|
+
regionthreads.each do |t|
|
210
|
+
t.join
|
211
|
+
end
|
212
|
+
}
|
213
|
+
end
|
214
|
+
private_class_method :cleanCloud
|
215
|
+
|
216
|
+
def self.cleanRegion(cloud, credset, region, global_vs_region_semaphore, global_done, habitats)
|
217
|
+
had_failures = false
|
218
|
+
cloudclass = MU::Cloud.cloudClass(cloud)
|
219
|
+
habitatclass = MU::Cloud.resourceClass(cloud, "Habitat")
|
220
|
+
|
221
|
+
if !habitats
|
222
|
+
habitats = []
|
223
|
+
if $MU_CFG and $MU_CFG[cloud.downcase] and
|
224
|
+
$MU_CFG[cloud.downcase][credset] and
|
225
|
+
$MU_CFG[cloud.downcase][credset]["project"]
|
226
|
+
# XXX GCP credential schema needs an array for projects
|
227
|
+
habitats << $MU_CFG[cloud.downcase][credset]["project"]
|
228
|
+
end
|
229
|
+
begin
|
230
|
+
habitats.concat(cloudclass.listHabitats(credset, use_cache: false))
|
231
|
+
rescue NoMethodError
|
395
232
|
end
|
396
233
|
end
|
397
234
|
|
398
|
-
if
|
399
|
-
|
400
|
-
|
401
|
-
|
402
|
-
|
403
|
-
|
235
|
+
if habitats == []
|
236
|
+
habitats << "" # dummy
|
237
|
+
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}", MU::NOTICE
|
238
|
+
end
|
239
|
+
habitats.uniq!
|
240
|
+
|
241
|
+
# We do these in an order that unrolls dependent resources
|
242
|
+
# sensibly, and we hit :Collection twice because AWS
|
243
|
+
# CloudFormation sometimes fails internally.
|
244
|
+
habitat_threads = []
|
245
|
+
habitats.each { |habitat|
|
246
|
+
if habitats and !habitats.empty? and habitat != ""
|
247
|
+
next if !habitats.include?(habitat)
|
404
248
|
end
|
405
|
-
if
|
406
|
-
|
407
|
-
MU::Cloud::Google.storage.delete_object(
|
408
|
-
MU.adminBucketName,
|
409
|
-
"#{MU.deploy_id}-secret"
|
410
|
-
)
|
411
|
-
rescue ::Google::Apis::ClientError => e
|
412
|
-
raise e if !e.message.match(/^notFound: /)
|
413
|
-
end
|
249
|
+
if @habitatsused and !@habitatsused.empty? and habitat != ""
|
250
|
+
next if !@habitatsused.include?(habitat)
|
414
251
|
end
|
415
|
-
if
|
416
|
-
|
252
|
+
next if !habitatclass.isLive?(habitat, credset)
|
253
|
+
|
254
|
+
habitat_threads << Thread.new {
|
255
|
+
Thread.current.thread_variable_set("name", "#{cloud}/#{credset}/#{habitat}/#{region}")
|
256
|
+
Thread.abort_on_exception = false
|
257
|
+
if !cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
|
258
|
+
had_failures = true
|
259
|
+
end
|
260
|
+
} # TYPES_IN_ORDER.each { |t|
|
261
|
+
} # habitats.each { |habitat|
|
262
|
+
|
263
|
+
last_checkin = Time.now
|
264
|
+
begin
|
265
|
+
deletia = []
|
266
|
+
habitat_threads.each { |t|
|
267
|
+
if !t.status
|
268
|
+
t.join
|
269
|
+
deletia << t
|
270
|
+
end
|
271
|
+
}
|
272
|
+
deletia.each { |t|
|
273
|
+
habitat_threads.delete(t)
|
274
|
+
}
|
275
|
+
if (Time.now - last_checkin) > 120
|
276
|
+
list = habitat_threads.map { |t|
|
277
|
+
t.thread_variable_get("name") + (t.thread_variable_get("type") ? "/"+t.thread_variable_get("type") : "")
|
278
|
+
}
|
279
|
+
MU.log "Waiting on #{habitat_threads.size.to_s} habitat#{habitat_threads.size > 1 ? "s" : ""} in region #{region}", MU::NOTICE, details: list
|
280
|
+
last_checkin = Time.now
|
417
281
|
end
|
418
|
-
|
282
|
+
sleep 10 if !habitat_threads.empty?
|
283
|
+
end while !habitat_threads.empty?
|
419
284
|
|
420
|
-
|
421
|
-
|
285
|
+
had_failures
|
286
|
+
end
|
287
|
+
private_class_method :cleanRegion
|
288
|
+
|
289
|
+
def self.cleanHabitat(cloud, credset, region, habitat, global_vs_region_semaphore, global_done)
|
290
|
+
had_failures = false
|
291
|
+
if habitat != ""
|
292
|
+
MU.log "Checking for #{cloud}/#{credset} resources from #{MU.deploy_id} in #{region}, habitat #{habitat}", MU::NOTICE
|
422
293
|
end
|
423
294
|
|
295
|
+
flags = {
|
296
|
+
"habitat" => habitat,
|
297
|
+
"onlycloud" => @onlycloud,
|
298
|
+
"skipsnapshots" => @skipsnapshots,
|
299
|
+
}
|
300
|
+
TYPES_IN_ORDER.each { |t|
|
301
|
+
begin
|
302
|
+
skipme = false
|
303
|
+
global_vs_region_semaphore.synchronize {
|
304
|
+
if MU::Cloud.resourceClass(cloud, t).isGlobal?
|
305
|
+
global_done[habitat] ||= []
|
306
|
+
if !global_done[habitat].include?(t)
|
307
|
+
global_done[habitat] << t
|
308
|
+
flags['global'] = true
|
309
|
+
else
|
310
|
+
skipme = true
|
311
|
+
end
|
312
|
+
end
|
313
|
+
}
|
314
|
+
next if skipme
|
315
|
+
rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented
|
316
|
+
next
|
317
|
+
rescue MU::MuError, NoMethodError => e
|
318
|
+
MU.log "While checking mu/providers/#{cloud.downcase}/#{cloudclass.cfg_name} for global-ness in cleanup: "+e.message, MU::WARN
|
319
|
+
next
|
320
|
+
rescue ::Aws::EC2::Errors::AuthFailure, ::Google::Apis::ClientError => e
|
321
|
+
MU.log e.message+" in "+region, MU::ERR
|
322
|
+
next
|
323
|
+
end
|
324
|
+
|
325
|
+
begin
|
326
|
+
if !call_cleanup(t, credset, cloud, flags, region)
|
327
|
+
had_failures = true
|
328
|
+
end
|
329
|
+
rescue MU::Cloud::MuDefunctHabitat, MU::Cloud::MuCloudResourceNotImplemented
|
330
|
+
next
|
331
|
+
end
|
332
|
+
}
|
333
|
+
|
334
|
+
had_failures
|
424
335
|
end
|
336
|
+
private_class_method :cleanHabitat
|
425
337
|
|
426
338
|
# Wrapper for dynamically invoking resource type cleanup methods.
|
427
339
|
# @param type [String]:
|
@@ -430,38 +342,37 @@ module MU
|
|
430
342
|
# @param flags [Hash]:
|
431
343
|
# @param region [String]:
|
432
344
|
def self.call_cleanup(type, credset, provider, flags, region)
|
345
|
+
Thread.current.thread_variable_set("type", type)
|
433
346
|
if @mommacat.nil? or @mommacat.numKittens(types: [type]) > 0
|
434
347
|
if @mommacat
|
348
|
+
|
435
349
|
found = @mommacat.findLitterMate(type: type, return_all: true, credentials: credset)
|
436
|
-
|
437
|
-
if found
|
438
|
-
found.
|
439
|
-
|
440
|
-
|
441
|
-
|
442
|
-
|
443
|
-
|
444
|
-
|
350
|
+
|
351
|
+
if found
|
352
|
+
flags['known'] = if found.is_a?(Array)
|
353
|
+
found.map { |k| k.cloud_id }
|
354
|
+
elsif found.is_a?(Hash)
|
355
|
+
found.each_value.map { |k| k.cloud_id }
|
356
|
+
else
|
357
|
+
[found.cloud_id]
|
358
|
+
end
|
445
359
|
end
|
446
360
|
end
|
447
|
-
|
448
|
-
|
449
|
-
|
450
|
-
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
456
|
-
|
457
|
-
)
|
458
|
-
# rescue ::Seahorse::Client::NetworkingError => e
|
459
|
-
# MU.log "Service not available in AWS region #{r}, skipping", MU::DEBUG, details: e.message
|
460
|
-
# end
|
361
|
+
|
362
|
+
MU::Cloud.loadBaseType(type).cleanup(
|
363
|
+
noop: @noop,
|
364
|
+
ignoremaster: @ignoremaster,
|
365
|
+
region: region,
|
366
|
+
cloud: provider,
|
367
|
+
flags: flags,
|
368
|
+
credentials: credset,
|
369
|
+
deploy_id: @deploy_id
|
370
|
+
)
|
461
371
|
else
|
462
372
|
true
|
463
373
|
end
|
464
|
-
|
465
374
|
end
|
375
|
+
private_class_method :call_cleanup
|
376
|
+
|
466
377
|
end #class
|
467
378
|
end #module
|