cloud-mu 3.1.5 → 3.3.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +5 -1
- data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
- data/ansible/roles/mu-windows/files/config.xml +76 -0
- data/ansible/roles/mu-windows/tasks/main.yml +16 -0
- data/bin/mu-adopt +16 -12
- data/bin/mu-azure-tests +57 -0
- data/bin/mu-cleanup +2 -4
- data/bin/mu-configure +52 -0
- data/bin/mu-deploy +3 -3
- data/bin/mu-findstray-tests +25 -0
- data/bin/mu-gen-docs +2 -4
- data/bin/mu-load-config.rb +2 -1
- data/bin/mu-node-manage +15 -16
- data/bin/mu-run-tests +37 -12
- data/cloud-mu.gemspec +3 -3
- data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
- data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
- data/cookbooks/mu-tools/libraries/helper.rb +1 -1
- data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
- data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
- data/cookbooks/mu-tools/recipes/eks.rb +2 -2
- data/cookbooks/mu-tools/recipes/windows-client.rb +25 -22
- data/extras/clean-stock-amis +25 -19
- data/extras/generate-stock-images +1 -0
- data/extras/image-generators/AWS/win2k12.yaml +2 -0
- data/extras/image-generators/AWS/win2k16.yaml +2 -0
- data/extras/image-generators/AWS/win2k19.yaml +2 -0
- data/modules/mommacat.ru +1 -1
- data/modules/mu.rb +86 -98
- data/modules/mu/adoption.rb +373 -58
- data/modules/mu/cleanup.rb +214 -303
- data/modules/mu/cloud.rb +128 -1733
- data/modules/mu/cloud/database.rb +49 -0
- data/modules/mu/cloud/dnszone.rb +44 -0
- data/modules/mu/cloud/machine_images.rb +212 -0
- data/modules/mu/cloud/providers.rb +81 -0
- data/modules/mu/cloud/resource_base.rb +929 -0
- data/modules/mu/cloud/server.rb +40 -0
- data/modules/mu/cloud/server_pool.rb +1 -0
- data/modules/mu/cloud/ssh_sessions.rb +228 -0
- data/modules/mu/cloud/winrm_sessions.rb +237 -0
- data/modules/mu/cloud/wrappers.rb +169 -0
- data/modules/mu/config.rb +123 -81
- data/modules/mu/config/alarm.rb +2 -6
- data/modules/mu/config/bucket.rb +32 -3
- data/modules/mu/config/cache_cluster.rb +2 -2
- data/modules/mu/config/cdn.rb +100 -0
- data/modules/mu/config/collection.rb +1 -1
- data/modules/mu/config/container_cluster.rb +7 -2
- data/modules/mu/config/database.rb +84 -105
- data/modules/mu/config/database.yml +1 -2
- data/modules/mu/config/dnszone.rb +5 -4
- data/modules/mu/config/doc_helpers.rb +5 -6
- data/modules/mu/config/endpoint.rb +2 -1
- data/modules/mu/config/firewall_rule.rb +3 -19
- data/modules/mu/config/folder.rb +1 -1
- data/modules/mu/config/function.rb +17 -8
- data/modules/mu/config/group.rb +1 -1
- data/modules/mu/config/habitat.rb +1 -1
- data/modules/mu/config/job.rb +89 -0
- data/modules/mu/config/loadbalancer.rb +57 -11
- data/modules/mu/config/log.rb +1 -1
- data/modules/mu/config/msg_queue.rb +1 -1
- data/modules/mu/config/nosqldb.rb +1 -1
- data/modules/mu/config/notifier.rb +8 -19
- data/modules/mu/config/ref.rb +92 -14
- data/modules/mu/config/role.rb +1 -1
- data/modules/mu/config/schema_helpers.rb +38 -37
- data/modules/mu/config/search_domain.rb +1 -1
- data/modules/mu/config/server.rb +12 -13
- data/modules/mu/config/server_pool.rb +3 -7
- data/modules/mu/config/storage_pool.rb +1 -1
- data/modules/mu/config/tail.rb +11 -0
- data/modules/mu/config/user.rb +1 -1
- data/modules/mu/config/vpc.rb +27 -23
- data/modules/mu/config/vpc.yml +0 -1
- data/modules/mu/defaults/AWS.yaml +90 -90
- data/modules/mu/defaults/Azure.yaml +1 -0
- data/modules/mu/defaults/Google.yaml +1 -0
- data/modules/mu/deploy.rb +34 -20
- data/modules/mu/groomer.rb +16 -1
- data/modules/mu/groomers/ansible.rb +69 -4
- data/modules/mu/groomers/chef.rb +51 -4
- data/modules/mu/logger.rb +120 -144
- data/modules/mu/master.rb +97 -4
- data/modules/mu/mommacat.rb +160 -874
- data/modules/mu/mommacat/daemon.rb +23 -14
- data/modules/mu/mommacat/naming.rb +110 -3
- data/modules/mu/mommacat/search.rb +497 -0
- data/modules/mu/mommacat/storage.rb +252 -194
- data/modules/mu/{clouds → providers}/README.md +1 -1
- data/modules/mu/{clouds → providers}/aws.rb +258 -57
- data/modules/mu/{clouds → providers}/aws/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/aws/bucket.rb +275 -41
- data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +14 -50
- data/modules/mu/providers/aws/cdn.rb +782 -0
- data/modules/mu/{clouds → providers}/aws/collection.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/container_cluster.rb +95 -84
- data/modules/mu/providers/aws/database.rb +1744 -0
- data/modules/mu/{clouds → providers}/aws/dnszone.rb +26 -12
- data/modules/mu/providers/aws/endpoint.rb +1072 -0
- data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +39 -32
- data/modules/mu/{clouds → providers}/aws/folder.rb +1 -1
- data/modules/mu/{clouds → providers}/aws/function.rb +289 -134
- data/modules/mu/{clouds → providers}/aws/group.rb +18 -20
- data/modules/mu/{clouds → providers}/aws/habitat.rb +3 -3
- data/modules/mu/providers/aws/job.rb +466 -0
- data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +77 -47
- data/modules/mu/{clouds → providers}/aws/log.rb +5 -5
- data/modules/mu/{clouds → providers}/aws/msg_queue.rb +14 -11
- data/modules/mu/{clouds → providers}/aws/nosqldb.rb +96 -5
- data/modules/mu/{clouds → providers}/aws/notifier.rb +135 -63
- data/modules/mu/{clouds → providers}/aws/role.rb +76 -48
- data/modules/mu/{clouds → providers}/aws/search_domain.rb +172 -41
- data/modules/mu/{clouds → providers}/aws/server.rb +66 -98
- data/modules/mu/{clouds → providers}/aws/server_pool.rb +42 -60
- data/modules/mu/{clouds → providers}/aws/storage_pool.rb +21 -38
- data/modules/mu/{clouds → providers}/aws/user.rb +12 -16
- data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
- data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/aws/vpc.rb +143 -74
- data/modules/mu/{clouds → providers}/aws/vpc_subnet.rb +0 -0
- data/modules/mu/{clouds → providers}/azure.rb +13 -0
- data/modules/mu/{clouds → providers}/azure/container_cluster.rb +1 -5
- data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +8 -1
- data/modules/mu/{clouds → providers}/azure/habitat.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/role.rb +0 -0
- data/modules/mu/{clouds → providers}/azure/server.rb +32 -24
- data/modules/mu/{clouds → providers}/azure/user.rb +1 -1
- data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/azure/vpc.rb +4 -6
- data/modules/mu/{clouds → providers}/cloudformation.rb +10 -0
- data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
- data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
- data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
- data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
- data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +3 -3
- data/modules/mu/{clouds → providers}/docker.rb +0 -0
- data/modules/mu/{clouds → providers}/google.rb +29 -6
- data/modules/mu/{clouds → providers}/google/bucket.rb +4 -4
- data/modules/mu/{clouds → providers}/google/container_cluster.rb +38 -20
- data/modules/mu/{clouds → providers}/google/database.rb +5 -12
- data/modules/mu/{clouds → providers}/google/firewall_rule.rb +5 -5
- data/modules/mu/{clouds → providers}/google/folder.rb +5 -9
- data/modules/mu/{clouds → providers}/google/function.rb +6 -6
- data/modules/mu/{clouds → providers}/google/group.rb +9 -17
- data/modules/mu/{clouds → providers}/google/habitat.rb +4 -8
- data/modules/mu/{clouds → providers}/google/loadbalancer.rb +5 -5
- data/modules/mu/{clouds → providers}/google/role.rb +50 -31
- data/modules/mu/{clouds → providers}/google/server.rb +41 -24
- data/modules/mu/{clouds → providers}/google/server_pool.rb +14 -14
- data/modules/mu/{clouds → providers}/google/user.rb +34 -24
- data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
- data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
- data/modules/mu/{clouds → providers}/google/vpc.rb +45 -14
- data/modules/tests/aws-jobs-functions.yaml +46 -0
- data/modules/tests/centos6.yaml +15 -0
- data/modules/tests/centos7.yaml +15 -0
- data/modules/tests/centos8.yaml +12 -0
- data/modules/tests/ecs.yaml +2 -2
- data/modules/tests/eks.yaml +1 -1
- data/modules/tests/functions/node-function/lambda_function.js +10 -0
- data/modules/tests/functions/python-function/lambda_function.py +12 -0
- data/modules/tests/microservice_app.yaml +288 -0
- data/modules/tests/rds.yaml +108 -0
- data/modules/tests/regrooms/rds.yaml +123 -0
- data/modules/tests/server-with-scrub-muisms.yaml +1 -1
- data/modules/tests/super_complex_bok.yml +2 -2
- data/modules/tests/super_simple_bok.yml +3 -5
- data/spec/mu/clouds/azure_spec.rb +2 -2
- metadata +122 -92
- data/modules/mu/clouds/aws/database.rb +0 -1974
- data/modules/mu/clouds/aws/endpoint.rb +0 -596
@@ -91,6 +91,7 @@ $opts[:clouds].each { |cloud|
|
|
91
91
|
end
|
92
92
|
next if !needed
|
93
93
|
end
|
94
|
+
MU.log "Loading "+bok_dir+"/"+cloud+"/"+platform+".yaml"
|
94
95
|
conf_engine = MU::Config.new(
|
95
96
|
bok_dir+"/"+cloud+"/"+platform+".yaml",
|
96
97
|
default_credentials: $opts[(cloud.downcase+"_creds").to_sym]
|
data/modules/mommacat.ru
CHANGED
data/modules/mu.rb
CHANGED
@@ -79,38 +79,40 @@ class Hash
|
|
79
79
|
}
|
80
80
|
return 0 if self == other # that was easy!
|
81
81
|
# compare elements and decide who's "bigger" based on their totals?
|
82
|
-
|
82
|
+
|
83
|
+
# fine, try some brute force and just hope everything implements to_s
|
84
|
+
self.flatten.map { |e| e.to_s }.join() <=> other.flatten.map { |e| e.to_s }.join()
|
83
85
|
end
|
84
86
|
|
85
|
-
# Recursively compare two hashes
|
86
|
-
def diff(with, on = self, level: 0, parents: [])
|
87
|
+
# Recursively compare two Mu Basket of Kittens hashes and report the differences
|
88
|
+
def diff(with, on = self, level: 0, parents: [], report: {}, habitat: nil)
|
87
89
|
return if with.nil? and on.nil?
|
88
90
|
if with.nil? or on.nil? or with.class != on.class
|
89
91
|
return # XXX ...however we're flagging differences
|
90
92
|
end
|
91
93
|
return if on == with
|
92
94
|
|
93
|
-
tree = ""
|
94
|
-
indentsize = 0
|
95
|
-
parents.each { |p|
|
96
|
-
tree += (" " * indentsize) + p + " => \n"
|
97
|
-
indentsize += 2
|
98
|
-
}
|
99
|
-
indent = (" " * indentsize)
|
100
|
-
|
101
95
|
changes = []
|
96
|
+
report ||= {}
|
102
97
|
if on.is_a?(Hash)
|
103
98
|
on_unique = (on.keys - with.keys)
|
104
99
|
with_unique = (with.keys - on.keys)
|
105
100
|
shared = (with.keys & on.keys)
|
106
101
|
shared.each { |k|
|
107
|
-
|
102
|
+
|
103
|
+
report_data = diff(with[k], on[k], level: level+1, parents: parents + [k], report: report[k], habitat: habitat)
|
104
|
+
if report_data and !report_data.empty?
|
105
|
+
report ||= {}
|
106
|
+
report[k] = report_data
|
107
|
+
end
|
108
108
|
}
|
109
109
|
on_unique.each { |k|
|
110
|
-
|
110
|
+
report[k] = { :action => :removed, :parents => parents, :value => on[k].clone }
|
111
|
+
report[k][:habitat] = habitat if habitat
|
111
112
|
}
|
112
113
|
with_unique.each { |k|
|
113
|
-
|
114
|
+
report[k] = { :action => :added, :parents => parents, :value => with[k].clone }
|
115
|
+
report[k][:habitat] = habitat if habitat
|
114
116
|
}
|
115
117
|
elsif on.is_a?(Array)
|
116
118
|
return if with == on
|
@@ -122,29 +124,27 @@ class Hash
|
|
122
124
|
# sorting arrays full of weird, non-primitive types.
|
123
125
|
done = []
|
124
126
|
on.sort.each { |elt|
|
125
|
-
if elt.is_a?(Hash) and elt
|
126
|
-
|
127
|
-
# Figure out what convention this thing is using for resource identification
|
128
|
-
compare_a, compare_b = if elt['name'].nil? and elt["id"].nil? and !elt["entity"].nil? and !other_elt["entity"].nil?
|
129
|
-
[elt["entity"], other_elt["entity"]]
|
130
|
-
else
|
131
|
-
[elt, other_elt]
|
132
|
-
end
|
127
|
+
if elt.is_a?(Hash) and !MU::MommaCat.getChunkName(elt).first.nil?
|
128
|
+
elt_namestr, elt_location, elt_location_list = MU::MommaCat.getChunkName(elt)
|
133
129
|
|
134
|
-
|
135
|
-
|
136
|
-
|
130
|
+
with.sort.each { |other_elt|
|
131
|
+
other_elt_namestr, other_elt_location, other_elt_location_list = MU::MommaCat.getChunkName(other_elt)
|
132
|
+
|
133
|
+
# Case 1: The array element exists in both version of this array
|
134
|
+
if elt_namestr and other_elt_namestr and
|
135
|
+
elt_namestr == other_elt_namestr and
|
136
|
+
(elt_location.nil? or other_elt_location.nil? or
|
137
|
+
elt_location == other_elt_location or
|
138
|
+
!(elt_location_list & other_elt_location_list).empty?
|
139
|
+
)
|
137
140
|
done << elt
|
138
141
|
done << other_elt
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
elt['entity']['id']
|
142
|
+
break if elt == other_elt # if they're identical, we're done
|
143
|
+
report_data = diff(other_elt, elt, level: level+1, parents: parents + [elt_namestr], habitat: (elt_location || habitat))
|
144
|
+
if report_data and !report_data.empty?
|
145
|
+
report ||= {}
|
146
|
+
report[elt_namestr] = report_data
|
145
147
|
end
|
146
|
-
|
147
|
-
diff(other_elt, elt, level: level+1, parents: parents + [namestr])
|
148
148
|
break
|
149
149
|
end
|
150
150
|
}
|
@@ -152,43 +152,34 @@ class Hash
|
|
152
152
|
}
|
153
153
|
on_unique = (on - with) - done
|
154
154
|
with_unique = (with - on) - done
|
155
|
-
|
156
|
-
#
|
157
|
-
# MU.log "A BEFORE", MU::NOTICE, details: before_a
|
158
|
-
# MU.log "A AFTER", MU::NOTICE, details: after_a
|
159
|
-
# end
|
160
|
-
# if before_b != after_b
|
161
|
-
# MU.log "B BEFORE", MU::NOTICE, details: before_b
|
162
|
-
# MU.log "B AFTER", MU::NOTICE, details: after_b
|
163
|
-
# end
|
164
|
-
# end
|
155
|
+
|
156
|
+
# Case 2: This array entry exists in the old version, but not the new one
|
165
157
|
on_unique.each { |e|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
158
|
+
namestr, loc = MU::MommaCat.getChunkName(e)
|
159
|
+
|
160
|
+
report ||= {}
|
161
|
+
report[namestr] = { :action => :removed, :parents => parents, :value => e.clone }
|
162
|
+
report[namestr][:habitat] = loc if loc
|
171
163
|
}
|
164
|
+
|
165
|
+
# Case 3: This array entry exists in the new version, but not the old one
|
172
166
|
with_unique.each { |e|
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
167
|
+
namestr, loc = MU::MommaCat.getChunkName(e)
|
168
|
+
|
169
|
+
report ||= {}
|
170
|
+
report[namestr] = { :action => :added, :parents => parents, :value => e.clone }
|
171
|
+
report[namestr][:habitat] = loc if loc
|
178
172
|
}
|
173
|
+
|
174
|
+
# A plain old leaf node of data
|
179
175
|
else
|
180
176
|
if on != with
|
181
|
-
|
182
|
-
|
177
|
+
report = { :action => :changed, :parents => parents, :oldvalue => on, :value => with.clone }
|
178
|
+
report[:habitat] = habitat if habitat
|
183
179
|
end
|
184
180
|
end
|
185
181
|
|
186
|
-
|
187
|
-
puts tree
|
188
|
-
changes.each { |c|
|
189
|
-
puts indent+c
|
190
|
-
}
|
191
|
-
end
|
182
|
+
report.freeze
|
192
183
|
end
|
193
184
|
|
194
185
|
# Implement a merge! that just updates each hash leaf as needed, not
|
@@ -212,8 +203,29 @@ class Hash
|
|
212
203
|
end
|
213
204
|
|
214
205
|
ENV['HOME'] = Etc.getpwuid(Process.uid).dir
|
206
|
+
module MU
|
207
|
+
|
208
|
+
# For log entries that should only be logged when we're in verbose mode
|
209
|
+
DEBUG = 0.freeze
|
210
|
+
# For ordinary log entries
|
211
|
+
INFO = 1.freeze
|
212
|
+
# For more interesting log entries which are not errors
|
213
|
+
NOTICE = 2.freeze
|
214
|
+
# Log entries for non-fatal errors
|
215
|
+
WARN = 3.freeze
|
216
|
+
# Log entries for non-fatal errors
|
217
|
+
WARNING = 3.freeze
|
218
|
+
# Log entries for fatal errors
|
219
|
+
ERR = 4.freeze
|
220
|
+
# Log entries for fatal errors
|
221
|
+
ERROR = 4.freeze
|
222
|
+
# Log entries that will be held and displayed/emailed at the end of deploy,
|
223
|
+
# cleanup, etc.
|
224
|
+
SUMMARY = 5.freeze
|
225
|
+
end
|
215
226
|
|
216
227
|
require 'mu/logger'
|
228
|
+
|
217
229
|
module MU
|
218
230
|
|
219
231
|
# Subclass core thread so we can gracefully handle it when we hit system
|
@@ -273,8 +285,9 @@ module MU
|
|
273
285
|
# Wrapper class for fatal Exceptions. Gives our internals something to
|
274
286
|
# inherit that will log an error message appropriately before bubbling up.
|
275
287
|
class MuError < StandardError
|
276
|
-
def initialize(message = nil)
|
277
|
-
|
288
|
+
def initialize(message = nil, silent: false, details: nil)
|
289
|
+
details ||= caller[2]
|
290
|
+
MU.log message, MU::ERR, details: details if !message.nil? and !silent
|
278
291
|
if MU.verbosity == MU::Logger::SILENT
|
279
292
|
super ""
|
280
293
|
else
|
@@ -286,8 +299,8 @@ module MU
|
|
286
299
|
# Wrapper class for temporary Exceptions. Gives our internals something to
|
287
300
|
# inherit that will log a notice message appropriately before bubbling up.
|
288
301
|
class MuNonFatal < StandardError
|
289
|
-
def initialize(message = nil)
|
290
|
-
MU.log message, MU::NOTICE if !message.nil?
|
302
|
+
def initialize(message = nil, silent: false, details: nil)
|
303
|
+
MU.log message, MU::NOTICE, details: details if !message.nil? and !silent
|
291
304
|
if MU.verbosity == MU::Logger::SILENT
|
292
305
|
super ""
|
293
306
|
else
|
@@ -598,9 +611,10 @@ module MU
|
|
598
611
|
end
|
599
612
|
|
600
613
|
# Shortcut to invoke {MU::Logger#log}
|
601
|
-
def self.log(msg, level = MU::INFO, details: nil, html: false, verbosity: nil, color: true)
|
614
|
+
def self.log(msg, level = MU::INFO, shorthand_details = nil, details: nil, html: false, verbosity: nil, color: true)
|
602
615
|
return if (level == MU::DEBUG and verbosity and verbosity <= MU::Logger::LOUD)
|
603
616
|
return if verbosity and verbosity == MU::Logger::SILENT
|
617
|
+
details ||= shorthand_details
|
604
618
|
|
605
619
|
if (level == MU::ERR or
|
606
620
|
level == MU::WARN or
|
@@ -619,25 +633,6 @@ module MU
|
|
619
633
|
@@logger.log(msg, level, details: details, html: html, verbosity: verbosity, color: color)
|
620
634
|
end
|
621
635
|
|
622
|
-
# For log entries that should only be logged when we're in verbose mode
|
623
|
-
DEBUG = 0.freeze
|
624
|
-
# For ordinary log entries
|
625
|
-
INFO = 1.freeze
|
626
|
-
# For more interesting log entries which are not errors
|
627
|
-
NOTICE = 2.freeze
|
628
|
-
# Log entries for non-fatal errors
|
629
|
-
WARN = 3.freeze
|
630
|
-
# Log entries for non-fatal errors
|
631
|
-
WARNING = 3.freeze
|
632
|
-
# Log entries for fatal errors
|
633
|
-
ERR = 4.freeze
|
634
|
-
# Log entries for fatal errors
|
635
|
-
ERROR = 4.freeze
|
636
|
-
# Log entries that will be held and displayed/emailed at the end of deploy,
|
637
|
-
# cleanup, etc.
|
638
|
-
SUMMARY = 5.freeze
|
639
|
-
|
640
|
-
|
641
636
|
autoload :Cleanup, 'mu/cleanup'
|
642
637
|
autoload :Deploy, 'mu/deploy'
|
643
638
|
autoload :MommaCat, 'mu/mommacat'
|
@@ -651,7 +646,7 @@ module MU
|
|
651
646
|
new_cfg = $MU_CFG.dup
|
652
647
|
examples = {}
|
653
648
|
MU::Cloud.supportedClouds.each { |cloud|
|
654
|
-
cloudclass =
|
649
|
+
cloudclass = MU::Cloud.cloudClass(cloud)
|
655
650
|
begin
|
656
651
|
if cloudclass.hosted? and !$MU_CFG[cloud.downcase]
|
657
652
|
cfg_blob = cloudclass.hosted_config
|
@@ -807,11 +802,7 @@ module MU
|
|
807
802
|
# @param groomer [String]: The grooming agent to load.
|
808
803
|
# @return [Class]: The class object implementing this groomer agent
|
809
804
|
def self.loadGroomer(groomer)
|
810
|
-
|
811
|
-
raise MuError, "Requested to use unsupported grooming agent #{groomer}"
|
812
|
-
end
|
813
|
-
require "mu/groomers/#{groomer.downcase}"
|
814
|
-
return Object.const_get("MU").const_get("Groomer").const_get(groomer)
|
805
|
+
MU::Groomer.loadGroomer(groomer)
|
815
806
|
end
|
816
807
|
|
817
808
|
@@myRegion_var = nil
|
@@ -965,8 +956,7 @@ module MU
|
|
965
956
|
|
966
957
|
@@myCloudDescriptor = nil
|
967
958
|
if MU.myCloud
|
968
|
-
|
969
|
-
found = svrclass.find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
|
959
|
+
found = MU::Cloud.resourceClass(MU.myCloud, "Server").find(cloud_id: @@myInstanceId, region: MU.myRegion) # XXX need habitat arg for google et al
|
970
960
|
# found = MU::MommaCat.findStray(MU.myCloud, "server", cloud_id: @@myInstanceId, dummy_ok: true, region: MU.myRegion)
|
971
961
|
if !found.nil? and found.size == 1
|
972
962
|
@@myCloudDescriptor = found.values.first
|
@@ -979,8 +969,7 @@ module MU
|
|
979
969
|
def self.myVPCObj
|
980
970
|
return nil if MU.myCloud.nil?
|
981
971
|
return @@myVPCObj_var if @@myVPCObj_var
|
982
|
-
|
983
|
-
@@myVPCObj_var ||= cloudclass.myVPCObj
|
972
|
+
@@myVPCObj_var ||= MU::Cloud.cloudClass(MU.myCloud).myVPCObj
|
984
973
|
@@myVPCObj_var
|
985
974
|
end
|
986
975
|
|
@@ -1105,10 +1094,9 @@ module MU
|
|
1105
1094
|
|
1106
1095
|
clouds = platform.nil? ? MU::Cloud.supportedClouds : [platform]
|
1107
1096
|
clouds.each { |cloud|
|
1108
|
-
|
1109
|
-
bucketname = cloudclass.adminBucketName(credentials)
|
1097
|
+
bucketname = MU::Cloud.cloudClass(cloud).adminBucketName(credentials)
|
1110
1098
|
begin
|
1111
|
-
if platform or (
|
1099
|
+
if platform or (MU::Cloud.cloudClass(cloud).hosted? and platform.nil?) or cloud == MU::Config.defaultCloud
|
1112
1100
|
return bucketname
|
1113
1101
|
end
|
1114
1102
|
end
|
data/modules/mu/adoption.rb
CHANGED
@@ -30,7 +30,7 @@ module MU
|
|
30
30
|
:omnibus => "Jam everything into one monolothic configuration"
|
31
31
|
}
|
32
32
|
|
33
|
-
def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [])
|
33
|
+
def initialize(clouds: MU::Cloud.supportedClouds, types: MU::Cloud.resource_types.keys, parent: nil, billing: nil, sources: nil, credentials: nil, group_by: :logical, savedeploys: false, diff: false, habitats: [], scrub_mu_isms: false, regions: [], merge: false, pattern: nil)
|
34
34
|
@scraped = {}
|
35
35
|
@clouds = clouds
|
36
36
|
@types = types
|
@@ -44,7 +44,11 @@ module MU
|
|
44
44
|
@savedeploys = savedeploys
|
45
45
|
@diff = diff
|
46
46
|
@habitats = habitats
|
47
|
+
@regions = regions
|
47
48
|
@habitats ||= []
|
49
|
+
@scrub_mu_isms = scrub_mu_isms
|
50
|
+
@merge = merge
|
51
|
+
@pattern = pattern
|
48
52
|
end
|
49
53
|
|
50
54
|
# Walk cloud providers with available credentials to discover resources
|
@@ -52,7 +56,7 @@ module MU
|
|
52
56
|
@default_parent = nil
|
53
57
|
|
54
58
|
@clouds.each { |cloud|
|
55
|
-
cloudclass =
|
59
|
+
cloudclass = MU::Cloud.cloudClass(cloud)
|
56
60
|
next if cloudclass.listCredentials.nil?
|
57
61
|
|
58
62
|
if cloud == "Google" and !@parent and @target_creds
|
@@ -64,6 +68,10 @@ module MU
|
|
64
68
|
|
65
69
|
cloudclass.listCredentials.each { |credset|
|
66
70
|
next if @sources and !@sources.include?(credset)
|
71
|
+
cfg = cloudclass.credConfig(credset)
|
72
|
+
if cfg and cfg['restrict_to_habitats']
|
73
|
+
cfg['restrict_to_habitats'] << cfg['project'] if cfg['project']
|
74
|
+
end
|
67
75
|
|
68
76
|
if @parent
|
69
77
|
# TODO handle different inputs (cloud_id, etc)
|
@@ -84,7 +92,7 @@ module MU
|
|
84
92
|
|
85
93
|
@types.each { |type|
|
86
94
|
begin
|
87
|
-
resclass =
|
95
|
+
resclass = MU::Cloud.resourceClass(cloud, type)
|
88
96
|
rescue ::MU::Cloud::MuCloudResourceNotImplemented
|
89
97
|
next
|
90
98
|
end
|
@@ -100,17 +108,31 @@ module MU
|
|
100
108
|
credentials: credset,
|
101
109
|
allow_multi: true,
|
102
110
|
habitats: @habitats.dup,
|
111
|
+
region: @regions,
|
103
112
|
dummy_ok: true,
|
104
|
-
|
105
|
-
|
113
|
+
skip_provider_owned: true,
|
114
|
+
# debug: false#,
|
106
115
|
)
|
107
116
|
|
108
117
|
|
109
118
|
if found and found.size > 0
|
119
|
+
if resclass.cfg_plural == "habitats"
|
120
|
+
found.reject! { |h|
|
121
|
+
!cloudclass.listHabitats(credset).include?(h.cloud_id)
|
122
|
+
}
|
123
|
+
end
|
110
124
|
MU.log "Found #{found.size.to_s} raw #{resclass.cfg_plural} in #{cloud}"
|
111
125
|
@scraped[type] ||= {}
|
112
126
|
found.each { |obj|
|
127
|
+
if obj.habitat and !cloudclass.listHabitats(credset).include?(obj.habitat)
|
128
|
+
next
|
129
|
+
end
|
130
|
+
|
113
131
|
# XXX apply any filters (e.g. MU-ID tags)
|
132
|
+
if obj.cloud_id.nil?
|
133
|
+
MU.log "This damn thing gave me no cloud id, what do I even do with that", MU::ERR, details: obj
|
134
|
+
exit
|
135
|
+
end
|
114
136
|
@scraped[type][obj.cloud_id] = obj
|
115
137
|
}
|
116
138
|
end
|
@@ -188,33 +210,62 @@ module MU
|
|
188
210
|
prefix = "mu" if prefix.empty? # so that appnames aren't ever empty
|
189
211
|
end
|
190
212
|
|
213
|
+
# Find any previous deploys with this particular profile, which we'll use
|
214
|
+
# later for --diff.
|
215
|
+
@existing_deploys = {}
|
216
|
+
@existing_deploys_by_id = {}
|
217
|
+
@origins = {}
|
218
|
+
@types_found_in = {}
|
191
219
|
groupings.each_pair { |appname, types|
|
192
|
-
bok = { "appname" => prefix+appname }
|
193
|
-
if @target_creds
|
194
|
-
bok["credentials"] = @target_creds
|
195
|
-
end
|
196
|
-
|
197
|
-
count = 0
|
198
220
|
allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
|
199
221
|
next if (types & allowed_types).size == 0
|
200
222
|
origin = {
|
201
|
-
"appname" =>
|
223
|
+
"appname" => prefix+appname,
|
202
224
|
"types" => (types & allowed_types).sort,
|
203
225
|
"habitats" => @habitats.sort,
|
204
226
|
"group_by" => @group_by.to_s
|
205
227
|
}
|
206
228
|
|
207
|
-
|
208
|
-
if @
|
209
|
-
|
210
|
-
|
229
|
+
@existing_deploys[appname] = MU::MommaCat.findMatchingDeploy(origin)
|
230
|
+
if @existing_deploys[appname]
|
231
|
+
@existing_deploys_by_id[@existing_deploys[appname].deploy_id] = @existing_deploys[appname]
|
232
|
+
@origins[appname] = origin
|
233
|
+
origin['types'].each { |t|
|
234
|
+
@types_found_in[t] = @existing_deploys[appname]
|
235
|
+
}
|
236
|
+
end
|
237
|
+
}
|
238
|
+
|
239
|
+
groupings.each_pair { |appname, types|
|
240
|
+
allowed_types = @types.map { |t| MU::Cloud.resource_types[t][:cfg_plural] }
|
241
|
+
next if (types & allowed_types).size == 0
|
242
|
+
|
243
|
+
bok = { "appname" => prefix+appname }
|
244
|
+
if @scrub_mu_isms
|
245
|
+
bok["scrub_mu_isms"] = true
|
246
|
+
end
|
247
|
+
if @target_creds
|
248
|
+
bok["credentials"] = @target_creds
|
249
|
+
end
|
250
|
+
|
251
|
+
count = 0
|
252
|
+
if @diff
|
253
|
+
if !@existing_deploys[appname]
|
254
|
+
MU.log "--diff was set but I failed to find a deploy like '#{appname}' to compare to (have #{@existing_deploys.keys.join(", ")})", MU::ERR, details: @origins[appname]
|
255
|
+
exit 1
|
256
|
+
else
|
257
|
+
MU.log "Will diff current live resources against #{@existing_deploys[appname].deploy_id}", MU::NOTICE, details: @origins[appname]
|
258
|
+
end
|
211
259
|
end
|
212
260
|
|
213
261
|
threads = []
|
262
|
+
timers = {}
|
263
|
+
walltimers = {}
|
214
264
|
@clouds.each { |cloud|
|
215
265
|
@scraped.each_pair { |type, resources|
|
266
|
+
typestart = Time.now
|
216
267
|
res_class = begin
|
217
|
-
MU::Cloud.
|
268
|
+
MU::Cloud.resourceClass(cloud, type)
|
218
269
|
rescue MU::Cloud::MuCloudResourceNotImplemented
|
219
270
|
# XXX I don't think this can actually happen
|
220
271
|
next
|
@@ -222,6 +273,7 @@ module MU
|
|
222
273
|
next if !types.include?(res_class.cfg_plural)
|
223
274
|
|
224
275
|
bok[res_class.cfg_plural] ||= []
|
276
|
+
timers[type] ||= {}
|
225
277
|
|
226
278
|
class_semaphore = Mutex.new
|
227
279
|
|
@@ -238,13 +290,18 @@ module MU
|
|
238
290
|
end
|
239
291
|
end
|
240
292
|
threads << Thread.new(obj_thr) { |obj|
|
293
|
+
start = Time.now
|
241
294
|
|
242
|
-
kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats)
|
243
|
-
if kitten_cfg
|
295
|
+
kitten_cfg = obj.toKitten(rootparent: @default_parent, billing: @billing, habitats: @habitats, types: @types)
|
296
|
+
if kitten_cfg and (!@pattern or @pattern.match(kitten_cfg['name']))
|
244
297
|
print "."
|
245
298
|
kitten_cfg.delete("credentials") if @target_creds
|
246
299
|
class_semaphore.synchronize {
|
247
300
|
bok[res_class.cfg_plural] << kitten_cfg
|
301
|
+
if !kitten_cfg['cloud_id']
|
302
|
+
MU.log "No cloud id in this #{res_class.cfg_name} kitten!", MU::ERR, details: kitten_cfg
|
303
|
+
end
|
304
|
+
timers[type][kitten_cfg['cloud_id']] = (Time.now - start)
|
248
305
|
}
|
249
306
|
count += 1
|
250
307
|
end
|
@@ -255,6 +312,7 @@ module MU
|
|
255
312
|
threads.each { |t|
|
256
313
|
t.join
|
257
314
|
}
|
315
|
+
|
258
316
|
puts ""
|
259
317
|
bok[res_class.cfg_plural].sort! { |a, b|
|
260
318
|
strs = [a, b].map { |x|
|
@@ -276,24 +334,30 @@ module MU
|
|
276
334
|
bok[res_class.cfg_plural].each { |sibling|
|
277
335
|
next if kitten_cfg == sibling
|
278
336
|
if sibling['name'] == kitten_cfg['name']
|
279
|
-
MU.
|
280
|
-
if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
|
281
|
-
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['parent'].id
|
282
|
-
elsif kitten_cfg['project']
|
283
|
-
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['project']
|
284
|
-
elsif kitten_cfg['region']
|
285
|
-
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['region']
|
286
|
-
elsif kitten_cfg['cloud_id']
|
287
|
-
kitten_cfg['name'] = kitten_cfg['name']+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
|
288
|
-
else
|
289
|
-
raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{sibling['name']} and couldn't come up with a good way to differentiate them"
|
290
|
-
end
|
337
|
+
MU::Adoption.deDuplicateName(kitten_cfg, res_class)
|
291
338
|
MU.log "De-duplication: Renamed #{res_class.cfg_name} name '#{sibling['name']}' => '#{kitten_cfg['name']}'", MU::NOTICE
|
292
339
|
break
|
293
340
|
end
|
294
341
|
}
|
295
342
|
}
|
343
|
+
walltimers[type] ||= 0
|
344
|
+
walltimers[type] += (Time.now - typestart)
|
345
|
+
}
|
346
|
+
}
|
347
|
+
|
348
|
+
timers.each_pair { |type, resources|
|
349
|
+
next if resources.empty?
|
350
|
+
total = resources.values.sum
|
351
|
+
top_5 = resources.keys.sort { |a, b|
|
352
|
+
resources[b] <=> resources[a]
|
353
|
+
}.slice(0, 5).map { |k|
|
354
|
+
k.to_s+": "+sprintf("%.2fs", resources[k])
|
296
355
|
}
|
356
|
+
if walltimers[type] < 45
|
357
|
+
MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])}"
|
358
|
+
else
|
359
|
+
MU.log "Kittened #{resources.size.to_s} eligible #{type}s in #{sprintf("%.2fs", walltimers[type])} (CPU time #{sprintf("%.2fs", total)}, avg #{sprintf("%.2fs", total/resources.size)}). Top 5:", MU::NOTICE, details: top_5
|
360
|
+
end
|
297
361
|
}
|
298
362
|
|
299
363
|
# No matching resources isn't necessarily an error
|
@@ -302,23 +366,36 @@ module MU
|
|
302
366
|
# Now walk through all of the Refs in these objects, resolve them, and minimize
|
303
367
|
# their config footprint
|
304
368
|
MU.log "Minimizing footprint of #{count.to_s} found resources", MU::DEBUG
|
305
|
-
@boks[bok['appname']] = vacuum(bok, origin: origin, save: @savedeploys)
|
306
369
|
|
307
|
-
|
370
|
+
generated_deploy = generateStubDeploy(bok)
|
371
|
+
@boks[bok['appname']] = vacuum(bok, origin: @origins[appname], deploy: generated_deploy, save: @savedeploys)
|
372
|
+
|
373
|
+
if @diff and !@existing_deploys[appname]
|
308
374
|
MU.log "diff flag set, but no comparable deploy provided for #{bok['appname']}", MU::ERR
|
309
375
|
exit 1
|
310
376
|
end
|
311
377
|
|
312
|
-
if
|
313
|
-
|
378
|
+
if @diff
|
379
|
+
prev_vacuumed = vacuum(@existing_deploys[appname].original_config, deploy: @existing_deploys[appname], keep_missing: true, copy_from: generated_deploy)
|
380
|
+
prevcfg = MU::Config.manxify(prev_vacuumed)
|
314
381
|
if !prevcfg
|
315
|
-
MU.log "#{
|
382
|
+
MU.log "#{@existing_deploys[appname].deploy_id} didn't have a working original config for me to compare", MU::ERR
|
316
383
|
exit 1
|
317
384
|
end
|
318
385
|
newcfg = MU::Config.manxify(@boks[bok['appname']])
|
386
|
+
report = prevcfg.diff(newcfg)
|
387
|
+
|
388
|
+
if report
|
389
|
+
|
390
|
+
if MU.muCfg['adopt_change_notify']
|
391
|
+
notifyChanges(@existing_deploys[appname], report.freeze)
|
392
|
+
end
|
393
|
+
if @merge
|
394
|
+
MU.log "Saving changes to #{@existing_deploys[appname].deploy_id}"
|
395
|
+
@existing_deploys[appname].updateBasketofKittens(newcfg, save_now: true)
|
396
|
+
end
|
397
|
+
end
|
319
398
|
|
320
|
-
prevcfg.diff(newcfg)
|
321
|
-
exit
|
322
399
|
end
|
323
400
|
}
|
324
401
|
@boks
|
@@ -326,6 +403,183 @@ module MU
|
|
326
403
|
|
327
404
|
private
|
328
405
|
|
406
|
+
# @param tier [Hash]
|
407
|
+
# @param parent_key [String]
|
408
|
+
def crawlChangeReport(tier, parent_key = nil, indent: "")
|
409
|
+
report = []
|
410
|
+
if tier.is_a?(Array)
|
411
|
+
tier.each { |a|
|
412
|
+
sub_report = crawlChangeReport(a, parent_key)
|
413
|
+
report.concat(sub_report) if sub_report and !sub_report.empty?
|
414
|
+
}
|
415
|
+
elsif tier.is_a?(Hash)
|
416
|
+
if tier[:action]
|
417
|
+
preposition = if tier[:action] == :added
|
418
|
+
"to"
|
419
|
+
elsif tier[:action] == :removed
|
420
|
+
"from"
|
421
|
+
else
|
422
|
+
"in"
|
423
|
+
end
|
424
|
+
|
425
|
+
name = ""
|
426
|
+
type_of = parent_key.sub(/s$|\[.*/, '') if parent_key
|
427
|
+
loc = tier[:habitat]
|
428
|
+
|
429
|
+
if tier[:value] and tier[:value].is_a?(Hash)
|
430
|
+
name, loc = MU::MommaCat.getChunkName(tier[:value], type_of)
|
431
|
+
elsif parent_key
|
432
|
+
name = parent_key
|
433
|
+
end
|
434
|
+
|
435
|
+
path_str = []
|
436
|
+
slack_path_str = ""
|
437
|
+
if tier[:parents] and tier[:parents].size > 2
|
438
|
+
path = tier[:parents].clone
|
439
|
+
slack_path_str += "#{preposition} \*"+path.join(" ⇨ ")+"\*" if path.size > 0
|
440
|
+
path.shift
|
441
|
+
path.shift
|
442
|
+
path.pop if path.last == name
|
443
|
+
for c in (0..(path.size-1)) do
|
444
|
+
path_str << (" " * (c+2)) + (path[c] || "<nil>")
|
445
|
+
end
|
446
|
+
end
|
447
|
+
path_str << "" if !path_str.empty?
|
448
|
+
|
449
|
+
plain = (name ? name : type_of) if name or type_of
|
450
|
+
plain ||= "" # XXX but this is a problem
|
451
|
+
slack = "`"+plain+"`"
|
452
|
+
|
453
|
+
plain += " ("+loc+")" if loc and !loc.empty?
|
454
|
+
color = plain
|
455
|
+
|
456
|
+
if tier[:action] == :added
|
457
|
+
color = "+ ".green + plain
|
458
|
+
plain = "+ " + plain
|
459
|
+
slack += " added"
|
460
|
+
elsif tier[:action] == :removed
|
461
|
+
color = "- ".red + plain
|
462
|
+
plain = "- " + plain
|
463
|
+
slack += " removed"
|
464
|
+
end
|
465
|
+
|
466
|
+
slack += " #{tier[:action]} #{preposition} \*#{loc}\*" if loc and !loc.empty? and [Array, Hash].include?(tier[:value].class)
|
467
|
+
|
468
|
+
plain = path_str.join(" => \n") + indent + plain
|
469
|
+
color = path_str.join(" => \n") + indent + color
|
470
|
+
|
471
|
+
slack += " "+slack_path_str if !slack_path_str.empty?
|
472
|
+
myreport = {
|
473
|
+
"slack" => slack,
|
474
|
+
"plain" => plain,
|
475
|
+
"color" => color
|
476
|
+
}
|
477
|
+
|
478
|
+
append = ""
|
479
|
+
if tier[:value] and (tier[:value].is_a?(Array) or tier[:value].is_a?(Hash))
|
480
|
+
if tier[:value].is_a?(Hash)
|
481
|
+
if name
|
482
|
+
tier[:value].delete("entity")
|
483
|
+
tier[:value].delete(name.sub(/\[.*/, '')) if name
|
484
|
+
end
|
485
|
+
if (tier[:value].keys - ["id", "name", "type"]).size > 0
|
486
|
+
myreport["details"] = tier[:value].clone
|
487
|
+
append = PP.pp(tier[:value], '').gsub(/(^|\n)/, '\1'+indent)
|
488
|
+
end
|
489
|
+
else
|
490
|
+
append = indent+"["+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s.light_blue }.join(", ")+"]"
|
491
|
+
slack += " #{tier[:action].to_s}: "+tier[:value].map { |v| MU::MommaCat.getChunkName(v, type_of).reverse.join("/") || v.to_s }.join(", ")
|
492
|
+
end
|
493
|
+
else
|
494
|
+
tier[:value] ||= "<nil>"
|
495
|
+
if ![:removed].include?(tier[:action])
|
496
|
+
myreport["slack"] += ". New #{tier[:field] ? "`"+tier[:field]+"`" : :value}: \*#{tier[:value]}\*"
|
497
|
+
else
|
498
|
+
myreport["slack"] += " (was \*#{tier[:value]}\*)"
|
499
|
+
end
|
500
|
+
append = tier[:value].to_s.bold
|
501
|
+
end
|
502
|
+
|
503
|
+
if append and !append.empty?
|
504
|
+
myreport["plain"] += " =>\n "+indent+append
|
505
|
+
myreport["color"] += " =>\n "+indent+append
|
506
|
+
end
|
507
|
+
|
508
|
+
report << myreport if tier[:action]
|
509
|
+
end
|
510
|
+
|
511
|
+
# Just because we've got changes at this level doesn't mean there aren't
|
512
|
+
# more further down.
|
513
|
+
tier.each_pair { |k, v|
|
514
|
+
next if !(v.is_a?(Hash) or v.is_a?(Array))
|
515
|
+
sub_report = crawlChangeReport(v, k, indent: indent+" ")
|
516
|
+
report.concat(sub_report) if sub_report and !sub_report.empty?
|
517
|
+
}
|
518
|
+
end
|
519
|
+
|
520
|
+
report
|
521
|
+
end
|
522
|
+
|
523
|
+
|
524
|
+
def notifyChanges(deploy, report)
|
525
|
+
snippet_threshold = (MU.muCfg['adopt_change_notify'] && MU.muCfg['adopt_change_notify']['slack_snippet_threshold']) || 5
|
526
|
+
|
527
|
+
report.each_pair { |res_type, resources|
|
528
|
+
shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(res_type, false)
|
529
|
+
next if !shortclass # we don't really care about Mu metadata changes
|
530
|
+
resources.each_pair { |name, data|
|
531
|
+
if MU::MommaCat.getChunkName(data[:value], res_type).first.nil?
|
532
|
+
symbol = if data[:action] == :added
|
533
|
+
"+".green
|
534
|
+
elsif data[:action] == :removed
|
535
|
+
"-".red
|
536
|
+
else
|
537
|
+
"~".yellow
|
538
|
+
end
|
539
|
+
puts (symbol+" "+res_type+"["+name+"]")
|
540
|
+
end
|
541
|
+
|
542
|
+
noun = shortclass ? shortclass.to_s : res_type.capitalize
|
543
|
+
verb = if data[:action]
|
544
|
+
data[:action].to_s
|
545
|
+
else
|
546
|
+
"modified"
|
547
|
+
end
|
548
|
+
|
549
|
+
changes = crawlChangeReport(data.freeze, res_type)
|
550
|
+
|
551
|
+
slacktext = "#{noun} \*#{name}\* was #{verb}"
|
552
|
+
if data[:habitat]
|
553
|
+
slacktext += " in \*#{data[:habitat]}\*"
|
554
|
+
end
|
555
|
+
snippets = []
|
556
|
+
|
557
|
+
if [:added, :removed].include?(data[:action]) and data[:value]
|
558
|
+
snippets << { text: "```"+JSON.pretty_generate(data[:value])+"```" }
|
559
|
+
else
|
560
|
+
changes.each { |c|
|
561
|
+
slacktext += "\n • "+c["slack"]
|
562
|
+
if c["details"]
|
563
|
+
details = JSON.pretty_generate(c["details"])
|
564
|
+
snippets << { text: "```"+JSON.pretty_generate(c["details"])+"```" }
|
565
|
+
end
|
566
|
+
}
|
567
|
+
end
|
568
|
+
|
569
|
+
changes.each { |c|
|
570
|
+
puts c["color"]
|
571
|
+
}
|
572
|
+
puts ""
|
573
|
+
|
574
|
+
if MU.muCfg['adopt_change_notify'] and MU.muCfg['adopt_change_notify']['slack']
|
575
|
+
deploy.sendAdminSlack(slacktext, scrub_mu_isms: MU.muCfg['adopt_scrub_mu_isms'], snippets: snippets, noop: false)
|
576
|
+
end
|
577
|
+
|
578
|
+
}
|
579
|
+
}
|
580
|
+
|
581
|
+
end
|
582
|
+
|
329
583
|
def scrubSchemaDefaults(conf_chunk, schema_chunk, depth = 0, type: nil)
|
330
584
|
return if schema_chunk.nil?
|
331
585
|
|
@@ -333,7 +587,7 @@ module MU
|
|
333
587
|
deletia = []
|
334
588
|
schema_chunk["properties"].each_pair { |key, subschema|
|
335
589
|
next if !conf_chunk[key]
|
336
|
-
shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key)
|
590
|
+
shortclass, _cfg_name, _cfg_plural, _classname = MU::Cloud.getResourceNames(key, false)
|
337
591
|
|
338
592
|
if subschema["default_if"]
|
339
593
|
subschema["default_if"].each { |cond|
|
@@ -357,8 +611,7 @@ module MU
|
|
357
611
|
# theory
|
358
612
|
realschema = if type and schema_chunk["items"] and schema_chunk["items"]["properties"] and item["cloud"] and MU::Cloud.supportedClouds.include?(item['cloud'])
|
359
613
|
|
360
|
-
|
361
|
-
_toplevel_required, cloudschema = cloudclass.schema(self)
|
614
|
+
_toplevel_required, cloudschema = MU::Cloud.resourceClass(item['cloud'], type).schema(self)
|
362
615
|
|
363
616
|
newschema = schema_chunk["items"].dup
|
364
617
|
newschema["properties"].merge!(cloudschema)
|
@@ -382,8 +635,7 @@ module MU
|
|
382
635
|
# Do the same for our main objects: if they all use the same credentials,
|
383
636
|
# for example, remove the explicit +credentials+ attributes and set that
|
384
637
|
# value globally, once.
|
385
|
-
def vacuum(bok, origin: nil, save: false, deploy: nil)
|
386
|
-
deploy ||= generateStubDeploy(bok)
|
638
|
+
def vacuum(bok, origin: nil, save: false, deploy: nil, copy_from: nil, keep_missing: false)
|
387
639
|
|
388
640
|
globals = {
|
389
641
|
'cloud' => {},
|
@@ -403,11 +655,24 @@ module MU
|
|
403
655
|
end
|
404
656
|
}
|
405
657
|
obj = deploy.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
|
658
|
+
inject_metadata = save
|
659
|
+
if obj.nil? and copy_from
|
660
|
+
obj = copy_from.findLitterMate(type: attrs[:cfg_plural], name: resource['name'])
|
661
|
+
if obj
|
662
|
+
inject_metadata = true
|
663
|
+
obj.intoDeploy(deploy, force: true)
|
664
|
+
end
|
665
|
+
end
|
666
|
+
|
406
667
|
begin
|
407
668
|
raise Incomplete if obj.nil?
|
669
|
+
if inject_metadata
|
670
|
+
deploydata = obj.notify
|
671
|
+
deploy.notify(attrs[:cfg_plural], resource['name'], deploydata, triggering_node: obj)
|
672
|
+
end
|
408
673
|
new_cfg = resolveReferences(resource, deploy, obj)
|
409
674
|
new_cfg.delete("cloud_id")
|
410
|
-
cred_cfg = MU::Cloud.
|
675
|
+
cred_cfg = MU::Cloud.cloudClass(obj.cloud).credConfig(obj.credentials)
|
411
676
|
if cred_cfg['region'] == new_cfg['region']
|
412
677
|
new_cfg.delete('region')
|
413
678
|
end
|
@@ -417,6 +682,11 @@ module MU
|
|
417
682
|
end
|
418
683
|
processed << new_cfg
|
419
684
|
rescue Incomplete
|
685
|
+
if keep_missing
|
686
|
+
processed << resource
|
687
|
+
else
|
688
|
+
MU.log "#{attrs[:cfg_name]} #{resource['name']} didn't show up from findLitterMate", MU::WARN, details: deploy.original_config[attrs[:cfg_plural]].reject { |r| r['name'] != "" }
|
689
|
+
end
|
420
690
|
end
|
421
691
|
}
|
422
692
|
|
@@ -427,24 +697,23 @@ module MU
|
|
427
697
|
|
428
698
|
# Pare out global values like +cloud+ or +region+ that appear to be
|
429
699
|
# universal in the deploy we're creating.
|
430
|
-
|
700
|
+
scrub_globals = Proc.new { |h, field|
|
431
701
|
if h.is_a?(Hash)
|
432
702
|
newhash = {}
|
433
703
|
h.each_pair { |k, v|
|
434
704
|
next if k == field
|
435
|
-
newhash[k] = scrub_globals(v, field)
|
705
|
+
newhash[k] = scrub_globals.call(v, field)
|
436
706
|
}
|
437
707
|
h = newhash
|
438
708
|
elsif h.is_a?(Array)
|
439
709
|
newarr = []
|
440
710
|
h.each { |v|
|
441
|
-
newarr << scrub_globals(v, field)
|
711
|
+
newarr << scrub_globals.call(v, field)
|
442
712
|
}
|
443
|
-
h = newarr
|
713
|
+
h = newarr.uniq
|
444
714
|
end
|
445
|
-
|
446
715
|
h
|
447
|
-
|
716
|
+
}
|
448
717
|
|
449
718
|
globals.each_pair { |field, counts|
|
450
719
|
next if counts.size != 1
|
@@ -454,7 +723,7 @@ module MU
|
|
454
723
|
if bok[attrs[:cfg_plural]]
|
455
724
|
new_resources = []
|
456
725
|
bok[attrs[:cfg_plural]].each { |resource|
|
457
|
-
new_resources << scrub_globals(resource, field)
|
726
|
+
new_resources << scrub_globals.call(resource, field)
|
458
727
|
}
|
459
728
|
bok[attrs[:cfg_plural]] = new_resources
|
460
729
|
end
|
@@ -472,11 +741,33 @@ module MU
|
|
472
741
|
end
|
473
742
|
|
474
743
|
def resolveReferences(cfg, deploy, parent)
|
744
|
+
mask_deploy_id = false
|
745
|
+
|
746
|
+
check_deploy_id = Proc.new { |cfgblob|
|
747
|
+
(deploy and
|
748
|
+
(cfgblob.is_a?(MU::Config::Ref) or cfgblob.is_a?(Hash)) and
|
749
|
+
cfgblob['deploy_id'] and
|
750
|
+
cfgblob['deploy_id'] != deploy.deploy_id and
|
751
|
+
@diff and
|
752
|
+
@types_found_in[cfgblob['type']] and
|
753
|
+
@types_found_in[cfgblob['type']].deploy_id == cfgblob['deploy_id']
|
754
|
+
)
|
755
|
+
}
|
756
|
+
|
757
|
+
mask_deploy_id = check_deploy_id.call(cfg)
|
758
|
+
|
475
759
|
if cfg.is_a?(MU::Config::Ref)
|
476
|
-
|
760
|
+
if mask_deploy_id
|
761
|
+
cfg.delete("deploy_id")
|
762
|
+
cfg.delete("mommacat")
|
763
|
+
cfg.kitten(deploy)
|
764
|
+
else
|
765
|
+
cfg.kitten(deploy) || cfg.kitten
|
766
|
+
end
|
767
|
+
|
477
768
|
hashcfg = cfg.to_h
|
478
769
|
|
479
|
-
if cfg.kitten
|
770
|
+
if cfg.kitten
|
480
771
|
littermate = deploy.findLitterMate(type: cfg.type, name: cfg.name, cloud_id: cfg.id, habitat: cfg.habitat)
|
481
772
|
|
482
773
|
if littermate and littermate.config['name']
|
@@ -501,13 +792,12 @@ module MU
|
|
501
792
|
elsif hashcfg["id"] and !hashcfg["name"]
|
502
793
|
hashcfg.delete("deploy_id")
|
503
794
|
else
|
504
|
-
|
505
|
-
raise Incomplete, "Failed to resolve reference on behalf of #{parent}"
|
795
|
+
raise Incomplete.new "Failed to resolve reference on behalf of #{parent}", details: hashcfg
|
506
796
|
end
|
507
797
|
hashcfg.delete("deploy_id") if hashcfg['deploy_id'] == deploy.deploy_id
|
508
798
|
|
509
799
|
if parent and parent.config
|
510
|
-
cred_cfg = MU::Cloud.
|
800
|
+
cred_cfg = MU::Cloud.cloudClass(parent.cloud).credConfig(parent.credentials)
|
511
801
|
|
512
802
|
if parent.config['region'] == hashcfg['region'] or
|
513
803
|
cred_cfg['region'] == hashcfg['region']
|
@@ -566,7 +856,12 @@ module MU
|
|
566
856
|
MU.log "Dropping unresolved value", MU::WARN, details: value
|
567
857
|
end
|
568
858
|
}
|
569
|
-
cfg = new_array
|
859
|
+
cfg = new_array.uniq
|
860
|
+
end
|
861
|
+
|
862
|
+
if mask_deploy_id or check_deploy_id.call(cfg)
|
863
|
+
cfg.delete("deploy_id")
|
864
|
+
MU.log "#{parent} in #{deploy.deploy_id} references something in #{@types_found_in[cfg['type']].deploy_id}, ditching extraneous deploy_id", MU::DEBUG, details: cfg.to_h
|
570
865
|
end
|
571
866
|
|
572
867
|
cfg
|
@@ -616,6 +911,10 @@ module MU
|
|
616
911
|
|
617
912
|
if !@scraped[typename][kitten['cloud_id']]
|
618
913
|
MU.log "No object in scraped tree for #{attrs[:cfg_name]} #{kitten['cloud_id']} (#{kitten['name']})", MU::ERR, details: kitten
|
914
|
+
if kitten['cloud_id'].nil?
|
915
|
+
pp caller
|
916
|
+
exit
|
917
|
+
end
|
619
918
|
next
|
620
919
|
end
|
621
920
|
|
@@ -626,7 +925,8 @@ module MU
|
|
626
925
|
deploy.addKitten(
|
627
926
|
attrs[:cfg_plural],
|
628
927
|
kitten['name'],
|
629
|
-
@scraped[typename][kitten['cloud_id']]
|
928
|
+
@scraped[typename][kitten['cloud_id']],
|
929
|
+
do_notify: true
|
630
930
|
)
|
631
931
|
}
|
632
932
|
end
|
@@ -635,6 +935,21 @@ module MU
|
|
635
935
|
deploy
|
636
936
|
end
|
637
937
|
|
938
|
+
def self.deDuplicateName(kitten_cfg, res_class)
|
939
|
+
orig_name = kitten_cfg['name'].dup
|
940
|
+
if kitten_cfg['parent'] and kitten_cfg['parent'].respond_to?(:id) and kitten_cfg['parent'].id
|
941
|
+
kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['parent'].id
|
942
|
+
elsif kitten_cfg['project']
|
943
|
+
kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['project']
|
944
|
+
elsif kitten_cfg['region']
|
945
|
+
kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['region']
|
946
|
+
elsif kitten_cfg['cloud_id']
|
947
|
+
kitten_cfg['name'] = kitten_cfg['name']+"-"+kitten_cfg['cloud_id'].gsub(/[^a-z0-9]/i, "-")
|
948
|
+
else
|
949
|
+
raise MU::Config::DuplicateNameError, "Saw duplicate #{res_class.cfg_name} name #{orig_name} and couldn't come up with a good way to differentiate them"
|
950
|
+
end
|
951
|
+
end
|
952
|
+
|
638
953
|
# Go through everything we've scraped and update our mappings of cloud ids
|
639
954
|
# and bare name fields, so that resources can reference one another
|
640
955
|
# portably by name.
|