cloud-mu 3.1.3 → 3.3.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (212) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +15 -3
  3. data/ansible/roles/mu-windows/README.md +33 -0
  4. data/ansible/roles/mu-windows/defaults/main.yml +2 -0
  5. data/ansible/roles/mu-windows/files/LaunchConfig.json +9 -0
  6. data/ansible/roles/mu-windows/files/config.xml +76 -0
  7. data/ansible/roles/mu-windows/handlers/main.yml +2 -0
  8. data/ansible/roles/mu-windows/meta/main.yml +53 -0
  9. data/ansible/roles/mu-windows/tasks/main.yml +36 -0
  10. data/ansible/roles/mu-windows/tests/inventory +2 -0
  11. data/ansible/roles/mu-windows/tests/test.yml +5 -0
  12. data/ansible/roles/mu-windows/vars/main.yml +2 -0
  13. data/bin/mu-adopt +21 -13
  14. data/bin/mu-azure-tests +57 -0
  15. data/bin/mu-cleanup +2 -4
  16. data/bin/mu-configure +52 -0
  17. data/bin/mu-deploy +3 -3
  18. data/bin/mu-findstray-tests +25 -0
  19. data/bin/mu-gen-docs +2 -4
  20. data/bin/mu-load-config.rb +4 -4
  21. data/bin/mu-node-manage +15 -16
  22. data/bin/mu-run-tests +147 -37
  23. data/cloud-mu.gemspec +22 -20
  24. data/cookbooks/mu-activedirectory/resources/domain.rb +4 -4
  25. data/cookbooks/mu-activedirectory/resources/domain_controller.rb +4 -4
  26. data/cookbooks/mu-tools/libraries/helper.rb +3 -2
  27. data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
  28. data/cookbooks/mu-tools/recipes/apply_security.rb +14 -14
  29. data/cookbooks/mu-tools/recipes/aws_api.rb +9 -0
  30. data/cookbooks/mu-tools/recipes/eks.rb +2 -2
  31. data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
  32. data/cookbooks/mu-tools/recipes/selinux.rb +2 -1
  33. data/cookbooks/mu-tools/recipes/windows-client.rb +163 -164
  34. data/cookbooks/mu-tools/resources/disk.rb +1 -1
  35. data/cookbooks/mu-tools/resources/windows_users.rb +44 -43
  36. data/extras/clean-stock-amis +25 -19
  37. data/extras/generate-stock-images +1 -0
  38. data/extras/image-generators/AWS/win2k12.yaml +18 -13
  39. data/extras/image-generators/AWS/win2k16.yaml +18 -13
  40. data/extras/image-generators/AWS/win2k19.yaml +21 -0
  41. data/extras/image-generators/Google/centos6.yaml +1 -0
  42. data/extras/image-generators/Google/centos7.yaml +1 -1
  43. data/modules/mommacat.ru +6 -16
  44. data/modules/mu.rb +158 -111
  45. data/modules/mu/adoption.rb +404 -71
  46. data/modules/mu/cleanup.rb +221 -306
  47. data/modules/mu/cloud.rb +129 -1633
  48. data/modules/mu/cloud/database.rb +49 -0
  49. data/modules/mu/cloud/dnszone.rb +44 -0
  50. data/modules/mu/cloud/machine_images.rb +212 -0
  51. data/modules/mu/cloud/providers.rb +81 -0
  52. data/modules/mu/cloud/resource_base.rb +926 -0
  53. data/modules/mu/cloud/server.rb +40 -0
  54. data/modules/mu/cloud/server_pool.rb +1 -0
  55. data/modules/mu/cloud/ssh_sessions.rb +228 -0
  56. data/modules/mu/cloud/winrm_sessions.rb +237 -0
  57. data/modules/mu/cloud/wrappers.rb +169 -0
  58. data/modules/mu/config.rb +171 -1767
  59. data/modules/mu/config/alarm.rb +2 -6
  60. data/modules/mu/config/bucket.rb +32 -3
  61. data/modules/mu/config/cache_cluster.rb +2 -2
  62. data/modules/mu/config/cdn.rb +100 -0
  63. data/modules/mu/config/collection.rb +4 -4
  64. data/modules/mu/config/container_cluster.rb +9 -4
  65. data/modules/mu/config/database.rb +84 -105
  66. data/modules/mu/config/database.yml +1 -2
  67. data/modules/mu/config/dnszone.rb +10 -9
  68. data/modules/mu/config/doc_helpers.rb +516 -0
  69. data/modules/mu/config/endpoint.rb +5 -4
  70. data/modules/mu/config/firewall_rule.rb +103 -4
  71. data/modules/mu/config/folder.rb +4 -4
  72. data/modules/mu/config/function.rb +19 -10
  73. data/modules/mu/config/group.rb +4 -4
  74. data/modules/mu/config/habitat.rb +4 -4
  75. data/modules/mu/config/job.rb +89 -0
  76. data/modules/mu/config/loadbalancer.rb +60 -14
  77. data/modules/mu/config/log.rb +4 -4
  78. data/modules/mu/config/msg_queue.rb +4 -4
  79. data/modules/mu/config/nosqldb.rb +4 -4
  80. data/modules/mu/config/notifier.rb +10 -21
  81. data/modules/mu/config/ref.rb +411 -0
  82. data/modules/mu/config/role.rb +4 -4
  83. data/modules/mu/config/schema_helpers.rb +509 -0
  84. data/modules/mu/config/search_domain.rb +4 -4
  85. data/modules/mu/config/server.rb +98 -71
  86. data/modules/mu/config/server.yml +1 -0
  87. data/modules/mu/config/server_pool.rb +5 -9
  88. data/modules/mu/config/storage_pool.rb +1 -1
  89. data/modules/mu/config/tail.rb +200 -0
  90. data/modules/mu/config/user.rb +4 -4
  91. data/modules/mu/config/vpc.rb +71 -27
  92. data/modules/mu/config/vpc.yml +0 -1
  93. data/modules/mu/defaults/AWS.yaml +91 -68
  94. data/modules/mu/defaults/Azure.yaml +1 -0
  95. data/modules/mu/defaults/Google.yaml +3 -2
  96. data/modules/mu/deploy.rb +43 -26
  97. data/modules/mu/groomer.rb +17 -2
  98. data/modules/mu/groomers/ansible.rb +188 -41
  99. data/modules/mu/groomers/chef.rb +116 -55
  100. data/modules/mu/logger.rb +127 -148
  101. data/modules/mu/master.rb +410 -2
  102. data/modules/mu/master/chef.rb +3 -4
  103. data/modules/mu/master/ldap.rb +3 -3
  104. data/modules/mu/master/ssl.rb +12 -3
  105. data/modules/mu/mommacat.rb +218 -2612
  106. data/modules/mu/mommacat/daemon.rb +403 -0
  107. data/modules/mu/mommacat/naming.rb +473 -0
  108. data/modules/mu/mommacat/search.rb +495 -0
  109. data/modules/mu/mommacat/storage.rb +722 -0
  110. data/modules/mu/{clouds → providers}/README.md +1 -1
  111. data/modules/mu/{clouds → providers}/aws.rb +380 -122
  112. data/modules/mu/{clouds → providers}/aws/alarm.rb +7 -5
  113. data/modules/mu/{clouds → providers}/aws/bucket.rb +297 -59
  114. data/modules/mu/{clouds → providers}/aws/cache_cluster.rb +37 -71
  115. data/modules/mu/providers/aws/cdn.rb +782 -0
  116. data/modules/mu/{clouds → providers}/aws/collection.rb +26 -25
  117. data/modules/mu/{clouds → providers}/aws/container_cluster.rb +724 -744
  118. data/modules/mu/providers/aws/database.rb +1744 -0
  119. data/modules/mu/{clouds → providers}/aws/dnszone.rb +88 -70
  120. data/modules/mu/providers/aws/endpoint.rb +1072 -0
  121. data/modules/mu/{clouds → providers}/aws/firewall_rule.rb +220 -247
  122. data/modules/mu/{clouds → providers}/aws/folder.rb +8 -8
  123. data/modules/mu/{clouds → providers}/aws/function.rb +300 -142
  124. data/modules/mu/{clouds → providers}/aws/group.rb +31 -29
  125. data/modules/mu/{clouds → providers}/aws/habitat.rb +18 -15
  126. data/modules/mu/providers/aws/job.rb +466 -0
  127. data/modules/mu/{clouds → providers}/aws/loadbalancer.rb +66 -56
  128. data/modules/mu/{clouds → providers}/aws/log.rb +17 -14
  129. data/modules/mu/{clouds → providers}/aws/msg_queue.rb +29 -19
  130. data/modules/mu/{clouds → providers}/aws/nosqldb.rb +114 -16
  131. data/modules/mu/{clouds → providers}/aws/notifier.rb +142 -65
  132. data/modules/mu/{clouds → providers}/aws/role.rb +158 -118
  133. data/modules/mu/{clouds → providers}/aws/search_domain.rb +201 -59
  134. data/modules/mu/{clouds → providers}/aws/server.rb +844 -1139
  135. data/modules/mu/{clouds → providers}/aws/server_pool.rb +74 -65
  136. data/modules/mu/{clouds → providers}/aws/storage_pool.rb +26 -44
  137. data/modules/mu/{clouds → providers}/aws/user.rb +24 -25
  138. data/modules/mu/{clouds → providers}/aws/userdata/README.md +0 -0
  139. data/modules/mu/{clouds → providers}/aws/userdata/linux.erb +5 -4
  140. data/modules/mu/{clouds → providers}/aws/userdata/windows.erb +2 -1
  141. data/modules/mu/{clouds → providers}/aws/vpc.rb +525 -931
  142. data/modules/mu/providers/aws/vpc_subnet.rb +286 -0
  143. data/modules/mu/{clouds → providers}/azure.rb +29 -9
  144. data/modules/mu/{clouds → providers}/azure/container_cluster.rb +3 -8
  145. data/modules/mu/{clouds → providers}/azure/firewall_rule.rb +18 -11
  146. data/modules/mu/{clouds → providers}/azure/habitat.rb +8 -6
  147. data/modules/mu/{clouds → providers}/azure/loadbalancer.rb +5 -5
  148. data/modules/mu/{clouds → providers}/azure/role.rb +8 -10
  149. data/modules/mu/{clouds → providers}/azure/server.rb +97 -49
  150. data/modules/mu/{clouds → providers}/azure/user.rb +6 -8
  151. data/modules/mu/{clouds → providers}/azure/userdata/README.md +0 -0
  152. data/modules/mu/{clouds → providers}/azure/userdata/linux.erb +0 -0
  153. data/modules/mu/{clouds → providers}/azure/userdata/windows.erb +0 -0
  154. data/modules/mu/{clouds → providers}/azure/vpc.rb +16 -21
  155. data/modules/mu/{clouds → providers}/cloudformation.rb +18 -7
  156. data/modules/mu/{clouds → providers}/cloudformation/alarm.rb +3 -3
  157. data/modules/mu/{clouds → providers}/cloudformation/cache_cluster.rb +3 -3
  158. data/modules/mu/{clouds → providers}/cloudformation/collection.rb +3 -3
  159. data/modules/mu/{clouds → providers}/cloudformation/database.rb +6 -17
  160. data/modules/mu/{clouds → providers}/cloudformation/dnszone.rb +3 -3
  161. data/modules/mu/{clouds → providers}/cloudformation/firewall_rule.rb +3 -3
  162. data/modules/mu/{clouds → providers}/cloudformation/loadbalancer.rb +3 -3
  163. data/modules/mu/{clouds → providers}/cloudformation/log.rb +3 -3
  164. data/modules/mu/{clouds → providers}/cloudformation/server.rb +7 -7
  165. data/modules/mu/{clouds → providers}/cloudformation/server_pool.rb +5 -5
  166. data/modules/mu/{clouds → providers}/cloudformation/vpc.rb +5 -7
  167. data/modules/mu/{clouds → providers}/docker.rb +0 -0
  168. data/modules/mu/{clouds → providers}/google.rb +68 -30
  169. data/modules/mu/{clouds → providers}/google/bucket.rb +13 -15
  170. data/modules/mu/{clouds → providers}/google/container_cluster.rb +85 -78
  171. data/modules/mu/{clouds → providers}/google/database.rb +11 -21
  172. data/modules/mu/{clouds → providers}/google/firewall_rule.rb +15 -14
  173. data/modules/mu/{clouds → providers}/google/folder.rb +20 -17
  174. data/modules/mu/{clouds → providers}/google/function.rb +140 -168
  175. data/modules/mu/{clouds → providers}/google/group.rb +29 -34
  176. data/modules/mu/{clouds → providers}/google/habitat.rb +21 -22
  177. data/modules/mu/{clouds → providers}/google/loadbalancer.rb +19 -21
  178. data/modules/mu/{clouds → providers}/google/role.rb +94 -58
  179. data/modules/mu/{clouds → providers}/google/server.rb +243 -156
  180. data/modules/mu/{clouds → providers}/google/server_pool.rb +26 -45
  181. data/modules/mu/{clouds → providers}/google/user.rb +95 -31
  182. data/modules/mu/{clouds → providers}/google/userdata/README.md +0 -0
  183. data/modules/mu/{clouds → providers}/google/userdata/linux.erb +0 -0
  184. data/modules/mu/{clouds → providers}/google/userdata/windows.erb +0 -0
  185. data/modules/mu/{clouds → providers}/google/vpc.rb +103 -79
  186. data/modules/tests/aws-jobs-functions.yaml +46 -0
  187. data/modules/tests/bucket.yml +4 -0
  188. data/modules/tests/centos6.yaml +15 -0
  189. data/modules/tests/centos7.yaml +15 -0
  190. data/modules/tests/centos8.yaml +12 -0
  191. data/modules/tests/ecs.yaml +23 -0
  192. data/modules/tests/eks.yaml +1 -1
  193. data/modules/tests/functions/node-function/lambda_function.js +10 -0
  194. data/modules/tests/functions/python-function/lambda_function.py +12 -0
  195. data/modules/tests/includes-and-params.yaml +2 -1
  196. data/modules/tests/microservice_app.yaml +288 -0
  197. data/modules/tests/rds.yaml +108 -0
  198. data/modules/tests/regrooms/aws-iam.yaml +201 -0
  199. data/modules/tests/regrooms/bucket.yml +19 -0
  200. data/modules/tests/regrooms/rds.yaml +123 -0
  201. data/modules/tests/server-with-scrub-muisms.yaml +2 -1
  202. data/modules/tests/super_complex_bok.yml +2 -2
  203. data/modules/tests/super_simple_bok.yml +3 -5
  204. data/modules/tests/win2k12.yaml +17 -5
  205. data/modules/tests/win2k16.yaml +25 -0
  206. data/modules/tests/win2k19.yaml +25 -0
  207. data/requirements.txt +1 -0
  208. data/spec/mu/clouds/azure_spec.rb +2 -2
  209. metadata +240 -154
  210. data/extras/image-generators/AWS/windows.yaml +0 -18
  211. data/modules/mu/clouds/aws/database.rb +0 -1985
  212. data/modules/mu/clouds/aws/endpoint.rb +0 -592
@@ -360,7 +360,7 @@ module MU
360
360
  # @param stamp [Integer]: The MS-style timestamp, e.g. 130838184558490696
361
361
  # @return [Time]
362
362
  def self.convertMicrosoftTime(stamp)
363
- ms_epoch = DateTime.new(1601,1,1).strftime("%Q").to_i
363
+ # ms_epoch = DateTime.new(1601,1,1).strftime("%Q").to_i
364
364
  unixtime = (stamp.to_i/10000) + DateTime.new(1601,1,1).strftime("%Q").to_i
365
365
  Time.at(unixtime/1000)
366
366
  end
@@ -629,7 +629,7 @@ module MU
629
629
  return true if !require_group
630
630
 
631
631
  shortuser = username.sub(/\@.*/, "")
632
- user = findUsers(search = [shortuser], exact: true)
632
+ user = findUsers([shortuser], exact: true)
633
633
  if user[shortuser]["memberOf"].is_a?(Array)
634
634
  user[shortuser]["memberOf"].each { |group|
635
635
  shortname = group.sub(/^CN=(.*?),.*/, '\1')
@@ -945,7 +945,7 @@ module MU
945
945
  cur_users[user]['realname'] = name
946
946
  end
947
947
  if disable
948
- user_props = findUsers([user], exact: true)
948
+ findUsers([user], exact: true)
949
949
  MU.log "Disabling #{user}", MU::WARN
950
950
  conn.replace_attribute(user_dn, :userAccountControl, AD_PW_ATTRS['disable'].to_i.to_s(2))
951
951
  elsif enable
@@ -90,7 +90,7 @@ module MU
90
90
 
91
91
  begin
92
92
  csr = OpenSSL::X509::Request.new File.read csr_path
93
- rescue Exception => e
93
+ rescue StandardError => e
94
94
  MU.log e.message, MU::ERR, details: File.read(csr_path)
95
95
  raise e
96
96
  end
@@ -160,7 +160,6 @@ module MU
160
160
 
161
161
  key = getKey(name, for_user: for_user)
162
162
 
163
- puts cn_str
164
163
  cn = OpenSSL::X509::Name.parse(cn_str)
165
164
 
166
165
  # If we're generating our local CA, we're not really doing a CSR, but
@@ -222,8 +221,12 @@ puts cn_str
222
221
  [cert, pfx_cert]
223
222
  end
224
223
 
225
- private
226
224
 
225
+ # Convert an x509 certificate to the .pfx thing Windows likes
226
+ # @param certfile [String]: Path to source certificate
227
+ # @param keyfile [String]: Path to source certificate's key
228
+ # @param pfxfile [String]: Path to output the new certificate
229
+ # @return [OpenSSL::PKCS12]
227
230
  def self.toPfx(certfile, keyfile, pfxfile)
228
231
  cacert = getCert("Mu_CA", ca: true).first
229
232
  cert = OpenSSL::X509::Certificate.new(File.read(certfile))
@@ -234,7 +237,12 @@ puts cn_str
234
237
  }
235
238
  pfx
236
239
  end
240
+ private_class_method :toPfx
237
241
 
242
+ # Given a list of strings that might be IPs or hostnames, format as a
243
+ # of Subject Alternative Names for use in a certificate.
244
+ # @param sans [Array<String>]
245
+ # @return [String]
238
246
  def self.formatSANS(sans)
239
247
  sans.map { |s|
240
248
  if s.match(/^\d+\.\d+\.\d+\.\d+$/)
@@ -244,6 +252,7 @@ puts cn_str
244
252
  end
245
253
  }.join(",")
246
254
  end
255
+ private_class_method :formatSANS
247
256
 
248
257
  end
249
258
  end
@@ -18,6 +18,10 @@ require 'json'
18
18
  require 'stringio'
19
19
  require 'securerandom'
20
20
  require 'timeout'
21
+ require 'mu/mommacat/storage'
22
+ require 'mu/mommacat/search'
23
+ require 'mu/mommacat/daemon'
24
+ require 'mu/mommacat/naming'
21
25
 
22
26
  module MU
23
27
 
@@ -42,60 +46,6 @@ module MU
42
46
  @@litters_loadtime = {}
43
47
  @@litter_semaphore = Mutex.new
44
48
 
45
- # Return a {MU::MommaCat} instance for an existing deploy. Use this instead
46
- # of using #initialize directly to avoid loading deploys multiple times or
47
- # stepping on the global context for the deployment you're really working
48
- # on..
49
- # @param deploy_id [String]: The deploy ID of the deploy to load.
50
- # @param set_context_to_me [Boolean]: Whether new MommaCat objects should overwrite any existing per-thread global deploy variables.
51
- # @param use_cache [Boolean]: If we have an existing object for this deploy, use that
52
- # @return [MU::MommaCat]
53
- def self.getLitter(deploy_id, set_context_to_me: false, use_cache: true)
54
- if deploy_id.nil? or deploy_id.empty?
55
- raise MuError, "Cannot fetch a deployment without a deploy_id"
56
- end
57
-
58
- # XXX this caching may be harmful, causing stale resource objects to stick
59
- # around. Have we fixed this? Sort of. Bad entries seem to have no kittens,
60
- # so force a reload if we see that. That's probably not the root problem.
61
- littercache = nil
62
- begin
63
- @@litter_semaphore.synchronize {
64
- littercache = @@litters.dup
65
- }
66
- if littercache[deploy_id] and @@litters_loadtime[deploy_id]
67
- deploy_root = File.expand_path(MU.dataDir+"/deployments")
68
- this_deploy_dir = deploy_root+"/"+deploy_id
69
- if File.exist?("#{this_deploy_dir}/deployment.json")
70
- lastmod = File.mtime("#{this_deploy_dir}/deployment.json")
71
- if lastmod > @@litters_loadtime[deploy_id]
72
- MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE
73
- use_cache = false
74
- end
75
- end
76
- end
77
- rescue ThreadError => e
78
- # already locked by a parent caller and this is a read op, so this is ok
79
- raise e if !e.message.match(/recursive locking/)
80
- littercache = @@litters.dup
81
- end
82
-
83
- if !use_cache or littercache[deploy_id].nil?
84
- need_gc = !littercache[deploy_id].nil?
85
- newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
86
- # This, we have to synchronize, as it's a write
87
- @@litter_semaphore.synchronize {
88
- @@litters[deploy_id] = newlitter
89
- @@litters_loadtime[deploy_id] = Time.now
90
- }
91
- GC.start if need_gc
92
- elsif set_context_to_me
93
- MU::MommaCat.setThreadContext(@@litters[deploy_id])
94
- end
95
- return @@litters[deploy_id]
96
- # MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
97
- end
98
-
99
49
  # Update the in-memory cache of a given deploy. This is intended for use by
100
50
  # {#save!}, primarily.
101
51
  # @param deploy_id [String]
@@ -127,21 +77,7 @@ module MU
127
77
  attr_reader :chef_user
128
78
  attr_reader :no_artifacts
129
79
  attr_accessor :kittens # really want a method only available to :Deploy
130
- @myhome = Etc.getpwuid(Process.uid).dir
131
- @nagios_home = "/opt/mu/var/nagios_user_home"
132
- @locks = Hash.new
133
- @deploy_cache = Hash.new
134
80
  @nocleanup = false
135
- # List the currently held flock() locks.
136
- def self.trapSafeLocks;
137
- @locks
138
- end
139
- # List the currently held flock() locks.
140
- def self.locks;
141
- @lock_semaphore.synchronize {
142
- @locks
143
- }
144
- end
145
81
 
146
82
  @@deploy_struct_semaphore = Mutex.new
147
83
  # Don't let things that modify the deploy struct Hash step on each other.
@@ -219,7 +155,7 @@ module MU
219
155
  if @mu_user == "root"
220
156
  @chef_user = "mu"
221
157
  else
222
- @chef_user = @mu_user.dup.gsub(/\./, "")
158
+ @chef_user = @mu_user.dup.delete(".")
223
159
  @mu_user = "root" if @mu_user == "mu"
224
160
  end
225
161
  @kitten_semaphore = Mutex.new
@@ -231,6 +167,7 @@ module MU
231
167
  @need_deploy_flush = false
232
168
  @node_cert_semaphore = Mutex.new
233
169
  @deployment = deployment_data
170
+
234
171
  @deployment['mu_public_ip'] = MU.mu_public_ip
235
172
  @private_key = nil
236
173
  @public_key = nil
@@ -246,60 +183,17 @@ module MU
246
183
  @appname ||= @original_config['name'] if @original_config
247
184
  @timestamp = timestamp
248
185
  @environment = environment
186
+ @original_config['environment'] ||= @environment if @original_config
249
187
 
250
188
  if set_context_to_me
251
189
  MU::MommaCat.setThreadContext(self)
252
190
  end
253
191
 
254
192
  if create and !@no_artifacts
255
- if !Dir.exist?(MU.dataDir+"/deployments")
256
- MU.log "Creating #{MU.dataDir}/deployments", MU::DEBUG
257
- Dir.mkdir(MU.dataDir+"/deployments", 0700)
258
- end
259
- path = File.expand_path(MU.dataDir+"/deployments")+"/"+@deploy_id
260
- if !Dir.exist?(path)
261
- MU.log "Creating #{path}", MU::DEBUG
262
- Dir.mkdir(path, 0700)
263
- end
264
- if @original_config.nil? or !@original_config.is_a?(Hash)
265
- raise DeployInitializeError, "New MommaCat repository requires config hash"
266
- end
267
- credsets = {}
268
-
269
- MU::Cloud.resource_types.values.each { |attrs|
270
- if !@original_config[attrs[:cfg_plural]].nil? and @original_config[attrs[:cfg_plural]].size > 0
271
- @original_config[attrs[:cfg_plural]].each { |resource|
272
-
273
- credsets[resource['cloud']] ||= []
274
- credsets[resource['cloud']] << resource['credentials']
275
- @clouds[resource['cloud']] = 0 if !@clouds.has_key?(resource['cloud'])
276
- @clouds[resource['cloud']] = @clouds[resource['cloud']] + 1
277
-
278
- }
279
- end
280
- }
281
-
282
- @ssh_key_name, @ssh_private_key, @ssh_public_key = self.SSHKey
283
- if !File.exist?(deploy_dir+"/private_key")
284
- @private_key, @public_key = createDeployKey
285
- end
286
- MU.log "Creating deploy secret for #{MU.deploy_id}"
287
- @deploy_secret = Password.random(256)
288
- if !@original_config['scrub_mu_isms'] and !@no_artifacts
289
- credsets.each_pair { |cloud, creds|
290
- creds.uniq!
291
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
292
- creds.each { |credentials|
293
- cloudclass.writeDeploySecret(@deploy_id, @deploy_secret, credentials: credentials)
294
- }
295
- }
296
- end
297
- if set_context_to_me
298
- MU::MommaCat.setThreadContext(self)
299
- end
300
-
193
+ initDeployDirectory
194
+ setDeploySecret
195
+ MU::MommaCat.setThreadContext(self) if set_context_to_me
301
196
  save!
302
-
303
197
  end
304
198
 
305
199
  @appname ||= MU.appname
@@ -307,10 +201,8 @@ module MU
307
201
  @environment ||= MU.environment
308
202
 
309
203
  loadDeploy(set_context_to_me: set_context_to_me)
310
- if !deploy_secret.nil?
311
- if !authKey(deploy_secret)
312
- raise DeployInitializeError, "Client request did not include a valid deploy authorization secret. Verify that userdata runs correctly?"
313
- end
204
+ if !deploy_secret.nil? and !authKey(deploy_secret)
205
+ raise DeployInitializeError, "Client request did not include a valid deploy authorization secret. Verify that userdata runs correctly?"
314
206
  end
315
207
 
316
208
 
@@ -322,86 +214,7 @@ module MU
322
214
  # deploy, IF it already exists, which is to say if we're loading an
323
215
  # existing deploy instead of creating a new one.
324
216
  if !create and @deployment and @original_config and !skip_resource_objects
325
-
326
- MU::Cloud.resource_types.each_pair { |res_type, attrs|
327
- type = attrs[:cfg_plural]
328
- if @deployment.has_key?(type)
329
-
330
- @deployment[type].each_pair { |res_name, data|
331
- orig_cfg = nil
332
- if @original_config.has_key?(type)
333
- @original_config[type].each { |resource|
334
- if resource["name"] == res_name
335
- orig_cfg = resource
336
- break
337
- end
338
- }
339
- end
340
-
341
- # Some Server objects originated from ServerPools, get their
342
- # configs from there
343
- if type == "servers" and orig_cfg.nil? and
344
- @original_config.has_key?("server_pools")
345
- @original_config["server_pools"].each { |resource|
346
- if resource["name"] == res_name
347
- orig_cfg = resource
348
- break
349
- end
350
- }
351
- end
352
-
353
- if orig_cfg.nil?
354
- MU.log "Failed to locate original config for #{attrs[:cfg_name]} #{res_name} in #{@deploy_id}", MU::WARN if !["firewall_rules", "databases", "storage_pools", "cache_clusters", "alarms"].include?(type) # XXX shaddap
355
- next
356
- end
357
-
358
- if orig_cfg['vpc'] and orig_cfg['vpc'].is_a?(Hash)
359
- ref = if orig_cfg['vpc']['id'] and orig_cfg['vpc']['id'].is_a?(Hash)
360
- orig_cfg['vpc']['id']['mommacat'] = self
361
- MU::Config::Ref.get(orig_cfg['vpc']['id'])
362
- else
363
- orig_cfg['vpc']['mommacat'] = self
364
- MU::Config::Ref.get(orig_cfg['vpc'])
365
- end
366
- orig_cfg['vpc'].delete('mommacat')
367
- orig_cfg['vpc'] = ref if ref.kitten(shallow: true)
368
- end
369
-
370
- begin
371
- # Load up MU::Cloud objects for all our kittens in this deploy
372
- orig_cfg['environment'] = @environment # not always set in old deploys
373
- if attrs[:has_multiples]
374
- data.keys.each { |mu_name|
375
- attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: mu_name, delay_descriptor_load: delay_descriptor_load)
376
- }
377
- else
378
- # XXX hack for old deployments, this can go away some day
379
- if data['mu_name'].nil? or data['mu_name'].empty?
380
- if res_type.to_s == "LoadBalancer" and !data['awsname'].nil?
381
- data['mu_name'] = data['awsname'].dup
382
- elsif res_type.to_s == "FirewallRule" and !data['group_name'].nil?
383
- data['mu_name'] = data['group_name'].dup
384
- elsif res_type.to_s == "Database" and !data['identifier'].nil?
385
- data['mu_name'] = data['identifier'].dup.upcase
386
- elsif res_type.to_s == "VPC"
387
- # VPC names are deterministic, just generate the things
388
- data['mu_name'] = getResourceName(data['name'])
389
- end
390
- end
391
- if data['mu_name'].nil?
392
- raise MuError, "Unable to find or guess a Mu name for #{res_type}: #{res_name} in #{@deploy_id}"
393
- end
394
- attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: data['mu_name'], cloud_id: data['cloud_id'])
395
- end
396
- rescue Exception => e
397
- if e.class != MU::Cloud::MuCloudResourceNotImplemented
398
- MU.log "Failed to load an existing resource of type '#{type}' in #{@deploy_id}: #{e.inspect}", MU::WARN, details: e.backtrace
399
- end
400
- end
401
- }
402
-
403
- end
404
- }
217
+ loadObjects(delay_descriptor_load)
405
218
  end
406
219
 
407
220
  @initializing = false
@@ -414,7 +227,7 @@ module MU
414
227
  def cloudsUsed
415
228
  seen = []
416
229
  seen << @original_config['cloud'] if @original_config['cloud']
417
- MU::Cloud.resource_types.values.each { |attrs|
230
+ MU::Cloud.resource_types.each_value { |attrs|
418
231
  type = attrs[:cfg_plural]
419
232
  if @original_config[type]
420
233
  @original_config[type].each { |resource|
@@ -434,19 +247,15 @@ module MU
434
247
  # clouds = []
435
248
  seen << @original_config['credentials'] if @original_config['credentials']
436
249
  # defaultcloud = @original_config['cloud']
437
- MU::Cloud.resource_types.values.each { |attrs|
250
+ MU::Cloud.resource_types.each_value { |attrs|
438
251
  type = attrs[:cfg_plural]
439
252
  if @original_config[type]
440
253
  @original_config[type].each { |resource|
441
254
  if resource['credentials']
442
255
  seen << resource['credentials']
443
256
  else
444
- cloudclass = if @original_config['cloud']
445
- Object.const_get("MU").const_get("Cloud").const_get(@original_config['cloud'])
446
- else
447
- Object.const_get("MU").const_get("Cloud").const_get(MU::Config.defaultCloud)
448
- end
449
- seen << cloudclass.credConfig(name_only: true)
257
+ cloudconst = @original_config['cloud'] ? @original_config['cloud'] : MU::Config.defaultCloud
258
+ seen << MU::Cloud.cloudClass(cloudconst).credConfig(name_only: true)
450
259
  end
451
260
  }
452
261
  end
@@ -455,6 +264,45 @@ module MU
455
264
  seen.uniq
456
265
  end
457
266
 
267
+ # List the accounts/projects/subscriptions used by each resource in our
268
+ # deploy.
269
+ # @return [Array<String>]
270
+ def habitatsUsed
271
+ return [] if !@original_config
272
+ habitats = []
273
+ habitats << @original_config['project'] if @original_config['project']
274
+ if @original_config['habitat']
275
+ hab_ref = MU::Config::Ref.get(@original_config['habitat'])
276
+ if hab_ref and hab_ref.id
277
+ habitats << hab_ref.id
278
+ end
279
+ end
280
+
281
+ MU::Cloud.resource_types.each_value { |attrs|
282
+ type = attrs[:cfg_plural]
283
+ if @original_config[type]
284
+ @original_config[type].each { |resource|
285
+ if resource['project']
286
+ habitats << resource['project']
287
+ elsif resource['habitat']
288
+ hab_ref = MU::Config::Ref.get(resource['habitat'])
289
+ if hab_ref and hab_ref.id
290
+ habitats << hab_ref.id
291
+ end
292
+ elsif resource['cloud']
293
+ # XXX this should be a general method implemented by each cloud
294
+ # provider
295
+ if resource['cloud'] == "Google"
296
+ habitats << MU::Cloud.cloudClass(resource['cloud']).defaultProject(resource['credentials'])
297
+ end
298
+ end
299
+ }
300
+ end
301
+ }
302
+
303
+ habitats.uniq!
304
+ end
305
+
458
306
  # List the regions used by each resource in our deploy. This will just be
459
307
  # a flat list of strings with no regard to which region belongs with what
460
308
  # cloud provider- things mostly use this as a lookup table so they can
@@ -469,13 +317,11 @@ module MU
469
317
  if @original_config[type]
470
318
  @original_config[type].each { |resource|
471
319
  if resource['cloud']
472
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(resource['cloud'])
473
- resclass = Object.const_get("MU").const_get("Cloud").const_get(resource['cloud']).const_get(res_type.to_s)
474
- if resclass.isGlobal?
475
- regions.concat(cloudclass.listRegions)
320
+ if MU::Cloud.resourceClass(resource['cloud'], res_type).isGlobal?
321
+ # XXX why was I doing this, urgh
476
322
  next
477
323
  elsif !resource['region']
478
- regions << cloudclass.myRegion
324
+ regions << MU::Cloud.cloudClass(resource['cloud']).myRegion(resource['credentials'])
479
325
  end
480
326
  end
481
327
  if resource['region']
@@ -506,7 +352,7 @@ module MU
506
352
  end
507
353
 
508
354
  count = 0
509
- MU::Cloud.resource_types.values.each { |data|
355
+ MU::Cloud.resource_types.each_value { |data|
510
356
  next if @original_config[data[:cfg_plural]].nil?
511
357
  next if realtypes.size > 0 and (!negate and !realtypes.include?(data[:cfg_plural]))
512
358
  @original_config[data[:cfg_plural]].each { |resource|
@@ -524,13 +370,13 @@ module MU
524
370
  raise MuError, "Nil arguments to removeKitten are not allowed"
525
371
  end
526
372
  @kitten_semaphore.synchronize {
527
- MU::Cloud.resource_types.values.each { |attrs|
373
+ MU::Cloud.resource_types.each_value { |attrs|
528
374
  type = attrs[:cfg_plural]
529
375
  next if !@kittens.has_key?(type)
530
376
  tmplitter = @kittens[type].values.dup
531
377
  tmplitter.each { |nodeclass, data|
532
378
  if data.is_a?(Hash)
533
- data.keys.each { |mu_name|
379
+ data.each_key { |mu_name|
534
380
  if data == object
535
381
  @kittens[type][nodeclass].delete(mu_name)
536
382
  return
@@ -548,215 +394,44 @@ module MU
548
394
  @kittens
549
395
  end
550
396
 
551
- # Overwrite this deployment's configuration with a new version. Save the
552
- # previous version as well.
553
- # @param new_conf [Hash]: A new configuration, fully resolved by {MU::Config}
554
- def updateBasketofKittens(new_conf)
555
- loadDeploy
556
- if new_conf == @original_config
557
- MU.log "#{@deploy_id}", MU::WARN
558
- return
559
- end
560
-
561
- backup = "#{deploy_dir}/basket_of_kittens.json.#{Time.now.to_i.to_s}"
562
- MU.log "Saving previous config of #{@deploy_id} to #{backup}"
563
- config = File.new(backup, File::CREAT|File::TRUNC|File::RDWR, 0600)
564
- config.flock(File::LOCK_EX)
565
- config.puts JSON.pretty_generate(@original_config)
566
- config.flock(File::LOCK_UN)
567
- config.close
568
-
569
- @original_config = new_conf
570
- # save! # XXX this will happen later, more sensibly
571
- MU.log "New config saved to #{deploy_dir}/basket_of_kittens.json"
572
- end
573
-
574
397
  # Keep tabs on a {MU::Cloud} object so that it can be found easily by
575
398
  # #findLitterMate.
576
399
  # @param type [String]:
577
400
  # @param name [String]:
578
401
  # @param object [MU::Cloud]:
579
- def addKitten(type, name, object)
402
+ def addKitten(type, name, object, do_notify: false)
580
403
  if !type or !name or !object or !object.mu_name
581
404
  raise MuError, "Nil arguments to addKitten are not allowed (got type: #{type}, name: #{name}, and '#{object}' to add)"
582
405
  end
583
406
 
584
407
  _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type)
585
- has_multiples = attrs[:has_multiples]
586
408
  object.intoDeploy(self)
587
409
 
588
- @kitten_semaphore.synchronize {
410
+ add_block = Proc.new {
589
411
  @kittens[type] ||= {}
590
412
  @kittens[type][object.habitat] ||= {}
591
- if has_multiples
413
+ if attrs[:has_multiples]
592
414
  @kittens[type][object.habitat][name] ||= {}
593
415
  @kittens[type][object.habitat][name][object.mu_name] = object
594
416
  else
595
417
  @kittens[type][object.habitat][name] = object
596
418
  end
597
- }
598
- end
599
-
600
- # Check a provided deploy key against our stored version. The instance has
601
- # in theory accessed a secret via S3 and encrypted it with the deploy's
602
- # public key. If it decrypts correctly, we assume this instance is indeed
603
- # one of ours.
604
- # @param ciphertext [String]: The text to decrypt.
605
- # return [Boolean]: Whether the provided text was encrypted with the correct key
606
- def authKey(ciphertext)
607
- if @private_key.nil? or @deploy_secret.nil?
608
- MU.log "Missing auth metadata, can't authorize node in authKey", MU::ERR
609
- return false
610
- end
611
- my_key = OpenSSL::PKey::RSA.new(@private_key)
612
-
613
- begin
614
- if my_key.private_decrypt(ciphertext).force_encoding("UTF-8") == @deploy_secret.force_encoding("UTF-8")
615
- MU.log "Matched ciphertext for #{MU.deploy_id}", MU::INFO
616
- return true
617
- else
618
- MU.log "Mis-matched ciphertext for #{MU.deploy_id}", MU::ERR
619
- return false
620
- end
621
- rescue OpenSSL::PKey::RSAError => e
622
- MU.log "Error decrypting provided ciphertext using private key from #{deploy_dir}/private_key: #{e.message}", MU::ERR, details: ciphertext
623
- return false
624
- end
625
- end
626
-
627
- # Generate a three-character string which can be used to unique-ify the
628
- # names of resources which might potentially collide, e.g. Windows local
629
- # hostnames, Amazon Elastic Load Balancers, or server pool instances.
630
- # @return [String]: A three-character string consisting of two alphnumeric
631
- # characters (uppercase) and one number.
632
- def self.genUniquenessString
633
- begin
634
- candidate = SecureRandom.base64(2).slice(0..1) + SecureRandom.random_number(9).to_s
635
- candidate.upcase!
636
- end while candidate.match(/[^A-Z0-9]/)
637
- return candidate
638
- end
639
-
640
- @unique_map_semaphore = Mutex.new
641
- @name_unique_str_map = {}
642
- # Keep a map of the uniqueness strings we assign to various full names, in
643
- # case we want to reuse them later.
644
- # @return [Hash<String>]
645
- def self.name_unique_str_map
646
- @name_unique_str_map
647
- end
648
-
649
- # Keep a map of the uniqueness strings we assign to various full names, in
650
- # case we want to reuse them later.
651
- # @return [Mutex]
652
- def self.unique_map_semaphore
653
- @unique_map_semaphore
654
- end
655
-
656
- # Generate a name string for a resource, incorporate the MU identifier
657
- # for this deployment. Will dynamically shorten the name to fit for
658
- # restrictive uses (e.g. Windows local hostnames, Amazon Elastic Load
659
- # Balancers).
660
- # @param name [String]: The shorthand name of the resource, usually the value of the "name" field in an Mu resource declaration.
661
- # @param max_length [Integer]: The maximum length of the resulting resource name.
662
- # @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated.
663
- # @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
664
- # @param disallowed_chars [Regexp]: A pattern of characters that are illegal for this resource name, such as +/[^a-zA-Z0-9-]/+
665
- # @return [String]: A full name string for this resource
666
- def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil)
667
- if name.nil?
668
- raise MuError, "Got no argument to MU::MommaCat.getResourceName"
669
- end
670
- if @appname.nil? or @environment.nil? or @timestamp.nil? or @seed.nil?
671
- MU.log "getResourceName: Missing global deploy variables in thread #{Thread.current.object_id}, using bare name '#{name}' (appname: #{@appname}, environment: #{@environment}, timestamp: #{@timestamp}, seed: #{@seed}, deploy_id: #{@deploy_id}", MU::WARN, details: caller
672
- return name
673
- end
674
- need_unique_string = false if scrub_mu_isms
675
-
676
- muname = nil
677
- if need_unique_string
678
- reserved = 4
679
- else
680
- reserved = 0
681
- end
682
-
683
- # First, pare down the base name string until it will fit
684
- basename = @appname.upcase + "-" + @environment.upcase + "-" + @timestamp + "-" + @seed.upcase + "-" + name.upcase
685
- if scrub_mu_isms
686
- basename = @appname.upcase + "-" + @environment.upcase + name.upcase
687
- end
688
-
689
- subchar = if disallowed_chars
690
- if "-".match(disallowed_chars)
691
- if !"_".match(disallowed_chars)
692
- "_"
693
- else
694
- ""
695
- end
696
- else
697
- "-"
419
+ if do_notify
420
+ notify(type, name, object.notify, triggering_node: object, delayed_save: true)
698
421
  end
699
- end
422
+ }
700
423
 
701
- if disallowed_chars
702
- basename.gsub!(disallowed_chars, subchar) if disallowed_chars
703
- end
704
- attempts = 0
705
424
  begin
706
- if (basename.length + reserved) > max_length
707
- MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG
708
- if basename == @appname.upcase + "-" + @seed.upcase + "-" + name.upcase
709
- # If we've run out of stuff to strip, truncate what's left and
710
- # just leave room for the deploy seed and uniqueness string. This
711
- # is the bare minimum, and probably what you'll see for most Windows
712
- # hostnames.
713
- basename = name.upcase + "-" + @appname.upcase
714
- basename.slice!((max_length-(reserved+3))..basename.length)
715
- basename.sub!(/-$/, "")
716
- basename = basename + "-" + @seed.upcase
717
- basename.gsub!(disallowed_chars, subchar) if disallowed_chars
718
- else
719
- # If we have to strip anything, assume we've lost uniqueness and
720
- # will have to compensate with #genUniquenessString.
721
- need_unique_string = true
722
- reserved = 4
723
- basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
724
- basename = basename + "-" + @seed.upcase + "-" + name.upcase
725
- basename.gsub!(disallowed_chars, subchar) if disallowed_chars
726
- end
727
- end
728
- attempts += 1
729
- raise MuError, "Failed to generate a reasonable name getResourceName(#{name}, max_length: #{max_length.to_s}, need_unique_string: #{need_unique_string.to_s}, use_unique_string: #{use_unique_string.to_s}, reuse_unique_string: #{reuse_unique_string.to_s}, scrub_mu_isms: #{scrub_mu_isms.to_s}, disallowed_chars: #{disallowed_chars})" if attempts > 10
730
- end while (basename.length + reserved) > max_length
731
-
732
- # Finally, apply our short random differentiator, if it's needed.
733
- if need_unique_string
734
- # Preferentially use a requested one, if it's not already in use.
735
- if !use_unique_string.nil?
736
- muname = basename + "-" + use_unique_string
737
- if !allocateUniqueResourceName(muname) and !reuse_unique_string
738
- MU.log "Requested to use #{use_unique_string} as differentiator when naming #{name}, but the name #{muname} is unavailable.", MU::WARN
739
- muname = nil
740
- end
741
- end
742
- if !muname
743
- begin
744
- unique_string = MU::MommaCat.genUniquenessString
745
- muname = basename + "-" + unique_string
746
- end while !allocateUniqueResourceName(muname)
747
- MU::MommaCat.unique_map_semaphore.synchronize {
748
- MU::MommaCat.name_unique_str_map[muname] = unique_string
749
- }
750
- end
751
- else
752
- muname = basename
425
+ @kitten_semaphore.synchronize {
426
+ add_block.call()
427
+ }
428
+ rescue ThreadError => e
429
+ # already locked by a parent call to this method, so this should be safe
430
+ raise e if !e.message.match(/recursive locking/)
431
+ add_block.call()
753
432
  end
754
- muname.gsub!(disallowed_chars, subchar) if disallowed_chars
755
-
756
- return muname
757
433
  end
758
434
 
759
-
760
435
  # Encrypt a string with the deployment's public key.
761
436
  # @param ciphertext [String]: The string to encrypt
762
437
  def encryptWithDeployKey(ciphertext)
@@ -771,7 +446,6 @@ module MU
771
446
  return my_private_key.private_decrypt(ciphertext)
772
447
  end
773
448
 
774
-
775
449
  # Save a string into deployment metadata for the current deployment,
776
450
  # encrypting it with our deploy key.
777
451
  # @param instance_id [String]: The cloud instance identifier with which this secret is associated.
@@ -786,7 +460,7 @@ module MU
786
460
  loadDeploy(true) # make sure we're not trampling deployment data
787
461
  @secret_semaphore.synchronize {
788
462
  if @secrets[type].nil?
789
- raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.to_s})"
463
+ raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.join(", ")})"
790
464
  end
791
465
  @secrets[type][instance_id] = encryptWithDeployKey(raw_secret)
792
466
  }
@@ -802,7 +476,7 @@ module MU
802
476
  @secret_semaphore.synchronize {
803
477
  if @secrets[type].nil?
804
478
  return nil if quiet
805
- raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.to_s})"
479
+ raise SecretError, "'#{type}' is not a valid secret type (valid types: #{@secrets.keys.join(", ")})"
806
480
  end
807
481
  if @secrets[type][instance_id].nil?
808
482
  return nil if quiet
@@ -812,157 +486,6 @@ module MU
812
486
  return decryptWithDeployKey(@secrets[type][instance_id])
813
487
  end
814
488
 
815
-
816
- # Run {MU::Cloud::Server#postBoot} and {MU::Cloud::Server#groom} on a node.
817
- # @param cloud_id [OpenStruct]: The cloud provider's identifier for this node.
818
- # @param name [String]: The MU resource name of the node being created.
819
- # @param mu_name [String]: The full #{MU::MommaCat.getResourceName} name of the server we're grooming, if it's been initialized already.
820
- # @param type [String]: The type of resource that created this node (either *server* or *serverpool*).
821
- def groomNode(cloud_id, name, type, mu_name: nil, reraise_fail: false, sync_wait: true)
822
- if cloud_id.nil?
823
- raise GroomError, "MU::MommaCat.groomNode requires a {MU::Cloud::Server} object"
824
- end
825
- if name.nil? or name.empty?
826
- raise GroomError, "MU::MommaCat.groomNode requires a resource name"
827
- end
828
- if type.nil? or type.empty?
829
- raise GroomError, "MU::MommaCat.groomNode requires a resource type"
830
- end
831
-
832
- if !MU::MommaCat.lock(cloud_id+"-mommagroom", true)
833
- MU.log "Instance #{cloud_id} on #{MU.deploy_id} (#{type}: #{name}) is already being groomed, ignoring this extra request.", MU::NOTICE
834
- MU::MommaCat.unlockAll
835
- if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
836
- puts "------------------------------"
837
- puts "Open flock() locks:"
838
- pp MU::MommaCat.locks
839
- puts "------------------------------"
840
- end
841
- return
842
- end
843
- loadDeploy
844
-
845
- # XXX this is to stop Net::SSH from killing our entire stack when it
846
- # throws an exception. See ECAP-139 in JIRA. Far as we can tell, it's
847
- # just not entirely thread safe.
848
- Thread.handle_interrupt(Net::SSH::Disconnect => :never) {
849
- begin
850
- Thread.handle_interrupt(Net::SSH::Disconnect => :immediate) {
851
- MU.log "(Probably harmless) Caught a Net::SSH::Disconnect in #{Thread.current.inspect}", MU::DEBUG, details: Thread.current.backtrace
852
- }
853
- ensure
854
- end
855
- }
856
-
857
- if @original_config[type+"s"].nil?
858
- raise GroomError, "I see no configured resources of type #{type} (bootstrap request for #{name} on #{@deploy_id})"
859
- end
860
- kitten = nil
861
-
862
- kitten = findLitterMate(type: "server", name: name, mu_name: mu_name, cloud_id: cloud_id)
863
- if !kitten.nil?
864
- MU.log "Re-grooming #{mu_name}", details: kitten.deploydata
865
- else
866
- first_groom = true
867
- @original_config[type+"s"].each { |svr|
868
- if svr['name'] == name
869
- svr["instance_id"] = cloud_id
870
-
871
- # This will almost always be true in server pools, but lets be safe. Somewhat problematic because we are only
872
- # looking at deploy_id, but we still know this is our DNS record and not a custom one.
873
- if svr['dns_records'] && !svr['dns_records'].empty?
874
- svr['dns_records'].each { |dnsrec|
875
- if dnsrec.has_key?("name") && dnsrec['name'].start_with?(MU.deploy_id.downcase)
876
- MU.log "DNS record for #{MU.deploy_id.downcase}, #{name} is probably wrong, deleting", MU::WARN, details: dnsrec
877
- dnsrec.delete('name')
878
- dnsrec.delete('target')
879
- end
880
- }
881
- end
882
-
883
- kitten = MU::Cloud::Server.new(mommacat: self, kitten_cfg: svr, cloud_id: cloud_id)
884
- mu_name = kitten.mu_name if mu_name.nil?
885
- MU.log "Grooming #{mu_name} for the first time", details: svr
886
- break
887
- end
888
- }
889
- end
890
-
891
- begin
892
- # This is a shared lock with MU::Cloud::AWS::Server.create, to keep from
893
- # stomping on synchronous deploys that are still running. This
894
- # means we're going to wait here if this instance is still being
895
- # bootstrapped by "regular" means.
896
- if !MU::MommaCat.lock(cloud_id+"-create", true)
897
- MU.log "#{mu_name} is still in mid-creation, skipping", MU::NOTICE
898
- MU::MommaCat.unlockAll
899
- if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
900
- puts "------------------------------"
901
- puts "Open flock() locks:"
902
- pp MU::MommaCat.locks
903
- puts "------------------------------"
904
- end
905
- return
906
- end
907
- MU::MommaCat.unlock(cloud_id+"-create")
908
-
909
- if !kitten.postBoot(cloud_id)
910
- MU.log "#{mu_name} is already being groomed, skipping", MU::NOTICE
911
- MU::MommaCat.unlockAll
912
- if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
913
- puts "------------------------------"
914
- puts "Open flock() locks:"
915
- pp MU::MommaCat.locks
916
- puts "------------------------------"
917
- end
918
- return
919
- end
920
-
921
- # This is a shared lock with MU::Deploy.createResources, simulating the
922
- # thread logic that tells MU::Cloud::AWS::Server.deploy to wait until
923
- # its dependencies are ready. We don't, for example, want to start
924
- # deploying if we rely on an RDS instance that isn't ready yet. We can
925
- # release this immediately, once we successfully grab it.
926
- MU::MommaCat.lock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
927
- MU::MommaCat.unlock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
928
-
929
- kitten.groom
930
- rescue Exception => e
931
- MU::MommaCat.unlockAll
932
- if e.class.name != "MU::Cloud::AWS::Server::BootstrapTempFail" and !File.exist?(deploy_dir+"/.cleanup."+cloud_id) and !File.exist?(deploy_dir+"/.cleanup")
933
- MU.log "Grooming FAILED for #{kitten.mu_name} (#{e.inspect})", MU::ERR, details: e.backtrace
934
- sendAdminSlack("Grooming FAILED for `#{kitten.mu_name}` with `#{e.message}` :crying_cat_face:", msg: e.backtrace.join("\n"))
935
- sendAdminMail("Grooming FAILED for #{kitten.mu_name} on #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id})",
936
- msg: e.inspect,
937
- data: e.backtrace,
938
- debug: true
939
- )
940
- raise e if reraise_fail
941
- else
942
- MU.log "Grooming of #{kitten.mu_name} interrupted by cleanup or planned reboot"
943
- end
944
- return
945
- end
946
-
947
- if !@deployment['servers'].nil? and !sync_wait
948
- syncLitter(@deployment["servers"].keys, triggering_node: kitten)
949
- end
950
- MU::MommaCat.unlock(cloud_id+"-mommagroom")
951
- if MU.myCloud == "AWS"
952
- MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
953
- end
954
- MU::MommaCat.getLitter(MU.deploy_id)
955
- MU::MommaCat.syncMonitoringConfig(false)
956
- MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})"
957
- FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt")
958
- MU::MommaCat.unlockAll
959
- if first_groom
960
- sendAdminSlack("Grooming complete for #{mu_name} :heart_eyes_cat:")
961
- sendAdminMail("Grooming complete for '#{name}' (#{mu_name}) on deploy \"#{MU.handle}\" (#{MU.deploy_id})")
962
- end
963
- return
964
- end
965
-
966
489
  # Return the parts and pieces of this deploy's node ssh key set. Generate
967
490
  # or load if that hasn't been done already.
968
491
  def SSHKey
@@ -1012,1251 +535,126 @@ module MU
1012
535
  return [@ssh_key_name, @ssh_private_key, @ssh_public_key]
1013
536
  end
1014
537
 
1015
- @lock_semaphore = Mutex.new
1016
- # Release all flock() locks held by the current thread.
1017
- def self.unlockAll
1018
- if !@locks.nil? and !@locks[Thread.current.object_id].nil?
1019
- # Work from a copy so we can iterate without worrying about contention
1020
- # in lock() or unlock(). We can't just wrap our iterator block in a
1021
- # semaphore here, because we're calling another method that uses the
1022
- # same semaphore.
1023
- @lock_semaphore.synchronize {
1024
- delete_list = []
1025
- @locks[Thread.current.object_id].keys.each { |id|
1026
- MU.log "Releasing lock on #{deploy_dir(MU.deploy_id)}/locks/#{id}.lock (thread #{Thread.current.object_id})", MU::DEBUG
1027
- begin
1028
- @locks[Thread.current.object_id][id].flock(File::LOCK_UN)
1029
- @locks[Thread.current.object_id][id].close
1030
- rescue IOError => e
1031
- MU.log "Got #{e.inspect} unlocking #{id} on #{Thread.current.object_id}", MU::WARN
1032
- end
1033
- delete_list << id
1034
- }
1035
- # We do this here because we can't mangle a Hash while we're iterating
1036
- # over it.
1037
- delete_list.each { |id|
1038
- @locks[Thread.current.object_id].delete(id)
1039
- }
1040
- if @locks[Thread.current.object_id].size == 0
1041
- @locks.delete(Thread.current.object_id)
1042
- end
1043
- }
1044
- end
1045
- end
1046
-
1047
- # Create/hold a flock() lock.
1048
- # @param id [String]: The lock identifier to release.
1049
- # @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
1050
- # return [false, nil]
1051
- def self.lock(id, nonblock = false, global = false)
1052
- raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
538
+ @@dummy_cache = {}
1053
539
 
1054
- if !global
1055
- lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
1056
- else
1057
- lockdir = File.expand_path(MU.dataDir+"/locks")
1058
- end
540
+ # Add or remove a resource's metadata to this deployment's structure and
541
+ # flush it to disk.
542
+ # @param type [String]: The type of resource (e.g. *server*, *database*).
543
+ # @param key [String]: The name field of this resource.
544
+ # @param mu_name [String]: The mu_name of this resource.
545
+ # @param data [Hash]: The resource's metadata.
546
+ # @param triggering_node [MU::Cloud]: A cloud object calling this notify, usually on behalf of itself
547
+ # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
548
+ # @return [void]
549
+ def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
1059
550
 
1060
- if !Dir.exist?(lockdir)
1061
- MU.log "Creating #{lockdir}", MU::DEBUG
1062
- Dir.mkdir(lockdir, 0700)
1063
- end
551
+ begin
552
+ MU::MommaCat.lock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
1064
553
 
1065
- @lock_semaphore.synchronize {
1066
- if @locks[Thread.current.object_id].nil?
1067
- @locks[Thread.current.object_id] = Hash.new
554
+ if !@need_deploy_flush or @deployment.nil? or @deployment.empty?
555
+ loadDeploy(true) # make sure we're saving the latest and greatest
1068
556
  end
1069
557
 
1070
- @locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
1071
- }
1072
- MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
1073
- begin
1074
- if nonblock
1075
- if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
1076
- return false
1077
- end
1078
- else
1079
- @locks[Thread.current.object_id][id].flock(File::LOCK_EX)
1080
- end
1081
- rescue IOError
1082
- raise MU::BootstrapTempFail, "Interrupted waiting for lock on thread #{Thread.current.object_id}, probably just a node rebooting as part of a synchronous install"
1083
- end
1084
- MU.log "Lock on #{lockdir}/#{id}.lock on thread #{Thread.current.object_id} acquired", MU::DEBUG
1085
- return true
1086
- end
558
+ _shortclass, _cfg_name, type, _classname, attrs = MU::Cloud.getResourceNames(type, false)
559
+ has_multiples = attrs[:has_multiples] ? true : false
1087
560
 
1088
- # Release a flock() lock.
1089
- # @param id [String]: The lock identifier to release.
1090
- def self.unlock(id, global = false)
1091
- raise MuError, "Can't pass a nil id to MU::MommaCat.unlock" if id.nil?
1092
- lockdir = nil
1093
- if !global
1094
- lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
1095
- else
1096
- lockdir = File.expand_path(MU.dataDir+"/locks")
1097
- end
1098
- @lock_semaphore.synchronize {
1099
- return if @locks.nil? or @locks[Thread.current.object_id].nil? or @locks[Thread.current.object_id][id].nil?
1100
- }
1101
- MU.log "Releasing lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})", MU::DEBUG
1102
- begin
1103
- @locks[Thread.current.object_id][id].flock(File::LOCK_UN)
1104
- @locks[Thread.current.object_id][id].close
1105
- if !@locks[Thread.current.object_id].nil?
1106
- @locks[Thread.current.object_id].delete(id)
561
+ mu_name ||= if !data.nil? and !data["mu_name"].nil?
562
+ data["mu_name"]
563
+ elsif !triggering_node.nil? and !triggering_node.mu_name.nil?
564
+ triggering_node.mu_name
1107
565
  end
1108
- if @locks[Thread.current.object_id].size == 0
1109
- @locks.delete(Thread.current.object_id)
566
+ if mu_name.nil? and has_multiples
567
+ MU.log "MU::MommaCat.notify called to modify deployment struct for a type (#{type}) with :has_multiples, but no mu_name available to look under #{key}. Call was #{caller(1..1)}", MU::WARN, details: data
568
+ return
1110
569
  end
1111
- rescue IOError => e
1112
- MU.log "Got #{e.inspect} unlocking #{id} on #{Thread.current.object_id}", MU::WARN
1113
- end
1114
- end
1115
570
 
1116
- # Remove a deployment's metadata.
1117
- # @param deploy_id [String]: The deployment identifier to remove.
1118
- def self.purge(deploy_id)
1119
- if deploy_id.nil? or deploy_id.empty?
1120
- raise MuError, "Got nil deploy_id in MU::MommaCat.purge"
1121
- end
1122
- # XXX archiving is better than annihilating
1123
- path = File.expand_path(MU.dataDir+"/deployments")
1124
- if Dir.exist?(path+"/"+deploy_id)
1125
- unlockAll
1126
- MU.log "Purging #{path}/#{deploy_id}" if File.exist?(path+"/"+deploy_id+"/deployment.json")
571
+ @need_deploy_flush = true
1127
572
 
1128
- FileUtils.rm_rf(path+"/"+deploy_id, :secure => true)
1129
- end
1130
- if File.exist?(path+"/unique_ids")
1131
- File.open(path+"/unique_ids", File::CREAT|File::RDWR, 0600) { |f|
1132
- newlines = []
1133
- f.flock(File::LOCK_EX)
1134
- f.readlines.each { |line|
1135
- newlines << line if !line.match(/:#{deploy_id}$/)
573
+ if !remove
574
+ if data.nil?
575
+ MU.log "MU::MommaCat.notify called to modify deployment struct, but no data provided", MU::WARN
576
+ return
577
+ end
578
+ @notify_semaphore.synchronize {
579
+ @deployment[type] ||= {}
1136
580
  }
1137
- f.rewind
1138
- f.truncate(0)
1139
- f.puts(newlines)
1140
- f.flush
1141
- f.flock(File::LOCK_UN)
1142
- }
1143
- end
1144
- end
1145
-
1146
- # Remove the metadata of the currently loaded deployment.
1147
- def purge!
1148
- MU::MommaCat.purge(MU.deploy_id)
1149
- end
1150
-
1151
- @cleanup_threads = []
1152
-
1153
- # Iterate over all known deployments and look for instances that have been
1154
- # terminated, but not yet cleaned up, then clean them up.
1155
- def self.cleanTerminatedInstances(debug = false)
1156
- loglevel = debug ? MU::NOTICE : MU::DEBUG
1157
- MU::MommaCat.lock("clean-terminated-instances", false, true)
1158
- MU.log "Checking for harvested instances in need of cleanup", loglevel
1159
- parent_thread_id = Thread.current.object_id
1160
- purged = 0
1161
-
1162
- MU::MommaCat.listDeploys.each { |deploy_id|
1163
- next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
1164
- MU.log "Checking for dead wood in #{deploy_id}", loglevel
1165
- need_reload = false
1166
- @cleanup_threads << Thread.new {
1167
- MU.dupGlobals(parent_thread_id)
1168
- deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
1169
- purged_this_deploy = 0
1170
- MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys
1171
- if deploy.kittens.has_key?("servers")
1172
- MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id
1173
- deploy.kittens["servers"].values.each { |nodeclasses|
1174
- nodeclasses.each_pair { |nodeclass, servers|
1175
- deletia = []
1176
- MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys
1177
- servers.each_pair { |mu_name, server|
1178
- server.describe
1179
- if !server.cloud_id
1180
- MU.log "Checking for presence of #{mu_name}, but unable to fetch its cloud_id", MU::WARN, details: server
1181
- elsif !server.active?
1182
- next if File.exist?(deploy_dir(deploy_id)+"/.cleanup-"+server.cloud_id)
1183
- deletia << mu_name
1184
- need_reload = true
1185
- MU.log "Cleaning up metadata for #{server} (#{nodeclass}), formerly #{server.cloud_id}, which appears to have been terminated", MU::NOTICE
1186
- begin
1187
- server.destroy
1188
- deploy.sendAdminMail("Retired metadata for terminated node #{mu_name}")
1189
- deploy.sendAdminSlack("Retired metadata for terminated node `#{mu_name}`")
1190
- rescue Exception => e
1191
- MU.log "Saw #{e.message} while retiring #{mu_name}", MU::ERR, details: e.backtrace
1192
- next
1193
- end
1194
- MU.log "Cleanup of metadata for #{server} (#{nodeclass}), formerly #{server.cloud_id} complete", MU::NOTICE
1195
- purged = purged + 1
1196
- purged_this_deploy = purged_this_deploy + 1
1197
- end
1198
- }
1199
- deletia.each { |mu_name|
1200
- servers.delete(mu_name)
1201
- }
1202
- if purged_this_deploy > 0
1203
- # XXX triggering_node needs to take more than one node name
1204
- deploy.syncLitter(servers.keys, triggering_node: deletia.first)
1205
- end
1206
- }
581
+ if has_multiples
582
+ @notify_semaphore.synchronize {
583
+ @deployment[type][key] ||= {}
1207
584
  }
585
+ @deployment[type][key][mu_name] = data
586
+ MU.log "Adding to @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: data
587
+ else
588
+ @deployment[type][key] = data
589
+ MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data
1208
590
  end
1209
- if need_reload
1210
- MU.log "Saving modified deploy #{deploy_id}", loglevel
1211
- deploy.save!
1212
- MU::MommaCat.getLitter(deploy_id)
591
+ save!(key) if !delayed_save and !@no_artifacts
592
+ else
593
+ have_deploy = true
594
+ if @deployment[type].nil? or @deployment[type][key].nil?
595
+ MU.log "MU::MommaCat.notify called to remove #{type} #{key}#{has_multiples ? " "+mu_name : ""} deployment struct, but no such data exist", MU::DEBUG
596
+ return
1213
597
  end
1214
- MU.purgeGlobals
1215
- }
1216
- }
1217
- @cleanup_threads.each { |t|
1218
- t.join
1219
- }
1220
- MU.log "cleanTerminatedInstances threads complete", loglevel
1221
- MU::MommaCat.unlock("clean-terminated-instances", true)
1222
- @cleanup_threads = []
1223
598
 
1224
- if purged > 0
1225
- if MU.myCloud == "AWS"
1226
- MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
599
+ if have_deploy
600
+ @notify_semaphore.synchronize {
601
+ if has_multiples
602
+ MU.log "Removing @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: @deployment[type][key][mu_name]
603
+ @deployment[type][key].delete(mu_name)
604
+ end
605
+
606
+ if @deployment[type][key].empty? or !has_multiples
607
+ MU.log "Removing @deployment[#{type}][#{key}]", MU::DEBUG, details: @deployment[type][key]
608
+ @deployment[type].delete(key)
609
+ end
610
+
611
+ if @deployment[type].empty?
612
+ @deployment.delete(type)
613
+ end
614
+ }
615
+ end
616
+ save! if !delayed_save and !@no_artifacts
1227
617
  end
1228
- MU::MommaCat.syncMonitoringConfig
1229
- GC.start
618
+ ensure
619
+ MU::MommaCat.unlock("deployment-notification", deploy_id: @deploy_id) if !@no_artifacts
1230
620
  end
1231
- MU.log "cleanTerminatedInstances returning", loglevel
1232
621
  end
1233
622
 
1234
- @@dummy_cache = {}
623
+ # Send a Slack notification to a deployment's administrators.
624
+ # @param subject [String]: The subject line of the message.
625
+ # @param msg [String]: The message body.
626
+ # @return [void]
627
+ def sendAdminSlack(subject, msg: "", scrub_mu_isms: true, snippets: [], noop: false)
628
+ if MU.muCfg['slack'] and MU.muCfg['slack']['webhook'] and
629
+ (!MU.muCfg['slack']['skip_environments'] or !MU.muCfg['slack']['skip_environments'].any?{ |s| s.casecmp(MU.environment)==0 })
630
+ require 'slack-notifier'
631
+ slackargs = nil
632
+ keyword_args = { channel: MU.muCfg['slack']['channel'] }
633
+ begin
634
+ slack = Slack::Notifier.new MU.muCfg['slack']['webhook']
635
+ prefix = scrub_mu_isms ? subject : "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}"
1235
636
 
1236
- # Locate a resource that's either a member of another deployment, or of no
1237
- # deployment at all, and return a {MU::Cloud} object for it.
1238
- # @param cloud [String]: The Cloud provider to use.
1239
- # @param type [String]: The resource type. Can be the full class name, symbolic name, or Basket of Kittens configuration shorthand for the resource type.
1240
- # @param deploy_id [String]: The identifier of an outside deploy to search.
1241
- # @param name [String]: The name of the resource as defined in its 'name' Basket of Kittens field, typically used in conjunction with deploy_id.
1242
- # @param mu_name [String]: The fully-resolved and deployed name of the resource, typically used in conjunction with deploy_id.
1243
- # @param cloud_id [String]: A cloud provider identifier for this resource.
1244
- # @param region [String]: The cloud provider region
1245
- # @param tag_key [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_value.
1246
- # @param tag_value [String]: A cloud provider tag to help identify the resource, used in conjunction with tag_key.
1247
- # @param allow_multi [Boolean]: Permit an array of matching resources to be returned (if applicable) instead of just one.
1248
- # @param dummy_ok [Boolean]: Permit return of a faked {MU::Cloud} object if we don't have enough information to identify a real live one.
1249
- # @param flags [Hash]: Other cloud or resource type specific options to pass to that resource's find() method
1250
- # @return [Array<MU::Cloud>]
1251
- def self.findStray(
1252
- cloud,
1253
- type,
1254
- deploy_id: nil,
1255
- name: nil,
1256
- mu_name: nil,
1257
- cloud_id: nil,
1258
- credentials: nil,
1259
- region: nil,
1260
- tag_key: nil,
1261
- tag_value: nil,
1262
- allow_multi: false,
1263
- calling_deploy: MU.mommacat,
1264
- flags: {},
1265
- habitats: [],
1266
- dummy_ok: false,
1267
- debug: false,
1268
- no_deploy_search: false
1269
- )
1270
- start = Time.now
1271
- callstr = "findStray(cloud: #{cloud}, type: #{type}, deploy_id: #{deploy_id}, calling_deploy: #{calling_deploy.deploy_id if !calling_deploy.nil?}, name: #{name}, cloud_id: #{cloud_id}, tag_key: #{tag_key}, tag_value: #{tag_value}, credentials: #{credentials}, habitats: #{habitats ? habitats.to_s : "[]"}, dummy_ok: #{dummy_ok.to_s}, flags: #{flags.to_s}) from #{caller[0]}"
1272
- callstack = caller.dup
1273
-
1274
- return nil if cloud == "CloudFormation" and !cloud_id.nil?
1275
- shortclass, _cfg_name, cfg_plural, classname, _attrs = MU::Cloud.getResourceNames(type)
1276
- if !MU::Cloud.supportedClouds.include?(cloud) or shortclass.nil?
1277
- MU.log "findStray was called with bogus cloud argument '#{cloud}'", MU::WARN, details: callstr
1278
- return nil
1279
- end
637
+ text = if msg and !msg.empty?
638
+ "#{prefix}:\n\n```#{msg}```"
639
+ else
640
+ prefix
641
+ end
1280
642
 
1281
- begin
1282
- # TODO this is dumb as hell, clean this up.. and while we're at it
1283
- # .dup everything so we don't mangle referenced values from the caller
1284
- deploy_id = deploy_id.to_s if deploy_id.class.to_s == "MU::Config::Tail"
1285
- name = name.to_s if name.class.to_s == "MU::Config::Tail"
1286
- cloud_id = cloud_id.to_s if !cloud_id.nil?
1287
- mu_name = mu_name.to_s if mu_name.class.to_s == "MU::Config::Tail"
1288
- tag_key = tag_key.to_s if tag_key.class.to_s == "MU::Config::Tail"
1289
- tag_value = tag_value.to_s if tag_value.class.to_s == "MU::Config::Tail"
1290
- type = cfg_plural
1291
- resourceclass = MU::Cloud.loadCloudType(cloud, shortclass)
1292
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(cloud)
1293
-
1294
- credlist = if credentials
1295
- [credentials]
1296
- else
1297
- cloudclass.listCredentials
1298
- end
1299
-
1300
- if (tag_key and !tag_value) or (!tag_key and tag_value)
1301
- raise MuError, "Can't call findStray with only one of tag_key and tag_value set, must be both or neither"
1302
- end
1303
- # Help ourselves by making more refined parameters out of mu_name, if
1304
- # they weren't passed explicitly
1305
- if mu_name
1306
- if !tag_key and !tag_value
1307
- # XXX "Name" is an AWS-ism, perhaps those plugins should do this bit?
1308
- tag_key="Name"
1309
- tag_value=mu_name
1310
- end
1311
- # We can extract a deploy_id from mu_name if we don't have one already
1312
- if !deploy_id and mu_name
1313
- deploy_id = mu_name.sub(/^(\w+-\w+-\d{10}-[A-Z]{2})-/, '\1')
1314
- end
1315
- end
1316
- loglevel = debug ? MU::NOTICE : MU::DEBUG
1317
-
1318
- MU.log callstr, loglevel, details: caller
1319
-
1320
- # See if the thing we're looking for is a member of the deploy that's
1321
- # asking after it.
1322
- if !deploy_id.nil? and !calling_deploy.nil? and
1323
- calling_deploy.deploy_id == deploy_id and (!name.nil? or !mu_name.nil?)
1324
- handle = calling_deploy.findLitterMate(type: type, name: name, mu_name: mu_name, cloud_id: cloud_id, credentials: credentials)
1325
- return [handle] if !handle.nil?
1326
- end
1327
-
1328
- kittens = {}
1329
- # Search our other deploys for matching resources
1330
- if !no_deploy_search and (deploy_id or name or mu_name or cloud_id)
1331
- MU.log "findStray: searching my deployments (#{cfg_plural}, name: #{name}, deploy_id: #{deploy_id}, mu_name: #{mu_name}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1332
-
1333
- # Check our in-memory cache of live deploys before resorting to
1334
- # metadata
1335
- littercache = nil
1336
- # Sometimes we're called inside a locked thread, sometimes not. Deal
1337
- # with locking gracefully.
1338
- begin
1339
- @@litter_semaphore.synchronize {
1340
- littercache = @@litters.dup
1341
- }
1342
- rescue ThreadError => e
1343
- raise e if !e.message.match(/recursive locking/)
1344
- littercache = @@litters.dup
1345
- end
1346
-
1347
- littercache.each_pair { |cur_deploy, momma|
1348
- next if deploy_id and deploy_id != cur_deploy
1349
-
1350
- straykitten = momma.findLitterMate(type: type, cloud_id: cloud_id, name: name, mu_name: mu_name, credentials: credentials, created_only: true)
1351
- if straykitten
1352
- MU.log "Found matching kitten #{straykitten.mu_name} in-memory - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1353
- # Peace out if we found the exact resource we want
1354
- if cloud_id and straykitten.cloud_id.to_s == cloud_id.to_s
1355
- return [straykitten]
1356
- elsif mu_name and straykitten.mu_name == mu_name
1357
- return [straykitten]
1358
- else
1359
- kittens[straykitten.cloud_id] ||= straykitten
1360
- end
1361
- end
1362
- }
1363
-
1364
- mu_descs = MU::MommaCat.getResourceMetadata(cfg_plural, name: name, deploy_id: deploy_id, mu_name: mu_name)
1365
- MU.log "findStray: #{mu_descs.size.to_s} deploys had matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1366
-
1367
- mu_descs.each_pair { |cur_deploy_id, matches|
1368
- MU.log "findStray: #{cur_deploy_id} had #{matches.size.to_s} initial matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1369
- next if matches.nil? or matches.size == 0
1370
-
1371
- momma = MU::MommaCat.getLitter(cur_deploy_id)
1372
-
1373
- straykitten = nil
1374
-
1375
- # If we found exactly one match in this deploy, use its metadata to
1376
- # guess at resource names we weren't told.
1377
- if matches.size > 1 and cloud_id
1378
- MU.log "findStray: attempting to narrow down multiple matches with cloud_id #{cloud_id} - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1379
- straykitten = momma.findLitterMate(type: type, cloud_id: cloud_id, credentials: credentials, created_only: true)
1380
- elsif matches.size == 1 and name.nil? and mu_name.nil?
1381
- if cloud_id.nil?
1382
- straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: matches.first["cloud_id"], credentials: credentials)
1383
- else
1384
- MU.log "findStray: fetching single match with cloud_id #{cloud_id} - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1385
- straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: cloud_id, credentials: credentials)
1386
- end
1387
- # elsif !flags.nil? and !flags.empty? # XXX eh, maybe later
1388
- # # see if we can narrow it down further with some flags
1389
- # filtered = []
1390
- # matches.each { |m|
1391
- # f = resourceclass.find(cloud_id: m['mu_name'], flags: flags)
1392
- # filtered << m if !f.nil? and f.size > 0
1393
- # MU.log "RESULT FROM find(cloud_id: #{m['mu_name']}, flags: #{flags})", MU::WARN, details: f
1394
- # }
1395
- # if filtered.size == 1
1396
- # straykitten = momma.findLitterMate(type: type, name: matches.first["name"], cloud_id: filtered.first['cloud_id'])
1397
- # end
1398
- else
1399
- # There's more than one of this type of resource in the target
1400
- # deploy, so see if findLitterMate can narrow it down for us
1401
- straykitten = momma.findLitterMate(type: type, name: name, mu_name: mu_name, cloud_id: cloud_id, credentials: credentials)
1402
- end
1403
-
1404
- next if straykitten.nil?
1405
- straykitten.intoDeploy(momma)
1406
-
1407
- if straykitten.cloud_id.nil?
1408
- MU.log "findStray: kitten #{straykitten.mu_name} came back with nil cloud_id", MU::WARN
1409
- next
1410
- end
1411
-
1412
- kittens[straykitten.cloud_id] ||= straykitten
1413
-
1414
- # Peace out if we found the exact resource we want
1415
- if cloud_id and straykitten.cloud_id.to_s == cloud_id.to_s
1416
- return [straykitten]
1417
- # ...or if we've validated our one possible match
1418
- elsif !cloud_id and mu_descs.size == 1 and matches.size == 1
1419
- return [straykitten]
1420
- elsif credentials and credlist.size == 1 and straykitten.credentials == credentials
1421
- return [straykitten]
1422
- end
1423
- }
1424
-
1425
-
1426
- # if !mu_descs.nil? and mu_descs.size > 0 and !deploy_id.nil? and !deploy_id.empty? and !mu_descs.first.empty?
1427
- # MU.log "I found descriptions that might match #{resourceclass.cfg_plural} name: #{name}, deploy_id: #{deploy_id}, mu_name: #{mu_name}, but couldn't isolate my target kitten", MU::WARN, details: caller
1428
- # puts File.read(deploy_dir(deploy_id)+"/deployment.json")
1429
- # end
1430
-
1431
- # We can't refine any further by asking the cloud provider...
1432
- if !cloud_id and !tag_key and !tag_value and kittens.size > 1
1433
- if !allow_multi
1434
- raise MuError, "Multiple matches in MU::MommaCat.findStray where none allowed from deploy_id: '#{deploy_id}', name: '#{name}', mu_name: '#{mu_name}' (#{caller[0]})"
1435
- else
1436
- return kittens.values
1437
- end
1438
- end
1439
- end
1440
-
1441
- matches = []
1442
-
1443
- found_the_thing = false
1444
- credlist.each { |creds|
1445
- break if found_the_thing
1446
- if cloud_id or (tag_key and tag_value) or !flags.empty? or allow_multi
1447
-
1448
- regions = begin
1449
- region ? [region] : cloudclass.listRegions(credentials: creds)
1450
- rescue NoMethodError # Not all cloud providers have regions
1451
- [nil]
1452
- end
1453
-
1454
- # ..not all resource types care about regions either
1455
- if resourceclass.isGlobal?
1456
- regions = [nil]
1457
- end
1458
-
1459
- # Decide what habitats (accounts/projects/subscriptions) we'll
1460
- # search, if applicable for this resource type.
1461
- habitats ||= []
1462
- begin
1463
- if flags["project"] # backwards-compat
1464
- habitats << flags["project"]
1465
- end
1466
- if habitats.empty?
1467
- if resourceclass.canLiveIn.include?(nil)
1468
- habitats << nil
1469
- end
1470
- if resourceclass.canLiveIn.include?(:Habitat)
1471
- habitats.concat(cloudclass.listProjects(creds))
1472
- end
1473
- end
1474
- rescue NoMethodError # we only expect this to work on Google atm
1475
- end
1476
-
1477
- if habitats.empty?
1478
- habitats << nil
1479
- end
1480
- habitats.uniq!
1481
-
1482
- habitat_threads = []
1483
- desc_semaphore = Mutex.new
1484
-
1485
- cloud_descs = {}
1486
- habitats.each { |hab|
1487
- begin
1488
- habitat_threads.each { |t| t.join(0.1) }
1489
- habitat_threads.reject! { |t| t.nil? or !t.status }
1490
- sleep 1 if habitat_threads.size > 5
1491
- end while habitat_threads.size > 5
1492
- habitat_threads << Thread.new(hab) { |p|
1493
- MU.log "findStray: Searching #{p} (#{habitat_threads.size.to_s} habitat threads running) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1494
- cloud_descs[p] = {}
1495
- region_threads = []
1496
- regions.each { |reg| region_threads << Thread.new(reg) { |r|
1497
- MU.log "findStray: Searching #{r} in #{p} (#{region_threads.size.to_s} region threads running) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1498
- MU.log "findStray: calling #{classname}.find(cloud_id: #{cloud_id}, region: #{r}, tag_key: #{tag_key}, tag_value: #{tag_value}, flags: #{flags}, credentials: #{creds}, project: #{p}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1499
- begin
1500
- found = resourceclass.find(cloud_id: cloud_id, region: r, tag_key: tag_key, tag_value: tag_value, flags: flags, credentials: creds, habitat: p)
1501
- MU.log "findStray: #{found ? found.size.to_s : "nil"} results - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1502
- rescue Exception => e
1503
- MU.log "#{e.class.name} THREW A FIND EXCEPTION "+e.message, MU::WARN, details: caller
1504
- pp e.backtrace
1505
- MU.log "#{callstr}", MU::WARN, details: callstack
1506
- exit
1507
- end
1508
- if found
1509
- desc_semaphore.synchronize {
1510
- cloud_descs[p][r] = found
1511
- }
1512
- end
1513
- # Stop if you found the thing by a specific cloud_id
1514
- if cloud_id and found and !found.empty?
1515
- found_the_thing = true
1516
- Thread.exit
1517
- end
1518
- } }
1519
- begin
1520
- region_threads.each { |t| t.join(0.1) }
1521
- region_threads.reject! { |t| t.nil? or !t.status }
1522
- if region_threads.size > 0
1523
- MU.log "#{region_threads.size.to_s} regions still running in #{p}", loglevel
1524
- sleep 3
1525
- end
1526
- end while region_threads.size > 0
1527
- }
1528
- }
1529
- begin
1530
- habitat_threads.each { |t| t.join(0.1) }
1531
- habitat_threads.reject! { |t| t.nil? or !t.status }
1532
- if habitat_threads.size > 0
1533
- MU.log "#{habitat_threads.size.to_s} habitats still running", loglevel
1534
- sleep 3
1535
- end
1536
- end while habitat_threads.size > 0
1537
-
1538
- habitat_threads = []
1539
- habitats.each { |hab| habitat_threads << Thread.new(hab) { |p|
1540
- region_threads = []
1541
- regions.each { |reg| region_threads << Thread.new(reg) { |r|
1542
- next if cloud_descs[p][r].nil?
1543
- cloud_descs[p][r].each_pair { |kitten_cloud_id, descriptor|
1544
-
1545
- # We already have a MU::Cloud object for this guy, use it
1546
- if kittens.has_key?(kitten_cloud_id)
1547
- desc_semaphore.synchronize {
1548
- matches << kittens[kitten_cloud_id]
1549
- }
1550
- elsif kittens.size == 0
1551
- if !dummy_ok
1552
- next
1553
- end
1554
-
1555
- # If we don't have a MU::Cloud object, manufacture a dummy
1556
- # one. Give it a fake name if we have to and have decided
1557
- # that's ok. Wild inferences from the cloud descriptor are
1558
- # ok to try here.
1559
- use_name = if (name.nil? or name.empty?)
1560
- if !dummy_ok
1561
- nil
1562
- elsif !mu_name.nil?
1563
- mu_name
1564
- else
1565
- try = nil
1566
- [:display_name, :name, (resourceclass.cfg_name+"_name").to_sym].each { |field|
1567
- if descriptor.respond_to?(field) and descriptor.send(field).is_a?(String)
1568
- try = descriptor.send(field)
1569
- break
1570
- end
1571
-
1572
- }
1573
- try ||= if !tag_value.nil?
1574
- tag_value
1575
- else
1576
- kitten_cloud_id
1577
- end
1578
- try
1579
- end
1580
- else
1581
- name
1582
- end
1583
- if use_name.nil?
1584
- MU.log "Found cloud provider data for #{cloud} #{type} #{kitten_cloud_id}, but without a name I can't manufacture a proper #{type} object to return - #{sprintf("%.2fs", (Time.now-start))}", loglevel, details: caller
1585
- next
1586
- end
1587
- cfg = {
1588
- "name" => use_name,
1589
- "cloud" => cloud,
1590
- "credentials" => creds
1591
- }
1592
- if !r.nil? and !resourceclass.isGlobal?
1593
- cfg["region"] = r
1594
- end
1595
-
1596
- if !p.nil? and resourceclass.canLiveIn.include?(:Habitat)
1597
- cfg["project"] = p
1598
- end
1599
- # If we can at least find the config from the deploy this will
1600
- # belong with, use that, even if it's an ungroomed resource.
1601
- if !calling_deploy.nil? and
1602
- !calling_deploy.original_config.nil? and
1603
- !calling_deploy.original_config[type+"s"].nil?
1604
- calling_deploy.original_config[type+"s"].each { |s|
1605
- if s["name"] == use_name
1606
- cfg = s.dup
1607
- break
1608
- end
1609
- }
1610
-
1611
- newkitten = resourceclass.new(mommacat: calling_deploy, kitten_cfg: cfg, cloud_id: kitten_cloud_id)
1612
- desc_semaphore.synchronize {
1613
- matches << newkitten
1614
- }
1615
- else
1616
- if !@@dummy_cache[cfg_plural] or !@@dummy_cache[cfg_plural][cfg.to_s]
1617
- MU.log "findStray: Generating dummy '#{resourceclass.to_s}' cloudobj with name: #{use_name}, cloud_id: #{kitten_cloud_id.to_s} - #{sprintf("%.2fs", (Time.now-start))}", loglevel, details: cfg
1618
- resourceclass.new(mu_name: use_name, kitten_cfg: cfg, cloud_id: kitten_cloud_id.to_s, from_cloud_desc: descriptor)
1619
- desc_semaphore.synchronize {
1620
- @@dummy_cache[cfg_plural] ||= {}
1621
- @@dummy_cache[cfg_plural][cfg.to_s] = resourceclass.new(mu_name: use_name, kitten_cfg: cfg, cloud_id: kitten_cloud_id.to_s, from_cloud_desc: descriptor)
1622
- MU.log "findStray: Finished generating dummy '#{resourceclass.to_s}' cloudobj - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1623
- }
1624
- end
1625
- desc_semaphore.synchronize {
1626
- matches << @@dummy_cache[cfg_plural][cfg.to_s]
1627
- }
1628
- end
1629
- end
1630
- }
1631
- } }
1632
- MU.log "findStray: tying up #{region_threads.size.to_s} region threads - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1633
- region_threads.each { |t|
1634
- t.join
1635
- }
1636
- } }
1637
- MU.log "findStray: tying up #{habitat_threads.size.to_s} habitat threads - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1638
- habitat_threads.each { |t|
1639
- t.join
1640
- }
1641
- end
1642
- }
1643
- rescue Exception => e
1644
- MU.log e.inspect, MU::ERR, details: e.backtrace
1645
- end
1646
- MU.log "findStray: returning #{matches ? matches.size.to_s : "0"} matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
1647
-
1648
- matches
1649
- end
1650
-
1651
- # Return the resource object of another member of this deployment
1652
- # @param type [String,Symbol]: The type of resource
1653
- # @param name [String]: The name of the resource as defined in its 'name' Basket of Kittens field
1654
- # @param mu_name [String]: The fully-resolved and deployed name of the resource
1655
- # @param cloud_id [String]: The cloud provider's unique identifier for this resource
1656
- # @param created_only [Boolean]: Only return the littermate if its cloud_id method returns a value
1657
- # @param return_all [Boolean]: Return a Hash of matching objects indexed by their mu_name, instead of a single match. Only valid for resource types where has_multiples is true.
1658
- # @return [MU::Cloud]
1659
- def findLitterMate(type: nil, name: nil, mu_name: nil, cloud_id: nil, created_only: false, return_all: false, credentials: nil, habitat: nil, debug: false, indent: "")
1660
- shortclass, cfg_name, cfg_plural, classname, attrs = MU::Cloud.getResourceNames(type)
1661
- type = cfg_plural
1662
- has_multiples = attrs[:has_multiples]
1663
-
1664
- loglevel = debug ? MU::NOTICE : MU::DEBUG
1665
-
1666
- argstring = [:type, :name, :mu_name, :cloud_id, :created_only, :credentials, :habitat, :has_multiples].reject { |a|
1667
- binding.local_variable_get(a).nil?
1668
- }.map { |v|
1669
- v.to_s+": "+binding.local_variable_get(v).to_s
1670
- }.join(", ")
1671
-
1672
- # Fun times: if we specified a habitat, which we may also have done by
1673
- # its shorthand sibling name, let's... call ourselves first to make sure
1674
- # we're fishing for the right thing.
1675
- if habitat
1676
- if habitat.is_a?(MU::Config::Ref) and habitat.id
1677
- habitat = habitat.id
1678
- else
1679
- MU.log indent+"findLitterMate(#{argstring}): Attempting to resolve habitat name #{habitat}", loglevel
1680
- realhabitat = findLitterMate(type: "habitat", name: habitat, debug: debug, credentials: credentials, indent: indent+" ")
1681
- if realhabitat and realhabitat.mu_name
1682
- MU.log indent+"findLitterMate: Resolved habitat name #{habitat} to #{realhabitat.mu_name}", loglevel, details: [realhabitat.mu_name, realhabitat.cloud_id, realhabitat.config.keys]
1683
- habitat = realhabitat.cloud_id
1684
- elsif debug
1685
- MU.log indent+"findLitterMate(#{argstring}): Failed to resolve habitat name #{habitat}", MU::WARN
1686
- end
1687
- end
1688
- end
1689
-
1690
-
1691
- @kitten_semaphore.synchronize {
1692
- if !@kittens.has_key?(type)
1693
- if debug
1694
- MU.log indent+"NO SUCH KEY #{type} findLitterMate(#{argstring})", MU::WARN, details: @kittens.keys
1695
- end
1696
- return nil
1697
- end
1698
- MU.log indent+"START findLitterMate(#{argstring}), caller: #{caller[2]}", loglevel, details: @kittens[type].keys.map { |hab| hab.to_s+": "+@kittens[type][hab].keys.join(", ") }
1699
- matches = []
1700
-
1701
- @kittens[type].each { |habitat_group, sib_classes|
1702
- next if habitat and habitat_group != habitat and !habitat_group.nil?
1703
- sib_classes.each_pair { |sib_class, data|
1704
- virtual_name = nil
1705
-
1706
- if !has_multiples and data and !data.is_a?(Hash) and data.config and data.config.is_a?(Hash) and data.config['virtual_name'] and name == data.config['virtual_name']
1707
- virtual_name = data.config['virtual_name']
1708
- elsif !name.nil? and name != sib_class
1709
- next
1710
- end
1711
- if has_multiples
1712
- if !name.nil?
1713
- if return_all
1714
- MU.log indent+"MULTI-MATCH RETURN_ALL findLitterMate(#{argstring})", loglevel, details: data.keys
1715
- return data.dup
1716
- end
1717
- if data.size == 1 and (cloud_id.nil? or data.values.first.cloud_id == cloud_id)
1718
- return data.values.first
1719
- elsif mu_name.nil? and cloud_id.nil?
1720
- MU.log indent+"#{@deploy_id}: Found multiple matches in findLitterMate based on #{type}: #{name}, and not enough info to narrow down further. Returning an arbitrary result. Caller: #{caller[2]}", MU::WARN, details: data.keys
1721
- return data.values.first
1722
- end
1723
- end
1724
- data.each_pair { |sib_mu_name, obj|
1725
- if (!mu_name.nil? and mu_name == sib_mu_name) or
1726
- (!cloud_id.nil? and cloud_id == obj.cloud_id) or
1727
- (!credentials.nil? and credentials == obj.credentials)
1728
- if !created_only or !obj.cloud_id.nil?
1729
- if return_all
1730
- MU.log indent+"MULTI-MATCH RETURN_ALL findLitterMate(#{argstring})", loglevel, details: data.keys
1731
- return data.dup
1732
- else
1733
- MU.log indent+"MULTI-MATCH findLitterMate(#{argstring})", loglevel, details: data.keys
1734
- return obj
1735
- end
1736
- end
1737
- end
1738
- }
1739
- else
1740
-
1741
- MU.log indent+"CHECKING AGAINST findLitterMate #{habitat_group}/#{type}/#{sib_class} data.cloud_id: #{data.cloud_id}, data.credentials: #{data.credentials}, sib_class: #{sib_class}, virtual_name: #{virtual_name}", loglevel, details: argstring
1742
-
1743
- data_cloud_id = data.cloud_id.nil? ? nil : data.cloud_id.to_s
1744
-
1745
- MU.log indent+"(name.nil? or sib_class == name or virtual_name == name)", loglevel, details: (name.nil? or sib_class == name or virtual_name == name).to_s
1746
- MU.log indent+"(cloud_id.nil? or cloud_id[#{cloud_id.class.name}:#{cloud_id.to_s}] == data_cloud_id[#{data_cloud_id.class.name}:#{data_cloud_id}])", loglevel, details: (cloud_id.nil? or cloud_id == data_cloud_id).to_s
1747
- MU.log indent+"(credentials.nil? or data.credentials.nil? or credentials[#{credentials.class.name}:#{credentials}] == data.credentials[#{data.credentials.class.name}:#{data.credentials}])", loglevel, details: (credentials.nil? or data.credentials.nil? or credentials == data.credentials).to_s
1748
-
1749
- if (name.nil? or sib_class == name.to_s or virtual_name == name.to_s) and
1750
- (cloud_id.nil? or cloud_id.to_s == data_cloud_id) and
1751
- (credentials.nil? or data.credentials.nil? or credentials.to_s == data.credentials.to_s)
1752
- MU.log indent+"OUTER MATCH PASSED, NEED !created_only (#{created_only.to_s}) or !data_cloud_id.nil? (#{data_cloud_id})", loglevel, details: (cloud_id.nil? or cloud_id == data_cloud_id).to_s
1753
- if !created_only or !data_cloud_id.nil?
1754
- MU.log indent+"SINGLE MATCH findLitterMate(#{argstring})", loglevel, details: [data.mu_name, data_cloud_id, data.config.keys]
1755
- matches << data
1756
- end
1757
- end
1758
- end
1759
- }
1760
- }
1761
-
1762
- return matches.first if matches.size == 1
1763
- if return_all and matches.size > 1
1764
- return matches
1765
- end
1766
- }
1767
-
1768
- MU.log indent+"NO MATCH findLitterMate(#{argstring})", loglevel
1769
-
1770
- return nil
1771
- end
1772
-
1773
- # Add or remove a resource's metadata to this deployment's structure and
1774
- # flush it to disk.
1775
- # @param type [String]: The type of resource (e.g. *server*, *database*).
1776
- # @param key [String]: The name field of this resource.
1777
- # @param data [Hash]: The resource's metadata.
1778
- # @param remove [Boolean]: Remove this resource from the deploy structure, instead of adding it.
1779
- # @return [void]
1780
- def notify(type, key, data, mu_name: nil, remove: false, triggering_node: nil, delayed_save: false)
1781
- return if @no_artifacts
1782
- MU::MommaCat.lock("deployment-notification")
1783
-
1784
- if !@need_deploy_flush or @deployment.nil? or @deployment.empty?
1785
- loadDeploy(true) # make sure we're saving the latest and greatest
1786
- end
1787
-
1788
- _shortclass, _cfg_name, cfg_plural, _classname, attrs = MU::Cloud.getResourceNames(type)
1789
- has_multiples = false
1790
-
1791
- # it's not always the case that we're logging data for a legal resource
1792
- # type, though that's what we're usually for
1793
- if cfg_plural
1794
- type = cfg_plural
1795
- has_multiples = attrs[:has_multiples]
1796
- end
1797
-
1798
- if mu_name.nil?
1799
- if !data.nil? and !data["mu_name"].nil?
1800
- mu_name = data["mu_name"]
1801
- elsif !triggering_node.nil? and !triggering_node.mu_name.nil?
1802
- mu_name = triggering_node.mu_name
1803
- end
1804
- if mu_name.nil? and has_multiples
1805
- MU.log "MU::MommaCat.notify called to modify deployment struct for a type (#{type}) with :has_multiples, but no mu_name available to look under #{key}. Call was #{caller[0]}", MU::WARN, details: data
1806
- MU::MommaCat.unlock("deployment-notification")
1807
- return
1808
- end
1809
- end
1810
-
1811
- @need_deploy_flush = true
1812
-
1813
- if !remove
1814
- if data.nil?
1815
- MU.log "MU::MommaCat.notify called to modify deployment struct, but no data provided", MU::WARN
1816
- MU::MommaCat.unlock("deployment-notification")
1817
- return
1818
- end
1819
- @notify_semaphore.synchronize {
1820
- @deployment[type] ||= {}
1821
- }
1822
- if has_multiples
1823
- @notify_semaphore.synchronize {
1824
- @deployment[type][key] ||= {}
1825
- }
1826
- # fix has_multiples classes that weren't tiered correctly
1827
- if @deployment[type][key].is_a?(Hash) and @deployment[type][key].has_key?("mu_name")
1828
- olddata = @deployment[type][key].dup
1829
- @deployment[type][key][olddata["mu_name"]] = olddata
1830
- end
1831
- @deployment[type][key][mu_name] = data
1832
- MU.log "Adding to @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: data
1833
- else
1834
- @deployment[type][key] = data
1835
- MU.log "Adding to @deployment[#{type}][#{key}]", MU::DEBUG, details: data
1836
- end
1837
- save!(key) if !delayed_save
1838
- else
1839
- have_deploy = true
1840
- if @deployment[type].nil? or @deployment[type][key].nil?
1841
-
1842
- if has_multiples
1843
- MU.log "MU::MommaCat.notify called to remove #{type} #{key} #{mu_name} deployment struct, but no such data exist", MU::DEBUG
1844
- else
1845
- MU.log "MU::MommaCat.notify called to remove #{type} #{key} deployment struct, but no such data exist", MU::DEBUG
1846
- end
1847
- MU::MommaCat.unlock("deployment-notification")
1848
-
1849
- return
1850
- end
1851
-
1852
- if have_deploy
1853
- @notify_semaphore.synchronize {
1854
- if has_multiples
1855
- MU.log "Removing @deployment[#{type}][#{key}][#{mu_name}]", MU::DEBUG, details: @deployment[type][key][mu_name]
1856
- @deployment[type][key].delete(mu_name)
1857
- if @deployment[type][key].size == 0
1858
- @deployment[type].delete(key)
1859
- end
1860
- else
1861
- MU.log "Removing @deployment[#{type}][#{key}]", MU::DEBUG, details: @deployment[type][key]
1862
- @deployment[type].delete(key)
1863
- end
1864
- if @deployment[type].size == 0
1865
- @deployment.delete(type)
1866
- end
1867
- }
1868
- end
1869
- save! if !delayed_save
1870
-
1871
- end
1872
-
1873
- MU::MommaCat.unlock("deployment-notification")
1874
- end
1875
-
1876
- # Tag a resource. Defaults to applying our MU deployment identifier, if no
1877
- # arguments other than the resource identifier are given.
1878
- # XXX this belongs in the cloud layer(s)
1879
- #
1880
- # @param resource [String]: The cloud provider identifier of the resource to tag
1881
- # @param tag_name [String]: The name of the tag to create
1882
- # @param tag_value [String]: The value of the tag
1883
- # @param region [String]: The cloud provider region
1884
- # @return [void]
1885
- def self.createTag(resource = nil,
1886
- tag_name="MU-ID",
1887
- tag_value=MU.deploy_id,
1888
- region: MU.curRegion,
1889
- credentials: nil)
1890
- attempts = 0
1891
-
1892
- if !MU::Cloud::CloudFormation.emitCloudFormation
1893
- begin
1894
- MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags(
1895
- resources: [resource],
1896
- tags: [
1897
- {
1898
- key: tag_name,
1899
- value: tag_value
1900
- }
1901
- ]
1902
- )
1903
- rescue Aws::EC2::Errors::ServiceError => e
1904
- MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1
1905
- if attempts < 5
1906
- attempts = attempts + 1
1907
- sleep 15
1908
- retry
1909
- else
1910
- raise e
1911
- end
1912
- end
1913
- MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG
1914
- else
1915
- return {
1916
- "Key" => tag_name,
1917
- "Value" => tag_value
1918
- }
1919
- end
1920
- end
1921
-
1922
- # List the name/value pairs for our mandatory standard set of resource tags, which
1923
- # should be applied to all taggable cloud provider resources.
1924
- # @return [Hash<String,String>]
1925
- def self.listStandardTags
1926
- return {} if !MU.deploy_id
1927
- {
1928
- "MU-ID" => MU.deploy_id,
1929
- "MU-APP" => MU.appname,
1930
- "MU-ENV" => MU.environment,
1931
- "MU-MASTER-IP" => MU.mu_public_ip
1932
- }
1933
- end
1934
- # List the name/value pairs for our mandatory standard set of resource tags
1935
- # for this deploy.
1936
- # @return [Hash<String,String>]
1937
- def listStandardTags
1938
- {
1939
- "MU-ID" => @deploy_id,
1940
- "MU-APP" => @appname,
1941
- "MU-ENV" => @environment,
1942
- "MU-MASTER-IP" => MU.mu_public_ip
1943
- }
1944
- end
1945
-
1946
- # List the name/value pairs of our optional set of resource tags which
1947
- # should be applied to all taggable cloud provider resources.
1948
- # @return [Hash<String,String>]
1949
- def self.listOptionalTags
1950
- return {
1951
- "MU-HANDLE" => MU.handle,
1952
- "MU-MASTER-NAME" => Socket.gethostname,
1953
- "MU-OWNER" => MU.mu_user
1954
- }
1955
- end
1956
-
1957
- # Clean an IP address out of ~/.ssh/known hosts
1958
- # @param ip [String]: The IP to remove
1959
- # @return [void]
1960
- def self.removeIPFromSSHKnownHosts(ip)
1961
- return if ip.nil?
1962
- sshdir = "#{@myhome}/.ssh"
1963
- knownhosts = "#{sshdir}/known_hosts"
1964
-
1965
- if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
1966
- MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
1967
- if !@noop
1968
- File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
1969
- f.flock(File::LOCK_EX)
1970
- newlines = Array.new
1971
- delete_block = false
1972
- f.readlines.each { |line|
1973
- next if line.match(/^#{Regexp.quote(ip)} /)
1974
- newlines << line
1975
- }
1976
- f.rewind
1977
- f.truncate(0)
1978
- f.puts(newlines)
1979
- f.flush
1980
- f.flock(File::LOCK_UN)
1981
- }
1982
- end
1983
- end
1984
- end
1985
-
1986
- # Clean a node's entries out of ~/.ssh/config
1987
- # @param nodename [String]: The node's name
1988
- # @return [void]
1989
- def self.removeHostFromSSHConfig(nodename)
1990
- sshdir = "#{@myhome}/.ssh"
1991
- sshconf = "#{sshdir}/config"
1992
-
1993
- if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /)
1994
- MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
1995
- if !@noop
1996
- File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
1997
- f.flock(File::LOCK_EX)
1998
- newlines = Array.new
1999
- delete_block = false
2000
- f.readlines.each { |line|
2001
- if line.match(/^Host #{nodename}(\s|$)/)
2002
- delete_block = true
2003
- elsif line.match(/^Host /)
2004
- delete_block = false
2005
- end
2006
- newlines << line if !delete_block
2007
- }
2008
- f.rewind
2009
- f.truncate(0)
2010
- f.puts(newlines)
2011
- f.flush
2012
- f.flock(File::LOCK_UN)
2013
- }
2014
- end
2015
- end
2016
-
2017
- end
2018
-
2019
- # Make sure the given node has proper DNS entries, /etc/hosts entries,
2020
- # SSH config entries, etc.
2021
- # @param server [MU::Cloud::Server]: The {MU::Cloud::Server} we'll be setting up.
2022
- # @param sync_wait [Boolean]: Whether to wait for DNS to fully synchronize before returning.
2023
- def self.nameKitten(server, sync_wait: false)
2024
- node, config, _deploydata = server.describe
2025
-
2026
- mu_zone = nil
2027
- # XXX GCP!
2028
- if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
2029
- zones = MU::Cloud::DNSZone.find(cloud_id: "platform-mu")
2030
- mu_zone = zones.values.first if !zones.nil?
2031
- end
2032
- if !mu_zone.nil?
2033
- MU::Cloud::DNSZone.genericMuDNSEntry(name: node, target: server.canonicalIP, cloudclass: MU::Cloud::Server, sync_wait: sync_wait)
2034
- else
2035
- MU::MommaCat.addInstanceToEtcHosts(server.canonicalIP, node)
2036
- end
2037
-
2038
- ## TO DO: Do DNS registration of "real" records as the last stage after the groomer completes
2039
- if config && config['dns_records'] && !config['dns_records'].empty?
2040
- dnscfg = config['dns_records'].dup
2041
- dnscfg.each { |dnsrec|
2042
- if !dnsrec.has_key?('name')
2043
- dnsrec['name'] = node.downcase
2044
- dnsrec['name'] = "#{dnsrec['name']}.#{MU.environment.downcase}" if dnsrec["append_environment_name"] && !dnsrec['name'].match(/\.#{MU.environment.downcase}$/)
2045
- end
2046
-
2047
- if !dnsrec.has_key?("target")
2048
- # Default to register public endpoint
2049
- public = true
2050
-
2051
- if dnsrec.has_key?("target_type")
2052
- # See if we have a preference for pubic/private endpoint
2053
- public = dnsrec["target_type"] == "private" ? false : true
2054
- end
2055
-
2056
- dnsrec["target"] =
2057
- if dnsrec["type"] == "CNAME"
2058
- if public
2059
- # Make sure we have a public canonical name to register. Use the private one if we don't
2060
- server.cloud_desc.public_dns_name.empty? ? server.cloud_desc.private_dns_name : server.cloud_desc.public_dns_name
2061
- else
2062
- # If we specifically requested to register the private canonical name lets use that
2063
- server.cloud_desc.private_dns_name
2064
- end
2065
- elsif dnsrec["type"] == "A"
2066
- if public
2067
- # Make sure we have a public IP address to register. Use the private one if we don't
2068
- server.cloud_desc.public_ip_address ? server.cloud_desc.public_ip_address : server.cloud_desc.private_ip_address
2069
- else
2070
- # If we specifically requested to register the private IP lets use that
2071
- server.cloud_desc.private_ip_address
2072
- end
2073
- end
2074
- end
2075
- }
2076
- if !MU::Cloud::AWS.isGovCloud?
2077
- MU::Cloud::DNSZone.createRecordsFromConfig(dnscfg)
2078
- end
2079
- end
2080
-
2081
- MU::MommaCat.removeHostFromSSHConfig(node)
2082
- if server and server.canonicalIP
2083
- MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP)
2084
- end
2085
- # XXX add names paramater with useful stuff
2086
- MU::MommaCat.addHostToSSHConfig(
2087
- server,
2088
- ssh_owner: server.deploy.mu_user,
2089
- ssh_dir: Etc.getpwnam(server.deploy.mu_user).dir+"/.ssh"
2090
- )
2091
- end
2092
-
2093
- @ssh_semaphore = Mutex.new
2094
- # Insert a definition for a node into our SSH config.
2095
- # @param server [MU::Cloud::Server]: The name of the node.
2096
- # @param names [Array<String>]: Other names that we'd like this host to be known by for SSH purposes
2097
- # @param ssh_dir [String]: The configuration directory of the SSH config to emit.
2098
- # @param ssh_conf [String]: A specific SSH configuration file to write entries into.
2099
- # @param ssh_owner [String]: The preferred owner of the SSH configuration files.
2100
- # @param timeout [Integer]: An alternate timeout value for connections to this server.
2101
- # @return [void]
2102
- def self.addHostToSSHConfig(server,
2103
- ssh_dir: "#{@myhome}/.ssh",
2104
- ssh_conf: "#{@myhome}/.ssh/config",
2105
- ssh_owner: Etc.getpwuid(Process.uid).name,
2106
- names: [],
2107
- timeout: 0
2108
- )
2109
- if server.nil?
2110
- MU.log "Called addHostToSSHConfig without a MU::Cloud::Server object", MU::ERR, details: caller
2111
- return nil
2112
- end
2113
-
2114
- _nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = begin
2115
- server.getSSHConfig
2116
- rescue MU::MuError
2117
- return
2118
- end
2119
-
2120
- if ssh_user.nil? or ssh_user.empty?
2121
- MU.log "Failed to extract ssh_user for #{server.mu_name} addHostToSSHConfig", MU::ERR
2122
- return
2123
- end
2124
- if canonical_ip.nil? or canonical_ip.empty?
2125
- MU.log "Failed to extract canonical_ip for #{server.mu_name} addHostToSSHConfig", MU::ERR
2126
- return
2127
- end
2128
- if ssh_key_name.nil? or ssh_key_name.empty?
2129
- MU.log "Failed to extract ssh_key_name for #{ssh_key_name.mu_name} in addHostToSSHConfig", MU::ERR
2130
- return
2131
- end
2132
-
2133
- @ssh_semaphore.synchronize {
2134
-
2135
- if File.exist?(ssh_conf)
2136
- File.readlines(ssh_conf).each { |line|
2137
- if line.match(/^Host #{server.mu_name} /)
2138
- MU.log("Attempt to add duplicate #{ssh_conf} entry for #{server.mu_name}", MU::WARN)
2139
- return
2140
- end
2141
- }
2142
- end
2143
-
2144
- File.open(ssh_conf, 'a', 0600) { |ssh_config|
2145
- ssh_config.flock(File::LOCK_EX)
2146
- host_str = "Host #{server.mu_name} #{server.canonicalIP}"
2147
- if !names.nil? and names.size > 0
2148
- host_str = host_str+" "+names.join(" ")
2149
- end
2150
- ssh_config.puts host_str
2151
- ssh_config.puts " Hostname #{server.canonicalIP}"
2152
- if !nat_ssh_host.nil? and server.canonicalIP != nat_ssh_host
2153
- ssh_config.puts " ProxyCommand ssh -W %h:%p #{nat_ssh_user}@#{nat_ssh_host}"
2154
- end
2155
- if timeout > 0
2156
- ssh_config.puts " ConnectTimeout #{timeout}"
2157
- end
2158
-
2159
- ssh_config.puts " User #{ssh_user}"
2160
- # XXX I'd rather add the host key to known_hosts, but Net::SSH is a little dumb
2161
- ssh_config.puts " StrictHostKeyChecking no"
2162
- ssh_config.puts " ServerAliveInterval 60"
2163
-
2164
- ssh_config.puts " IdentityFile #{ssh_dir}/#{ssh_key_name}"
2165
- if !File.exist?("#{ssh_dir}/#{ssh_key_name}")
2166
- MU.log "#{server.mu_name} - ssh private key #{ssh_dir}/#{ssh_key_name} does not exist", MU::WARN
2167
- end
2168
-
2169
- ssh_config.flock(File::LOCK_UN)
2170
- ssh_config.chown(Etc.getpwnam(ssh_owner).uid, Etc.getpwnam(ssh_owner).gid)
2171
- }
2172
- MU.log "Wrote #{server.mu_name} ssh key to #{ssh_dir}/config", MU::DEBUG
2173
- return "#{ssh_dir}/#{ssh_key_name}"
2174
- }
2175
- end
2176
-
2177
- # Clean a node's entries out of /etc/hosts
2178
- # @param node [String]: The node's name
2179
- # @return [void]
2180
- def self.removeInstanceFromEtcHosts(node)
2181
- return if MU.mu_user != "mu"
2182
- hostsfile = "/etc/hosts"
2183
- FileUtils.copy(hostsfile, "#{hostsfile}.bak-#{MU.deploy_id}")
2184
- File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
2185
- f.flock(File::LOCK_EX)
2186
- newlines = Array.new
2187
- f.readlines.each { |line|
2188
- newlines << line if !line.match(/ #{node}(\s|$)/)
2189
- }
2190
- f.rewind
2191
- f.truncate(0)
2192
- f.puts(newlines)
2193
- f.flush
2194
-
2195
- f.flock(File::LOCK_UN)
2196
- }
2197
- end
2198
-
2199
-
2200
- # Insert node names associated with a new instance into /etc/hosts so we
2201
- # can treat them as if they were real DNS entries. Especially helpful when
2202
- # Chef/Ohai mistake the proper hostname, e.g. when bootstrapping Windows.
2203
- # @param public_ip [String]: The node's IP address
2204
- # @param chef_name [String]: The node's Chef node name
2205
- # @param system_name [String]: The node's local system name
2206
- # @return [void]
2207
- def self.addInstanceToEtcHosts(public_ip, chef_name = nil, system_name = nil)
2208
-
2209
- # XXX cover ipv6 case
2210
- if public_ip.nil? or !public_ip.match(/^\d+\.\d+\.\d+\.\d+$/) or (chef_name.nil? and system_name.nil?)
2211
- raise MuError, "addInstanceToEtcHosts requires public_ip and one or both of chef_name and system_name!"
2212
- end
2213
- if chef_name == "localhost" or system_name == "localhost"
2214
- raise MuError, "Can't set localhost as a name in addInstanceToEtcHosts"
2215
- end
2216
-
2217
- if !["mu", "root"].include?(MU.mu_user)
2218
- response = nil
2219
- begin
2220
- response = open("https://127.0.0.1:#{MU.mommaCatPort.to_s}/rest/hosts_add/#{chef_name}/#{public_ip}").read
2221
- rescue Errno::ECONNRESET, Errno::ECONNREFUSED
2222
- end
2223
- if response != "ok"
2224
- MU.log "Error adding #{public_ip} to /etc/hosts via MommaCat request", MU::ERR
2225
- end
2226
- return
2227
- end
2228
-
2229
- File.readlines("/etc/hosts").each { |line|
2230
- if line.match(/^#{public_ip} /) or (chef_name != nil and line.match(/ #{chef_name}(\s|$)/)) or (system_name != nil and line.match(/ #{system_name}(\s|$)/))
2231
- MU.log "Ignoring attempt to add duplicate /etc/hosts entry: #{public_ip} #{chef_name} #{system_name}", MU::DEBUG
2232
- return
2233
- end
2234
- }
2235
- File.open("/etc/hosts", 'a') { |etc_hosts|
2236
- etc_hosts.flock(File::LOCK_EX)
2237
- etc_hosts.puts("#{public_ip} #{chef_name} #{system_name}")
2238
- etc_hosts.flock(File::LOCK_UN)
2239
- }
2240
- MU.log("Added to /etc/hosts: #{public_ip} #{chef_name} #{system_name}")
2241
- end
2242
-
2243
-
2244
- # Send a Slack notification to a deployment's administrators.
2245
- # @param subject [String]: The subject line of the message.
2246
- # @param msg [String]: The message body.
2247
- # @return [void]
2248
- def sendAdminSlack(subject, msg: "")
2249
- if $MU_CFG['slack'] and $MU_CFG['slack']['webhook'] and
2250
- (!$MU_CFG['slack']['skip_environments'] or !$MU_CFG['slack']['skip_environments'].any?{ |s| s.casecmp(MU.environment)==0 })
2251
- require 'slack-notifier'
2252
- slack = Slack::Notifier.new $MU_CFG['slack']['webhook']
643
+ if snippets and snippets.size > 0
644
+ keyword_args[:attachments] = snippets
645
+ end
2253
646
 
2254
- if msg and !msg.empty?
2255
- slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}:\n\n```#{msg}\n```", channel: $MU_CFG['slack']['channel']
2256
- else
2257
- slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}", channel: $MU_CFG['slack']['channel']
647
+ if !noop
648
+ slack.ping(text, **keyword_args)
649
+ else
650
+ MU.log "Would send to #{MU.muCfg['slack']['channel']}", MU::NOTICE, details: [ text, keyword_args ]
651
+ end
652
+ rescue Slack::Notifier::APIError => e
653
+ MU.log "Failed to send message to slack: #{e.message}", MU::ERR, details: keyword_args
654
+ return false
2258
655
  end
2259
656
  end
657
+ true
2260
658
  end
2261
659
 
2262
660
  # Send an email notification to a deployment's administrators.
@@ -2277,13 +675,13 @@ end
2277
675
  to << "#{admin['name']} <#{admin['email']}>"
2278
676
  }
2279
677
  end
2280
- message = <<MESSAGE_END
678
+ message = <<MAIL_HEAD_END
2281
679
  From: #{MU.handle} <root@localhost>
2282
680
  To: #{to.join(",")}
2283
681
  Subject: #{subject}
2284
682
 
2285
683
  #{msg}
2286
- MESSAGE_END
684
+ MAIL_HEAD_END
2287
685
  if !kitten.nil? and kitten.kind_of?(MU::Cloud)
2288
686
  message = message + "\n\n**** #{kitten}:\n"
2289
687
  if !kitten.report.nil?
@@ -2308,205 +706,6 @@ MESSAGE_END
2308
706
  end
2309
707
  end
2310
708
 
2311
- # Manufactures a human-readable deployment name from the random
2312
- # two-character seed in MU-ID. Cat-themed when possible.
2313
- # @param seed [String]: A two-character seed from which we'll generate a name.
2314
- # @return [String]: Two words
2315
- def self.generateHandle(seed)
2316
- word_one=word_two=nil
2317
-
2318
- # Unless we've got two letters that don't have corresponding cat-themed
2319
- # words, we'll insist that our generated handle have at least one cat
2320
- # element to it.
2321
- require_cat_words = true
2322
- if @catwords.select { |word| word.match(/^#{seed[0]}/i) }.size == 0 and
2323
- @catwords.select { |word| word.match(/^#{seed[1]}/i) }.size == 0
2324
- require_cat_words = false
2325
- MU.log "Got an annoying pair of letters #{seed}, not forcing cat-theming", MU::DEBUG
2326
- end
2327
- allnouns = @catnouns + @jaegernouns
2328
- alladjs = @catadjs + @jaegeradjs
2329
-
2330
- tries = 0
2331
- begin
2332
- # Try to avoid picking something "nouny" for the first word
2333
- source = @catadjs + @catmixed + @jaegeradjs + @jaegermixed
2334
- first_ltr = source.select { |word| word.match(/^#{seed[0]}/i) }
2335
- if !first_ltr or first_ltr.size == 0
2336
- first_ltr = @words.select { |word| word.match(/^#{seed[0]}/i) }
2337
- end
2338
- word_one = first_ltr.shuffle.first
2339
-
2340
- # If we got a paired set that happen to match our letters, go with it
2341
- if !word_one.nil? and word_one.match(/-#{seed[1]}/i)
2342
- word_one, word_two = word_one.split(/-/)
2343
- else
2344
- source = @words
2345
- if @catwords.include?(word_one)
2346
- source = @jaegerwords
2347
- elsif require_cat_words
2348
- source = @catwords
2349
- end
2350
- second_ltr = source.select { |word| word.match(/^#{seed[1]}/i) and !word.match(/-/i) }
2351
- word_two = second_ltr.shuffle.first
2352
- end
2353
- tries = tries + 1
2354
- end while tries < 50 and (word_one.nil? or word_two.nil? or word_one.match(/-/) or word_one == word_two or (allnouns.include?(word_one) and allnouns.include?(word_two)) or (alladjs.include?(word_one) and alladjs.include?(word_two)) or (require_cat_words and !@catwords.include?(word_one) and !@catwords.include?(word_two)))
2355
-
2356
- if tries >= 50 and (word_one.nil? or word_two.nil?)
2357
- MU.log "I failed to generated a valid handle, faking it", MU::ERR
2358
- return "#{seed[0].capitalize} #{seed[1].capitalize}"
2359
- end
2360
-
2361
- return "#{word_one.capitalize} #{word_two.capitalize}"
2362
- end
2363
-
2364
- # Ensure that the Nagios configuration local to the MU master has been
2365
- # updated, and make sure Nagios has all of the ssh keys it needs to tunnel
2366
- # to client nodes.
2367
- # @return [void]
2368
- def self.syncMonitoringConfig(blocking = true)
2369
- return if Etc.getpwuid(Process.uid).name != "root" or (MU.mu_user != "mu" and MU.mu_user != "root")
2370
- parent_thread_id = Thread.current.object_id
2371
- nagios_threads = []
2372
- nagios_threads << Thread.new {
2373
- MU.dupGlobals(parent_thread_id)
2374
- realhome = Etc.getpwnam("nagios").dir
2375
- [@nagios_home, "#{@nagios_home}/.ssh"].each { |dir|
2376
- Dir.mkdir(dir, 0711) if !Dir.exist?(dir)
2377
- File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, dir)
2378
- }
2379
- if realhome != @nagios_home and Dir.exist?(realhome) and !File.symlink?("#{realhome}/.ssh")
2380
- File.rename("#{realhome}/.ssh", "#{realhome}/.ssh.#{$$}") if Dir.exist?("#{realhome}/.ssh")
2381
- File.symlink("#{@nagios_home}/.ssh", Etc.getpwnam("nagios").dir+"/.ssh")
2382
- end
2383
- MU.log "Updating #{@nagios_home}/.ssh/config..."
2384
- ssh_lock = File.new("#{@nagios_home}/.ssh/config.mu.lock", File::CREAT|File::TRUNC|File::RDWR, 0600)
2385
- ssh_lock.flock(File::LOCK_EX)
2386
- ssh_conf = File.new("#{@nagios_home}/.ssh/config.tmp", File::CREAT|File::TRUNC|File::RDWR, 0600)
2387
- ssh_conf.puts "Host MU-MASTER localhost"
2388
- ssh_conf.puts " Hostname localhost"
2389
- ssh_conf.puts " User root"
2390
- ssh_conf.puts " IdentityFile #{@nagios_home}/.ssh/id_rsa"
2391
- ssh_conf.puts " StrictHostKeyChecking no"
2392
- ssh_conf.close
2393
- FileUtils.cp("#{@myhome}/.ssh/id_rsa", "#{@nagios_home}/.ssh/id_rsa")
2394
- File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/id_rsa")
2395
- threads = []
2396
-
2397
- parent_thread_id = Thread.current.object_id
2398
- MU::MommaCat.listDeploys.sort.each { |deploy_id|
2399
- begin
2400
- # We don't want to use cached litter information here because this is also called by cleanTerminatedInstances.
2401
- deploy = MU::MommaCat.getLitter(deploy_id)
2402
- if deploy.ssh_key_name.nil? or deploy.ssh_key_name.empty?
2403
- MU.log "Failed to extract ssh key name from #{deploy_id} in syncMonitoringConfig", MU::ERR if deploy.kittens.has_key?("servers")
2404
- next
2405
- end
2406
- FileUtils.cp("#{@myhome}/.ssh/#{deploy.ssh_key_name}", "#{@nagios_home}/.ssh/#{deploy.ssh_key_name}")
2407
- File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/#{deploy.ssh_key_name}")
2408
- if deploy.kittens.has_key?("servers")
2409
- deploy.kittens["servers"].values.each { |nodeclasses|
2410
- nodeclasses.values.each { |nodes|
2411
- nodes.values.each { |server|
2412
- next if !server.cloud_desc
2413
- MU.dupGlobals(parent_thread_id)
2414
- threads << Thread.new {
2415
- MU::MommaCat.setThreadContext(deploy)
2416
- MU.log "Adding #{server.mu_name} to #{@nagios_home}/.ssh/config", MU::DEBUG
2417
- MU::MommaCat.addHostToSSHConfig(
2418
- server,
2419
- ssh_dir: "#{@nagios_home}/.ssh",
2420
- ssh_conf: "#{@nagios_home}/.ssh/config.tmp",
2421
- ssh_owner: "nagios"
2422
- )
2423
- MU.purgeGlobals
2424
- }
2425
- }
2426
- }
2427
- }
2428
- end
2429
- rescue Exception => e
2430
- MU.log "#{e.inspect} while generating Nagios SSH config in #{deploy_id}", MU::ERR, details: e.backtrace
2431
- end
2432
- }
2433
- threads.each { |t|
2434
- t.join
2435
- }
2436
- ssh_lock.flock(File::LOCK_UN)
2437
- ssh_lock.close
2438
- File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/config.tmp")
2439
- File.rename("#{@nagios_home}/.ssh/config.tmp", "#{@nagios_home}/.ssh/config")
2440
-
2441
- MU.log "Updating Nagios monitoring config, this may take a while..."
2442
- output = nil
2443
- if $MU_CFG and !$MU_CFG['master_runlist_extras'].nil?
2444
- output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only],#{$MU_CFG['master_runlist_extras'].join(",")}' 2>&1}
2445
- else
2446
- output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only]' 2>&1}
2447
- end
2448
-
2449
- if $?.exitstatus != 0
2450
- MU.log "Nagios monitoring config update returned a non-zero exit code!", MU::ERR, details: output
2451
- else
2452
- MU.log "Nagios monitoring config update complete."
2453
- end
2454
- }
2455
-
2456
- if blocking
2457
- nagios_threads.each { |t|
2458
- t.join
2459
- }
2460
- end
2461
- end
2462
-
2463
- # Return a list of all currently active deploy identifiers.
2464
- # @return [Array<String>]
2465
- def self.listDeploys
2466
- return [] if !Dir.exist?("#{MU.dataDir}/deployments")
2467
- deploys = []
2468
- Dir.entries("#{MU.dataDir}/deployments").reverse_each { |muid|
2469
- next if !Dir.exist?("#{MU.dataDir}/deployments/#{muid}") or muid == "." or muid == ".."
2470
- deploys << muid
2471
- }
2472
- return deploys
2473
- end
2474
-
2475
- # Return a list of all nodes in all deployments. Does so without loading
2476
- # deployments fully.
2477
- # @return [Hash]
2478
- def self.listAllNodes
2479
- nodes = Hash.new
2480
- MU::MommaCat.deploy_struct_semaphore.synchronize {
2481
- MU::MommaCat.listDeploys.each { |deploy|
2482
- if !Dir.exist?(MU::MommaCat.deploy_dir(deploy)) or
2483
- !File.size?("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json")
2484
- MU.log "Didn't see deployment metadata for '#{deploy}'", MU::WARN
2485
- next
2486
- end
2487
- data = File.open("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json", File::RDONLY)
2488
- MU.log "Getting lock to read #{MU::MommaCat.deploy_dir(deploy)}/deployment.json", MU::DEBUG
2489
- data.flock(File::LOCK_EX)
2490
- begin
2491
- deployment = JSON.parse(File.read("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json"))
2492
- deployment["deploy_id"] = deploy
2493
- if deployment.has_key?("servers")
2494
- deployment["servers"].each_key { |nodeclass|
2495
- deployment["servers"][nodeclass].each_pair { |mu_name, metadata|
2496
- nodes[mu_name] = metadata
2497
- }
2498
- }
2499
- end
2500
- rescue JSON::ParserError => e
2501
- MU.log "JSON parse failed on #{MU::MommaCat.deploy_dir(deploy)}/deployment.json", MU::ERR, details: e.message
2502
- end
2503
- data.flock(File::LOCK_UN)
2504
- data.close
2505
- }
2506
- }
2507
- return nodes
2508
- end
2509
-
2510
709
  # Return a list of all nodes associated with the current deployment.
2511
710
  # @return [Hash]
2512
711
  def listNodes
@@ -2570,124 +769,59 @@ MESSAGE_END
2570
769
  MU::Master::SSL.sign(csr_path, sans, for_user: MU.mu_user)
2571
770
  end
2572
771
 
2573
- # Make sure deployment data is synchronized to/from each node in the
772
+ # Make sure deployment data is synchronized to/from each +Server+ in the
2574
773
  # currently-loaded deployment.
774
+ # @param nodeclasses [Array<String>]
775
+ # @param triggering_node [String,MU::Cloud::Server]
776
+ # @param save_only [Boolean]
2575
777
  def syncLitter(nodeclasses = [], triggering_node: nil, save_only: false)
2576
- # XXX take some config logic to decide what nodeclasses to hit? like, make
2577
- # inferences from dependencies or something?
2578
-
2579
- return if MU.syncLitterThread
778
+ return if MU.syncLitterThread # don't run recursively by accident
2580
779
  return if !Dir.exist?(deploy_dir)
2581
- svrs = MU::Cloud.resource_types[:Server][:cfg_plural] # legibility shorthand
2582
- if !triggering_node.nil? and nodeclasses.size > 0
2583
- nodeclasses.reject! { |n| n == triggering_node.to_s }
2584
- return if nodeclasses.size == 0
2585
- end
2586
-
2587
- @kitten_semaphore.synchronize {
2588
- if @kittens.nil? or
2589
- @kittens[svrs].nil?
2590
- MU.log "No #{svrs} as yet available in #{@deploy_id}", MU::DEBUG, details: @kittens
2591
- return
2592
- end
2593
780
 
781
+ if !triggering_node.nil? and triggering_node.is_a?(MU::Cloud::Server)
782
+ triggering_node = triggering_node.mu_name
783
+ end
2594
784
 
2595
- MU.log "Updating these node classes in #{@deploy_id}", MU::DEBUG, details: nodeclasses
2596
- }
785
+ siblings = findLitterMate(type: "server", return_all: true)
786
+ return if siblings.nil? or (siblings.respond_to?(:empty?) and siblings.empty?)
2597
787
 
2598
788
  update_servers = []
2599
- if nodeclasses.nil? or nodeclasses.size == 0
2600
- litter = findLitterMate(type: "server", return_all: true)
2601
- return if litter.nil?
2602
- litter.each_pair { |mu_name, node|
2603
- if !triggering_node.nil? and (
2604
- (triggering_node.is_a?(MU::Cloud::Server) and mu_name == triggering_node.mu_name) or
2605
- (triggering_node.is_a?(String) and mu_name == triggering_node)
2606
- )
2607
- next
2608
- end
2609
-
2610
- if !node.groomer.nil?
2611
- update_servers << node
2612
- end
2613
- }
2614
- else
2615
- litter = {}
2616
- nodeclasses.each { |nodeclass|
2617
- mates = findLitterMate(type: "server", name: nodeclass, return_all: true)
2618
- litter.merge!(mates) if mates
2619
- }
2620
- litter.each_pair { |mu_name, node|
2621
- if !triggering_node.nil? and (
2622
- (triggering_node.is_a?(MU::Cloud::Server) and mu_name == triggering_node.mu_name) or
2623
- (triggering_node.is_a?(String) and mu_name == triggering_node)
2624
- )
2625
- next
2626
- end
789
+ siblings.each_pair { |mu_name, node|
790
+ next if mu_name == triggering_node or node.groomer.nil?
791
+ next if nodeclasses.size > 0 and !nodeclasses.include?(node.config['name'])
792
+ if !node.deploydata or !node.deploydata['nodename']
793
+ MU.log "#{mu_name} deploy data is missing (possibly retired or mid-bootstrap), so not syncing it", MU::NOTICE
794
+ next
795
+ end
2627
796
 
2628
- if !node.deploydata or !node.deploydata.keys.include?('nodename')
2629
- details = node.deploydata ? node.deploydata.keys : nil
2630
- MU.log "#{mu_name} deploy data is missing (possibly retired or mid-bootstrap), so not syncing it", MU::WARN, details: details
2631
- else
2632
- update_servers << node
2633
- end
2634
- }
2635
- end
2636
- return if update_servers.size == 0
2637
-
2638
- MU.log "Updating these nodes in #{@deploy_id}", MU::DEBUG, details: update_servers.map { |n| n.mu_name }
2639
-
2640
- update_servers.each { |node|
2641
- # Not clear where this pollution comes from, but let's stick a temp
2642
- # fix in here.
2643
- if node.deploydata['nodename'] != node.mu_name and
2644
- !node.deploydata['nodename'].nil? and !node.deploydata['nodename'].emty?
2645
- MU.log "Node #{node.mu_name} had wrong or missing nodename (#{node.deploydata['nodename']}), correcting", MU::WARN
2646
- node.deploydata['nodename'] = node.mu_name
2647
- if @deployment[svrs] and @deployment[svrs][node.config['name']] and
2648
- @deployment[svrs][node.config['name']][node.mu_name]
2649
- @deployment[svrs][node.config['name']][node.mu_name]['nodename'] = node.mu_name
2650
- end
2651
- save!
797
+ if @deployment["servers"][node.config['name']][node.mu_name].nil? or
798
+ @deployment["servers"][node.config['name']][node.mu_name] != node.deploydata
799
+ @deployment["servers"][node.config['name']][node.mu_name] = node.deploydata
800
+ elsif !save_only
801
+ # Don't bother running grooms on nodes that don't need to be updated,
802
+ # unless we're just going to do a save.
803
+ next
2652
804
  end
805
+ update_servers << node
2653
806
  }
2654
807
 
2655
- # Merge everyone's deploydata together
2656
- if !save_only
2657
- skip = []
2658
- update_servers.each { |node|
2659
- if node.mu_name.nil? or node.deploydata.nil? or node.config.nil?
2660
- MU.log "Missing mu_name #{node.mu_name}, deploydata, or config from #{node} in syncLitter", MU::ERR, details: node.deploydata
2661
- next
2662
- end
808
+ return if update_servers.empty?
2663
809
 
2664
- if !@deployment[svrs][node.config['name']].has_key?(node.mu_name) or @deployment[svrs][node.config['name']][node.mu_name] != node.deploydata
2665
- @deployment[svrs][node.config['name']][node.mu_name] = node.deploydata
2666
- else
2667
- skip << node
2668
- end
2669
- }
2670
- update_servers = update_servers - skip
2671
- end
810
+ MU.log "Updating nodes in #{@deploy_id}", MU::DEBUG, details: update_servers.map { |n| n.mu_name }
2672
811
 
2673
- return if MU.inGem? || update_servers.size < 1
2674
812
  threads = []
2675
- parent_thread_id = Thread.current.object_id
2676
813
  update_servers.each { |sibling|
814
+ next if sibling.config.has_key?("groom") and !sibling.config["groom"]
2677
815
  threads << Thread.new {
2678
816
  Thread.abort_on_exception = true
2679
- MU.dupGlobals(parent_thread_id)
2680
817
  Thread.current.thread_variable_set("name", "sync-"+sibling.mu_name.downcase)
2681
818
  MU.setVar("syncLitterThread", true)
2682
819
  begin
2683
- if sibling.config['groom'].nil? or sibling.config['groom']
2684
- sibling.groomer.saveDeployData
2685
- sibling.groomer.run(purpose: "Synchronizing sibling kittens") if !save_only
2686
- end
820
+ sibling.groomer.saveDeployData
821
+ sibling.groomer.run(purpose: "Synchronizing sibling kittens") if !save_only
2687
822
  rescue MU::Groomer::RunError => e
2688
- MU.log "Sync of #{sibling.mu_name} failed: #{e.inspect}", MU::WARN
823
+ MU.log "Sync of #{sibling.mu_name} failed", MU::WARN, details: e.inspect
2689
824
  end
2690
- MU.purgeGlobals
2691
825
  }
2692
826
  }
2693
827
 
@@ -2720,6 +854,7 @@ MESSAGE_END
2720
854
  MU::Master::SSL.bootstrap
2721
855
  sans = []
2722
856
  sans << canonical_ip if canonical_ip
857
+ sans << resource.mu_name.downcase if resource.mu_name and resource.mu_name != cert_cn
2723
858
  # XXX were there other names we wanted to include?
2724
859
  key = MU::Master::SSL.getKey(cert_cn, keysize: keysize)
2725
860
  cert, pfx_cert = MU::Master::SSL.getCert(cert_cn, "/CN=#{cert_cn}/O=Mu/C=US", sans: sans, pfx: is_windows)
@@ -2733,7 +868,7 @@ MESSAGE_END
2733
868
  end
2734
869
 
2735
870
  if resource and resource.config and resource.config['cloud']
2736
- cloudclass = Object.const_get("MU").const_get("Cloud").const_get(resource.config['cloud'])
871
+ cloudclass = MU::Cloud.cloudClass(resource.config['cloud'])
2737
872
 
2738
873
  cloudclass.writeDeploySecret(@deploy_id, cert.to_pem, cert_cn+".crt", credentials: resource.config['credentials'])
2739
874
  cloudclass.writeDeploySecret(@deploy_id, key.to_pem, cert_cn+".key", credentials: resource.config['credentials'])
@@ -2750,559 +885,30 @@ MESSAGE_END
2750
885
  results[cert_cn]
2751
886
  end
2752
887
 
2753
- # @return [String]: The Mu Master filesystem directory holding metadata for the current deployment
2754
- def deploy_dir
2755
- MU::MommaCat.deploy_dir(@deploy_id)
2756
- end
2757
-
2758
- # Path to the log file used by the Momma Cat daemon
2759
- # @return [String]
2760
- def self.daemonLogFile
2761
- base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
2762
- "#{base}/log/mu-momma-cat.log"
2763
- end
2764
-
2765
- # Path to the PID file used by the Momma Cat daemon
2766
- # @return [String]
2767
- def self.daemonPidFile
2768
- base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
2769
- "#{base}/run/mommacat.pid"
2770
- end
2771
-
2772
- # Start the Momma Cat daemon and return the exit status of the command used
2773
- # @return [Integer]
2774
- def self.start
2775
- if MU.inGem? and MU.muCfg['disable_mommacat']
2776
- return
2777
- end
2778
- base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
2779
- [base, "#{base}/log", "#{base}/run"].each { |dir|
2780
- if !Dir.exist?(dir)
2781
- MU.log "Creating #{dir}"
2782
- Dir.mkdir(dir)
2783
- end
2784
- }
2785
- return 0 if status
2786
-
2787
- MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}"
2788
- origdir = Dir.getwd
2789
- Dir.chdir(MU.myRoot+"/modules")
2790
-
2791
- # XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
2792
- if MU.inGem?
2793
- cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2794
- else
2795
- cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2796
- end
2797
-
2798
- MU.log cmd, MU::NOTICE
2799
-
2800
- retries = 0
2801
- begin
2802
- output = %x{#{cmd}}
2803
- sleep 1
2804
- retries += 1
2805
- if retries >= 10
2806
- MU.log "MommaCat failed to start (command was #{cmd}, working directory #{MU.myRoot}/modules)", MU::WARN, details: output
2807
- pp caller
2808
- return $?.exitstatus
2809
- end
2810
- end while !status
2811
-
2812
- Dir.chdir(origdir)
2813
-
2814
- if $?.exitstatus != 0
2815
- exit 1
2816
- end
2817
-
2818
- return $?.exitstatus
2819
- end
2820
-
2821
- # Return true if the Momma Cat daemon appears to be running
2822
- # @return [Boolean]
2823
- def self.status
2824
- if MU.inGem? and MU.muCfg['disable_mommacat']
2825
- return true
2826
- end
2827
- if File.exist?(daemonPidFile)
2828
- pid = File.read(daemonPidFile).chomp.to_i
2829
- begin
2830
- Process.getpgid(pid)
2831
- MU.log "Momma Cat running with pid #{pid.to_s}"
2832
- return true
2833
- rescue Errno::ESRCH
2834
- end
2835
- end
2836
- MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile
2837
- false
2838
- end
2839
-
2840
- # Stop the Momma Cat daemon, if it's running
2841
- def self.stop
2842
- if File.exist?(daemonPidFile)
2843
- pid = File.read(daemonPidFile).chomp.to_i
2844
- MU.log "Stopping Momma Cat with pid #{pid.to_s}"
2845
- Process.kill("INT", pid)
2846
- killed = false
2847
- begin
2848
- Process.getpgid(pid)
2849
- sleep 1
2850
- rescue Errno::ESRCH
2851
- killed = true
2852
- end while killed
2853
- MU.log "Momma Cat with pid #{pid.to_s} stopped", MU::DEBUG, details: daemonPidFile
2854
-
2855
- begin
2856
- File.unlink(daemonPidFile)
2857
- rescue Errno::ENOENT
2858
- end
2859
- end
2860
- end
2861
-
2862
- # (Re)start the Momma Cat daemon and return the exit status of the start command
2863
- # @return [Integer]
2864
- def self.restart
2865
- stop
2866
- start
2867
- end
2868
-
2869
- # Locate and return the deploy, if any, which matches the provided origin
2870
- # description
2871
- # @param origin [Hash]
2872
- def self.findMatchingDeploy(origin)
2873
- MU::MommaCat.listDeploys.each { |deploy_id|
2874
- o_path = deploy_dir(deploy_id)+"/origin.json"
2875
- next if !File.exist?(o_path)
2876
- this_origin = JSON.parse(File.read(o_path))
2877
- if origin == this_origin
2878
- MU.log "Deploy #{deploy_id} matches origin hash, loading", details: origin
2879
- return MU::MommaCat.new(deploy_id)
2880
- end
2881
- }
2882
- nil
2883
- end
2884
-
2885
- # Synchronize all in-memory information related to this to deployment to
2886
- # disk.
2887
- # @param triggering_node [MU::Cloud::Server]: If we're being triggered by the addition/removal/update of a node, this allows us to notify any sibling or dependent nodes of changes
2888
- # @param force [Boolean]: Save even if +no_artifacts+ is set
2889
- # @param origin [Hash]: Optional blob of data indicating how this deploy was created
2890
- def save!(triggering_node = nil, force: false, origin: nil)
2891
-
2892
- return if @no_artifacts and !force
2893
-
2894
- MU::MommaCat.deploy_struct_semaphore.synchronize {
2895
- MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG
2896
-
2897
- if !Dir.exist?(deploy_dir)
2898
- MU.log "Creating #{deploy_dir}", MU::DEBUG
2899
- Dir.mkdir(deploy_dir, 0700)
2900
- end
2901
-
2902
- if !origin.nil?
2903
- o_file = File.new("#{deploy_dir}/origin.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
2904
- o_file.puts JSON.pretty_generate(origin)
2905
- o_file.close
2906
- end
2907
-
2908
- if !@private_key.nil?
2909
- privkey = File.new("#{deploy_dir}/private_key", File::CREAT|File::TRUNC|File::RDWR, 0600)
2910
- privkey.puts @private_key
2911
- privkey.close
2912
- end
2913
-
2914
- if !@public_key.nil?
2915
- pubkey = File.new("#{deploy_dir}/public_key", File::CREAT|File::TRUNC|File::RDWR, 0600)
2916
- pubkey.puts @public_key
2917
- pubkey.close
2918
- end
2919
-
2920
- if !@deployment.nil? and @deployment.size > 0
2921
- @deployment['handle'] = MU.handle if @deployment['handle'].nil? and !MU.handle.nil?
2922
- @deployment['public_key'] = @public_key
2923
- @deployment['timestamp'] ||= @timestamp
2924
- @deployment['seed'] ||= @seed
2925
- @deployment['appname'] ||= @appname
2926
- @deployment['handle'] ||= @handle
2927
- @deployment['ssh_public_key'] ||= @ssh_public_key if @ssh_public_key
2928
- begin
2929
- # XXX doing this to trigger JSON errors before stomping the stored
2930
- # file...
2931
- JSON.pretty_generate(@deployment, max_nesting: false)
2932
- deploy = File.new("#{deploy_dir}/deployment.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
2933
- MU.log "Getting lock to write #{deploy_dir}/deployment.json", MU::DEBUG
2934
- deploy.flock(File::LOCK_EX)
2935
- deploy.puts JSON.pretty_generate(@deployment, max_nesting: false)
2936
- rescue JSON::NestingError => e
2937
- MU.log e.inspect, MU::ERR, details: @deployment
2938
- raise MuError, "Got #{e.message} trying to save deployment"
2939
- rescue Encoding::UndefinedConversionError => e
2940
- MU.log e.inspect, MU::ERR, details: @deployment
2941
- raise MuError, "Got #{e.message} at #{e.error_char.dump} (#{e.source_encoding_name} => #{e.destination_encoding_name}) trying to save deployment"
2942
- end
2943
- deploy.flock(File::LOCK_UN)
2944
- deploy.close
2945
- @need_deploy_flush = false
2946
- MU::MommaCat.updateLitter(@deploy_id, self)
2947
- end
2948
-
2949
- if !@original_config.nil? and @original_config.is_a?(Hash)
2950
- config = File.new("#{deploy_dir}/basket_of_kittens.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
2951
- config.puts JSON.pretty_generate(MU::Config.manxify(@original_config))
2952
- config.close
2953
- end
2954
-
2955
- if !@ssh_private_key.nil?
2956
- key = File.new("#{deploy_dir}/node_ssh.key", File::CREAT|File::TRUNC|File::RDWR, 0600)
2957
- key.puts @ssh_private_key
2958
- key.close
2959
- end
2960
- if !@ssh_public_key.nil?
2961
- key = File.new("#{deploy_dir}/node_ssh.pub", File::CREAT|File::TRUNC|File::RDWR, 0600)
2962
- key.puts @ssh_public_key
2963
- key.close
2964
- end
2965
- if !@ssh_key_name.nil?
2966
- key = File.new("#{deploy_dir}/ssh_key_name", File::CREAT|File::TRUNC|File::RDWR, 0600)
2967
- key.puts @ssh_key_name
2968
- key.close
2969
- end
2970
- if !@environment.nil?
2971
- env = File.new("#{deploy_dir}/environment_name", File::CREAT|File::TRUNC|File::RDWR, 0600)
2972
- env.puts @environment
2973
- env.close
2974
- end
2975
- if !@deploy_secret.nil?
2976
- secret = File.new("#{deploy_dir}/deploy_secret", File::CREAT|File::TRUNC|File::RDWR, 0600)
2977
- secret.print @deploy_secret
2978
- secret.close
2979
- end
2980
- if !@secrets.nil?
2981
- secretdir = "#{deploy_dir}/secrets"
2982
- if !Dir.exist?(secretdir)
2983
- MU.log "Creating #{secretdir}", MU::DEBUG
2984
- Dir.mkdir(secretdir, 0700)
2985
- end
2986
- @secrets.each_pair { |type, servers|
2987
- servers.each_pair { |server, svr_secret|
2988
- key = File.new("#{secretdir}/#{type}.#{server}", File::CREAT|File::TRUNC|File::RDWR, 0600)
2989
- key.puts svr_secret
2990
- key.close
2991
- }
2992
- }
2993
- end
2994
- }
2995
-
2996
- # Update groomer copies of this metadata
2997
- syncLitter(@deployment['servers'].keys, triggering_node: triggering_node, save_only: true) if @deployment.has_key?("servers")
2998
- end
2999
-
3000
- # Find one or more resources by their Mu resource name, and return
3001
- # MommaCat objects for their containing deploys, their BoK config data,
3002
- # and their deployment data.
3003
- #
3004
- # @param type [String]: The type of resource, e.g. "vpc" or "server."
3005
- # @param name [String]: The Mu resource class, typically the name field of a Basket of Kittens resource declaration.
3006
- # @param mu_name [String]: The fully-expanded Mu resource name, e.g. MGMT-PROD-2015040115-FR-ADMGMT2
3007
-
3008
888
  private
3009
889
 
3010
- # Check to see whether a given resource name is unique across all
3011
- # deployments on this Mu server. We only enforce this for certain classes
3012
- # of names. If the name in question is available, add it to our cache of
3013
- # said names. See #{MU::MommaCat.getResourceName}
3014
- # @param name [String]: The name to attempt to allocate.
3015
- # @return [Boolean]: True if allocation was successful.
3016
- def allocateUniqueResourceName(name)
3017
- raise MuError, "Cannot call allocateUniqueResourceName without an active deployment" if @deploy_id.nil?
3018
- path = File.expand_path(MU.dataDir+"/deployments")
3019
- File.open(path+"/unique_ids", File::CREAT|File::RDWR, 0600) { |f|
3020
- existing = []
3021
- f.flock(File::LOCK_EX)
3022
- f.readlines.each { |line|
3023
- existing << line.chomp
3024
- }
3025
- begin
3026
- existing.each { |used|
3027
- if used.match(/^#{name}:/)
3028
- if !used.match(/^#{name}:#{@deploy_id}$/)
3029
- MU.log "#{name} is already reserved by another resource on this Mu server.", MU::WARN, details: caller
3030
- return false
3031
- else
3032
- return true
3033
- end
3034
- end
3035
- }
3036
- f.puts name+":"+@deploy_id
3037
- return true
3038
- ensure
3039
- f.flock(File::LOCK_UN)
3040
- end
3041
- }
3042
- end
3043
-
3044
- ###########################################################################
3045
- ###########################################################################
3046
- def self.deploy_dir(deploy_id)
3047
- raise MuError, "deploy_dir must get a deploy_id if called as class method (from #{caller[0]}; #{caller[1]})" if deploy_id.nil?
3048
- # XXX this will blow up if someone sticks MU in /
3049
- path = File.expand_path(MU.dataDir+"/deployments")
3050
- if !Dir.exist?(path)
3051
- MU.log "Creating #{path}", MU::DEBUG
3052
- Dir.mkdir(path, 0700)
3053
- end
3054
- path = path+"/"+deploy_id
3055
- return path
3056
- end
3057
-
3058
- def self.deploy_exists?(deploy_id)
3059
- if deploy_id.nil? or deploy_id.empty?
3060
- MU.log "Got nil deploy_id in MU::MommaCat.deploy_exists?", MU::WARN
3061
- return
3062
- end
3063
- path = File.expand_path(MU.dataDir+"/deployments")
3064
- if !Dir.exist?(path)
3065
- Dir.mkdir(path, 0700)
3066
- end
3067
- deploy_path = File.expand_path(path+"/"+deploy_id)
3068
- return Dir.exist?(deploy_path)
3069
- end
3070
-
3071
-
3072
890
  def createDeployKey
3073
891
  key = OpenSSL::PKey::RSA.generate(4096)
3074
892
  MU.log "Generated deploy key for #{MU.deploy_id}", MU::DEBUG, details: key.public_key.export
3075
893
  return [key.export, key.public_key.export]
3076
894
  end
3077
895
 
3078
- # @param deploy_id [String]: The deployment to search. Will search all deployments if not specified.
3079
- # @return [Hash,Array<Hash>]
3080
- def self.getResourceMetadata(type, name: nil, deploy_id: nil, use_cache: true, mu_name: nil)
3081
- if type.nil?
3082
- raise MuError, "Can't call getResourceMetadata without a type argument"
3083
- end
3084
- _shortclass, _cfg_name, type, _classname = MU::Cloud.getResourceNames(type)
3085
-
3086
- # first, check our in-memory deploys, which may or may not have been
3087
- # written to disk yet.
3088
- littercache = nil
3089
- begin
3090
- @@litter_semaphore.synchronize {
3091
- littercache = @@litters.dup
3092
- }
3093
- rescue ThreadError => e
3094
- # already locked by a parent caller and this is a read op, so this is ok
3095
- raise e if !e.message.match(/recursive locking/)
3096
- littercache = @@litters.dup
3097
- end
3098
- littercache.each_pair { |deploy, momma|
3099
- @@deploy_struct_semaphore.synchronize {
3100
- @deploy_cache[deploy] = {
3101
- "mtime" => Time.now,
3102
- "data" => momma.deployment
3103
- }
3104
- }
3105
- }
3106
-
3107
- deploy_root = File.expand_path(MU.dataDir+"/deployments")
3108
- MU::MommaCat.deploy_struct_semaphore.synchronize {
3109
- if Dir.exist?(deploy_root)
3110
- Dir.entries(deploy_root).each { |deploy|
3111
- this_deploy_dir = deploy_root+"/"+deploy
3112
- next if deploy == "." or deploy == ".." or !Dir.exist?(this_deploy_dir)
3113
- next if deploy_id and deploy_id != deploy
3114
-
3115
- if !File.size?(this_deploy_dir+"/deployment.json")
3116
- MU.log "#{this_deploy_dir}/deployment.json doesn't exist, skipping when loading cache", MU::DEBUG
3117
- next
3118
- end
3119
- if @deploy_cache[deploy].nil? or !use_cache
3120
- @deploy_cache[deploy] = Hash.new
3121
- elsif @deploy_cache[deploy]['mtime'] == File.mtime("#{this_deploy_dir}/deployment.json")
3122
- MU.log "Using cached copy of deploy #{deploy} from #{@deploy_cache[deploy]['mtime']}", MU::DEBUG
3123
-
3124
- next
3125
- end
3126
-
3127
- @deploy_cache[deploy] = Hash.new if !@deploy_cache.has_key?(deploy)
3128
- MU.log "Caching deploy #{deploy}", MU::DEBUG
3129
- lock = File.open("#{this_deploy_dir}/deployment.json", File::RDONLY)
3130
- lock.flock(File::LOCK_EX)
3131
- @deploy_cache[deploy]['mtime'] = File.mtime("#{this_deploy_dir}/deployment.json")
3132
-
3133
- begin
3134
- @deploy_cache[deploy]['data'] = JSON.parse(File.read("#{this_deploy_dir}/deployment.json"))
3135
- lock.flock(File::LOCK_UN)
3136
-
3137
- next if @deploy_cache[deploy].nil? or @deploy_cache[deploy]['data'].nil?
3138
- # Populate some generable entries that should be in the deploy
3139
- # data. Also, bounce out if we realize we've found exactly what
3140
- # we needed already.
3141
- MU::Cloud.resource_types.values.each { |attrs|
3142
-
3143
- next if @deploy_cache[deploy]['data'][attrs[:cfg_plural]].nil?
3144
- if !attrs[:has_multiples]
3145
- @deploy_cache[deploy]['data'][attrs[:cfg_plural]].each_pair { |nodename, data|
3146
- # XXX we don't actually store node names for some resources, need to farm them
3147
- # and fix metadata
3148
- # if !mu_name.nil? and nodename == mu_name
3149
- # return { deploy => [data] }
3150
- # end
3151
- }
3152
- else
3153
- @deploy_cache[deploy]['data'][attrs[:cfg_plural]].each_pair { |node_class, nodes|
3154
- next if nodes.nil? or !nodes.is_a?(Hash)
3155
- nodes.each_pair { |nodename, data|
3156
- next if !data.is_a?(Hash)
3157
- data['#MU_NODE_CLASS'] = node_class
3158
- if !data.has_key?("cloud") # XXX kludge until old metadata gets fixed
3159
- data["cloud"] = MU::Config.defaultCloud
3160
- end
3161
- data['#MU_NAME'] = nodename
3162
- if !mu_name.nil? and nodename == mu_name
3163
- return {deploy => [data]} if deploy_id && deploy == deploy_id
3164
- end
3165
- }
3166
- }
3167
- end
3168
- }
3169
- rescue JSON::ParserError => e
3170
- raise MuError, "JSON parse failed on #{this_deploy_dir}/deployment.json\n\n"+File.read("#{this_deploy_dir}/deployment.json")
3171
- end
3172
- lock.flock(File::LOCK_UN)
3173
- lock.close
3174
- }
3175
- end
3176
- }
3177
-
3178
- matches = {}
3179
-
3180
- if deploy_id.nil?
3181
- @deploy_cache.each_key { |deploy|
3182
- next if !@deploy_cache[deploy].has_key?('data')
3183
- next if !@deploy_cache[deploy]['data'].has_key?(type)
3184
- if !name.nil?
3185
- next if @deploy_cache[deploy]['data'][type][name].nil?
3186
- matches[deploy] ||= []
3187
- matches[deploy] << @deploy_cache[deploy]['data'][type][name].dup
3188
- else
3189
- matches[deploy] ||= []
3190
- matches[deploy].concat(@deploy_cache[deploy]['data'][type].values)
3191
- end
3192
- }
3193
- return matches
3194
- elsif !@deploy_cache[deploy_id].nil?
3195
- if !@deploy_cache[deploy_id]['data'].nil? and
3196
- !@deploy_cache[deploy_id]['data'][type].nil?
3197
- if !name.nil?
3198
- if !@deploy_cache[deploy_id]['data'][type][name].nil?
3199
- matches[deploy_id] ||= []
3200
- matches[deploy_id] << @deploy_cache[deploy_id]['data'][type][name].dup
3201
- else
3202
- return matches # nothing, actually
3203
- end
3204
- else
3205
- matches[deploy_id] = @deploy_cache[deploy_id]['data'][type].values
3206
- end
3207
- end
3208
- end
3209
-
3210
- return matches
3211
- end
3212
-
3213
896
  ###########################################################################
3214
897
  ###########################################################################
3215
- def loadDeploy(deployment_json_only = false, set_context_to_me: true)
3216
- MU::MommaCat.deploy_struct_semaphore.synchronize {
3217
- if File.size?(deploy_dir+"/deployment.json")
3218
- deploy = File.open("#{deploy_dir}/deployment.json", File::RDONLY)
3219
- MU.log "Getting lock to read #{deploy_dir}/deployment.json", MU::DEBUG
3220
- # deploy.flock(File::LOCK_EX)
3221
- begin
3222
- Timeout::timeout(90) {deploy.flock(File::LOCK_EX)}
3223
- rescue Timeout::Error
3224
- raise MuError, "Timed out trying to get an exclusive lock on #{deploy_dir}/deployment.json"
3225
- end
3226
-
3227
- begin
3228
- @deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
3229
- rescue JSON::ParserError => e
3230
- MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
3231
- end
3232
-
3233
- deploy.flock(File::LOCK_UN)
3234
- deploy.close
3235
- if set_context_to_me
3236
- ["appname", "environment", "timestamp", "seed", "handle"].each { |var|
3237
- @deployment[var] ||= instance_variable_get("@#{var}".to_sym)
3238
- if @deployment[var]
3239
- if var != "handle"
3240
- MU.setVar(var, @deployment[var].upcase)
3241
- else
3242
- MU.setVar(var, @deployment[var])
3243
- end
3244
- else
3245
- MU.log "Missing global variable #{var} for #{MU.deploy_id}", MU::ERR
3246
- end
3247
- }
3248
- end
3249
- @timestamp = @deployment['timestamp']
3250
- @seed = @deployment['seed']
3251
- @appname = @deployment['appname']
3252
- @handle = @deployment['handle']
3253
-
3254
- return if deployment_json_only
3255
- end
3256
- if File.exist?(deploy_dir+"/private_key")
3257
- @private_key = File.read("#{deploy_dir}/private_key")
3258
- @public_key = File.read("#{deploy_dir}/public_key")
3259
- end
3260
- if File.exist?(deploy_dir+"/basket_of_kittens.json")
3261
- begin
3262
- @original_config = JSON.parse(File.read("#{deploy_dir}/basket_of_kittens.json"))
3263
- rescue JSON::ParserError => e
3264
- MU.log "JSON parse failed on #{deploy_dir}/basket_of_kittens.json", MU::ERR, details: e.message
898
+ def setThreadContextToMe
899
+ ["appname", "environment", "timestamp", "seed", "handle"].each { |var|
900
+ @deployment[var] ||= instance_variable_get("@#{var}".to_sym)
901
+ if @deployment[var]
902
+ if var != "handle"
903
+ MU.setVar(var, @deployment[var].upcase)
904
+ else
905
+ MU.setVar(var, @deployment[var])
3265
906
  end
3266
- end
3267
- if File.exist?(deploy_dir+"/ssh_key_name")
3268
- @ssh_key_name = File.read("#{deploy_dir}/ssh_key_name").chomp!
3269
- end
3270
- if File.exist?(deploy_dir+"/node_ssh.key")
3271
- @ssh_private_key = File.read("#{deploy_dir}/node_ssh.key")
3272
- end
3273
- if File.exist?(deploy_dir+"/node_ssh.pub")
3274
- @ssh_public_key = File.read("#{deploy_dir}/node_ssh.pub")
3275
- end
3276
- if File.exist?(deploy_dir+"/environment_name")
3277
- @environment = File.read("#{deploy_dir}/environment_name").chomp!
3278
- end
3279
- if File.exist?(deploy_dir+"/deploy_secret")
3280
- @deploy_secret = File.read("#{deploy_dir}/deploy_secret")
3281
- end
3282
- if Dir.exist?("#{deploy_dir}/secrets")
3283
- @secrets.each_key { |type|
3284
- Dir.glob("#{deploy_dir}/secrets/#{type}.*") { |filename|
3285
- server = File.basename(filename).split(/\./)[1]
3286
-
3287
- @secrets[type][server] = File.read(filename).chomp!
3288
- }
3289
- }
907
+ else
908
+ MU.log "Missing global variable #{var} for #{MU.deploy_id}", MU::ERR
3290
909
  end
3291
910
  }
3292
911
  end
3293
912
 
3294
- # 2019-06-03 adding things from https://aiweirdness.com/post/185339301987/once-again-a-neural-net-tries-to-name-cats
3295
- @catadjs = %w{fuzzy ginger lilac chocolate xanthic wiggly itty chonky norty slonky floofy}
3296
- @catnouns = %w{bastet biscuits bobcat catnip cheetah chonk dot felix hamb jaguar kitty leopard lion lynx maru mittens moggy neko nip ocelot panther patches paws phoebe purr queen roar saber sekhmet skogkatt socks sphinx spot tail tiger tom whiskers wildcat yowl floof beans ailurophile dander dewclaw grimalkin kibble quick tuft misty simba slonk mew quat eek ziggy whiskeridoo cromch monch screm}
3297
- @catmixed = %w{abyssinian angora bengal birman bobtail bombay burmese calico chartreux cheshire cornish-rex curl devon egyptian-mau feline furever fumbs havana himilayan japanese-bobtail javanese khao-manee maine-coon manx marmalade mau munchkin norwegian pallas persian peterbald polydactyl ragdoll russian-blue savannah scottish-fold serengeti shorthair siamese siberian singapura snowshoe stray tabby tonkinese tortoiseshell turkish-van tuxedo uncia caterwaul lilac-point chocolate-point mackerel maltese knead whitenose vorpal chewie-bean chicken-whiskey fish-especially thelonious-monsieur tom-glitter serendipitous-kill sparky-buttons}
3298
- @catwords = @catadjs + @catnouns + @catmixed
3299
-
3300
- @jaegeradjs = %w{azure fearless lucky olive vivid electric grey yarely violet ivory jade cinnamon crimson tacit umber mammoth ultra iron zodiac}
3301
- @jaegernouns = %w{horizon hulk ultimatum yardarm watchman whilrwind wright rhythm ocean enigma eruption typhoon jaeger brawler blaze vandal excalibur paladin juliet kaleidoscope romeo}
3302
- @jaegermixed = %w{alpha ajax amber avenger brave bravo charlie chocolate chrome corinthian dancer danger dash delta duet echo edge elite eureka foxtrot guardian gold hyperion illusion imperative india intercept kilo lancer night nova november oscar omega pacer quickstrike rogue ronin striker tango titan valor victor vulcan warder xenomorph xenon xray xylem yankee yell yukon zeal zero zoner zodiac}
3303
- @jaegerwords = @jaegeradjs + @jaegernouns + @jaegermixed
3304
-
3305
- @words = @catwords + @jaegerwords
3306
-
3307
913
  end #class
3308
914
  end #module