cloud-mu 3.1.3 → 3.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Dockerfile +10 -2
- data/bin/mu-adopt +5 -1
- data/bin/mu-load-config.rb +2 -3
- data/bin/mu-run-tests +112 -27
- data/cloud-mu.gemspec +20 -20
- data/cookbooks/mu-tools/libraries/helper.rb +2 -1
- data/cookbooks/mu-tools/libraries/monkey.rb +35 -0
- data/cookbooks/mu-tools/recipes/google_api.rb +2 -2
- data/cookbooks/mu-tools/resources/disk.rb +1 -1
- data/extras/image-generators/Google/centos6.yaml +1 -0
- data/extras/image-generators/Google/centos7.yaml +1 -1
- data/modules/mommacat.ru +5 -15
- data/modules/mu.rb +10 -14
- data/modules/mu/adoption.rb +20 -14
- data/modules/mu/cleanup.rb +13 -9
- data/modules/mu/cloud.rb +26 -26
- data/modules/mu/clouds/aws.rb +100 -59
- data/modules/mu/clouds/aws/alarm.rb +4 -2
- data/modules/mu/clouds/aws/bucket.rb +25 -21
- data/modules/mu/clouds/aws/cache_cluster.rb +25 -23
- data/modules/mu/clouds/aws/collection.rb +21 -20
- data/modules/mu/clouds/aws/container_cluster.rb +47 -26
- data/modules/mu/clouds/aws/database.rb +57 -68
- data/modules/mu/clouds/aws/dnszone.rb +14 -14
- data/modules/mu/clouds/aws/endpoint.rb +20 -16
- data/modules/mu/clouds/aws/firewall_rule.rb +19 -16
- data/modules/mu/clouds/aws/folder.rb +7 -7
- data/modules/mu/clouds/aws/function.rb +15 -12
- data/modules/mu/clouds/aws/group.rb +14 -10
- data/modules/mu/clouds/aws/habitat.rb +16 -13
- data/modules/mu/clouds/aws/loadbalancer.rb +16 -15
- data/modules/mu/clouds/aws/log.rb +13 -10
- data/modules/mu/clouds/aws/msg_queue.rb +15 -8
- data/modules/mu/clouds/aws/nosqldb.rb +18 -11
- data/modules/mu/clouds/aws/notifier.rb +11 -6
- data/modules/mu/clouds/aws/role.rb +87 -70
- data/modules/mu/clouds/aws/search_domain.rb +30 -19
- data/modules/mu/clouds/aws/server.rb +102 -72
- data/modules/mu/clouds/aws/server_pool.rb +47 -28
- data/modules/mu/clouds/aws/storage_pool.rb +5 -6
- data/modules/mu/clouds/aws/user.rb +13 -10
- data/modules/mu/clouds/aws/vpc.rb +135 -121
- data/modules/mu/clouds/azure.rb +16 -9
- data/modules/mu/clouds/azure/container_cluster.rb +2 -3
- data/modules/mu/clouds/azure/firewall_rule.rb +10 -10
- data/modules/mu/clouds/azure/habitat.rb +8 -6
- data/modules/mu/clouds/azure/loadbalancer.rb +5 -5
- data/modules/mu/clouds/azure/role.rb +8 -10
- data/modules/mu/clouds/azure/server.rb +65 -25
- data/modules/mu/clouds/azure/user.rb +5 -7
- data/modules/mu/clouds/azure/vpc.rb +12 -15
- data/modules/mu/clouds/cloudformation.rb +8 -7
- data/modules/mu/clouds/cloudformation/vpc.rb +2 -4
- data/modules/mu/clouds/google.rb +39 -24
- data/modules/mu/clouds/google/bucket.rb +9 -11
- data/modules/mu/clouds/google/container_cluster.rb +27 -42
- data/modules/mu/clouds/google/database.rb +6 -9
- data/modules/mu/clouds/google/firewall_rule.rb +11 -10
- data/modules/mu/clouds/google/folder.rb +16 -9
- data/modules/mu/clouds/google/function.rb +127 -161
- data/modules/mu/clouds/google/group.rb +21 -18
- data/modules/mu/clouds/google/habitat.rb +18 -15
- data/modules/mu/clouds/google/loadbalancer.rb +14 -16
- data/modules/mu/clouds/google/role.rb +48 -31
- data/modules/mu/clouds/google/server.rb +105 -105
- data/modules/mu/clouds/google/server_pool.rb +12 -31
- data/modules/mu/clouds/google/user.rb +67 -13
- data/modules/mu/clouds/google/vpc.rb +58 -65
- data/modules/mu/config.rb +89 -1738
- data/modules/mu/config/bucket.rb +3 -3
- data/modules/mu/config/collection.rb +3 -3
- data/modules/mu/config/container_cluster.rb +2 -2
- data/modules/mu/config/dnszone.rb +5 -5
- data/modules/mu/config/doc_helpers.rb +517 -0
- data/modules/mu/config/endpoint.rb +3 -3
- data/modules/mu/config/firewall_rule.rb +118 -3
- data/modules/mu/config/folder.rb +3 -3
- data/modules/mu/config/function.rb +2 -2
- data/modules/mu/config/group.rb +3 -3
- data/modules/mu/config/habitat.rb +3 -3
- data/modules/mu/config/loadbalancer.rb +3 -3
- data/modules/mu/config/log.rb +3 -3
- data/modules/mu/config/msg_queue.rb +3 -3
- data/modules/mu/config/nosqldb.rb +3 -3
- data/modules/mu/config/notifier.rb +2 -2
- data/modules/mu/config/ref.rb +333 -0
- data/modules/mu/config/role.rb +3 -3
- data/modules/mu/config/schema_helpers.rb +508 -0
- data/modules/mu/config/search_domain.rb +3 -3
- data/modules/mu/config/server.rb +86 -58
- data/modules/mu/config/server_pool.rb +2 -2
- data/modules/mu/config/tail.rb +189 -0
- data/modules/mu/config/user.rb +3 -3
- data/modules/mu/config/vpc.rb +44 -4
- data/modules/mu/defaults/Google.yaml +2 -2
- data/modules/mu/deploy.rb +13 -10
- data/modules/mu/groomer.rb +1 -1
- data/modules/mu/groomers/ansible.rb +69 -24
- data/modules/mu/groomers/chef.rb +52 -44
- data/modules/mu/logger.rb +17 -14
- data/modules/mu/master.rb +317 -2
- data/modules/mu/master/chef.rb +3 -4
- data/modules/mu/master/ldap.rb +3 -3
- data/modules/mu/master/ssl.rb +12 -2
- data/modules/mu/mommacat.rb +85 -1766
- data/modules/mu/mommacat/daemon.rb +394 -0
- data/modules/mu/mommacat/naming.rb +366 -0
- data/modules/mu/mommacat/storage.rb +689 -0
- data/modules/tests/bucket.yml +4 -0
- data/modules/tests/{win2k12.yaml → needwork/win2k12.yaml} +0 -0
- data/modules/tests/regrooms/aws-iam.yaml +201 -0
- data/modules/tests/regrooms/bucket.yml +19 -0
- metadata +112 -102
data/modules/mu/master/ldap.rb
CHANGED
|
@@ -360,7 +360,7 @@ module MU
|
|
|
360
360
|
# @param stamp [Integer]: The MS-style timestamp, e.g. 130838184558490696
|
|
361
361
|
# @return [Time]
|
|
362
362
|
def self.convertMicrosoftTime(stamp)
|
|
363
|
-
ms_epoch = DateTime.new(1601,1,1).strftime("%Q").to_i
|
|
363
|
+
# ms_epoch = DateTime.new(1601,1,1).strftime("%Q").to_i
|
|
364
364
|
unixtime = (stamp.to_i/10000) + DateTime.new(1601,1,1).strftime("%Q").to_i
|
|
365
365
|
Time.at(unixtime/1000)
|
|
366
366
|
end
|
|
@@ -629,7 +629,7 @@ module MU
|
|
|
629
629
|
return true if !require_group
|
|
630
630
|
|
|
631
631
|
shortuser = username.sub(/\@.*/, "")
|
|
632
|
-
user = findUsers(
|
|
632
|
+
user = findUsers([shortuser], exact: true)
|
|
633
633
|
if user[shortuser]["memberOf"].is_a?(Array)
|
|
634
634
|
user[shortuser]["memberOf"].each { |group|
|
|
635
635
|
shortname = group.sub(/^CN=(.*?),.*/, '\1')
|
|
@@ -945,7 +945,7 @@ module MU
|
|
|
945
945
|
cur_users[user]['realname'] = name
|
|
946
946
|
end
|
|
947
947
|
if disable
|
|
948
|
-
|
|
948
|
+
findUsers([user], exact: true)
|
|
949
949
|
MU.log "Disabling #{user}", MU::WARN
|
|
950
950
|
conn.replace_attribute(user_dn, :userAccountControl, AD_PW_ATTRS['disable'].to_i.to_s(2))
|
|
951
951
|
elsif enable
|
data/modules/mu/master/ssl.rb
CHANGED
|
@@ -90,7 +90,7 @@ module MU
|
|
|
90
90
|
|
|
91
91
|
begin
|
|
92
92
|
csr = OpenSSL::X509::Request.new File.read csr_path
|
|
93
|
-
rescue
|
|
93
|
+
rescue StandardError => e
|
|
94
94
|
MU.log e.message, MU::ERR, details: File.read(csr_path)
|
|
95
95
|
raise e
|
|
96
96
|
end
|
|
@@ -222,8 +222,12 @@ puts cn_str
|
|
|
222
222
|
[cert, pfx_cert]
|
|
223
223
|
end
|
|
224
224
|
|
|
225
|
-
private
|
|
226
225
|
|
|
226
|
+
# Convert an x509 certificate to the .pfx thing Windows likes
|
|
227
|
+
# @param certfile [String]: Path to source certificate
|
|
228
|
+
# @param keyfile [String]: Path to source certificate's key
|
|
229
|
+
# @param pfxfile [String]: Path to output the new certificate
|
|
230
|
+
# @return [OpenSSL::PKCS12]
|
|
227
231
|
def self.toPfx(certfile, keyfile, pfxfile)
|
|
228
232
|
cacert = getCert("Mu_CA", ca: true).first
|
|
229
233
|
cert = OpenSSL::X509::Certificate.new(File.read(certfile))
|
|
@@ -234,7 +238,12 @@ puts cn_str
|
|
|
234
238
|
}
|
|
235
239
|
pfx
|
|
236
240
|
end
|
|
241
|
+
private_class_method :toPfx
|
|
237
242
|
|
|
243
|
+
# Given a list of strings that might be IPs or hostnames, format as a
|
|
244
|
+
# of Subject Alternative Names for use in a certificate.
|
|
245
|
+
# @param sans [Array<String>]
|
|
246
|
+
# @return [String]
|
|
238
247
|
def self.formatSANS(sans)
|
|
239
248
|
sans.map { |s|
|
|
240
249
|
if s.match(/^\d+\.\d+\.\d+\.\d+$/)
|
|
@@ -244,6 +253,7 @@ puts cn_str
|
|
|
244
253
|
end
|
|
245
254
|
}.join(",")
|
|
246
255
|
end
|
|
256
|
+
private_class_method :formatSANS
|
|
247
257
|
|
|
248
258
|
end
|
|
249
259
|
end
|
data/modules/mu/mommacat.rb
CHANGED
|
@@ -18,6 +18,9 @@ require 'json'
|
|
|
18
18
|
require 'stringio'
|
|
19
19
|
require 'securerandom'
|
|
20
20
|
require 'timeout'
|
|
21
|
+
require 'mu/mommacat/storage'
|
|
22
|
+
require 'mu/mommacat/daemon'
|
|
23
|
+
require 'mu/mommacat/naming'
|
|
21
24
|
|
|
22
25
|
module MU
|
|
23
26
|
|
|
@@ -42,60 +45,6 @@ module MU
|
|
|
42
45
|
@@litters_loadtime = {}
|
|
43
46
|
@@litter_semaphore = Mutex.new
|
|
44
47
|
|
|
45
|
-
# Return a {MU::MommaCat} instance for an existing deploy. Use this instead
|
|
46
|
-
# of using #initialize directly to avoid loading deploys multiple times or
|
|
47
|
-
# stepping on the global context for the deployment you're really working
|
|
48
|
-
# on..
|
|
49
|
-
# @param deploy_id [String]: The deploy ID of the deploy to load.
|
|
50
|
-
# @param set_context_to_me [Boolean]: Whether new MommaCat objects should overwrite any existing per-thread global deploy variables.
|
|
51
|
-
# @param use_cache [Boolean]: If we have an existing object for this deploy, use that
|
|
52
|
-
# @return [MU::MommaCat]
|
|
53
|
-
def self.getLitter(deploy_id, set_context_to_me: false, use_cache: true)
|
|
54
|
-
if deploy_id.nil? or deploy_id.empty?
|
|
55
|
-
raise MuError, "Cannot fetch a deployment without a deploy_id"
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
# XXX this caching may be harmful, causing stale resource objects to stick
|
|
59
|
-
# around. Have we fixed this? Sort of. Bad entries seem to have no kittens,
|
|
60
|
-
# so force a reload if we see that. That's probably not the root problem.
|
|
61
|
-
littercache = nil
|
|
62
|
-
begin
|
|
63
|
-
@@litter_semaphore.synchronize {
|
|
64
|
-
littercache = @@litters.dup
|
|
65
|
-
}
|
|
66
|
-
if littercache[deploy_id] and @@litters_loadtime[deploy_id]
|
|
67
|
-
deploy_root = File.expand_path(MU.dataDir+"/deployments")
|
|
68
|
-
this_deploy_dir = deploy_root+"/"+deploy_id
|
|
69
|
-
if File.exist?("#{this_deploy_dir}/deployment.json")
|
|
70
|
-
lastmod = File.mtime("#{this_deploy_dir}/deployment.json")
|
|
71
|
-
if lastmod > @@litters_loadtime[deploy_id]
|
|
72
|
-
MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE
|
|
73
|
-
use_cache = false
|
|
74
|
-
end
|
|
75
|
-
end
|
|
76
|
-
end
|
|
77
|
-
rescue ThreadError => e
|
|
78
|
-
# already locked by a parent caller and this is a read op, so this is ok
|
|
79
|
-
raise e if !e.message.match(/recursive locking/)
|
|
80
|
-
littercache = @@litters.dup
|
|
81
|
-
end
|
|
82
|
-
|
|
83
|
-
if !use_cache or littercache[deploy_id].nil?
|
|
84
|
-
need_gc = !littercache[deploy_id].nil?
|
|
85
|
-
newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
|
|
86
|
-
# This, we have to synchronize, as it's a write
|
|
87
|
-
@@litter_semaphore.synchronize {
|
|
88
|
-
@@litters[deploy_id] = newlitter
|
|
89
|
-
@@litters_loadtime[deploy_id] = Time.now
|
|
90
|
-
}
|
|
91
|
-
GC.start if need_gc
|
|
92
|
-
elsif set_context_to_me
|
|
93
|
-
MU::MommaCat.setThreadContext(@@litters[deploy_id])
|
|
94
|
-
end
|
|
95
|
-
return @@litters[deploy_id]
|
|
96
|
-
# MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
|
|
97
|
-
end
|
|
98
|
-
|
|
99
48
|
# Update the in-memory cache of a given deploy. This is intended for use by
|
|
100
49
|
# {#save!}, primarily.
|
|
101
50
|
# @param deploy_id [String]
|
|
@@ -127,21 +76,7 @@ module MU
|
|
|
127
76
|
attr_reader :chef_user
|
|
128
77
|
attr_reader :no_artifacts
|
|
129
78
|
attr_accessor :kittens # really want a method only available to :Deploy
|
|
130
|
-
@myhome = Etc.getpwuid(Process.uid).dir
|
|
131
|
-
@nagios_home = "/opt/mu/var/nagios_user_home"
|
|
132
|
-
@locks = Hash.new
|
|
133
|
-
@deploy_cache = Hash.new
|
|
134
79
|
@nocleanup = false
|
|
135
|
-
# List the currently held flock() locks.
|
|
136
|
-
def self.trapSafeLocks;
|
|
137
|
-
@locks
|
|
138
|
-
end
|
|
139
|
-
# List the currently held flock() locks.
|
|
140
|
-
def self.locks;
|
|
141
|
-
@lock_semaphore.synchronize {
|
|
142
|
-
@locks
|
|
143
|
-
}
|
|
144
|
-
end
|
|
145
80
|
|
|
146
81
|
@@deploy_struct_semaphore = Mutex.new
|
|
147
82
|
# Don't let things that modify the deploy struct Hash step on each other.
|
|
@@ -393,7 +328,7 @@ module MU
|
|
|
393
328
|
end
|
|
394
329
|
attrs[:interface].new(mommacat: self, kitten_cfg: orig_cfg, mu_name: data['mu_name'], cloud_id: data['cloud_id'])
|
|
395
330
|
end
|
|
396
|
-
rescue
|
|
331
|
+
rescue StandardError => e
|
|
397
332
|
if e.class != MU::Cloud::MuCloudResourceNotImplemented
|
|
398
333
|
MU.log "Failed to load an existing resource of type '#{type}' in #{@deploy_id}: #{e.inspect}", MU::WARN, details: e.backtrace
|
|
399
334
|
end
|
|
@@ -455,6 +390,46 @@ module MU
|
|
|
455
390
|
seen.uniq
|
|
456
391
|
end
|
|
457
392
|
|
|
393
|
+
# List the accounts/projects/subscriptions used by each resource in our
|
|
394
|
+
# deploy.
|
|
395
|
+
# @return [Array<String>]
|
|
396
|
+
def habitatsUsed
|
|
397
|
+
return [] if !@original_config
|
|
398
|
+
habitats = []
|
|
399
|
+
habitats << @original_config['project'] if @original_config['project']
|
|
400
|
+
if @original_config['habitat']
|
|
401
|
+
hab_ref = MU::Config::Ref.get(@original_config['habitat'])
|
|
402
|
+
if hab_ref and hab_ref.id
|
|
403
|
+
habitats << hab_ref.id
|
|
404
|
+
end
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
MU::Cloud.resource_types.values.each { |attrs|
|
|
408
|
+
type = attrs[:cfg_plural]
|
|
409
|
+
if @original_config[type]
|
|
410
|
+
@original_config[type].each { |resource|
|
|
411
|
+
if resource['project']
|
|
412
|
+
habitats << resource['project']
|
|
413
|
+
elsif resource['habitat']
|
|
414
|
+
hab_ref = MU::Config::Ref.get(resource['habitat'])
|
|
415
|
+
if hab_ref and hab_ref.id
|
|
416
|
+
habitats << hab_ref.id
|
|
417
|
+
end
|
|
418
|
+
elsif resource['cloud']
|
|
419
|
+
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(resource['cloud'])
|
|
420
|
+
# XXX this should be a general method implemented by each cloud
|
|
421
|
+
# provider
|
|
422
|
+
if resource['cloud'] == "Google"
|
|
423
|
+
habitats << cloudclass.defaultProject(resource['credentials'])
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
}
|
|
427
|
+
end
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
habitats.uniq!
|
|
431
|
+
end
|
|
432
|
+
|
|
458
433
|
# List the regions used by each resource in our deploy. This will just be
|
|
459
434
|
# a flat list of strings with no regard to which region belongs with what
|
|
460
435
|
# cloud provider- things mostly use this as a lookup table so they can
|
|
@@ -472,7 +447,7 @@ module MU
|
|
|
472
447
|
cloudclass = Object.const_get("MU").const_get("Cloud").const_get(resource['cloud'])
|
|
473
448
|
resclass = Object.const_get("MU").const_get("Cloud").const_get(resource['cloud']).const_get(res_type.to_s)
|
|
474
449
|
if resclass.isGlobal?
|
|
475
|
-
|
|
450
|
+
# XXX why was I doing this, urgh
|
|
476
451
|
next
|
|
477
452
|
elsif !resource['region']
|
|
478
453
|
regions << cloudclass.myRegion
|
|
@@ -548,29 +523,6 @@ module MU
|
|
|
548
523
|
@kittens
|
|
549
524
|
end
|
|
550
525
|
|
|
551
|
-
# Overwrite this deployment's configuration with a new version. Save the
|
|
552
|
-
# previous version as well.
|
|
553
|
-
# @param new_conf [Hash]: A new configuration, fully resolved by {MU::Config}
|
|
554
|
-
def updateBasketofKittens(new_conf)
|
|
555
|
-
loadDeploy
|
|
556
|
-
if new_conf == @original_config
|
|
557
|
-
MU.log "#{@deploy_id}", MU::WARN
|
|
558
|
-
return
|
|
559
|
-
end
|
|
560
|
-
|
|
561
|
-
backup = "#{deploy_dir}/basket_of_kittens.json.#{Time.now.to_i.to_s}"
|
|
562
|
-
MU.log "Saving previous config of #{@deploy_id} to #{backup}"
|
|
563
|
-
config = File.new(backup, File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
564
|
-
config.flock(File::LOCK_EX)
|
|
565
|
-
config.puts JSON.pretty_generate(@original_config)
|
|
566
|
-
config.flock(File::LOCK_UN)
|
|
567
|
-
config.close
|
|
568
|
-
|
|
569
|
-
@original_config = new_conf
|
|
570
|
-
# save! # XXX this will happen later, more sensibly
|
|
571
|
-
MU.log "New config saved to #{deploy_dir}/basket_of_kittens.json"
|
|
572
|
-
end
|
|
573
|
-
|
|
574
526
|
# Keep tabs on a {MU::Cloud} object so that it can be found easily by
|
|
575
527
|
# #findLitterMate.
|
|
576
528
|
# @param type [String]:
|
|
@@ -597,166 +549,6 @@ module MU
|
|
|
597
549
|
}
|
|
598
550
|
end
|
|
599
551
|
|
|
600
|
-
# Check a provided deploy key against our stored version. The instance has
|
|
601
|
-
# in theory accessed a secret via S3 and encrypted it with the deploy's
|
|
602
|
-
# public key. If it decrypts correctly, we assume this instance is indeed
|
|
603
|
-
# one of ours.
|
|
604
|
-
# @param ciphertext [String]: The text to decrypt.
|
|
605
|
-
# return [Boolean]: Whether the provided text was encrypted with the correct key
|
|
606
|
-
def authKey(ciphertext)
|
|
607
|
-
if @private_key.nil? or @deploy_secret.nil?
|
|
608
|
-
MU.log "Missing auth metadata, can't authorize node in authKey", MU::ERR
|
|
609
|
-
return false
|
|
610
|
-
end
|
|
611
|
-
my_key = OpenSSL::PKey::RSA.new(@private_key)
|
|
612
|
-
|
|
613
|
-
begin
|
|
614
|
-
if my_key.private_decrypt(ciphertext).force_encoding("UTF-8") == @deploy_secret.force_encoding("UTF-8")
|
|
615
|
-
MU.log "Matched ciphertext for #{MU.deploy_id}", MU::INFO
|
|
616
|
-
return true
|
|
617
|
-
else
|
|
618
|
-
MU.log "Mis-matched ciphertext for #{MU.deploy_id}", MU::ERR
|
|
619
|
-
return false
|
|
620
|
-
end
|
|
621
|
-
rescue OpenSSL::PKey::RSAError => e
|
|
622
|
-
MU.log "Error decrypting provided ciphertext using private key from #{deploy_dir}/private_key: #{e.message}", MU::ERR, details: ciphertext
|
|
623
|
-
return false
|
|
624
|
-
end
|
|
625
|
-
end
|
|
626
|
-
|
|
627
|
-
# Generate a three-character string which can be used to unique-ify the
|
|
628
|
-
# names of resources which might potentially collide, e.g. Windows local
|
|
629
|
-
# hostnames, Amazon Elastic Load Balancers, or server pool instances.
|
|
630
|
-
# @return [String]: A three-character string consisting of two alphnumeric
|
|
631
|
-
# characters (uppercase) and one number.
|
|
632
|
-
def self.genUniquenessString
|
|
633
|
-
begin
|
|
634
|
-
candidate = SecureRandom.base64(2).slice(0..1) + SecureRandom.random_number(9).to_s
|
|
635
|
-
candidate.upcase!
|
|
636
|
-
end while candidate.match(/[^A-Z0-9]/)
|
|
637
|
-
return candidate
|
|
638
|
-
end
|
|
639
|
-
|
|
640
|
-
@unique_map_semaphore = Mutex.new
|
|
641
|
-
@name_unique_str_map = {}
|
|
642
|
-
# Keep a map of the uniqueness strings we assign to various full names, in
|
|
643
|
-
# case we want to reuse them later.
|
|
644
|
-
# @return [Hash<String>]
|
|
645
|
-
def self.name_unique_str_map
|
|
646
|
-
@name_unique_str_map
|
|
647
|
-
end
|
|
648
|
-
|
|
649
|
-
# Keep a map of the uniqueness strings we assign to various full names, in
|
|
650
|
-
# case we want to reuse them later.
|
|
651
|
-
# @return [Mutex]
|
|
652
|
-
def self.unique_map_semaphore
|
|
653
|
-
@unique_map_semaphore
|
|
654
|
-
end
|
|
655
|
-
|
|
656
|
-
# Generate a name string for a resource, incorporate the MU identifier
|
|
657
|
-
# for this deployment. Will dynamically shorten the name to fit for
|
|
658
|
-
# restrictive uses (e.g. Windows local hostnames, Amazon Elastic Load
|
|
659
|
-
# Balancers).
|
|
660
|
-
# @param name [String]: The shorthand name of the resource, usually the value of the "name" field in an Mu resource declaration.
|
|
661
|
-
# @param max_length [Integer]: The maximum length of the resulting resource name.
|
|
662
|
-
# @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated.
|
|
663
|
-
# @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
|
|
664
|
-
# @param disallowed_chars [Regexp]: A pattern of characters that are illegal for this resource name, such as +/[^a-zA-Z0-9-]/+
|
|
665
|
-
# @return [String]: A full name string for this resource
|
|
666
|
-
def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], disallowed_chars: nil)
|
|
667
|
-
if name.nil?
|
|
668
|
-
raise MuError, "Got no argument to MU::MommaCat.getResourceName"
|
|
669
|
-
end
|
|
670
|
-
if @appname.nil? or @environment.nil? or @timestamp.nil? or @seed.nil?
|
|
671
|
-
MU.log "getResourceName: Missing global deploy variables in thread #{Thread.current.object_id}, using bare name '#{name}' (appname: #{@appname}, environment: #{@environment}, timestamp: #{@timestamp}, seed: #{@seed}, deploy_id: #{@deploy_id}", MU::WARN, details: caller
|
|
672
|
-
return name
|
|
673
|
-
end
|
|
674
|
-
need_unique_string = false if scrub_mu_isms
|
|
675
|
-
|
|
676
|
-
muname = nil
|
|
677
|
-
if need_unique_string
|
|
678
|
-
reserved = 4
|
|
679
|
-
else
|
|
680
|
-
reserved = 0
|
|
681
|
-
end
|
|
682
|
-
|
|
683
|
-
# First, pare down the base name string until it will fit
|
|
684
|
-
basename = @appname.upcase + "-" + @environment.upcase + "-" + @timestamp + "-" + @seed.upcase + "-" + name.upcase
|
|
685
|
-
if scrub_mu_isms
|
|
686
|
-
basename = @appname.upcase + "-" + @environment.upcase + name.upcase
|
|
687
|
-
end
|
|
688
|
-
|
|
689
|
-
subchar = if disallowed_chars
|
|
690
|
-
if "-".match(disallowed_chars)
|
|
691
|
-
if !"_".match(disallowed_chars)
|
|
692
|
-
"_"
|
|
693
|
-
else
|
|
694
|
-
""
|
|
695
|
-
end
|
|
696
|
-
else
|
|
697
|
-
"-"
|
|
698
|
-
end
|
|
699
|
-
end
|
|
700
|
-
|
|
701
|
-
if disallowed_chars
|
|
702
|
-
basename.gsub!(disallowed_chars, subchar) if disallowed_chars
|
|
703
|
-
end
|
|
704
|
-
attempts = 0
|
|
705
|
-
begin
|
|
706
|
-
if (basename.length + reserved) > max_length
|
|
707
|
-
MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG
|
|
708
|
-
if basename == @appname.upcase + "-" + @seed.upcase + "-" + name.upcase
|
|
709
|
-
# If we've run out of stuff to strip, truncate what's left and
|
|
710
|
-
# just leave room for the deploy seed and uniqueness string. This
|
|
711
|
-
# is the bare minimum, and probably what you'll see for most Windows
|
|
712
|
-
# hostnames.
|
|
713
|
-
basename = name.upcase + "-" + @appname.upcase
|
|
714
|
-
basename.slice!((max_length-(reserved+3))..basename.length)
|
|
715
|
-
basename.sub!(/-$/, "")
|
|
716
|
-
basename = basename + "-" + @seed.upcase
|
|
717
|
-
basename.gsub!(disallowed_chars, subchar) if disallowed_chars
|
|
718
|
-
else
|
|
719
|
-
# If we have to strip anything, assume we've lost uniqueness and
|
|
720
|
-
# will have to compensate with #genUniquenessString.
|
|
721
|
-
need_unique_string = true
|
|
722
|
-
reserved = 4
|
|
723
|
-
basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
|
|
724
|
-
basename = basename + "-" + @seed.upcase + "-" + name.upcase
|
|
725
|
-
basename.gsub!(disallowed_chars, subchar) if disallowed_chars
|
|
726
|
-
end
|
|
727
|
-
end
|
|
728
|
-
attempts += 1
|
|
729
|
-
raise MuError, "Failed to generate a reasonable name getResourceName(#{name}, max_length: #{max_length.to_s}, need_unique_string: #{need_unique_string.to_s}, use_unique_string: #{use_unique_string.to_s}, reuse_unique_string: #{reuse_unique_string.to_s}, scrub_mu_isms: #{scrub_mu_isms.to_s}, disallowed_chars: #{disallowed_chars})" if attempts > 10
|
|
730
|
-
end while (basename.length + reserved) > max_length
|
|
731
|
-
|
|
732
|
-
# Finally, apply our short random differentiator, if it's needed.
|
|
733
|
-
if need_unique_string
|
|
734
|
-
# Preferentially use a requested one, if it's not already in use.
|
|
735
|
-
if !use_unique_string.nil?
|
|
736
|
-
muname = basename + "-" + use_unique_string
|
|
737
|
-
if !allocateUniqueResourceName(muname) and !reuse_unique_string
|
|
738
|
-
MU.log "Requested to use #{use_unique_string} as differentiator when naming #{name}, but the name #{muname} is unavailable.", MU::WARN
|
|
739
|
-
muname = nil
|
|
740
|
-
end
|
|
741
|
-
end
|
|
742
|
-
if !muname
|
|
743
|
-
begin
|
|
744
|
-
unique_string = MU::MommaCat.genUniquenessString
|
|
745
|
-
muname = basename + "-" + unique_string
|
|
746
|
-
end while !allocateUniqueResourceName(muname)
|
|
747
|
-
MU::MommaCat.unique_map_semaphore.synchronize {
|
|
748
|
-
MU::MommaCat.name_unique_str_map[muname] = unique_string
|
|
749
|
-
}
|
|
750
|
-
end
|
|
751
|
-
else
|
|
752
|
-
muname = basename
|
|
753
|
-
end
|
|
754
|
-
muname.gsub!(disallowed_chars, subchar) if disallowed_chars
|
|
755
|
-
|
|
756
|
-
return muname
|
|
757
|
-
end
|
|
758
|
-
|
|
759
|
-
|
|
760
552
|
# Encrypt a string with the deployment's public key.
|
|
761
553
|
# @param ciphertext [String]: The string to encrypt
|
|
762
554
|
def encryptWithDeployKey(ciphertext)
|
|
@@ -771,7 +563,6 @@ module MU
|
|
|
771
563
|
return my_private_key.private_decrypt(ciphertext)
|
|
772
564
|
end
|
|
773
565
|
|
|
774
|
-
|
|
775
566
|
# Save a string into deployment metadata for the current deployment,
|
|
776
567
|
# encrypting it with our deploy key.
|
|
777
568
|
# @param instance_id [String]: The cloud instance identifier with which this secret is associated.
|
|
@@ -812,157 +603,6 @@ module MU
|
|
|
812
603
|
return decryptWithDeployKey(@secrets[type][instance_id])
|
|
813
604
|
end
|
|
814
605
|
|
|
815
|
-
|
|
816
|
-
# Run {MU::Cloud::Server#postBoot} and {MU::Cloud::Server#groom} on a node.
|
|
817
|
-
# @param cloud_id [OpenStruct]: The cloud provider's identifier for this node.
|
|
818
|
-
# @param name [String]: The MU resource name of the node being created.
|
|
819
|
-
# @param mu_name [String]: The full #{MU::MommaCat.getResourceName} name of the server we're grooming, if it's been initialized already.
|
|
820
|
-
# @param type [String]: The type of resource that created this node (either *server* or *serverpool*).
|
|
821
|
-
def groomNode(cloud_id, name, type, mu_name: nil, reraise_fail: false, sync_wait: true)
|
|
822
|
-
if cloud_id.nil?
|
|
823
|
-
raise GroomError, "MU::MommaCat.groomNode requires a {MU::Cloud::Server} object"
|
|
824
|
-
end
|
|
825
|
-
if name.nil? or name.empty?
|
|
826
|
-
raise GroomError, "MU::MommaCat.groomNode requires a resource name"
|
|
827
|
-
end
|
|
828
|
-
if type.nil? or type.empty?
|
|
829
|
-
raise GroomError, "MU::MommaCat.groomNode requires a resource type"
|
|
830
|
-
end
|
|
831
|
-
|
|
832
|
-
if !MU::MommaCat.lock(cloud_id+"-mommagroom", true)
|
|
833
|
-
MU.log "Instance #{cloud_id} on #{MU.deploy_id} (#{type}: #{name}) is already being groomed, ignoring this extra request.", MU::NOTICE
|
|
834
|
-
MU::MommaCat.unlockAll
|
|
835
|
-
if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
|
|
836
|
-
puts "------------------------------"
|
|
837
|
-
puts "Open flock() locks:"
|
|
838
|
-
pp MU::MommaCat.locks
|
|
839
|
-
puts "------------------------------"
|
|
840
|
-
end
|
|
841
|
-
return
|
|
842
|
-
end
|
|
843
|
-
loadDeploy
|
|
844
|
-
|
|
845
|
-
# XXX this is to stop Net::SSH from killing our entire stack when it
|
|
846
|
-
# throws an exception. See ECAP-139 in JIRA. Far as we can tell, it's
|
|
847
|
-
# just not entirely thread safe.
|
|
848
|
-
Thread.handle_interrupt(Net::SSH::Disconnect => :never) {
|
|
849
|
-
begin
|
|
850
|
-
Thread.handle_interrupt(Net::SSH::Disconnect => :immediate) {
|
|
851
|
-
MU.log "(Probably harmless) Caught a Net::SSH::Disconnect in #{Thread.current.inspect}", MU::DEBUG, details: Thread.current.backtrace
|
|
852
|
-
}
|
|
853
|
-
ensure
|
|
854
|
-
end
|
|
855
|
-
}
|
|
856
|
-
|
|
857
|
-
if @original_config[type+"s"].nil?
|
|
858
|
-
raise GroomError, "I see no configured resources of type #{type} (bootstrap request for #{name} on #{@deploy_id})"
|
|
859
|
-
end
|
|
860
|
-
kitten = nil
|
|
861
|
-
|
|
862
|
-
kitten = findLitterMate(type: "server", name: name, mu_name: mu_name, cloud_id: cloud_id)
|
|
863
|
-
if !kitten.nil?
|
|
864
|
-
MU.log "Re-grooming #{mu_name}", details: kitten.deploydata
|
|
865
|
-
else
|
|
866
|
-
first_groom = true
|
|
867
|
-
@original_config[type+"s"].each { |svr|
|
|
868
|
-
if svr['name'] == name
|
|
869
|
-
svr["instance_id"] = cloud_id
|
|
870
|
-
|
|
871
|
-
# This will almost always be true in server pools, but lets be safe. Somewhat problematic because we are only
|
|
872
|
-
# looking at deploy_id, but we still know this is our DNS record and not a custom one.
|
|
873
|
-
if svr['dns_records'] && !svr['dns_records'].empty?
|
|
874
|
-
svr['dns_records'].each { |dnsrec|
|
|
875
|
-
if dnsrec.has_key?("name") && dnsrec['name'].start_with?(MU.deploy_id.downcase)
|
|
876
|
-
MU.log "DNS record for #{MU.deploy_id.downcase}, #{name} is probably wrong, deleting", MU::WARN, details: dnsrec
|
|
877
|
-
dnsrec.delete('name')
|
|
878
|
-
dnsrec.delete('target')
|
|
879
|
-
end
|
|
880
|
-
}
|
|
881
|
-
end
|
|
882
|
-
|
|
883
|
-
kitten = MU::Cloud::Server.new(mommacat: self, kitten_cfg: svr, cloud_id: cloud_id)
|
|
884
|
-
mu_name = kitten.mu_name if mu_name.nil?
|
|
885
|
-
MU.log "Grooming #{mu_name} for the first time", details: svr
|
|
886
|
-
break
|
|
887
|
-
end
|
|
888
|
-
}
|
|
889
|
-
end
|
|
890
|
-
|
|
891
|
-
begin
|
|
892
|
-
# This is a shared lock with MU::Cloud::AWS::Server.create, to keep from
|
|
893
|
-
# stomping on synchronous deploys that are still running. This
|
|
894
|
-
# means we're going to wait here if this instance is still being
|
|
895
|
-
# bootstrapped by "regular" means.
|
|
896
|
-
if !MU::MommaCat.lock(cloud_id+"-create", true)
|
|
897
|
-
MU.log "#{mu_name} is still in mid-creation, skipping", MU::NOTICE
|
|
898
|
-
MU::MommaCat.unlockAll
|
|
899
|
-
if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
|
|
900
|
-
puts "------------------------------"
|
|
901
|
-
puts "Open flock() locks:"
|
|
902
|
-
pp MU::MommaCat.locks
|
|
903
|
-
puts "------------------------------"
|
|
904
|
-
end
|
|
905
|
-
return
|
|
906
|
-
end
|
|
907
|
-
MU::MommaCat.unlock(cloud_id+"-create")
|
|
908
|
-
|
|
909
|
-
if !kitten.postBoot(cloud_id)
|
|
910
|
-
MU.log "#{mu_name} is already being groomed, skipping", MU::NOTICE
|
|
911
|
-
MU::MommaCat.unlockAll
|
|
912
|
-
if !MU::MommaCat.locks.nil? and MU::MommaCat.locks.size > 0
|
|
913
|
-
puts "------------------------------"
|
|
914
|
-
puts "Open flock() locks:"
|
|
915
|
-
pp MU::MommaCat.locks
|
|
916
|
-
puts "------------------------------"
|
|
917
|
-
end
|
|
918
|
-
return
|
|
919
|
-
end
|
|
920
|
-
|
|
921
|
-
# This is a shared lock with MU::Deploy.createResources, simulating the
|
|
922
|
-
# thread logic that tells MU::Cloud::AWS::Server.deploy to wait until
|
|
923
|
-
# its dependencies are ready. We don't, for example, want to start
|
|
924
|
-
# deploying if we rely on an RDS instance that isn't ready yet. We can
|
|
925
|
-
# release this immediately, once we successfully grab it.
|
|
926
|
-
MU::MommaCat.lock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
|
|
927
|
-
MU::MommaCat.unlock("#{kitten.cloudclass.name}_#{kitten.config["name"]}-dependencies")
|
|
928
|
-
|
|
929
|
-
kitten.groom
|
|
930
|
-
rescue Exception => e
|
|
931
|
-
MU::MommaCat.unlockAll
|
|
932
|
-
if e.class.name != "MU::Cloud::AWS::Server::BootstrapTempFail" and !File.exist?(deploy_dir+"/.cleanup."+cloud_id) and !File.exist?(deploy_dir+"/.cleanup")
|
|
933
|
-
MU.log "Grooming FAILED for #{kitten.mu_name} (#{e.inspect})", MU::ERR, details: e.backtrace
|
|
934
|
-
sendAdminSlack("Grooming FAILED for `#{kitten.mu_name}` with `#{e.message}` :crying_cat_face:", msg: e.backtrace.join("\n"))
|
|
935
|
-
sendAdminMail("Grooming FAILED for #{kitten.mu_name} on #{MU.appname} \"#{MU.handle}\" (#{MU.deploy_id})",
|
|
936
|
-
msg: e.inspect,
|
|
937
|
-
data: e.backtrace,
|
|
938
|
-
debug: true
|
|
939
|
-
)
|
|
940
|
-
raise e if reraise_fail
|
|
941
|
-
else
|
|
942
|
-
MU.log "Grooming of #{kitten.mu_name} interrupted by cleanup or planned reboot"
|
|
943
|
-
end
|
|
944
|
-
return
|
|
945
|
-
end
|
|
946
|
-
|
|
947
|
-
if !@deployment['servers'].nil? and !sync_wait
|
|
948
|
-
syncLitter(@deployment["servers"].keys, triggering_node: kitten)
|
|
949
|
-
end
|
|
950
|
-
MU::MommaCat.unlock(cloud_id+"-mommagroom")
|
|
951
|
-
if MU.myCloud == "AWS"
|
|
952
|
-
MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
|
|
953
|
-
end
|
|
954
|
-
MU::MommaCat.getLitter(MU.deploy_id)
|
|
955
|
-
MU::MommaCat.syncMonitoringConfig(false)
|
|
956
|
-
MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})"
|
|
957
|
-
FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt")
|
|
958
|
-
MU::MommaCat.unlockAll
|
|
959
|
-
if first_groom
|
|
960
|
-
sendAdminSlack("Grooming complete for #{mu_name} :heart_eyes_cat:")
|
|
961
|
-
sendAdminMail("Grooming complete for '#{name}' (#{mu_name}) on deploy \"#{MU.handle}\" (#{MU.deploy_id})")
|
|
962
|
-
end
|
|
963
|
-
return
|
|
964
|
-
end
|
|
965
|
-
|
|
966
606
|
# Return the parts and pieces of this deploy's node ssh key set. Generate
|
|
967
607
|
# or load if that hasn't been done already.
|
|
968
608
|
def SSHKey
|
|
@@ -1012,225 +652,6 @@ module MU
|
|
|
1012
652
|
return [@ssh_key_name, @ssh_private_key, @ssh_public_key]
|
|
1013
653
|
end
|
|
1014
654
|
|
|
1015
|
-
@lock_semaphore = Mutex.new
|
|
1016
|
-
# Release all flock() locks held by the current thread.
|
|
1017
|
-
def self.unlockAll
|
|
1018
|
-
if !@locks.nil? and !@locks[Thread.current.object_id].nil?
|
|
1019
|
-
# Work from a copy so we can iterate without worrying about contention
|
|
1020
|
-
# in lock() or unlock(). We can't just wrap our iterator block in a
|
|
1021
|
-
# semaphore here, because we're calling another method that uses the
|
|
1022
|
-
# same semaphore.
|
|
1023
|
-
@lock_semaphore.synchronize {
|
|
1024
|
-
delete_list = []
|
|
1025
|
-
@locks[Thread.current.object_id].keys.each { |id|
|
|
1026
|
-
MU.log "Releasing lock on #{deploy_dir(MU.deploy_id)}/locks/#{id}.lock (thread #{Thread.current.object_id})", MU::DEBUG
|
|
1027
|
-
begin
|
|
1028
|
-
@locks[Thread.current.object_id][id].flock(File::LOCK_UN)
|
|
1029
|
-
@locks[Thread.current.object_id][id].close
|
|
1030
|
-
rescue IOError => e
|
|
1031
|
-
MU.log "Got #{e.inspect} unlocking #{id} on #{Thread.current.object_id}", MU::WARN
|
|
1032
|
-
end
|
|
1033
|
-
delete_list << id
|
|
1034
|
-
}
|
|
1035
|
-
# We do this here because we can't mangle a Hash while we're iterating
|
|
1036
|
-
# over it.
|
|
1037
|
-
delete_list.each { |id|
|
|
1038
|
-
@locks[Thread.current.object_id].delete(id)
|
|
1039
|
-
}
|
|
1040
|
-
if @locks[Thread.current.object_id].size == 0
|
|
1041
|
-
@locks.delete(Thread.current.object_id)
|
|
1042
|
-
end
|
|
1043
|
-
}
|
|
1044
|
-
end
|
|
1045
|
-
end
|
|
1046
|
-
|
|
1047
|
-
# Create/hold a flock() lock.
|
|
1048
|
-
# @param id [String]: The lock identifier to release.
|
|
1049
|
-
# @param nonblock [Boolean]: Whether to block while waiting for the lock. In non-blocking mode, we simply return false if the lock is not available.
|
|
1050
|
-
# return [false, nil]
|
|
1051
|
-
def self.lock(id, nonblock = false, global = false)
|
|
1052
|
-
raise MuError, "Can't pass a nil id to MU::MommaCat.lock" if id.nil?
|
|
1053
|
-
|
|
1054
|
-
if !global
|
|
1055
|
-
lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
|
|
1056
|
-
else
|
|
1057
|
-
lockdir = File.expand_path(MU.dataDir+"/locks")
|
|
1058
|
-
end
|
|
1059
|
-
|
|
1060
|
-
if !Dir.exist?(lockdir)
|
|
1061
|
-
MU.log "Creating #{lockdir}", MU::DEBUG
|
|
1062
|
-
Dir.mkdir(lockdir, 0700)
|
|
1063
|
-
end
|
|
1064
|
-
|
|
1065
|
-
@lock_semaphore.synchronize {
|
|
1066
|
-
if @locks[Thread.current.object_id].nil?
|
|
1067
|
-
@locks[Thread.current.object_id] = Hash.new
|
|
1068
|
-
end
|
|
1069
|
-
|
|
1070
|
-
@locks[Thread.current.object_id][id] = File.open("#{lockdir}/#{id}.lock", File::CREAT|File::RDWR, 0600)
|
|
1071
|
-
}
|
|
1072
|
-
MU.log "Getting a lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})...", MU::DEBUG
|
|
1073
|
-
begin
|
|
1074
|
-
if nonblock
|
|
1075
|
-
if !@locks[Thread.current.object_id][id].flock(File::LOCK_EX|File::LOCK_NB)
|
|
1076
|
-
return false
|
|
1077
|
-
end
|
|
1078
|
-
else
|
|
1079
|
-
@locks[Thread.current.object_id][id].flock(File::LOCK_EX)
|
|
1080
|
-
end
|
|
1081
|
-
rescue IOError
|
|
1082
|
-
raise MU::BootstrapTempFail, "Interrupted waiting for lock on thread #{Thread.current.object_id}, probably just a node rebooting as part of a synchronous install"
|
|
1083
|
-
end
|
|
1084
|
-
MU.log "Lock on #{lockdir}/#{id}.lock on thread #{Thread.current.object_id} acquired", MU::DEBUG
|
|
1085
|
-
return true
|
|
1086
|
-
end
|
|
1087
|
-
|
|
1088
|
-
# Release a flock() lock.
|
|
1089
|
-
# @param id [String]: The lock identifier to release.
|
|
1090
|
-
def self.unlock(id, global = false)
|
|
1091
|
-
raise MuError, "Can't pass a nil id to MU::MommaCat.unlock" if id.nil?
|
|
1092
|
-
lockdir = nil
|
|
1093
|
-
if !global
|
|
1094
|
-
lockdir = "#{deploy_dir(MU.deploy_id)}/locks"
|
|
1095
|
-
else
|
|
1096
|
-
lockdir = File.expand_path(MU.dataDir+"/locks")
|
|
1097
|
-
end
|
|
1098
|
-
@lock_semaphore.synchronize {
|
|
1099
|
-
return if @locks.nil? or @locks[Thread.current.object_id].nil? or @locks[Thread.current.object_id][id].nil?
|
|
1100
|
-
}
|
|
1101
|
-
MU.log "Releasing lock on #{lockdir}/#{id}.lock (thread #{Thread.current.object_id})", MU::DEBUG
|
|
1102
|
-
begin
|
|
1103
|
-
@locks[Thread.current.object_id][id].flock(File::LOCK_UN)
|
|
1104
|
-
@locks[Thread.current.object_id][id].close
|
|
1105
|
-
if !@locks[Thread.current.object_id].nil?
|
|
1106
|
-
@locks[Thread.current.object_id].delete(id)
|
|
1107
|
-
end
|
|
1108
|
-
if @locks[Thread.current.object_id].size == 0
|
|
1109
|
-
@locks.delete(Thread.current.object_id)
|
|
1110
|
-
end
|
|
1111
|
-
rescue IOError => e
|
|
1112
|
-
MU.log "Got #{e.inspect} unlocking #{id} on #{Thread.current.object_id}", MU::WARN
|
|
1113
|
-
end
|
|
1114
|
-
end
|
|
1115
|
-
|
|
1116
|
-
# Remove a deployment's metadata.
|
|
1117
|
-
# @param deploy_id [String]: The deployment identifier to remove.
|
|
1118
|
-
def self.purge(deploy_id)
|
|
1119
|
-
if deploy_id.nil? or deploy_id.empty?
|
|
1120
|
-
raise MuError, "Got nil deploy_id in MU::MommaCat.purge"
|
|
1121
|
-
end
|
|
1122
|
-
# XXX archiving is better than annihilating
|
|
1123
|
-
path = File.expand_path(MU.dataDir+"/deployments")
|
|
1124
|
-
if Dir.exist?(path+"/"+deploy_id)
|
|
1125
|
-
unlockAll
|
|
1126
|
-
MU.log "Purging #{path}/#{deploy_id}" if File.exist?(path+"/"+deploy_id+"/deployment.json")
|
|
1127
|
-
|
|
1128
|
-
FileUtils.rm_rf(path+"/"+deploy_id, :secure => true)
|
|
1129
|
-
end
|
|
1130
|
-
if File.exist?(path+"/unique_ids")
|
|
1131
|
-
File.open(path+"/unique_ids", File::CREAT|File::RDWR, 0600) { |f|
|
|
1132
|
-
newlines = []
|
|
1133
|
-
f.flock(File::LOCK_EX)
|
|
1134
|
-
f.readlines.each { |line|
|
|
1135
|
-
newlines << line if !line.match(/:#{deploy_id}$/)
|
|
1136
|
-
}
|
|
1137
|
-
f.rewind
|
|
1138
|
-
f.truncate(0)
|
|
1139
|
-
f.puts(newlines)
|
|
1140
|
-
f.flush
|
|
1141
|
-
f.flock(File::LOCK_UN)
|
|
1142
|
-
}
|
|
1143
|
-
end
|
|
1144
|
-
end
|
|
1145
|
-
|
|
1146
|
-
# Remove the metadata of the currently loaded deployment.
|
|
1147
|
-
def purge!
|
|
1148
|
-
MU::MommaCat.purge(MU.deploy_id)
|
|
1149
|
-
end
|
|
1150
|
-
|
|
1151
|
-
@cleanup_threads = []
|
|
1152
|
-
|
|
1153
|
-
# Iterate over all known deployments and look for instances that have been
|
|
1154
|
-
# terminated, but not yet cleaned up, then clean them up.
|
|
1155
|
-
def self.cleanTerminatedInstances(debug = false)
|
|
1156
|
-
loglevel = debug ? MU::NOTICE : MU::DEBUG
|
|
1157
|
-
MU::MommaCat.lock("clean-terminated-instances", false, true)
|
|
1158
|
-
MU.log "Checking for harvested instances in need of cleanup", loglevel
|
|
1159
|
-
parent_thread_id = Thread.current.object_id
|
|
1160
|
-
purged = 0
|
|
1161
|
-
|
|
1162
|
-
MU::MommaCat.listDeploys.each { |deploy_id|
|
|
1163
|
-
next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
|
|
1164
|
-
MU.log "Checking for dead wood in #{deploy_id}", loglevel
|
|
1165
|
-
need_reload = false
|
|
1166
|
-
@cleanup_threads << Thread.new {
|
|
1167
|
-
MU.dupGlobals(parent_thread_id)
|
|
1168
|
-
deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
|
|
1169
|
-
purged_this_deploy = 0
|
|
1170
|
-
MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys
|
|
1171
|
-
if deploy.kittens.has_key?("servers")
|
|
1172
|
-
MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id
|
|
1173
|
-
deploy.kittens["servers"].values.each { |nodeclasses|
|
|
1174
|
-
nodeclasses.each_pair { |nodeclass, servers|
|
|
1175
|
-
deletia = []
|
|
1176
|
-
MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys
|
|
1177
|
-
servers.each_pair { |mu_name, server|
|
|
1178
|
-
server.describe
|
|
1179
|
-
if !server.cloud_id
|
|
1180
|
-
MU.log "Checking for presence of #{mu_name}, but unable to fetch its cloud_id", MU::WARN, details: server
|
|
1181
|
-
elsif !server.active?
|
|
1182
|
-
next if File.exist?(deploy_dir(deploy_id)+"/.cleanup-"+server.cloud_id)
|
|
1183
|
-
deletia << mu_name
|
|
1184
|
-
need_reload = true
|
|
1185
|
-
MU.log "Cleaning up metadata for #{server} (#{nodeclass}), formerly #{server.cloud_id}, which appears to have been terminated", MU::NOTICE
|
|
1186
|
-
begin
|
|
1187
|
-
server.destroy
|
|
1188
|
-
deploy.sendAdminMail("Retired metadata for terminated node #{mu_name}")
|
|
1189
|
-
deploy.sendAdminSlack("Retired metadata for terminated node `#{mu_name}`")
|
|
1190
|
-
rescue Exception => e
|
|
1191
|
-
MU.log "Saw #{e.message} while retiring #{mu_name}", MU::ERR, details: e.backtrace
|
|
1192
|
-
next
|
|
1193
|
-
end
|
|
1194
|
-
MU.log "Cleanup of metadata for #{server} (#{nodeclass}), formerly #{server.cloud_id} complete", MU::NOTICE
|
|
1195
|
-
purged = purged + 1
|
|
1196
|
-
purged_this_deploy = purged_this_deploy + 1
|
|
1197
|
-
end
|
|
1198
|
-
}
|
|
1199
|
-
deletia.each { |mu_name|
|
|
1200
|
-
servers.delete(mu_name)
|
|
1201
|
-
}
|
|
1202
|
-
if purged_this_deploy > 0
|
|
1203
|
-
# XXX triggering_node needs to take more than one node name
|
|
1204
|
-
deploy.syncLitter(servers.keys, triggering_node: deletia.first)
|
|
1205
|
-
end
|
|
1206
|
-
}
|
|
1207
|
-
}
|
|
1208
|
-
end
|
|
1209
|
-
if need_reload
|
|
1210
|
-
MU.log "Saving modified deploy #{deploy_id}", loglevel
|
|
1211
|
-
deploy.save!
|
|
1212
|
-
MU::MommaCat.getLitter(deploy_id)
|
|
1213
|
-
end
|
|
1214
|
-
MU.purgeGlobals
|
|
1215
|
-
}
|
|
1216
|
-
}
|
|
1217
|
-
@cleanup_threads.each { |t|
|
|
1218
|
-
t.join
|
|
1219
|
-
}
|
|
1220
|
-
MU.log "cleanTerminatedInstances threads complete", loglevel
|
|
1221
|
-
MU::MommaCat.unlock("clean-terminated-instances", true)
|
|
1222
|
-
@cleanup_threads = []
|
|
1223
|
-
|
|
1224
|
-
if purged > 0
|
|
1225
|
-
if MU.myCloud == "AWS"
|
|
1226
|
-
MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
|
|
1227
|
-
end
|
|
1228
|
-
MU::MommaCat.syncMonitoringConfig
|
|
1229
|
-
GC.start
|
|
1230
|
-
end
|
|
1231
|
-
MU.log "cleanTerminatedInstances returning", loglevel
|
|
1232
|
-
end
|
|
1233
|
-
|
|
1234
655
|
@@dummy_cache = {}
|
|
1235
656
|
|
|
1236
657
|
# Locate a resource that's either a member of another deployment, or of no
|
|
@@ -1269,7 +690,7 @@ module MU
|
|
|
1269
690
|
)
|
|
1270
691
|
start = Time.now
|
|
1271
692
|
callstr = "findStray(cloud: #{cloud}, type: #{type}, deploy_id: #{deploy_id}, calling_deploy: #{calling_deploy.deploy_id if !calling_deploy.nil?}, name: #{name}, cloud_id: #{cloud_id}, tag_key: #{tag_key}, tag_value: #{tag_value}, credentials: #{credentials}, habitats: #{habitats ? habitats.to_s : "[]"}, dummy_ok: #{dummy_ok.to_s}, flags: #{flags.to_s}) from #{caller[0]}"
|
|
1272
|
-
callstack = caller.dup
|
|
693
|
+
# callstack = caller.dup
|
|
1273
694
|
|
|
1274
695
|
return nil if cloud == "CloudFormation" and !cloud_id.nil?
|
|
1275
696
|
shortclass, _cfg_name, cfg_plural, classname, _attrs = MU::Cloud.getResourceNames(type)
|
|
@@ -1496,15 +917,9 @@ module MU
|
|
|
1496
917
|
regions.each { |reg| region_threads << Thread.new(reg) { |r|
|
|
1497
918
|
MU.log "findStray: Searching #{r} in #{p} (#{region_threads.size.to_s} region threads running) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
|
|
1498
919
|
MU.log "findStray: calling #{classname}.find(cloud_id: #{cloud_id}, region: #{r}, tag_key: #{tag_key}, tag_value: #{tag_value}, flags: #{flags}, credentials: #{creds}, project: #{p}) - #{sprintf("%.2fs", (Time.now-start))}", loglevel
|
|
1499
|
-
begin
|
|
1500
920
|
found = resourceclass.find(cloud_id: cloud_id, region: r, tag_key: tag_key, tag_value: tag_value, flags: flags, credentials: creds, habitat: p)
|
|
1501
921
|
MU.log "findStray: #{found ? found.size.to_s : "nil"} results - #{sprintf("%.2fs", (Time.now-start))}", loglevel
|
|
1502
|
-
|
|
1503
|
-
MU.log "#{e.class.name} THREW A FIND EXCEPTION "+e.message, MU::WARN, details: caller
|
|
1504
|
-
pp e.backtrace
|
|
1505
|
-
MU.log "#{callstr}", MU::WARN, details: callstack
|
|
1506
|
-
exit
|
|
1507
|
-
end
|
|
922
|
+
|
|
1508
923
|
if found
|
|
1509
924
|
desc_semaphore.synchronize {
|
|
1510
925
|
cloud_descs[p][r] = found
|
|
@@ -1640,7 +1055,7 @@ end
|
|
|
1640
1055
|
}
|
|
1641
1056
|
end
|
|
1642
1057
|
}
|
|
1643
|
-
rescue
|
|
1058
|
+
rescue StandardError => e
|
|
1644
1059
|
MU.log e.inspect, MU::ERR, details: e.backtrace
|
|
1645
1060
|
end
|
|
1646
1061
|
MU.log "findStray: returning #{matches ? matches.size.to_s : "0"} matches - #{sprintf("%.2fs", (Time.now-start))}", loglevel
|
|
@@ -1873,409 +1288,41 @@ end
|
|
|
1873
1288
|
MU::MommaCat.unlock("deployment-notification")
|
|
1874
1289
|
end
|
|
1875
1290
|
|
|
1876
|
-
#
|
|
1877
|
-
#
|
|
1878
|
-
#
|
|
1879
|
-
#
|
|
1880
|
-
# @param resource [String]: The cloud provider identifier of the resource to tag
|
|
1881
|
-
# @param tag_name [String]: The name of the tag to create
|
|
1882
|
-
# @param tag_value [String]: The value of the tag
|
|
1883
|
-
# @param region [String]: The cloud provider region
|
|
1291
|
+
# Send a Slack notification to a deployment's administrators.
|
|
1292
|
+
# @param subject [String]: The subject line of the message.
|
|
1293
|
+
# @param msg [String]: The message body.
|
|
1884
1294
|
# @return [void]
|
|
1885
|
-
def
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
1890
|
-
attempts = 0
|
|
1891
|
-
|
|
1892
|
-
if !MU::Cloud::CloudFormation.emitCloudFormation
|
|
1893
|
-
begin
|
|
1894
|
-
MU::Cloud::AWS.ec2(credentials: credentials, region: region).create_tags(
|
|
1895
|
-
resources: [resource],
|
|
1896
|
-
tags: [
|
|
1897
|
-
{
|
|
1898
|
-
key: tag_name,
|
|
1899
|
-
value: tag_value
|
|
1900
|
-
}
|
|
1901
|
-
]
|
|
1902
|
-
)
|
|
1903
|
-
rescue Aws::EC2::Errors::ServiceError => e
|
|
1904
|
-
MU.log "Got #{e.inspect} tagging #{resource} with #{tag_name}=#{tag_value}", MU::WARN if attempts > 1
|
|
1905
|
-
if attempts < 5
|
|
1906
|
-
attempts = attempts + 1
|
|
1907
|
-
sleep 15
|
|
1908
|
-
retry
|
|
1909
|
-
else
|
|
1910
|
-
raise e
|
|
1911
|
-
end
|
|
1912
|
-
end
|
|
1913
|
-
MU.log "Created tag #{tag_name} with value #{tag_value} for resource #{resource}", MU::DEBUG
|
|
1914
|
-
else
|
|
1915
|
-
return {
|
|
1916
|
-
"Key" => tag_name,
|
|
1917
|
-
"Value" => tag_value
|
|
1918
|
-
}
|
|
1919
|
-
end
|
|
1920
|
-
end
|
|
1921
|
-
|
|
1922
|
-
# List the name/value pairs for our mandatory standard set of resource tags, which
|
|
1923
|
-
# should be applied to all taggable cloud provider resources.
|
|
1924
|
-
# @return [Hash<String,String>]
|
|
1925
|
-
def self.listStandardTags
|
|
1926
|
-
return {} if !MU.deploy_id
|
|
1927
|
-
{
|
|
1928
|
-
"MU-ID" => MU.deploy_id,
|
|
1929
|
-
"MU-APP" => MU.appname,
|
|
1930
|
-
"MU-ENV" => MU.environment,
|
|
1931
|
-
"MU-MASTER-IP" => MU.mu_public_ip
|
|
1932
|
-
}
|
|
1933
|
-
end
|
|
1934
|
-
# List the name/value pairs for our mandatory standard set of resource tags
|
|
1935
|
-
# for this deploy.
|
|
1936
|
-
# @return [Hash<String,String>]
|
|
1937
|
-
def listStandardTags
|
|
1938
|
-
{
|
|
1939
|
-
"MU-ID" => @deploy_id,
|
|
1940
|
-
"MU-APP" => @appname,
|
|
1941
|
-
"MU-ENV" => @environment,
|
|
1942
|
-
"MU-MASTER-IP" => MU.mu_public_ip
|
|
1943
|
-
}
|
|
1944
|
-
end
|
|
1945
|
-
|
|
1946
|
-
# List the name/value pairs of our optional set of resource tags which
|
|
1947
|
-
# should be applied to all taggable cloud provider resources.
|
|
1948
|
-
# @return [Hash<String,String>]
|
|
1949
|
-
def self.listOptionalTags
|
|
1950
|
-
return {
|
|
1951
|
-
"MU-HANDLE" => MU.handle,
|
|
1952
|
-
"MU-MASTER-NAME" => Socket.gethostname,
|
|
1953
|
-
"MU-OWNER" => MU.mu_user
|
|
1954
|
-
}
|
|
1955
|
-
end
|
|
1295
|
+
def sendAdminSlack(subject, msg: "")
|
|
1296
|
+
if $MU_CFG['slack'] and $MU_CFG['slack']['webhook'] and
|
|
1297
|
+
(!$MU_CFG['slack']['skip_environments'] or !$MU_CFG['slack']['skip_environments'].any?{ |s| s.casecmp(MU.environment)==0 })
|
|
1298
|
+
require 'slack-notifier'
|
|
1299
|
+
slack = Slack::Notifier.new $MU_CFG['slack']['webhook']
|
|
1956
1300
|
|
|
1957
|
-
|
|
1958
|
-
|
|
1959
|
-
|
|
1960
|
-
|
|
1961
|
-
return if ip.nil?
|
|
1962
|
-
sshdir = "#{@myhome}/.ssh"
|
|
1963
|
-
knownhosts = "#{sshdir}/known_hosts"
|
|
1964
|
-
|
|
1965
|
-
if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
|
|
1966
|
-
MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
|
|
1967
|
-
if !@noop
|
|
1968
|
-
File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
|
|
1969
|
-
f.flock(File::LOCK_EX)
|
|
1970
|
-
newlines = Array.new
|
|
1971
|
-
delete_block = false
|
|
1972
|
-
f.readlines.each { |line|
|
|
1973
|
-
next if line.match(/^#{Regexp.quote(ip)} /)
|
|
1974
|
-
newlines << line
|
|
1975
|
-
}
|
|
1976
|
-
f.rewind
|
|
1977
|
-
f.truncate(0)
|
|
1978
|
-
f.puts(newlines)
|
|
1979
|
-
f.flush
|
|
1980
|
-
f.flock(File::LOCK_UN)
|
|
1981
|
-
}
|
|
1301
|
+
if msg and !msg.empty?
|
|
1302
|
+
slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}:\n\n```#{msg}\n```", channel: $MU_CFG['slack']['channel']
|
|
1303
|
+
else
|
|
1304
|
+
slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}", channel: $MU_CFG['slack']['channel']
|
|
1982
1305
|
end
|
|
1983
1306
|
end
|
|
1984
1307
|
end
|
|
1985
1308
|
|
|
1986
|
-
#
|
|
1987
|
-
# @param
|
|
1309
|
+
# Send an email notification to a deployment's administrators.
|
|
1310
|
+
# @param subject [String]: The subject line of the message.
|
|
1311
|
+
# @param msg [String]: The message body.
|
|
1312
|
+
# @param data [Array]: Supplemental data to add to the message body.
|
|
1313
|
+
# @param debug [Boolean]: If set, will include the full deployment structure and original {MU::Config}-parsed configuration.
|
|
1988
1314
|
# @return [void]
|
|
1989
|
-
def
|
|
1990
|
-
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
1994
|
-
MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
|
|
1995
|
-
if !@noop
|
|
1996
|
-
File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
|
|
1997
|
-
f.flock(File::LOCK_EX)
|
|
1998
|
-
newlines = Array.new
|
|
1999
|
-
delete_block = false
|
|
2000
|
-
f.readlines.each { |line|
|
|
2001
|
-
if line.match(/^Host #{nodename}(\s|$)/)
|
|
2002
|
-
delete_block = true
|
|
2003
|
-
elsif line.match(/^Host /)
|
|
2004
|
-
delete_block = false
|
|
2005
|
-
end
|
|
2006
|
-
newlines << line if !delete_block
|
|
2007
|
-
}
|
|
2008
|
-
f.rewind
|
|
2009
|
-
f.truncate(0)
|
|
2010
|
-
f.puts(newlines)
|
|
2011
|
-
f.flush
|
|
2012
|
-
f.flock(File::LOCK_UN)
|
|
2013
|
-
}
|
|
2014
|
-
end
|
|
2015
|
-
end
|
|
2016
|
-
|
|
2017
|
-
end
|
|
2018
|
-
|
|
2019
|
-
# Make sure the given node has proper DNS entries, /etc/hosts entries,
|
|
2020
|
-
# SSH config entries, etc.
|
|
2021
|
-
# @param server [MU::Cloud::Server]: The {MU::Cloud::Server} we'll be setting up.
|
|
2022
|
-
# @param sync_wait [Boolean]: Whether to wait for DNS to fully synchronize before returning.
|
|
2023
|
-
def self.nameKitten(server, sync_wait: false)
|
|
2024
|
-
node, config, _deploydata = server.describe
|
|
2025
|
-
|
|
2026
|
-
mu_zone = nil
|
|
2027
|
-
# XXX GCP!
|
|
2028
|
-
if MU::Cloud::AWS.hosted? and !MU::Cloud::AWS.isGovCloud?
|
|
2029
|
-
zones = MU::Cloud::DNSZone.find(cloud_id: "platform-mu")
|
|
2030
|
-
mu_zone = zones.values.first if !zones.nil?
|
|
1315
|
+
def sendAdminMail(subject, msg: "", kitten: nil, data: nil, debug: false)
|
|
1316
|
+
require 'net/smtp'
|
|
1317
|
+
if @deployment.nil?
|
|
1318
|
+
MU.log "Can't send admin mail without a loaded deployment", MU::ERR
|
|
1319
|
+
return
|
|
2031
1320
|
end
|
|
2032
|
-
|
|
2033
|
-
|
|
2034
|
-
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
## TO DO: Do DNS registration of "real" records as the last stage after the groomer completes
|
|
2039
|
-
if config && config['dns_records'] && !config['dns_records'].empty?
|
|
2040
|
-
dnscfg = config['dns_records'].dup
|
|
2041
|
-
dnscfg.each { |dnsrec|
|
|
2042
|
-
if !dnsrec.has_key?('name')
|
|
2043
|
-
dnsrec['name'] = node.downcase
|
|
2044
|
-
dnsrec['name'] = "#{dnsrec['name']}.#{MU.environment.downcase}" if dnsrec["append_environment_name"] && !dnsrec['name'].match(/\.#{MU.environment.downcase}$/)
|
|
2045
|
-
end
|
|
2046
|
-
|
|
2047
|
-
if !dnsrec.has_key?("target")
|
|
2048
|
-
# Default to register public endpoint
|
|
2049
|
-
public = true
|
|
2050
|
-
|
|
2051
|
-
if dnsrec.has_key?("target_type")
|
|
2052
|
-
# See if we have a preference for pubic/private endpoint
|
|
2053
|
-
public = dnsrec["target_type"] == "private" ? false : true
|
|
2054
|
-
end
|
|
2055
|
-
|
|
2056
|
-
dnsrec["target"] =
|
|
2057
|
-
if dnsrec["type"] == "CNAME"
|
|
2058
|
-
if public
|
|
2059
|
-
# Make sure we have a public canonical name to register. Use the private one if we don't
|
|
2060
|
-
server.cloud_desc.public_dns_name.empty? ? server.cloud_desc.private_dns_name : server.cloud_desc.public_dns_name
|
|
2061
|
-
else
|
|
2062
|
-
# If we specifically requested to register the private canonical name lets use that
|
|
2063
|
-
server.cloud_desc.private_dns_name
|
|
2064
|
-
end
|
|
2065
|
-
elsif dnsrec["type"] == "A"
|
|
2066
|
-
if public
|
|
2067
|
-
# Make sure we have a public IP address to register. Use the private one if we don't
|
|
2068
|
-
server.cloud_desc.public_ip_address ? server.cloud_desc.public_ip_address : server.cloud_desc.private_ip_address
|
|
2069
|
-
else
|
|
2070
|
-
# If we specifically requested to register the private IP lets use that
|
|
2071
|
-
server.cloud_desc.private_ip_address
|
|
2072
|
-
end
|
|
2073
|
-
end
|
|
2074
|
-
end
|
|
2075
|
-
}
|
|
2076
|
-
if !MU::Cloud::AWS.isGovCloud?
|
|
2077
|
-
MU::Cloud::DNSZone.createRecordsFromConfig(dnscfg)
|
|
2078
|
-
end
|
|
2079
|
-
end
|
|
2080
|
-
|
|
2081
|
-
MU::MommaCat.removeHostFromSSHConfig(node)
|
|
2082
|
-
if server and server.canonicalIP
|
|
2083
|
-
MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP)
|
|
2084
|
-
end
|
|
2085
|
-
# XXX add names paramater with useful stuff
|
|
2086
|
-
MU::MommaCat.addHostToSSHConfig(
|
|
2087
|
-
server,
|
|
2088
|
-
ssh_owner: server.deploy.mu_user,
|
|
2089
|
-
ssh_dir: Etc.getpwnam(server.deploy.mu_user).dir+"/.ssh"
|
|
2090
|
-
)
|
|
2091
|
-
end
|
|
2092
|
-
|
|
2093
|
-
@ssh_semaphore = Mutex.new
|
|
2094
|
-
# Insert a definition for a node into our SSH config.
|
|
2095
|
-
# @param server [MU::Cloud::Server]: The name of the node.
|
|
2096
|
-
# @param names [Array<String>]: Other names that we'd like this host to be known by for SSH purposes
|
|
2097
|
-
# @param ssh_dir [String]: The configuration directory of the SSH config to emit.
|
|
2098
|
-
# @param ssh_conf [String]: A specific SSH configuration file to write entries into.
|
|
2099
|
-
# @param ssh_owner [String]: The preferred owner of the SSH configuration files.
|
|
2100
|
-
# @param timeout [Integer]: An alternate timeout value for connections to this server.
|
|
2101
|
-
# @return [void]
|
|
2102
|
-
def self.addHostToSSHConfig(server,
|
|
2103
|
-
ssh_dir: "#{@myhome}/.ssh",
|
|
2104
|
-
ssh_conf: "#{@myhome}/.ssh/config",
|
|
2105
|
-
ssh_owner: Etc.getpwuid(Process.uid).name,
|
|
2106
|
-
names: [],
|
|
2107
|
-
timeout: 0
|
|
2108
|
-
)
|
|
2109
|
-
if server.nil?
|
|
2110
|
-
MU.log "Called addHostToSSHConfig without a MU::Cloud::Server object", MU::ERR, details: caller
|
|
2111
|
-
return nil
|
|
2112
|
-
end
|
|
2113
|
-
|
|
2114
|
-
_nat_ssh_key, nat_ssh_user, nat_ssh_host, canonical_ip, ssh_user, ssh_key_name = begin
|
|
2115
|
-
server.getSSHConfig
|
|
2116
|
-
rescue MU::MuError
|
|
2117
|
-
return
|
|
2118
|
-
end
|
|
2119
|
-
|
|
2120
|
-
if ssh_user.nil? or ssh_user.empty?
|
|
2121
|
-
MU.log "Failed to extract ssh_user for #{server.mu_name} addHostToSSHConfig", MU::ERR
|
|
2122
|
-
return
|
|
2123
|
-
end
|
|
2124
|
-
if canonical_ip.nil? or canonical_ip.empty?
|
|
2125
|
-
MU.log "Failed to extract canonical_ip for #{server.mu_name} addHostToSSHConfig", MU::ERR
|
|
2126
|
-
return
|
|
2127
|
-
end
|
|
2128
|
-
if ssh_key_name.nil? or ssh_key_name.empty?
|
|
2129
|
-
MU.log "Failed to extract ssh_key_name for #{ssh_key_name.mu_name} in addHostToSSHConfig", MU::ERR
|
|
2130
|
-
return
|
|
2131
|
-
end
|
|
2132
|
-
|
|
2133
|
-
@ssh_semaphore.synchronize {
|
|
2134
|
-
|
|
2135
|
-
if File.exist?(ssh_conf)
|
|
2136
|
-
File.readlines(ssh_conf).each { |line|
|
|
2137
|
-
if line.match(/^Host #{server.mu_name} /)
|
|
2138
|
-
MU.log("Attempt to add duplicate #{ssh_conf} entry for #{server.mu_name}", MU::WARN)
|
|
2139
|
-
return
|
|
2140
|
-
end
|
|
2141
|
-
}
|
|
2142
|
-
end
|
|
2143
|
-
|
|
2144
|
-
File.open(ssh_conf, 'a', 0600) { |ssh_config|
|
|
2145
|
-
ssh_config.flock(File::LOCK_EX)
|
|
2146
|
-
host_str = "Host #{server.mu_name} #{server.canonicalIP}"
|
|
2147
|
-
if !names.nil? and names.size > 0
|
|
2148
|
-
host_str = host_str+" "+names.join(" ")
|
|
2149
|
-
end
|
|
2150
|
-
ssh_config.puts host_str
|
|
2151
|
-
ssh_config.puts " Hostname #{server.canonicalIP}"
|
|
2152
|
-
if !nat_ssh_host.nil? and server.canonicalIP != nat_ssh_host
|
|
2153
|
-
ssh_config.puts " ProxyCommand ssh -W %h:%p #{nat_ssh_user}@#{nat_ssh_host}"
|
|
2154
|
-
end
|
|
2155
|
-
if timeout > 0
|
|
2156
|
-
ssh_config.puts " ConnectTimeout #{timeout}"
|
|
2157
|
-
end
|
|
2158
|
-
|
|
2159
|
-
ssh_config.puts " User #{ssh_user}"
|
|
2160
|
-
# XXX I'd rather add the host key to known_hosts, but Net::SSH is a little dumb
|
|
2161
|
-
ssh_config.puts " StrictHostKeyChecking no"
|
|
2162
|
-
ssh_config.puts " ServerAliveInterval 60"
|
|
2163
|
-
|
|
2164
|
-
ssh_config.puts " IdentityFile #{ssh_dir}/#{ssh_key_name}"
|
|
2165
|
-
if !File.exist?("#{ssh_dir}/#{ssh_key_name}")
|
|
2166
|
-
MU.log "#{server.mu_name} - ssh private key #{ssh_dir}/#{ssh_key_name} does not exist", MU::WARN
|
|
2167
|
-
end
|
|
2168
|
-
|
|
2169
|
-
ssh_config.flock(File::LOCK_UN)
|
|
2170
|
-
ssh_config.chown(Etc.getpwnam(ssh_owner).uid, Etc.getpwnam(ssh_owner).gid)
|
|
2171
|
-
}
|
|
2172
|
-
MU.log "Wrote #{server.mu_name} ssh key to #{ssh_dir}/config", MU::DEBUG
|
|
2173
|
-
return "#{ssh_dir}/#{ssh_key_name}"
|
|
2174
|
-
}
|
|
2175
|
-
end
|
|
2176
|
-
|
|
2177
|
-
# Clean a node's entries out of /etc/hosts
|
|
2178
|
-
# @param node [String]: The node's name
|
|
2179
|
-
# @return [void]
|
|
2180
|
-
def self.removeInstanceFromEtcHosts(node)
|
|
2181
|
-
return if MU.mu_user != "mu"
|
|
2182
|
-
hostsfile = "/etc/hosts"
|
|
2183
|
-
FileUtils.copy(hostsfile, "#{hostsfile}.bak-#{MU.deploy_id}")
|
|
2184
|
-
File.open(hostsfile, File::CREAT|File::RDWR, 0644) { |f|
|
|
2185
|
-
f.flock(File::LOCK_EX)
|
|
2186
|
-
newlines = Array.new
|
|
2187
|
-
f.readlines.each { |line|
|
|
2188
|
-
newlines << line if !line.match(/ #{node}(\s|$)/)
|
|
2189
|
-
}
|
|
2190
|
-
f.rewind
|
|
2191
|
-
f.truncate(0)
|
|
2192
|
-
f.puts(newlines)
|
|
2193
|
-
f.flush
|
|
2194
|
-
|
|
2195
|
-
f.flock(File::LOCK_UN)
|
|
2196
|
-
}
|
|
2197
|
-
end
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
# Insert node names associated with a new instance into /etc/hosts so we
|
|
2201
|
-
# can treat them as if they were real DNS entries. Especially helpful when
|
|
2202
|
-
# Chef/Ohai mistake the proper hostname, e.g. when bootstrapping Windows.
|
|
2203
|
-
# @param public_ip [String]: The node's IP address
|
|
2204
|
-
# @param chef_name [String]: The node's Chef node name
|
|
2205
|
-
# @param system_name [String]: The node's local system name
|
|
2206
|
-
# @return [void]
|
|
2207
|
-
def self.addInstanceToEtcHosts(public_ip, chef_name = nil, system_name = nil)
|
|
2208
|
-
|
|
2209
|
-
# XXX cover ipv6 case
|
|
2210
|
-
if public_ip.nil? or !public_ip.match(/^\d+\.\d+\.\d+\.\d+$/) or (chef_name.nil? and system_name.nil?)
|
|
2211
|
-
raise MuError, "addInstanceToEtcHosts requires public_ip and one or both of chef_name and system_name!"
|
|
2212
|
-
end
|
|
2213
|
-
if chef_name == "localhost" or system_name == "localhost"
|
|
2214
|
-
raise MuError, "Can't set localhost as a name in addInstanceToEtcHosts"
|
|
2215
|
-
end
|
|
2216
|
-
|
|
2217
|
-
if !["mu", "root"].include?(MU.mu_user)
|
|
2218
|
-
response = nil
|
|
2219
|
-
begin
|
|
2220
|
-
response = open("https://127.0.0.1:#{MU.mommaCatPort.to_s}/rest/hosts_add/#{chef_name}/#{public_ip}").read
|
|
2221
|
-
rescue Errno::ECONNRESET, Errno::ECONNREFUSED
|
|
2222
|
-
end
|
|
2223
|
-
if response != "ok"
|
|
2224
|
-
MU.log "Error adding #{public_ip} to /etc/hosts via MommaCat request", MU::ERR
|
|
2225
|
-
end
|
|
2226
|
-
return
|
|
2227
|
-
end
|
|
2228
|
-
|
|
2229
|
-
File.readlines("/etc/hosts").each { |line|
|
|
2230
|
-
if line.match(/^#{public_ip} /) or (chef_name != nil and line.match(/ #{chef_name}(\s|$)/)) or (system_name != nil and line.match(/ #{system_name}(\s|$)/))
|
|
2231
|
-
MU.log "Ignoring attempt to add duplicate /etc/hosts entry: #{public_ip} #{chef_name} #{system_name}", MU::DEBUG
|
|
2232
|
-
return
|
|
2233
|
-
end
|
|
2234
|
-
}
|
|
2235
|
-
File.open("/etc/hosts", 'a') { |etc_hosts|
|
|
2236
|
-
etc_hosts.flock(File::LOCK_EX)
|
|
2237
|
-
etc_hosts.puts("#{public_ip} #{chef_name} #{system_name}")
|
|
2238
|
-
etc_hosts.flock(File::LOCK_UN)
|
|
2239
|
-
}
|
|
2240
|
-
MU.log("Added to /etc/hosts: #{public_ip} #{chef_name} #{system_name}")
|
|
2241
|
-
end
|
|
2242
|
-
|
|
2243
|
-
|
|
2244
|
-
# Send a Slack notification to a deployment's administrators.
|
|
2245
|
-
# @param subject [String]: The subject line of the message.
|
|
2246
|
-
# @param msg [String]: The message body.
|
|
2247
|
-
# @return [void]
|
|
2248
|
-
def sendAdminSlack(subject, msg: "")
|
|
2249
|
-
if $MU_CFG['slack'] and $MU_CFG['slack']['webhook'] and
|
|
2250
|
-
(!$MU_CFG['slack']['skip_environments'] or !$MU_CFG['slack']['skip_environments'].any?{ |s| s.casecmp(MU.environment)==0 })
|
|
2251
|
-
require 'slack-notifier'
|
|
2252
|
-
slack = Slack::Notifier.new $MU_CFG['slack']['webhook']
|
|
2253
|
-
|
|
2254
|
-
if msg and !msg.empty?
|
|
2255
|
-
slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}:\n\n```#{msg}\n```", channel: $MU_CFG['slack']['channel']
|
|
2256
|
-
else
|
|
2257
|
-
slack.ping "#{MU.appname} \*\"#{MU.handle}\"\* (`#{MU.deploy_id}`) - #{subject}", channel: $MU_CFG['slack']['channel']
|
|
2258
|
-
end
|
|
2259
|
-
end
|
|
2260
|
-
end
|
|
2261
|
-
|
|
2262
|
-
# Send an email notification to a deployment's administrators.
|
|
2263
|
-
# @param subject [String]: The subject line of the message.
|
|
2264
|
-
# @param msg [String]: The message body.
|
|
2265
|
-
# @param data [Array]: Supplemental data to add to the message body.
|
|
2266
|
-
# @param debug [Boolean]: If set, will include the full deployment structure and original {MU::Config}-parsed configuration.
|
|
2267
|
-
# @return [void]
|
|
2268
|
-
def sendAdminMail(subject, msg: "", kitten: nil, data: nil, debug: false)
|
|
2269
|
-
require 'net/smtp'
|
|
2270
|
-
if @deployment.nil?
|
|
2271
|
-
MU.log "Can't send admin mail without a loaded deployment", MU::ERR
|
|
2272
|
-
return
|
|
2273
|
-
end
|
|
2274
|
-
to = Array.new
|
|
2275
|
-
if !@original_config.nil?
|
|
2276
|
-
@original_config['admins'].each { |admin|
|
|
2277
|
-
to << "#{admin['name']} <#{admin['email']}>"
|
|
2278
|
-
}
|
|
1321
|
+
to = Array.new
|
|
1322
|
+
if !@original_config.nil?
|
|
1323
|
+
@original_config['admins'].each { |admin|
|
|
1324
|
+
to << "#{admin['name']} <#{admin['email']}>"
|
|
1325
|
+
}
|
|
2279
1326
|
end
|
|
2280
1327
|
message = <<MESSAGE_END
|
|
2281
1328
|
From: #{MU.handle} <root@localhost>
|
|
@@ -2308,205 +1355,6 @@ MESSAGE_END
|
|
|
2308
1355
|
end
|
|
2309
1356
|
end
|
|
2310
1357
|
|
|
2311
|
-
# Manufactures a human-readable deployment name from the random
|
|
2312
|
-
# two-character seed in MU-ID. Cat-themed when possible.
|
|
2313
|
-
# @param seed [String]: A two-character seed from which we'll generate a name.
|
|
2314
|
-
# @return [String]: Two words
|
|
2315
|
-
def self.generateHandle(seed)
|
|
2316
|
-
word_one=word_two=nil
|
|
2317
|
-
|
|
2318
|
-
# Unless we've got two letters that don't have corresponding cat-themed
|
|
2319
|
-
# words, we'll insist that our generated handle have at least one cat
|
|
2320
|
-
# element to it.
|
|
2321
|
-
require_cat_words = true
|
|
2322
|
-
if @catwords.select { |word| word.match(/^#{seed[0]}/i) }.size == 0 and
|
|
2323
|
-
@catwords.select { |word| word.match(/^#{seed[1]}/i) }.size == 0
|
|
2324
|
-
require_cat_words = false
|
|
2325
|
-
MU.log "Got an annoying pair of letters #{seed}, not forcing cat-theming", MU::DEBUG
|
|
2326
|
-
end
|
|
2327
|
-
allnouns = @catnouns + @jaegernouns
|
|
2328
|
-
alladjs = @catadjs + @jaegeradjs
|
|
2329
|
-
|
|
2330
|
-
tries = 0
|
|
2331
|
-
begin
|
|
2332
|
-
# Try to avoid picking something "nouny" for the first word
|
|
2333
|
-
source = @catadjs + @catmixed + @jaegeradjs + @jaegermixed
|
|
2334
|
-
first_ltr = source.select { |word| word.match(/^#{seed[0]}/i) }
|
|
2335
|
-
if !first_ltr or first_ltr.size == 0
|
|
2336
|
-
first_ltr = @words.select { |word| word.match(/^#{seed[0]}/i) }
|
|
2337
|
-
end
|
|
2338
|
-
word_one = first_ltr.shuffle.first
|
|
2339
|
-
|
|
2340
|
-
# If we got a paired set that happen to match our letters, go with it
|
|
2341
|
-
if !word_one.nil? and word_one.match(/-#{seed[1]}/i)
|
|
2342
|
-
word_one, word_two = word_one.split(/-/)
|
|
2343
|
-
else
|
|
2344
|
-
source = @words
|
|
2345
|
-
if @catwords.include?(word_one)
|
|
2346
|
-
source = @jaegerwords
|
|
2347
|
-
elsif require_cat_words
|
|
2348
|
-
source = @catwords
|
|
2349
|
-
end
|
|
2350
|
-
second_ltr = source.select { |word| word.match(/^#{seed[1]}/i) and !word.match(/-/i) }
|
|
2351
|
-
word_two = second_ltr.shuffle.first
|
|
2352
|
-
end
|
|
2353
|
-
tries = tries + 1
|
|
2354
|
-
end while tries < 50 and (word_one.nil? or word_two.nil? or word_one.match(/-/) or word_one == word_two or (allnouns.include?(word_one) and allnouns.include?(word_two)) or (alladjs.include?(word_one) and alladjs.include?(word_two)) or (require_cat_words and !@catwords.include?(word_one) and !@catwords.include?(word_two)))
|
|
2355
|
-
|
|
2356
|
-
if tries >= 50 and (word_one.nil? or word_two.nil?)
|
|
2357
|
-
MU.log "I failed to generated a valid handle, faking it", MU::ERR
|
|
2358
|
-
return "#{seed[0].capitalize} #{seed[1].capitalize}"
|
|
2359
|
-
end
|
|
2360
|
-
|
|
2361
|
-
return "#{word_one.capitalize} #{word_two.capitalize}"
|
|
2362
|
-
end
|
|
2363
|
-
|
|
2364
|
-
# Ensure that the Nagios configuration local to the MU master has been
|
|
2365
|
-
# updated, and make sure Nagios has all of the ssh keys it needs to tunnel
|
|
2366
|
-
# to client nodes.
|
|
2367
|
-
# @return [void]
|
|
2368
|
-
def self.syncMonitoringConfig(blocking = true)
|
|
2369
|
-
return if Etc.getpwuid(Process.uid).name != "root" or (MU.mu_user != "mu" and MU.mu_user != "root")
|
|
2370
|
-
parent_thread_id = Thread.current.object_id
|
|
2371
|
-
nagios_threads = []
|
|
2372
|
-
nagios_threads << Thread.new {
|
|
2373
|
-
MU.dupGlobals(parent_thread_id)
|
|
2374
|
-
realhome = Etc.getpwnam("nagios").dir
|
|
2375
|
-
[@nagios_home, "#{@nagios_home}/.ssh"].each { |dir|
|
|
2376
|
-
Dir.mkdir(dir, 0711) if !Dir.exist?(dir)
|
|
2377
|
-
File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, dir)
|
|
2378
|
-
}
|
|
2379
|
-
if realhome != @nagios_home and Dir.exist?(realhome) and !File.symlink?("#{realhome}/.ssh")
|
|
2380
|
-
File.rename("#{realhome}/.ssh", "#{realhome}/.ssh.#{$$}") if Dir.exist?("#{realhome}/.ssh")
|
|
2381
|
-
File.symlink("#{@nagios_home}/.ssh", Etc.getpwnam("nagios").dir+"/.ssh")
|
|
2382
|
-
end
|
|
2383
|
-
MU.log "Updating #{@nagios_home}/.ssh/config..."
|
|
2384
|
-
ssh_lock = File.new("#{@nagios_home}/.ssh/config.mu.lock", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2385
|
-
ssh_lock.flock(File::LOCK_EX)
|
|
2386
|
-
ssh_conf = File.new("#{@nagios_home}/.ssh/config.tmp", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2387
|
-
ssh_conf.puts "Host MU-MASTER localhost"
|
|
2388
|
-
ssh_conf.puts " Hostname localhost"
|
|
2389
|
-
ssh_conf.puts " User root"
|
|
2390
|
-
ssh_conf.puts " IdentityFile #{@nagios_home}/.ssh/id_rsa"
|
|
2391
|
-
ssh_conf.puts " StrictHostKeyChecking no"
|
|
2392
|
-
ssh_conf.close
|
|
2393
|
-
FileUtils.cp("#{@myhome}/.ssh/id_rsa", "#{@nagios_home}/.ssh/id_rsa")
|
|
2394
|
-
File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/id_rsa")
|
|
2395
|
-
threads = []
|
|
2396
|
-
|
|
2397
|
-
parent_thread_id = Thread.current.object_id
|
|
2398
|
-
MU::MommaCat.listDeploys.sort.each { |deploy_id|
|
|
2399
|
-
begin
|
|
2400
|
-
# We don't want to use cached litter information here because this is also called by cleanTerminatedInstances.
|
|
2401
|
-
deploy = MU::MommaCat.getLitter(deploy_id)
|
|
2402
|
-
if deploy.ssh_key_name.nil? or deploy.ssh_key_name.empty?
|
|
2403
|
-
MU.log "Failed to extract ssh key name from #{deploy_id} in syncMonitoringConfig", MU::ERR if deploy.kittens.has_key?("servers")
|
|
2404
|
-
next
|
|
2405
|
-
end
|
|
2406
|
-
FileUtils.cp("#{@myhome}/.ssh/#{deploy.ssh_key_name}", "#{@nagios_home}/.ssh/#{deploy.ssh_key_name}")
|
|
2407
|
-
File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/#{deploy.ssh_key_name}")
|
|
2408
|
-
if deploy.kittens.has_key?("servers")
|
|
2409
|
-
deploy.kittens["servers"].values.each { |nodeclasses|
|
|
2410
|
-
nodeclasses.values.each { |nodes|
|
|
2411
|
-
nodes.values.each { |server|
|
|
2412
|
-
next if !server.cloud_desc
|
|
2413
|
-
MU.dupGlobals(parent_thread_id)
|
|
2414
|
-
threads << Thread.new {
|
|
2415
|
-
MU::MommaCat.setThreadContext(deploy)
|
|
2416
|
-
MU.log "Adding #{server.mu_name} to #{@nagios_home}/.ssh/config", MU::DEBUG
|
|
2417
|
-
MU::MommaCat.addHostToSSHConfig(
|
|
2418
|
-
server,
|
|
2419
|
-
ssh_dir: "#{@nagios_home}/.ssh",
|
|
2420
|
-
ssh_conf: "#{@nagios_home}/.ssh/config.tmp",
|
|
2421
|
-
ssh_owner: "nagios"
|
|
2422
|
-
)
|
|
2423
|
-
MU.purgeGlobals
|
|
2424
|
-
}
|
|
2425
|
-
}
|
|
2426
|
-
}
|
|
2427
|
-
}
|
|
2428
|
-
end
|
|
2429
|
-
rescue Exception => e
|
|
2430
|
-
MU.log "#{e.inspect} while generating Nagios SSH config in #{deploy_id}", MU::ERR, details: e.backtrace
|
|
2431
|
-
end
|
|
2432
|
-
}
|
|
2433
|
-
threads.each { |t|
|
|
2434
|
-
t.join
|
|
2435
|
-
}
|
|
2436
|
-
ssh_lock.flock(File::LOCK_UN)
|
|
2437
|
-
ssh_lock.close
|
|
2438
|
-
File.chown(Etc.getpwnam("nagios").uid, Etc.getpwnam("nagios").gid, "#{@nagios_home}/.ssh/config.tmp")
|
|
2439
|
-
File.rename("#{@nagios_home}/.ssh/config.tmp", "#{@nagios_home}/.ssh/config")
|
|
2440
|
-
|
|
2441
|
-
MU.log "Updating Nagios monitoring config, this may take a while..."
|
|
2442
|
-
output = nil
|
|
2443
|
-
if $MU_CFG and !$MU_CFG['master_runlist_extras'].nil?
|
|
2444
|
-
output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only],#{$MU_CFG['master_runlist_extras'].join(",")}' 2>&1}
|
|
2445
|
-
else
|
|
2446
|
-
output = %x{#{MU::Groomer::Chef.chefclient} -o 'role[mu-master-nagios-only]' 2>&1}
|
|
2447
|
-
end
|
|
2448
|
-
|
|
2449
|
-
if $?.exitstatus != 0
|
|
2450
|
-
MU.log "Nagios monitoring config update returned a non-zero exit code!", MU::ERR, details: output
|
|
2451
|
-
else
|
|
2452
|
-
MU.log "Nagios monitoring config update complete."
|
|
2453
|
-
end
|
|
2454
|
-
}
|
|
2455
|
-
|
|
2456
|
-
if blocking
|
|
2457
|
-
nagios_threads.each { |t|
|
|
2458
|
-
t.join
|
|
2459
|
-
}
|
|
2460
|
-
end
|
|
2461
|
-
end
|
|
2462
|
-
|
|
2463
|
-
# Return a list of all currently active deploy identifiers.
|
|
2464
|
-
# @return [Array<String>]
|
|
2465
|
-
def self.listDeploys
|
|
2466
|
-
return [] if !Dir.exist?("#{MU.dataDir}/deployments")
|
|
2467
|
-
deploys = []
|
|
2468
|
-
Dir.entries("#{MU.dataDir}/deployments").reverse_each { |muid|
|
|
2469
|
-
next if !Dir.exist?("#{MU.dataDir}/deployments/#{muid}") or muid == "." or muid == ".."
|
|
2470
|
-
deploys << muid
|
|
2471
|
-
}
|
|
2472
|
-
return deploys
|
|
2473
|
-
end
|
|
2474
|
-
|
|
2475
|
-
# Return a list of all nodes in all deployments. Does so without loading
|
|
2476
|
-
# deployments fully.
|
|
2477
|
-
# @return [Hash]
|
|
2478
|
-
def self.listAllNodes
|
|
2479
|
-
nodes = Hash.new
|
|
2480
|
-
MU::MommaCat.deploy_struct_semaphore.synchronize {
|
|
2481
|
-
MU::MommaCat.listDeploys.each { |deploy|
|
|
2482
|
-
if !Dir.exist?(MU::MommaCat.deploy_dir(deploy)) or
|
|
2483
|
-
!File.size?("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json")
|
|
2484
|
-
MU.log "Didn't see deployment metadata for '#{deploy}'", MU::WARN
|
|
2485
|
-
next
|
|
2486
|
-
end
|
|
2487
|
-
data = File.open("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json", File::RDONLY)
|
|
2488
|
-
MU.log "Getting lock to read #{MU::MommaCat.deploy_dir(deploy)}/deployment.json", MU::DEBUG
|
|
2489
|
-
data.flock(File::LOCK_EX)
|
|
2490
|
-
begin
|
|
2491
|
-
deployment = JSON.parse(File.read("#{MU::MommaCat.deploy_dir(deploy)}/deployment.json"))
|
|
2492
|
-
deployment["deploy_id"] = deploy
|
|
2493
|
-
if deployment.has_key?("servers")
|
|
2494
|
-
deployment["servers"].each_key { |nodeclass|
|
|
2495
|
-
deployment["servers"][nodeclass].each_pair { |mu_name, metadata|
|
|
2496
|
-
nodes[mu_name] = metadata
|
|
2497
|
-
}
|
|
2498
|
-
}
|
|
2499
|
-
end
|
|
2500
|
-
rescue JSON::ParserError => e
|
|
2501
|
-
MU.log "JSON parse failed on #{MU::MommaCat.deploy_dir(deploy)}/deployment.json", MU::ERR, details: e.message
|
|
2502
|
-
end
|
|
2503
|
-
data.flock(File::LOCK_UN)
|
|
2504
|
-
data.close
|
|
2505
|
-
}
|
|
2506
|
-
}
|
|
2507
|
-
return nodes
|
|
2508
|
-
end
|
|
2509
|
-
|
|
2510
1358
|
# Return a list of all nodes associated with the current deployment.
|
|
2511
1359
|
# @return [Hash]
|
|
2512
1360
|
def listNodes
|
|
@@ -2750,559 +1598,30 @@ MESSAGE_END
|
|
|
2750
1598
|
results[cert_cn]
|
|
2751
1599
|
end
|
|
2752
1600
|
|
|
2753
|
-
# @return [String]: The Mu Master filesystem directory holding metadata for the current deployment
|
|
2754
|
-
def deploy_dir
|
|
2755
|
-
MU::MommaCat.deploy_dir(@deploy_id)
|
|
2756
|
-
end
|
|
2757
|
-
|
|
2758
|
-
# Path to the log file used by the Momma Cat daemon
|
|
2759
|
-
# @return [String]
|
|
2760
|
-
def self.daemonLogFile
|
|
2761
|
-
base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
|
|
2762
|
-
"#{base}/log/mu-momma-cat.log"
|
|
2763
|
-
end
|
|
2764
|
-
|
|
2765
|
-
# Path to the PID file used by the Momma Cat daemon
|
|
2766
|
-
# @return [String]
|
|
2767
|
-
def self.daemonPidFile
|
|
2768
|
-
base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
|
|
2769
|
-
"#{base}/run/mommacat.pid"
|
|
2770
|
-
end
|
|
2771
|
-
|
|
2772
|
-
# Start the Momma Cat daemon and return the exit status of the command used
|
|
2773
|
-
# @return [Integer]
|
|
2774
|
-
def self.start
|
|
2775
|
-
if MU.inGem? and MU.muCfg['disable_mommacat']
|
|
2776
|
-
return
|
|
2777
|
-
end
|
|
2778
|
-
base = (Process.uid == 0 and !MU.localOnly) ? "/var" : MU.dataDir
|
|
2779
|
-
[base, "#{base}/log", "#{base}/run"].each { |dir|
|
|
2780
|
-
if !Dir.exist?(dir)
|
|
2781
|
-
MU.log "Creating #{dir}"
|
|
2782
|
-
Dir.mkdir(dir)
|
|
2783
|
-
end
|
|
2784
|
-
}
|
|
2785
|
-
return 0 if status
|
|
2786
|
-
|
|
2787
|
-
MU.log "Starting Momma Cat on port #{MU.mommaCatPort}, logging to #{daemonLogFile}, PID file #{daemonPidFile}"
|
|
2788
|
-
origdir = Dir.getwd
|
|
2789
|
-
Dir.chdir(MU.myRoot+"/modules")
|
|
2790
|
-
|
|
2791
|
-
# XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
|
|
2792
|
-
if MU.inGem?
|
|
2793
|
-
cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
|
|
2794
|
-
else
|
|
2795
|
-
cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
|
|
2796
|
-
end
|
|
2797
|
-
|
|
2798
|
-
MU.log cmd, MU::NOTICE
|
|
2799
|
-
|
|
2800
|
-
retries = 0
|
|
2801
|
-
begin
|
|
2802
|
-
output = %x{#{cmd}}
|
|
2803
|
-
sleep 1
|
|
2804
|
-
retries += 1
|
|
2805
|
-
if retries >= 10
|
|
2806
|
-
MU.log "MommaCat failed to start (command was #{cmd}, working directory #{MU.myRoot}/modules)", MU::WARN, details: output
|
|
2807
|
-
pp caller
|
|
2808
|
-
return $?.exitstatus
|
|
2809
|
-
end
|
|
2810
|
-
end while !status
|
|
2811
|
-
|
|
2812
|
-
Dir.chdir(origdir)
|
|
2813
|
-
|
|
2814
|
-
if $?.exitstatus != 0
|
|
2815
|
-
exit 1
|
|
2816
|
-
end
|
|
2817
|
-
|
|
2818
|
-
return $?.exitstatus
|
|
2819
|
-
end
|
|
2820
|
-
|
|
2821
|
-
# Return true if the Momma Cat daemon appears to be running
|
|
2822
|
-
# @return [Boolean]
|
|
2823
|
-
def self.status
|
|
2824
|
-
if MU.inGem? and MU.muCfg['disable_mommacat']
|
|
2825
|
-
return true
|
|
2826
|
-
end
|
|
2827
|
-
if File.exist?(daemonPidFile)
|
|
2828
|
-
pid = File.read(daemonPidFile).chomp.to_i
|
|
2829
|
-
begin
|
|
2830
|
-
Process.getpgid(pid)
|
|
2831
|
-
MU.log "Momma Cat running with pid #{pid.to_s}"
|
|
2832
|
-
return true
|
|
2833
|
-
rescue Errno::ESRCH
|
|
2834
|
-
end
|
|
2835
|
-
end
|
|
2836
|
-
MU.log "Momma Cat daemon not running", MU::NOTICE, details: daemonPidFile
|
|
2837
|
-
false
|
|
2838
|
-
end
|
|
2839
|
-
|
|
2840
|
-
# Stop the Momma Cat daemon, if it's running
|
|
2841
|
-
def self.stop
|
|
2842
|
-
if File.exist?(daemonPidFile)
|
|
2843
|
-
pid = File.read(daemonPidFile).chomp.to_i
|
|
2844
|
-
MU.log "Stopping Momma Cat with pid #{pid.to_s}"
|
|
2845
|
-
Process.kill("INT", pid)
|
|
2846
|
-
killed = false
|
|
2847
|
-
begin
|
|
2848
|
-
Process.getpgid(pid)
|
|
2849
|
-
sleep 1
|
|
2850
|
-
rescue Errno::ESRCH
|
|
2851
|
-
killed = true
|
|
2852
|
-
end while killed
|
|
2853
|
-
MU.log "Momma Cat with pid #{pid.to_s} stopped", MU::DEBUG, details: daemonPidFile
|
|
2854
|
-
|
|
2855
|
-
begin
|
|
2856
|
-
File.unlink(daemonPidFile)
|
|
2857
|
-
rescue Errno::ENOENT
|
|
2858
|
-
end
|
|
2859
|
-
end
|
|
2860
|
-
end
|
|
2861
|
-
|
|
2862
|
-
# (Re)start the Momma Cat daemon and return the exit status of the start command
|
|
2863
|
-
# @return [Integer]
|
|
2864
|
-
def self.restart
|
|
2865
|
-
stop
|
|
2866
|
-
start
|
|
2867
|
-
end
|
|
2868
|
-
|
|
2869
|
-
# Locate and return the deploy, if any, which matches the provided origin
|
|
2870
|
-
# description
|
|
2871
|
-
# @param origin [Hash]
|
|
2872
|
-
def self.findMatchingDeploy(origin)
|
|
2873
|
-
MU::MommaCat.listDeploys.each { |deploy_id|
|
|
2874
|
-
o_path = deploy_dir(deploy_id)+"/origin.json"
|
|
2875
|
-
next if !File.exist?(o_path)
|
|
2876
|
-
this_origin = JSON.parse(File.read(o_path))
|
|
2877
|
-
if origin == this_origin
|
|
2878
|
-
MU.log "Deploy #{deploy_id} matches origin hash, loading", details: origin
|
|
2879
|
-
return MU::MommaCat.new(deploy_id)
|
|
2880
|
-
end
|
|
2881
|
-
}
|
|
2882
|
-
nil
|
|
2883
|
-
end
|
|
2884
|
-
|
|
2885
|
-
# Synchronize all in-memory information related to this to deployment to
|
|
2886
|
-
# disk.
|
|
2887
|
-
# @param triggering_node [MU::Cloud::Server]: If we're being triggered by the addition/removal/update of a node, this allows us to notify any sibling or dependent nodes of changes
|
|
2888
|
-
# @param force [Boolean]: Save even if +no_artifacts+ is set
|
|
2889
|
-
# @param origin [Hash]: Optional blob of data indicating how this deploy was created
|
|
2890
|
-
def save!(triggering_node = nil, force: false, origin: nil)
|
|
2891
|
-
|
|
2892
|
-
return if @no_artifacts and !force
|
|
2893
|
-
|
|
2894
|
-
MU::MommaCat.deploy_struct_semaphore.synchronize {
|
|
2895
|
-
MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG
|
|
2896
|
-
|
|
2897
|
-
if !Dir.exist?(deploy_dir)
|
|
2898
|
-
MU.log "Creating #{deploy_dir}", MU::DEBUG
|
|
2899
|
-
Dir.mkdir(deploy_dir, 0700)
|
|
2900
|
-
end
|
|
2901
|
-
|
|
2902
|
-
if !origin.nil?
|
|
2903
|
-
o_file = File.new("#{deploy_dir}/origin.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2904
|
-
o_file.puts JSON.pretty_generate(origin)
|
|
2905
|
-
o_file.close
|
|
2906
|
-
end
|
|
2907
|
-
|
|
2908
|
-
if !@private_key.nil?
|
|
2909
|
-
privkey = File.new("#{deploy_dir}/private_key", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2910
|
-
privkey.puts @private_key
|
|
2911
|
-
privkey.close
|
|
2912
|
-
end
|
|
2913
|
-
|
|
2914
|
-
if !@public_key.nil?
|
|
2915
|
-
pubkey = File.new("#{deploy_dir}/public_key", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2916
|
-
pubkey.puts @public_key
|
|
2917
|
-
pubkey.close
|
|
2918
|
-
end
|
|
2919
|
-
|
|
2920
|
-
if !@deployment.nil? and @deployment.size > 0
|
|
2921
|
-
@deployment['handle'] = MU.handle if @deployment['handle'].nil? and !MU.handle.nil?
|
|
2922
|
-
@deployment['public_key'] = @public_key
|
|
2923
|
-
@deployment['timestamp'] ||= @timestamp
|
|
2924
|
-
@deployment['seed'] ||= @seed
|
|
2925
|
-
@deployment['appname'] ||= @appname
|
|
2926
|
-
@deployment['handle'] ||= @handle
|
|
2927
|
-
@deployment['ssh_public_key'] ||= @ssh_public_key if @ssh_public_key
|
|
2928
|
-
begin
|
|
2929
|
-
# XXX doing this to trigger JSON errors before stomping the stored
|
|
2930
|
-
# file...
|
|
2931
|
-
JSON.pretty_generate(@deployment, max_nesting: false)
|
|
2932
|
-
deploy = File.new("#{deploy_dir}/deployment.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2933
|
-
MU.log "Getting lock to write #{deploy_dir}/deployment.json", MU::DEBUG
|
|
2934
|
-
deploy.flock(File::LOCK_EX)
|
|
2935
|
-
deploy.puts JSON.pretty_generate(@deployment, max_nesting: false)
|
|
2936
|
-
rescue JSON::NestingError => e
|
|
2937
|
-
MU.log e.inspect, MU::ERR, details: @deployment
|
|
2938
|
-
raise MuError, "Got #{e.message} trying to save deployment"
|
|
2939
|
-
rescue Encoding::UndefinedConversionError => e
|
|
2940
|
-
MU.log e.inspect, MU::ERR, details: @deployment
|
|
2941
|
-
raise MuError, "Got #{e.message} at #{e.error_char.dump} (#{e.source_encoding_name} => #{e.destination_encoding_name}) trying to save deployment"
|
|
2942
|
-
end
|
|
2943
|
-
deploy.flock(File::LOCK_UN)
|
|
2944
|
-
deploy.close
|
|
2945
|
-
@need_deploy_flush = false
|
|
2946
|
-
MU::MommaCat.updateLitter(@deploy_id, self)
|
|
2947
|
-
end
|
|
2948
|
-
|
|
2949
|
-
if !@original_config.nil? and @original_config.is_a?(Hash)
|
|
2950
|
-
config = File.new("#{deploy_dir}/basket_of_kittens.json", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2951
|
-
config.puts JSON.pretty_generate(MU::Config.manxify(@original_config))
|
|
2952
|
-
config.close
|
|
2953
|
-
end
|
|
2954
|
-
|
|
2955
|
-
if !@ssh_private_key.nil?
|
|
2956
|
-
key = File.new("#{deploy_dir}/node_ssh.key", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2957
|
-
key.puts @ssh_private_key
|
|
2958
|
-
key.close
|
|
2959
|
-
end
|
|
2960
|
-
if !@ssh_public_key.nil?
|
|
2961
|
-
key = File.new("#{deploy_dir}/node_ssh.pub", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2962
|
-
key.puts @ssh_public_key
|
|
2963
|
-
key.close
|
|
2964
|
-
end
|
|
2965
|
-
if !@ssh_key_name.nil?
|
|
2966
|
-
key = File.new("#{deploy_dir}/ssh_key_name", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2967
|
-
key.puts @ssh_key_name
|
|
2968
|
-
key.close
|
|
2969
|
-
end
|
|
2970
|
-
if !@environment.nil?
|
|
2971
|
-
env = File.new("#{deploy_dir}/environment_name", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2972
|
-
env.puts @environment
|
|
2973
|
-
env.close
|
|
2974
|
-
end
|
|
2975
|
-
if !@deploy_secret.nil?
|
|
2976
|
-
secret = File.new("#{deploy_dir}/deploy_secret", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2977
|
-
secret.print @deploy_secret
|
|
2978
|
-
secret.close
|
|
2979
|
-
end
|
|
2980
|
-
if !@secrets.nil?
|
|
2981
|
-
secretdir = "#{deploy_dir}/secrets"
|
|
2982
|
-
if !Dir.exist?(secretdir)
|
|
2983
|
-
MU.log "Creating #{secretdir}", MU::DEBUG
|
|
2984
|
-
Dir.mkdir(secretdir, 0700)
|
|
2985
|
-
end
|
|
2986
|
-
@secrets.each_pair { |type, servers|
|
|
2987
|
-
servers.each_pair { |server, svr_secret|
|
|
2988
|
-
key = File.new("#{secretdir}/#{type}.#{server}", File::CREAT|File::TRUNC|File::RDWR, 0600)
|
|
2989
|
-
key.puts svr_secret
|
|
2990
|
-
key.close
|
|
2991
|
-
}
|
|
2992
|
-
}
|
|
2993
|
-
end
|
|
2994
|
-
}
|
|
2995
|
-
|
|
2996
|
-
# Update groomer copies of this metadata
|
|
2997
|
-
syncLitter(@deployment['servers'].keys, triggering_node: triggering_node, save_only: true) if @deployment.has_key?("servers")
|
|
2998
|
-
end
|
|
2999
|
-
|
|
3000
|
-
# Find one or more resources by their Mu resource name, and return
|
|
3001
|
-
# MommaCat objects for their containing deploys, their BoK config data,
|
|
3002
|
-
# and their deployment data.
|
|
3003
|
-
#
|
|
3004
|
-
# @param type [String]: The type of resource, e.g. "vpc" or "server."
|
|
3005
|
-
# @param name [String]: The Mu resource class, typically the name field of a Basket of Kittens resource declaration.
|
|
3006
|
-
# @param mu_name [String]: The fully-expanded Mu resource name, e.g. MGMT-PROD-2015040115-FR-ADMGMT2
|
|
3007
|
-
|
|
3008
1601
|
private
|
|
3009
1602
|
|
|
3010
|
-
# Check to see whether a given resource name is unique across all
|
|
3011
|
-
# deployments on this Mu server. We only enforce this for certain classes
|
|
3012
|
-
# of names. If the name in question is available, add it to our cache of
|
|
3013
|
-
# said names. See #{MU::MommaCat.getResourceName}
|
|
3014
|
-
# @param name [String]: The name to attempt to allocate.
|
|
3015
|
-
# @return [Boolean]: True if allocation was successful.
|
|
3016
|
-
def allocateUniqueResourceName(name)
|
|
3017
|
-
raise MuError, "Cannot call allocateUniqueResourceName without an active deployment" if @deploy_id.nil?
|
|
3018
|
-
path = File.expand_path(MU.dataDir+"/deployments")
|
|
3019
|
-
File.open(path+"/unique_ids", File::CREAT|File::RDWR, 0600) { |f|
|
|
3020
|
-
existing = []
|
|
3021
|
-
f.flock(File::LOCK_EX)
|
|
3022
|
-
f.readlines.each { |line|
|
|
3023
|
-
existing << line.chomp
|
|
3024
|
-
}
|
|
3025
|
-
begin
|
|
3026
|
-
existing.each { |used|
|
|
3027
|
-
if used.match(/^#{name}:/)
|
|
3028
|
-
if !used.match(/^#{name}:#{@deploy_id}$/)
|
|
3029
|
-
MU.log "#{name} is already reserved by another resource on this Mu server.", MU::WARN, details: caller
|
|
3030
|
-
return false
|
|
3031
|
-
else
|
|
3032
|
-
return true
|
|
3033
|
-
end
|
|
3034
|
-
end
|
|
3035
|
-
}
|
|
3036
|
-
f.puts name+":"+@deploy_id
|
|
3037
|
-
return true
|
|
3038
|
-
ensure
|
|
3039
|
-
f.flock(File::LOCK_UN)
|
|
3040
|
-
end
|
|
3041
|
-
}
|
|
3042
|
-
end
|
|
3043
|
-
|
|
3044
|
-
###########################################################################
|
|
3045
|
-
###########################################################################
|
|
3046
|
-
def self.deploy_dir(deploy_id)
|
|
3047
|
-
raise MuError, "deploy_dir must get a deploy_id if called as class method (from #{caller[0]}; #{caller[1]})" if deploy_id.nil?
|
|
3048
|
-
# XXX this will blow up if someone sticks MU in /
|
|
3049
|
-
path = File.expand_path(MU.dataDir+"/deployments")
|
|
3050
|
-
if !Dir.exist?(path)
|
|
3051
|
-
MU.log "Creating #{path}", MU::DEBUG
|
|
3052
|
-
Dir.mkdir(path, 0700)
|
|
3053
|
-
end
|
|
3054
|
-
path = path+"/"+deploy_id
|
|
3055
|
-
return path
|
|
3056
|
-
end
|
|
3057
|
-
|
|
3058
|
-
def self.deploy_exists?(deploy_id)
|
|
3059
|
-
if deploy_id.nil? or deploy_id.empty?
|
|
3060
|
-
MU.log "Got nil deploy_id in MU::MommaCat.deploy_exists?", MU::WARN
|
|
3061
|
-
return
|
|
3062
|
-
end
|
|
3063
|
-
path = File.expand_path(MU.dataDir+"/deployments")
|
|
3064
|
-
if !Dir.exist?(path)
|
|
3065
|
-
Dir.mkdir(path, 0700)
|
|
3066
|
-
end
|
|
3067
|
-
deploy_path = File.expand_path(path+"/"+deploy_id)
|
|
3068
|
-
return Dir.exist?(deploy_path)
|
|
3069
|
-
end
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
1603
|
def createDeployKey
|
|
3073
1604
|
key = OpenSSL::PKey::RSA.generate(4096)
|
|
3074
1605
|
MU.log "Generated deploy key for #{MU.deploy_id}", MU::DEBUG, details: key.public_key.export
|
|
3075
1606
|
return [key.export, key.public_key.export]
|
|
3076
1607
|
end
|
|
3077
1608
|
|
|
3078
|
-
# @param deploy_id [String]: The deployment to search. Will search all deployments if not specified.
|
|
3079
|
-
# @return [Hash,Array<Hash>]
|
|
3080
|
-
def self.getResourceMetadata(type, name: nil, deploy_id: nil, use_cache: true, mu_name: nil)
|
|
3081
|
-
if type.nil?
|
|
3082
|
-
raise MuError, "Can't call getResourceMetadata without a type argument"
|
|
3083
|
-
end
|
|
3084
|
-
_shortclass, _cfg_name, type, _classname = MU::Cloud.getResourceNames(type)
|
|
3085
|
-
|
|
3086
|
-
# first, check our in-memory deploys, which may or may not have been
|
|
3087
|
-
# written to disk yet.
|
|
3088
|
-
littercache = nil
|
|
3089
|
-
begin
|
|
3090
|
-
@@litter_semaphore.synchronize {
|
|
3091
|
-
littercache = @@litters.dup
|
|
3092
|
-
}
|
|
3093
|
-
rescue ThreadError => e
|
|
3094
|
-
# already locked by a parent caller and this is a read op, so this is ok
|
|
3095
|
-
raise e if !e.message.match(/recursive locking/)
|
|
3096
|
-
littercache = @@litters.dup
|
|
3097
|
-
end
|
|
3098
|
-
littercache.each_pair { |deploy, momma|
|
|
3099
|
-
@@deploy_struct_semaphore.synchronize {
|
|
3100
|
-
@deploy_cache[deploy] = {
|
|
3101
|
-
"mtime" => Time.now,
|
|
3102
|
-
"data" => momma.deployment
|
|
3103
|
-
}
|
|
3104
|
-
}
|
|
3105
|
-
}
|
|
3106
|
-
|
|
3107
|
-
deploy_root = File.expand_path(MU.dataDir+"/deployments")
|
|
3108
|
-
MU::MommaCat.deploy_struct_semaphore.synchronize {
|
|
3109
|
-
if Dir.exist?(deploy_root)
|
|
3110
|
-
Dir.entries(deploy_root).each { |deploy|
|
|
3111
|
-
this_deploy_dir = deploy_root+"/"+deploy
|
|
3112
|
-
next if deploy == "." or deploy == ".." or !Dir.exist?(this_deploy_dir)
|
|
3113
|
-
next if deploy_id and deploy_id != deploy
|
|
3114
|
-
|
|
3115
|
-
if !File.size?(this_deploy_dir+"/deployment.json")
|
|
3116
|
-
MU.log "#{this_deploy_dir}/deployment.json doesn't exist, skipping when loading cache", MU::DEBUG
|
|
3117
|
-
next
|
|
3118
|
-
end
|
|
3119
|
-
if @deploy_cache[deploy].nil? or !use_cache
|
|
3120
|
-
@deploy_cache[deploy] = Hash.new
|
|
3121
|
-
elsif @deploy_cache[deploy]['mtime'] == File.mtime("#{this_deploy_dir}/deployment.json")
|
|
3122
|
-
MU.log "Using cached copy of deploy #{deploy} from #{@deploy_cache[deploy]['mtime']}", MU::DEBUG
|
|
3123
|
-
|
|
3124
|
-
next
|
|
3125
|
-
end
|
|
3126
|
-
|
|
3127
|
-
@deploy_cache[deploy] = Hash.new if !@deploy_cache.has_key?(deploy)
|
|
3128
|
-
MU.log "Caching deploy #{deploy}", MU::DEBUG
|
|
3129
|
-
lock = File.open("#{this_deploy_dir}/deployment.json", File::RDONLY)
|
|
3130
|
-
lock.flock(File::LOCK_EX)
|
|
3131
|
-
@deploy_cache[deploy]['mtime'] = File.mtime("#{this_deploy_dir}/deployment.json")
|
|
3132
|
-
|
|
3133
|
-
begin
|
|
3134
|
-
@deploy_cache[deploy]['data'] = JSON.parse(File.read("#{this_deploy_dir}/deployment.json"))
|
|
3135
|
-
lock.flock(File::LOCK_UN)
|
|
3136
|
-
|
|
3137
|
-
next if @deploy_cache[deploy].nil? or @deploy_cache[deploy]['data'].nil?
|
|
3138
|
-
# Populate some generable entries that should be in the deploy
|
|
3139
|
-
# data. Also, bounce out if we realize we've found exactly what
|
|
3140
|
-
# we needed already.
|
|
3141
|
-
MU::Cloud.resource_types.values.each { |attrs|
|
|
3142
|
-
|
|
3143
|
-
next if @deploy_cache[deploy]['data'][attrs[:cfg_plural]].nil?
|
|
3144
|
-
if !attrs[:has_multiples]
|
|
3145
|
-
@deploy_cache[deploy]['data'][attrs[:cfg_plural]].each_pair { |nodename, data|
|
|
3146
|
-
# XXX we don't actually store node names for some resources, need to farm them
|
|
3147
|
-
# and fix metadata
|
|
3148
|
-
# if !mu_name.nil? and nodename == mu_name
|
|
3149
|
-
# return { deploy => [data] }
|
|
3150
|
-
# end
|
|
3151
|
-
}
|
|
3152
|
-
else
|
|
3153
|
-
@deploy_cache[deploy]['data'][attrs[:cfg_plural]].each_pair { |node_class, nodes|
|
|
3154
|
-
next if nodes.nil? or !nodes.is_a?(Hash)
|
|
3155
|
-
nodes.each_pair { |nodename, data|
|
|
3156
|
-
next if !data.is_a?(Hash)
|
|
3157
|
-
data['#MU_NODE_CLASS'] = node_class
|
|
3158
|
-
if !data.has_key?("cloud") # XXX kludge until old metadata gets fixed
|
|
3159
|
-
data["cloud"] = MU::Config.defaultCloud
|
|
3160
|
-
end
|
|
3161
|
-
data['#MU_NAME'] = nodename
|
|
3162
|
-
if !mu_name.nil? and nodename == mu_name
|
|
3163
|
-
return {deploy => [data]} if deploy_id && deploy == deploy_id
|
|
3164
|
-
end
|
|
3165
|
-
}
|
|
3166
|
-
}
|
|
3167
|
-
end
|
|
3168
|
-
}
|
|
3169
|
-
rescue JSON::ParserError => e
|
|
3170
|
-
raise MuError, "JSON parse failed on #{this_deploy_dir}/deployment.json\n\n"+File.read("#{this_deploy_dir}/deployment.json")
|
|
3171
|
-
end
|
|
3172
|
-
lock.flock(File::LOCK_UN)
|
|
3173
|
-
lock.close
|
|
3174
|
-
}
|
|
3175
|
-
end
|
|
3176
|
-
}
|
|
3177
|
-
|
|
3178
|
-
matches = {}
|
|
3179
|
-
|
|
3180
|
-
if deploy_id.nil?
|
|
3181
|
-
@deploy_cache.each_key { |deploy|
|
|
3182
|
-
next if !@deploy_cache[deploy].has_key?('data')
|
|
3183
|
-
next if !@deploy_cache[deploy]['data'].has_key?(type)
|
|
3184
|
-
if !name.nil?
|
|
3185
|
-
next if @deploy_cache[deploy]['data'][type][name].nil?
|
|
3186
|
-
matches[deploy] ||= []
|
|
3187
|
-
matches[deploy] << @deploy_cache[deploy]['data'][type][name].dup
|
|
3188
|
-
else
|
|
3189
|
-
matches[deploy] ||= []
|
|
3190
|
-
matches[deploy].concat(@deploy_cache[deploy]['data'][type].values)
|
|
3191
|
-
end
|
|
3192
|
-
}
|
|
3193
|
-
return matches
|
|
3194
|
-
elsif !@deploy_cache[deploy_id].nil?
|
|
3195
|
-
if !@deploy_cache[deploy_id]['data'].nil? and
|
|
3196
|
-
!@deploy_cache[deploy_id]['data'][type].nil?
|
|
3197
|
-
if !name.nil?
|
|
3198
|
-
if !@deploy_cache[deploy_id]['data'][type][name].nil?
|
|
3199
|
-
matches[deploy_id] ||= []
|
|
3200
|
-
matches[deploy_id] << @deploy_cache[deploy_id]['data'][type][name].dup
|
|
3201
|
-
else
|
|
3202
|
-
return matches # nothing, actually
|
|
3203
|
-
end
|
|
3204
|
-
else
|
|
3205
|
-
matches[deploy_id] = @deploy_cache[deploy_id]['data'][type].values
|
|
3206
|
-
end
|
|
3207
|
-
end
|
|
3208
|
-
end
|
|
3209
|
-
|
|
3210
|
-
return matches
|
|
3211
|
-
end
|
|
3212
|
-
|
|
3213
1609
|
###########################################################################
|
|
3214
1610
|
###########################################################################
|
|
3215
|
-
def
|
|
3216
|
-
|
|
3217
|
-
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
rescue Timeout::Error
|
|
3224
|
-
raise MuError, "Timed out trying to get an exclusive lock on #{deploy_dir}/deployment.json"
|
|
3225
|
-
end
|
|
3226
|
-
|
|
3227
|
-
begin
|
|
3228
|
-
@deployment = JSON.parse(File.read("#{deploy_dir}/deployment.json"))
|
|
3229
|
-
rescue JSON::ParserError => e
|
|
3230
|
-
MU.log "JSON parse failed on #{deploy_dir}/deployment.json", MU::ERR, details: e.message
|
|
3231
|
-
end
|
|
3232
|
-
|
|
3233
|
-
deploy.flock(File::LOCK_UN)
|
|
3234
|
-
deploy.close
|
|
3235
|
-
if set_context_to_me
|
|
3236
|
-
["appname", "environment", "timestamp", "seed", "handle"].each { |var|
|
|
3237
|
-
@deployment[var] ||= instance_variable_get("@#{var}".to_sym)
|
|
3238
|
-
if @deployment[var]
|
|
3239
|
-
if var != "handle"
|
|
3240
|
-
MU.setVar(var, @deployment[var].upcase)
|
|
3241
|
-
else
|
|
3242
|
-
MU.setVar(var, @deployment[var])
|
|
3243
|
-
end
|
|
3244
|
-
else
|
|
3245
|
-
MU.log "Missing global variable #{var} for #{MU.deploy_id}", MU::ERR
|
|
3246
|
-
end
|
|
3247
|
-
}
|
|
3248
|
-
end
|
|
3249
|
-
@timestamp = @deployment['timestamp']
|
|
3250
|
-
@seed = @deployment['seed']
|
|
3251
|
-
@appname = @deployment['appname']
|
|
3252
|
-
@handle = @deployment['handle']
|
|
3253
|
-
|
|
3254
|
-
return if deployment_json_only
|
|
3255
|
-
end
|
|
3256
|
-
if File.exist?(deploy_dir+"/private_key")
|
|
3257
|
-
@private_key = File.read("#{deploy_dir}/private_key")
|
|
3258
|
-
@public_key = File.read("#{deploy_dir}/public_key")
|
|
3259
|
-
end
|
|
3260
|
-
if File.exist?(deploy_dir+"/basket_of_kittens.json")
|
|
3261
|
-
begin
|
|
3262
|
-
@original_config = JSON.parse(File.read("#{deploy_dir}/basket_of_kittens.json"))
|
|
3263
|
-
rescue JSON::ParserError => e
|
|
3264
|
-
MU.log "JSON parse failed on #{deploy_dir}/basket_of_kittens.json", MU::ERR, details: e.message
|
|
1611
|
+
def setThreadContextToMe
|
|
1612
|
+
["appname", "environment", "timestamp", "seed", "handle"].each { |var|
|
|
1613
|
+
@deployment[var] ||= instance_variable_get("@#{var}".to_sym)
|
|
1614
|
+
if @deployment[var]
|
|
1615
|
+
if var != "handle"
|
|
1616
|
+
MU.setVar(var, @deployment[var].upcase)
|
|
1617
|
+
else
|
|
1618
|
+
MU.setVar(var, @deployment[var])
|
|
3265
1619
|
end
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
@ssh_key_name = File.read("#{deploy_dir}/ssh_key_name").chomp!
|
|
3269
|
-
end
|
|
3270
|
-
if File.exist?(deploy_dir+"/node_ssh.key")
|
|
3271
|
-
@ssh_private_key = File.read("#{deploy_dir}/node_ssh.key")
|
|
3272
|
-
end
|
|
3273
|
-
if File.exist?(deploy_dir+"/node_ssh.pub")
|
|
3274
|
-
@ssh_public_key = File.read("#{deploy_dir}/node_ssh.pub")
|
|
3275
|
-
end
|
|
3276
|
-
if File.exist?(deploy_dir+"/environment_name")
|
|
3277
|
-
@environment = File.read("#{deploy_dir}/environment_name").chomp!
|
|
3278
|
-
end
|
|
3279
|
-
if File.exist?(deploy_dir+"/deploy_secret")
|
|
3280
|
-
@deploy_secret = File.read("#{deploy_dir}/deploy_secret")
|
|
3281
|
-
end
|
|
3282
|
-
if Dir.exist?("#{deploy_dir}/secrets")
|
|
3283
|
-
@secrets.each_key { |type|
|
|
3284
|
-
Dir.glob("#{deploy_dir}/secrets/#{type}.*") { |filename|
|
|
3285
|
-
server = File.basename(filename).split(/\./)[1]
|
|
3286
|
-
|
|
3287
|
-
@secrets[type][server] = File.read(filename).chomp!
|
|
3288
|
-
}
|
|
3289
|
-
}
|
|
1620
|
+
else
|
|
1621
|
+
MU.log "Missing global variable #{var} for #{MU.deploy_id}", MU::ERR
|
|
3290
1622
|
end
|
|
3291
1623
|
}
|
|
3292
1624
|
end
|
|
3293
1625
|
|
|
3294
|
-
# 2019-06-03 adding things from https://aiweirdness.com/post/185339301987/once-again-a-neural-net-tries-to-name-cats
|
|
3295
|
-
@catadjs = %w{fuzzy ginger lilac chocolate xanthic wiggly itty chonky norty slonky floofy}
|
|
3296
|
-
@catnouns = %w{bastet biscuits bobcat catnip cheetah chonk dot felix hamb jaguar kitty leopard lion lynx maru mittens moggy neko nip ocelot panther patches paws phoebe purr queen roar saber sekhmet skogkatt socks sphinx spot tail tiger tom whiskers wildcat yowl floof beans ailurophile dander dewclaw grimalkin kibble quick tuft misty simba slonk mew quat eek ziggy whiskeridoo cromch monch screm}
|
|
3297
|
-
@catmixed = %w{abyssinian angora bengal birman bobtail bombay burmese calico chartreux cheshire cornish-rex curl devon egyptian-mau feline furever fumbs havana himilayan japanese-bobtail javanese khao-manee maine-coon manx marmalade mau munchkin norwegian pallas persian peterbald polydactyl ragdoll russian-blue savannah scottish-fold serengeti shorthair siamese siberian singapura snowshoe stray tabby tonkinese tortoiseshell turkish-van tuxedo uncia caterwaul lilac-point chocolate-point mackerel maltese knead whitenose vorpal chewie-bean chicken-whiskey fish-especially thelonious-monsieur tom-glitter serendipitous-kill sparky-buttons}
|
|
3298
|
-
@catwords = @catadjs + @catnouns + @catmixed
|
|
3299
|
-
|
|
3300
|
-
@jaegeradjs = %w{azure fearless lucky olive vivid electric grey yarely violet ivory jade cinnamon crimson tacit umber mammoth ultra iron zodiac}
|
|
3301
|
-
@jaegernouns = %w{horizon hulk ultimatum yardarm watchman whilrwind wright rhythm ocean enigma eruption typhoon jaeger brawler blaze vandal excalibur paladin juliet kaleidoscope romeo}
|
|
3302
|
-
@jaegermixed = %w{alpha ajax amber avenger brave bravo charlie chocolate chrome corinthian dancer danger dash delta duet echo edge elite eureka foxtrot guardian gold hyperion illusion imperative india intercept kilo lancer night nova november oscar omega pacer quickstrike rogue ronin striker tango titan valor victor vulcan warder xenomorph xenon xray xylem yankee yell yukon zeal zero zoner zodiac}
|
|
3303
|
-
@jaegerwords = @jaegeradjs + @jaegernouns + @jaegermixed
|
|
3304
|
-
|
|
3305
|
-
@words = @catwords + @jaegerwords
|
|
3306
|
-
|
|
3307
1626
|
end #class
|
|
3308
1627
|
end #module
|