cloud-mu 3.0.0 → 3.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Dockerfile +25 -0
- data/README.md +2 -2
- data/bin/mu-node-manage +1 -1
- data/chefignore +1 -0
- data/cloud-mu.gemspec +2 -2
- data/extras/clean-stock-amis +0 -0
- data/extras/generate-stock-images +0 -0
- data/extras/list-stock-amis +0 -0
- data/extras/vault_tools/export_vaults.sh +0 -0
- data/extras/vault_tools/recreate_vaults.sh +0 -0
- data/extras/vault_tools/test_vaults.sh +0 -0
- data/kitchen.yml +97 -0
- data/modules/mu.rb +14 -2
- data/modules/mu/cloud.rb +26 -0
- data/modules/mu/clouds/aws.rb +1 -2
- data/modules/mu/clouds/aws/container_cluster.rb +2 -1
- data/modules/mu/clouds/aws/database.rb +2 -2
- data/modules/mu/clouds/aws/loadbalancer.rb +1 -1
- data/modules/mu/clouds/aws/search_domain.rb +1 -1
- data/modules/mu/clouds/aws/server.rb +9 -1
- data/modules/mu/clouds/aws/vpc.rb +1 -2
- data/modules/mu/config.rb +83 -3
- data/modules/mu/config/bucket.yml +1 -1
- data/modules/mu/config/cache_cluster.yml +1 -9
- data/modules/mu/config/container_cluster.yml +1 -1
- data/modules/mu/config/database.yml +3 -3
- data/modules/mu/config/log.yml +8 -3
- data/modules/mu/config/msg_queue.yml +1 -1
- data/modules/mu/config/nosqldb.yml +1 -1
- data/modules/mu/config/notifier.yml +1 -1
- data/modules/mu/config/search_domain.yml +2 -2
- data/modules/mu/config/server.yml +23 -3
- data/modules/mu/config/server_pool.yml +5 -2
- data/modules/mu/config/storage_pool.yml +1 -1
- data/modules/mu/config/vpc.rb +6 -6
- data/modules/mu/config/vpc.yml +54 -3
- data/modules/mu/groomers/chef.rb +27 -0
- data/modules/mu/mommacat.rb +113 -18
- metadata +18 -22
- data/Berksfile.lock +0 -179
- data/bin/mu-azure-tests +0 -43
- data/cookbooks/mu-tools/files/default/Mu_CA.pem +0 -33
- data/modules/mu/kittens.rb +0 -20651
- data/modules/mu/mu.yaml.rb +0 -276
- data/modules/scratchpad.erb +0 -1
@@ -1,17 +1,9 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == "complex" %>
|
2
2
|
name: redis
|
3
3
|
engine: redis
|
4
4
|
creation_style: new
|
5
5
|
size: cache.t2.medium
|
6
|
-
name: memcache
|
7
|
-
creation_style: new
|
8
|
-
engine: memcached
|
9
|
-
size: cache.t2.medium
|
10
6
|
<% else %>
|
11
|
-
name: redis
|
12
|
-
engine: redis
|
13
|
-
creation_style: new
|
14
|
-
size: cache.t2.medium
|
15
7
|
name: memcache
|
16
8
|
creation_style: new
|
17
9
|
engine: memcached
|
@@ -1,4 +1,4 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == "complex" %>
|
2
2
|
|
3
3
|
name: database-complex
|
4
4
|
size: db.r4.large
|
@@ -12,7 +12,7 @@ backup_retention_period: 10
|
|
12
12
|
cluster_node_count: 2
|
13
13
|
create_cluster: true
|
14
14
|
vpc:
|
15
|
-
|
15
|
+
name: <%= vpc_name %>
|
16
16
|
create_read_replica: true
|
17
17
|
master_user: Bob
|
18
18
|
multi_az_on_create: true
|
@@ -21,7 +21,7 @@ multi_az_on_create: true
|
|
21
21
|
|
22
22
|
name: database-simple
|
23
23
|
vpc:
|
24
|
-
|
24
|
+
name: <%= vpc_name %>
|
25
25
|
size: <%= db_size %>
|
26
26
|
engine: mariadb
|
27
27
|
storage: 5
|
data/modules/mu/config/log.yml
CHANGED
@@ -1,6 +1,11 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == 'complex' %>
|
2
2
|
name: <%= logs_name %>
|
3
|
-
|
3
|
+
filters:
|
4
|
+
- name: myfilter
|
5
|
+
metric_name: LogMetrics/myfilter
|
6
|
+
namespace: ok
|
7
|
+
search_pattern: failed
|
8
|
+
value: yes
|
4
9
|
<% else %>
|
5
10
|
name: <%= logs_name %>
|
6
|
-
<% end %>
|
11
|
+
<% end %>
|
@@ -1,4 +1,4 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == "complex" %>
|
2
2
|
|
3
3
|
name: searchdomain-complex
|
4
4
|
instance_type: t2.small.elasticsearch
|
@@ -17,7 +17,7 @@ advanced_options:
|
|
17
17
|
# user_pool_id: "us-east-1_eSwWA1VYQ"
|
18
18
|
slow_logs: <%= logs_name %>
|
19
19
|
vpc:
|
20
|
-
|
20
|
+
name: <%= vpc_name %>
|
21
21
|
|
22
22
|
<% else %> # IF NOT COMPLEX THEN ASSUME SIMPLE
|
23
23
|
|
@@ -1,11 +1,31 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == 'complex' %>
|
2
2
|
name: servercomplex
|
3
3
|
size: <%= instance_type %>
|
4
4
|
vpc:
|
5
|
-
|
5
|
+
name: <%= vpc_name %>
|
6
|
+
subnet_pref: public
|
7
|
+
platform: ubuntu
|
8
|
+
ssh_user: ubuntu
|
9
|
+
associate_public_ip: true
|
10
|
+
canned_iam_policies:
|
11
|
+
- AmazonDynamoDBReadOnlyAccess
|
12
|
+
- AmazonElastiCacheFullAccess
|
13
|
+
- AWSLambdaExecute
|
14
|
+
groomer: Ansible
|
15
|
+
run_list:
|
16
|
+
- geerlingguy.java
|
17
|
+
- geerlingguy.nginx
|
18
|
+
- sensu.sensu
|
19
|
+
tags:
|
20
|
+
- key: ThisIsATag
|
21
|
+
value: ThisIsAValue
|
22
|
+
src_dst_check: false
|
23
|
+
storage:
|
24
|
+
- device: /dev/xvdg
|
25
|
+
size: 50
|
6
26
|
<% else %>
|
7
27
|
name: serversimple
|
8
28
|
size: <%= instance_type %>
|
9
29
|
vpc:
|
10
|
-
|
30
|
+
name: <%= vpc_name %>
|
11
31
|
<% end %>
|
@@ -1,8 +1,8 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == 'complex' %>
|
2
2
|
name: <%= server_pools_name %>
|
3
3
|
cloud: AWS
|
4
4
|
vpc:
|
5
|
-
|
5
|
+
name: <%= vpc_name %>
|
6
6
|
alarms:
|
7
7
|
- comparison_operator: "GreaterThanThreshold"
|
8
8
|
metric_name: "HTTPCode_Target_5XX_Count"
|
@@ -66,6 +66,9 @@ basis:
|
|
66
66
|
name: <%= server_pools_name %>
|
67
67
|
min_size: 1
|
68
68
|
max_size: 1
|
69
|
+
vpc:
|
70
|
+
name: <%= vpc_name %>
|
71
|
+
subnet_pref: public
|
69
72
|
basis:
|
70
73
|
launch_config:
|
71
74
|
name: <%= server_pools_name %>
|
data/modules/mu/config/vpc.rb
CHANGED
@@ -390,16 +390,16 @@ module MU
|
|
390
390
|
"description" => "The ID of a VPN, NAT, or Internet gateway attached to your VPC. #INTERNET will refer to this VPC's default internet gateway, if one exists. #NAT will refer to a this VPC's NAT gateway, and will implicitly create one if none exists. #DENY will ensure that the subnets associated with this route do *not* have a route outside of the VPC's local address space (primarily for Google Cloud, where we must explicitly disable egress to the internet)."
|
391
391
|
},
|
392
392
|
"nat_host_id" => {
|
393
|
-
|
394
|
-
|
393
|
+
"type" => "string",
|
394
|
+
"description" => "The instance id of a NAT host in this VPC."
|
395
395
|
},
|
396
396
|
"nat_host_name" => {
|
397
|
-
|
398
|
-
|
397
|
+
"type" => "string",
|
398
|
+
"description" => "The MU resource name or Name tag of a NAT host in this VPC."
|
399
399
|
},
|
400
400
|
"interface" => {
|
401
|
-
|
402
|
-
|
401
|
+
"type" => "string",
|
402
|
+
"description" => "A network interface over which to route."
|
403
403
|
}
|
404
404
|
}
|
405
405
|
}
|
data/modules/mu/config/vpc.yml
CHANGED
@@ -1,6 +1,57 @@
|
|
1
|
-
<% if
|
1
|
+
<% if complexity == 'complex' %>
|
2
2
|
name: <%= vpc_name %>
|
3
|
-
|
3
|
+
create_nat_gateway: true
|
4
|
+
ip_block: 10.231.0.0/16
|
5
|
+
enable_traffic_logging: true
|
6
|
+
region: us-east-2
|
7
|
+
availability_zones:
|
8
|
+
- us-east-2a
|
9
|
+
- us-east-2c
|
10
|
+
- us-east-2e
|
11
|
+
route_tables:
|
12
|
+
- name: public
|
13
|
+
routes:
|
14
|
+
- destination_network: 0.0.0.0/0
|
15
|
+
gateway: "#INTERNET"
|
16
|
+
- name: private
|
17
|
+
routes:
|
18
|
+
- destination_network: 0.0.0.0/0
|
19
|
+
gateway: "#NAT"
|
20
|
+
subnets:
|
21
|
+
- name: Subnet0Internet
|
22
|
+
availability_zone: us-east-2a
|
23
|
+
ip_block: 10.0.0.0/19
|
24
|
+
route_table: internet
|
25
|
+
map_public_ips: true
|
26
|
+
create_nat_gateway: true
|
27
|
+
- name: Subnet0Private
|
28
|
+
availability_zone: us-east-2a
|
29
|
+
ip_block: 10.0.32.0/19
|
30
|
+
route_table: private
|
31
|
+
- name: Subnet1Internet
|
32
|
+
availability_zone: us-east-2c
|
33
|
+
ip_block: 10.0.64.0/19
|
34
|
+
route_table: internet
|
35
|
+
map_public_ips: true
|
36
|
+
- name: Subnet1Private
|
37
|
+
availability_zone: us-east-2c
|
38
|
+
ip_block: 10.0.96.0/19
|
39
|
+
route_table: private
|
40
|
+
- name: Subnet2Internet
|
41
|
+
availability_zone: us-east-2e
|
42
|
+
ip_block: 10.0.128.0/19
|
43
|
+
route_table: internet
|
44
|
+
map_public_ips: true
|
45
|
+
- name: Subnet2Private
|
46
|
+
availability_zone: us-east-2e
|
47
|
+
route_table: private
|
48
|
+
ip_block: 10.0.160.0/19
|
49
|
+
- name: NonRoutable1
|
50
|
+
availability_zone: us-east-2a
|
51
|
+
ip_block: 10.0.192.0/19
|
52
|
+
- name: NonRoutable2
|
53
|
+
availability_zone: us-east-2c
|
54
|
+
ip_block: 10.0.224.0/19
|
4
55
|
<% else %>
|
5
56
|
name: <%= vpc_name %>
|
6
|
-
<% end %>
|
57
|
+
<% end %>
|
data/modules/mu/groomers/chef.rb
CHANGED
@@ -48,11 +48,13 @@ module MU
|
|
48
48
|
require 'chef'
|
49
49
|
require 'chef/api_client_v1'
|
50
50
|
require 'chef/knife'
|
51
|
+
require 'chef/application/knife'
|
51
52
|
require 'chef/knife/ssh'
|
52
53
|
require 'chef/knife/bootstrap'
|
53
54
|
require 'chef/knife/node_delete'
|
54
55
|
require 'chef/knife/client_delete'
|
55
56
|
require 'chef/knife/data_bag_delete'
|
57
|
+
require 'chef/knife/data_bag_show'
|
56
58
|
require 'chef/knife/vault_delete'
|
57
59
|
require 'chef/scan_access_control'
|
58
60
|
require 'chef/file_access_control/unix'
|
@@ -827,6 +829,31 @@ retry
|
|
827
829
|
|
828
830
|
return if nodeonly
|
829
831
|
|
832
|
+
vaults_to_clean.each { |vault|
|
833
|
+
MU::MommaCat.lock("vault-#{vault['vault']}", false, true)
|
834
|
+
MU.log "Purging unknown clients from #{vault['vault']} #{vault['item']}", MU::DEBUG
|
835
|
+
output = %x{#{@knife} data bag show "#{vault['vault']}" "#{vault['item']}_keys" --format json}
|
836
|
+
# This is an ugly workaround for --clean-unknown-clients, which in
|
837
|
+
# fact cleans known clients.
|
838
|
+
if output
|
839
|
+
begin
|
840
|
+
vault_cfg = JSON.parse(output)
|
841
|
+
if vault_cfg['clients']
|
842
|
+
searchstr = vault_cfg['clients'].map { |c| "name:"+c }.join(" OR ")
|
843
|
+
MU.log "Preserving client list for vault #{vault['vault']} #{vault['item']}", MU::DEBUG, details: vault_cfg['clients']
|
844
|
+
if !noop
|
845
|
+
::Chef::Knife.run(['vault', 'rotate', 'keys', vault['vault'], vault['item'], "--clean-unknown-clients"])
|
846
|
+
::Chef::Knife.run(['vault', 'update', vault['vault'], vault['item'], "--search", searchstr])
|
847
|
+
::Chef::Knife.run(['vault', 'refresh', vault['vault'], vault['item']])
|
848
|
+
end
|
849
|
+
end
|
850
|
+
rescue JSON::ParserError => e
|
851
|
+
MU.log "Error parsing JSON from data bag #{vault['vault']} #{vault['item']}_keys, skipping vault client cleanse", MU::WARN
|
852
|
+
end
|
853
|
+
end
|
854
|
+
MU::MommaCat.unlock("vault-#{vault['vault']}")
|
855
|
+
}
|
856
|
+
|
830
857
|
begin
|
831
858
|
deleteSecret(vault: node) if !noop
|
832
859
|
rescue MuNoSuchSecret
|
data/modules/mu/mommacat.rb
CHANGED
@@ -39,6 +39,7 @@ module MU
|
|
39
39
|
end
|
40
40
|
|
41
41
|
@@litters = {}
|
42
|
+
@@litters_loadtime = {}
|
42
43
|
@@litter_semaphore = Mutex.new
|
43
44
|
|
44
45
|
# Return a {MU::MommaCat} instance for an existing deploy. Use this instead
|
@@ -62,17 +63,32 @@ module MU
|
|
62
63
|
@@litter_semaphore.synchronize {
|
63
64
|
littercache = @@litters.dup
|
64
65
|
}
|
66
|
+
if littercache[deploy_id] and @@litters_loadtime[deploy_id]
|
67
|
+
deploy_root = File.expand_path(MU.dataDir+"/deployments")
|
68
|
+
this_deploy_dir = deploy_root+"/"+deploy_id
|
69
|
+
if File.exist?("#{this_deploy_dir}/deployment.json")
|
70
|
+
lastmod = File.mtime("#{this_deploy_dir}/deployment.json")
|
71
|
+
if lastmod > @@litters_loadtime[deploy_id]
|
72
|
+
MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE
|
73
|
+
use_cache = false
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
65
77
|
rescue ThreadError => e
|
66
78
|
# already locked by a parent caller and this is a read op, so this is ok
|
67
79
|
raise e if !e.message.match(/recursive locking/)
|
68
80
|
littercache = @@litters.dup
|
69
81
|
end
|
82
|
+
|
70
83
|
if !use_cache or littercache[deploy_id].nil?
|
84
|
+
need_gc = !littercache[deploy_id].nil?
|
71
85
|
newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
|
72
86
|
# This, we have to synchronize, as it's a write
|
73
87
|
@@litter_semaphore.synchronize {
|
74
|
-
@@litters[deploy_id]
|
88
|
+
@@litters[deploy_id] = newlitter
|
89
|
+
@@litters_loadtime[deploy_id] = Time.now
|
75
90
|
}
|
91
|
+
GC.start if need_gc
|
76
92
|
elsif set_context_to_me
|
77
93
|
MU::MommaCat.setThreadContext(@@litters[deploy_id])
|
78
94
|
end
|
@@ -80,6 +96,18 @@ module MU
|
|
80
96
|
# MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
|
81
97
|
end
|
82
98
|
|
99
|
+
# Update the in-memory cache of a given deploy. This is intended for use by
|
100
|
+
# {#save!}, primarily.
|
101
|
+
# @param deploy_id [String]
|
102
|
+
# @param litter [MU::MommaCat]
|
103
|
+
def self.updateLitter(deploy_id, litter)
|
104
|
+
return if litter.nil?
|
105
|
+
@@litter_semaphore.synchronize {
|
106
|
+
@@litters[deploy_id] = litter
|
107
|
+
@@litters_loadtime[deploy_id] = Time.now
|
108
|
+
}
|
109
|
+
end
|
110
|
+
|
83
111
|
attr_reader :initializing
|
84
112
|
attr_reader :public_key
|
85
113
|
attr_reader :deploy_secret
|
@@ -633,8 +661,9 @@ module MU
|
|
633
661
|
# @param max_length [Integer]: The maximum length of the resulting resource name.
|
634
662
|
# @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated.
|
635
663
|
# @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
|
664
|
+
# @param allowed_chars [Regexp]: A pattern of characters that are legal for this resource name, such as +/[a-zA-Z0-9-]/+
|
636
665
|
# @return [String]: A full name string for this resource
|
637
|
-
def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'])
|
666
|
+
def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], allowed_chars: nil)
|
638
667
|
if name.nil?
|
639
668
|
raise MuError, "Got no argument to MU::MommaCat.getResourceName"
|
640
669
|
end
|
@@ -657,6 +686,19 @@ module MU
|
|
657
686
|
basename = @appname.upcase + "-" + @environment.upcase + name.upcase
|
658
687
|
end
|
659
688
|
|
689
|
+
subchar = if allowed_chars
|
690
|
+
if !"-".match(allowed_chars)
|
691
|
+
if "_".match(allowed_chars)
|
692
|
+
"_"
|
693
|
+
else
|
694
|
+
""
|
695
|
+
end
|
696
|
+
else
|
697
|
+
"-"
|
698
|
+
end
|
699
|
+
end
|
700
|
+
|
701
|
+
basename.gsub!(allowed_chars, subchar) if allowed_chars
|
660
702
|
begin
|
661
703
|
if (basename.length + reserved) > max_length
|
662
704
|
MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG
|
@@ -669,6 +711,7 @@ module MU
|
|
669
711
|
basename.slice!((max_length-(reserved+3))..basename.length)
|
670
712
|
basename.sub!(/-$/, "")
|
671
713
|
basename = basename + "-" + @seed.upcase
|
714
|
+
basename.gsub!(allowed_chars, subchar) if allowed_chars
|
672
715
|
else
|
673
716
|
# If we have to strip anything, assume we've lost uniqueness and
|
674
717
|
# will have to compensate with #genUniquenessString.
|
@@ -676,6 +719,7 @@ module MU
|
|
676
719
|
reserved = 4
|
677
720
|
basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
|
678
721
|
basename = basename + "-" + @seed.upcase + "-" + name.upcase
|
722
|
+
basename.gsub!(allowed_chars, subchar) if allowed_chars
|
679
723
|
end
|
680
724
|
end
|
681
725
|
end while (basename.length + reserved) > max_length
|
@@ -702,6 +746,7 @@ module MU
|
|
702
746
|
else
|
703
747
|
muname = basename
|
704
748
|
end
|
749
|
+
muname.gsub!(allowed_chars, subchar) if allowed_chars
|
705
750
|
|
706
751
|
return muname
|
707
752
|
end
|
@@ -901,7 +946,7 @@ module MU
|
|
901
946
|
if MU.myCloud == "AWS"
|
902
947
|
MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
|
903
948
|
end
|
904
|
-
MU::MommaCat.getLitter(MU.deploy_id
|
949
|
+
MU::MommaCat.getLitter(MU.deploy_id)
|
905
950
|
MU::MommaCat.syncMonitoringConfig(false)
|
906
951
|
MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})"
|
907
952
|
FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt")
|
@@ -927,7 +972,7 @@ module MU
|
|
927
972
|
MU.log "Creating #{ssh_dir}", MU::DEBUG
|
928
973
|
Dir.mkdir(ssh_dir, 0700)
|
929
974
|
if Process.uid == 0 and @mu_user != "mu"
|
930
|
-
|
975
|
+
FileUtils.chown Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir
|
931
976
|
end
|
932
977
|
end
|
933
978
|
if !File.exist?("#{ssh_dir}/#{@ssh_key_name}")
|
@@ -1102,23 +1147,28 @@ module MU
|
|
1102
1147
|
|
1103
1148
|
# Iterate over all known deployments and look for instances that have been
|
1104
1149
|
# terminated, but not yet cleaned up, then clean them up.
|
1105
|
-
def self.cleanTerminatedInstances
|
1150
|
+
def self.cleanTerminatedInstances(debug = false)
|
1151
|
+
loglevel = debug ? MU::NOTICE : MU::DEBUG
|
1106
1152
|
MU::MommaCat.lock("clean-terminated-instances", false, true)
|
1107
|
-
MU.log "Checking for harvested instances in need of cleanup",
|
1153
|
+
MU.log "Checking for harvested instances in need of cleanup", loglevel
|
1108
1154
|
parent_thread_id = Thread.current.object_id
|
1109
1155
|
purged = 0
|
1156
|
+
|
1110
1157
|
MU::MommaCat.listDeploys.each { |deploy_id|
|
1111
1158
|
next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
|
1112
|
-
MU.log "Checking for dead wood in #{deploy_id}",
|
1159
|
+
MU.log "Checking for dead wood in #{deploy_id}", loglevel
|
1113
1160
|
need_reload = false
|
1114
1161
|
@cleanup_threads << Thread.new {
|
1115
1162
|
MU.dupGlobals(parent_thread_id)
|
1116
1163
|
deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
|
1117
1164
|
purged_this_deploy = 0
|
1165
|
+
MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys
|
1118
1166
|
if deploy.kittens.has_key?("servers")
|
1167
|
+
MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id
|
1119
1168
|
deploy.kittens["servers"].values.each { |nodeclasses|
|
1120
1169
|
nodeclasses.each_pair { |nodeclass, servers|
|
1121
1170
|
deletia = []
|
1171
|
+
MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys
|
1122
1172
|
servers.each_pair { |mu_name, server|
|
1123
1173
|
server.describe
|
1124
1174
|
if !server.cloud_id
|
@@ -1145,15 +1195,16 @@ module MU
|
|
1145
1195
|
servers.delete(mu_name)
|
1146
1196
|
}
|
1147
1197
|
if purged_this_deploy > 0
|
1148
|
-
# XXX
|
1149
|
-
deploy.syncLitter(servers.keys)
|
1198
|
+
# XXX triggering_node needs to take more than one node name
|
1199
|
+
deploy.syncLitter(servers.keys, triggering_node: deletia.first)
|
1150
1200
|
end
|
1151
1201
|
}
|
1152
1202
|
}
|
1153
1203
|
end
|
1154
1204
|
if need_reload
|
1205
|
+
MU.log "Saving modified deploy #{deploy_id}", loglevel
|
1155
1206
|
deploy.save!
|
1156
|
-
MU::MommaCat.getLitter(deploy_id
|
1207
|
+
MU::MommaCat.getLitter(deploy_id)
|
1157
1208
|
end
|
1158
1209
|
MU.purgeGlobals
|
1159
1210
|
}
|
@@ -1161,6 +1212,8 @@ module MU
|
|
1161
1212
|
@cleanup_threads.each { |t|
|
1162
1213
|
t.join
|
1163
1214
|
}
|
1215
|
+
MU.log "cleanTerminatedInstances threads complete", loglevel
|
1216
|
+
MU::MommaCat.unlock("clean-terminated-instances", true)
|
1164
1217
|
@cleanup_threads = []
|
1165
1218
|
|
1166
1219
|
if purged > 0
|
@@ -1168,8 +1221,9 @@ module MU
|
|
1168
1221
|
MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
|
1169
1222
|
end
|
1170
1223
|
MU::MommaCat.syncMonitoringConfig
|
1224
|
+
GC.start
|
1171
1225
|
end
|
1172
|
-
MU
|
1226
|
+
MU.log "cleanTerminatedInstances returning", loglevel
|
1173
1227
|
end
|
1174
1228
|
|
1175
1229
|
@@dummy_cache = {}
|
@@ -1890,22 +1944,51 @@ end
|
|
1890
1944
|
}
|
1891
1945
|
end
|
1892
1946
|
|
1947
|
+
# Clean an IP address out of ~/.ssh/known hosts
|
1948
|
+
# @param ip [String]: The IP to remove
|
1949
|
+
# @return [void]
|
1950
|
+
def self.removeIPFromSSHKnownHosts(ip)
|
1951
|
+
return if ip.nil?
|
1952
|
+
sshdir = "#{@myhome}/.ssh"
|
1953
|
+
knownhosts = "#{sshdir}/known_hosts"
|
1954
|
+
|
1955
|
+
if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
|
1956
|
+
MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
|
1957
|
+
if !@noop
|
1958
|
+
File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
|
1959
|
+
f.flock(File::LOCK_EX)
|
1960
|
+
newlines = Array.new
|
1961
|
+
delete_block = false
|
1962
|
+
f.readlines.each { |line|
|
1963
|
+
next if line.match(/^#{Regexp.quote(ip)} /)
|
1964
|
+
newlines << line
|
1965
|
+
}
|
1966
|
+
f.rewind
|
1967
|
+
f.truncate(0)
|
1968
|
+
f.puts(newlines)
|
1969
|
+
f.flush
|
1970
|
+
f.flock(File::LOCK_UN)
|
1971
|
+
}
|
1972
|
+
end
|
1973
|
+
end
|
1974
|
+
end
|
1975
|
+
|
1893
1976
|
# Clean a node's entries out of ~/.ssh/config
|
1894
|
-
# @param
|
1977
|
+
# @param nodename [String]: The node's name
|
1895
1978
|
# @return [void]
|
1896
|
-
def self.removeHostFromSSHConfig(
|
1979
|
+
def self.removeHostFromSSHConfig(nodename)
|
1897
1980
|
sshdir = "#{@myhome}/.ssh"
|
1898
1981
|
sshconf = "#{sshdir}/config"
|
1899
1982
|
|
1900
|
-
if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{
|
1901
|
-
MU.log "Expunging old #{
|
1983
|
+
if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /)
|
1984
|
+
MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
|
1902
1985
|
if !@noop
|
1903
1986
|
File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
|
1904
1987
|
f.flock(File::LOCK_EX)
|
1905
1988
|
newlines = Array.new
|
1906
1989
|
delete_block = false
|
1907
1990
|
f.readlines.each { |line|
|
1908
|
-
if line.match(/^Host #{
|
1991
|
+
if line.match(/^Host #{nodename}(\s|$)/)
|
1909
1992
|
delete_block = true
|
1910
1993
|
elsif line.match(/^Host /)
|
1911
1994
|
delete_block = false
|
@@ -1986,6 +2069,9 @@ end
|
|
1986
2069
|
end
|
1987
2070
|
|
1988
2071
|
MU::MommaCat.removeHostFromSSHConfig(node)
|
2072
|
+
if server and server.canonicalIP
|
2073
|
+
MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP)
|
2074
|
+
end
|
1989
2075
|
# XXX add names paramater with useful stuff
|
1990
2076
|
MU::MommaCat.addHostToSSHConfig(
|
1991
2077
|
server,
|
@@ -2574,7 +2660,7 @@ MESSAGE_END
|
|
2574
2660
|
update_servers = update_servers - skip
|
2575
2661
|
end
|
2576
2662
|
|
2577
|
-
return if update_servers.size < 1
|
2663
|
+
return if MU.inGem? || update_servers.size < 1
|
2578
2664
|
threads = []
|
2579
2665
|
parent_thread_id = Thread.current.object_id
|
2580
2666
|
update_servers.each { |sibling|
|
@@ -2690,9 +2776,16 @@ MESSAGE_END
|
|
2690
2776
|
Dir.chdir(MU.myRoot+"/modules")
|
2691
2777
|
|
2692
2778
|
# XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
|
2693
|
-
|
2779
|
+
if MU.inGem?
|
2780
|
+
cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
|
2781
|
+
else
|
2782
|
+
cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
|
2783
|
+
end
|
2784
|
+
|
2694
2785
|
MU.log cmd, MU::NOTICE
|
2786
|
+
|
2695
2787
|
output = %x{#{cmd}}
|
2788
|
+
|
2696
2789
|
Dir.chdir(origdir)
|
2697
2790
|
|
2698
2791
|
retries = 0
|
@@ -2782,6 +2875,7 @@ MESSAGE_END
|
|
2782
2875
|
def save!(triggering_node = nil, force: false, origin: nil)
|
2783
2876
|
|
2784
2877
|
return if @no_artifacts and !force
|
2878
|
+
|
2785
2879
|
MU::MommaCat.deploy_struct_semaphore.synchronize {
|
2786
2880
|
MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG
|
2787
2881
|
|
@@ -2834,6 +2928,7 @@ MESSAGE_END
|
|
2834
2928
|
deploy.flock(File::LOCK_UN)
|
2835
2929
|
deploy.close
|
2836
2930
|
@need_deploy_flush = false
|
2931
|
+
MU::MommaCat.updateLitter(@deploy_id, self)
|
2837
2932
|
end
|
2838
2933
|
|
2839
2934
|
if !@original_config.nil? and @original_config.is_a?(Hash)
|