cloud-mu 3.0.0 → 3.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +25 -0
  3. data/README.md +2 -2
  4. data/bin/mu-node-manage +1 -1
  5. data/chefignore +1 -0
  6. data/cloud-mu.gemspec +2 -2
  7. data/extras/clean-stock-amis +0 -0
  8. data/extras/generate-stock-images +0 -0
  9. data/extras/list-stock-amis +0 -0
  10. data/extras/vault_tools/export_vaults.sh +0 -0
  11. data/extras/vault_tools/recreate_vaults.sh +0 -0
  12. data/extras/vault_tools/test_vaults.sh +0 -0
  13. data/kitchen.yml +97 -0
  14. data/modules/mu.rb +14 -2
  15. data/modules/mu/cloud.rb +26 -0
  16. data/modules/mu/clouds/aws.rb +1 -2
  17. data/modules/mu/clouds/aws/container_cluster.rb +2 -1
  18. data/modules/mu/clouds/aws/database.rb +2 -2
  19. data/modules/mu/clouds/aws/loadbalancer.rb +1 -1
  20. data/modules/mu/clouds/aws/search_domain.rb +1 -1
  21. data/modules/mu/clouds/aws/server.rb +9 -1
  22. data/modules/mu/clouds/aws/vpc.rb +1 -2
  23. data/modules/mu/config.rb +83 -3
  24. data/modules/mu/config/bucket.yml +1 -1
  25. data/modules/mu/config/cache_cluster.yml +1 -9
  26. data/modules/mu/config/container_cluster.yml +1 -1
  27. data/modules/mu/config/database.yml +3 -3
  28. data/modules/mu/config/log.yml +8 -3
  29. data/modules/mu/config/msg_queue.yml +1 -1
  30. data/modules/mu/config/nosqldb.yml +1 -1
  31. data/modules/mu/config/notifier.yml +1 -1
  32. data/modules/mu/config/search_domain.yml +2 -2
  33. data/modules/mu/config/server.yml +23 -3
  34. data/modules/mu/config/server_pool.yml +5 -2
  35. data/modules/mu/config/storage_pool.yml +1 -1
  36. data/modules/mu/config/vpc.rb +6 -6
  37. data/modules/mu/config/vpc.yml +54 -3
  38. data/modules/mu/groomers/chef.rb +27 -0
  39. data/modules/mu/mommacat.rb +113 -18
  40. metadata +18 -22
  41. data/Berksfile.lock +0 -179
  42. data/bin/mu-azure-tests +0 -43
  43. data/cookbooks/mu-tools/files/default/Mu_CA.pem +0 -33
  44. data/modules/mu/kittens.rb +0 -20651
  45. data/modules/mu/mu.yaml.rb +0 -276
  46. data/modules/scratchpad.erb +0 -1
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: bucket
3
3
  acl: public-read
4
4
  versioning: true
@@ -1,17 +1,9 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  name: redis
3
3
  engine: redis
4
4
  creation_style: new
5
5
  size: cache.t2.medium
6
- name: memcache
7
- creation_style: new
8
- engine: memcached
9
- size: cache.t2.medium
10
6
  <% else %>
11
- name: redis
12
- engine: redis
13
- creation_style: new
14
- size: cache.t2.medium
15
7
  name: memcache
16
8
  creation_style: new
17
9
  engine: memcached
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  name: k8s
3
3
  flavor: EKS
4
4
  instance_type: t2.medium
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
 
3
3
  name: database-complex
4
4
  size: db.r4.large
@@ -12,7 +12,7 @@ backup_retention_period: 10
12
12
  cluster_node_count: 2
13
13
  create_cluster: true
14
14
  vpc:
15
- vpc_name: <%= vpc_name %>
15
+ name: <%= vpc_name %>
16
16
  create_read_replica: true
17
17
  master_user: Bob
18
18
  multi_az_on_create: true
@@ -21,7 +21,7 @@ multi_az_on_create: true
21
21
 
22
22
  name: database-simple
23
23
  vpc:
24
- vpc_name: <%= vpc_name %>
24
+ name: <%= vpc_name %>
25
25
  size: <%= db_size %>
26
26
  engine: mariadb
27
27
  storage: 5
@@ -1,6 +1,11 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= logs_name %>
3
- # TODO: BUILD OUT COMPLEX EXAMPLE
3
+ filters:
4
+ - name: myfilter
5
+ metric_name: LogMetrics/myfilter
6
+ namespace: ok
7
+ search_pattern: failed
8
+ value: yes
4
9
  <% else %>
5
10
  name: <%= logs_name %>
6
- <% end %>
11
+ <% end %>
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= queues_name %>
3
3
  visibility_timeout: "1 hour"
4
4
  failqueue:
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: complex
3
3
  stream: NEW_IMAGE
4
4
  read_capacity: 50
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: notifier
3
3
  subscriptions:
4
4
  - endpoint: admin@example.com
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
 
3
3
  name: searchdomain-complex
4
4
  instance_type: t2.small.elasticsearch
@@ -17,7 +17,7 @@ advanced_options:
17
17
  # user_pool_id: "us-east-1_eSwWA1VYQ"
18
18
  slow_logs: <%= logs_name %>
19
19
  vpc:
20
- vpc_name: <%= vpc_name %>
20
+ name: <%= vpc_name %>
21
21
 
22
22
  <% else %> # IF NOT COMPLEX THEN ASSUME SIMPLE
23
23
 
@@ -1,11 +1,31 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: servercomplex
3
3
  size: <%= instance_type %>
4
4
  vpc:
5
- vpc_name: <%= vpc_name %>
5
+ name: <%= vpc_name %>
6
+ subnet_pref: public
7
+ platform: ubuntu
8
+ ssh_user: ubuntu
9
+ associate_public_ip: true
10
+ canned_iam_policies:
11
+ - AmazonDynamoDBReadOnlyAccess
12
+ - AmazonElastiCacheFullAccess
13
+ - AWSLambdaExecute
14
+ groomer: Ansible
15
+ run_list:
16
+ - geerlingguy.java
17
+ - geerlingguy.nginx
18
+ - sensu.sensu
19
+ tags:
20
+ - key: ThisIsATag
21
+ value: ThisIsAValue
22
+ src_dst_check: false
23
+ storage:
24
+ - device: /dev/xvdg
25
+ size: 50
6
26
  <% else %>
7
27
  name: serversimple
8
28
  size: <%= instance_type %>
9
29
  vpc:
10
- vpc_name: <%= vpc_name %>
30
+ name: <%= vpc_name %>
11
31
  <% end %>
@@ -1,8 +1,8 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= server_pools_name %>
3
3
  cloud: AWS
4
4
  vpc:
5
- vpc_name: <%= vpc_name %>
5
+ name: <%= vpc_name %>
6
6
  alarms:
7
7
  - comparison_operator: "GreaterThanThreshold"
8
8
  metric_name: "HTTPCode_Target_5XX_Count"
@@ -66,6 +66,9 @@ basis:
66
66
  name: <%= server_pools_name %>
67
67
  min_size: 1
68
68
  max_size: 1
69
+ vpc:
70
+ name: <%= vpc_name %>
71
+ subnet_pref: public
69
72
  basis:
70
73
  launch_config:
71
74
  name: <%= server_pools_name %>
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  # XXX glue to other resources in test BoKs, if you want to test complexity
3
3
  name: efs
4
4
  mount_points:
@@ -390,16 +390,16 @@ module MU
390
390
  "description" => "The ID of a VPN, NAT, or Internet gateway attached to your VPC. #INTERNET will refer to this VPC's default internet gateway, if one exists. #NAT will refer to a this VPC's NAT gateway, and will implicitly create one if none exists. #DENY will ensure that the subnets associated with this route do *not* have a route outside of the VPC's local address space (primarily for Google Cloud, where we must explicitly disable egress to the internet)."
391
391
  },
392
392
  "nat_host_id" => {
393
- "type" => "string",
394
- "description" => "The instance id of a NAT host in this VPN."
393
+ "type" => "string",
394
+ "description" => "The instance id of a NAT host in this VPC."
395
395
  },
396
396
  "nat_host_name" => {
397
- "type" => "string",
398
- "description" => "The MU resource name or Name tag of a NAT host in this VPN."
397
+ "type" => "string",
398
+ "description" => "The MU resource name or Name tag of a NAT host in this VPC."
399
399
  },
400
400
  "interface" => {
401
- "type" => "string",
402
- "description" => "A network interface over which to route."
401
+ "type" => "string",
402
+ "description" => "A network interface over which to route."
403
403
  }
404
404
  }
405
405
  }
@@ -1,6 +1,57 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= vpc_name %>
3
- # TODO: BUILD OUT COMPLEX EXAMPLE
3
+ create_nat_gateway: true
4
+ ip_block: 10.231.0.0/16
5
+ enable_traffic_logging: true
6
+ region: us-east-2
7
+ availability_zones:
8
+ - us-east-2a
9
+ - us-east-2c
10
+ - us-east-2e
11
+ route_tables:
12
+ - name: public
13
+ routes:
14
+ - destination_network: 0.0.0.0/0
15
+ gateway: "#INTERNET"
16
+ - name: private
17
+ routes:
18
+ - destination_network: 0.0.0.0/0
19
+ gateway: "#NAT"
20
+ subnets:
21
+ - name: Subnet0Internet
22
+ availability_zone: us-east-2a
23
+ ip_block: 10.0.0.0/19
24
+ route_table: internet
25
+ map_public_ips: true
26
+ create_nat_gateway: true
27
+ - name: Subnet0Private
28
+ availability_zone: us-east-2a
29
+ ip_block: 10.0.32.0/19
30
+ route_table: private
31
+ - name: Subnet1Internet
32
+ availability_zone: us-east-2c
33
+ ip_block: 10.0.64.0/19
34
+ route_table: internet
35
+ map_public_ips: true
36
+ - name: Subnet1Private
37
+ availability_zone: us-east-2c
38
+ ip_block: 10.0.96.0/19
39
+ route_table: private
40
+ - name: Subnet2Internet
41
+ availability_zone: us-east-2e
42
+ ip_block: 10.0.128.0/19
43
+ route_table: internet
44
+ map_public_ips: true
45
+ - name: Subnet2Private
46
+ availability_zone: us-east-2e
47
+ route_table: private
48
+ ip_block: 10.0.160.0/19
49
+ - name: NonRoutable1
50
+ availability_zone: us-east-2a
51
+ ip_block: 10.0.192.0/19
52
+ - name: NonRoutable2
53
+ availability_zone: us-east-2c
54
+ ip_block: 10.0.224.0/19
4
55
  <% else %>
5
56
  name: <%= vpc_name %>
6
- <% end %>
57
+ <% end %>
@@ -48,11 +48,13 @@ module MU
48
48
  require 'chef'
49
49
  require 'chef/api_client_v1'
50
50
  require 'chef/knife'
51
+ require 'chef/application/knife'
51
52
  require 'chef/knife/ssh'
52
53
  require 'chef/knife/bootstrap'
53
54
  require 'chef/knife/node_delete'
54
55
  require 'chef/knife/client_delete'
55
56
  require 'chef/knife/data_bag_delete'
57
+ require 'chef/knife/data_bag_show'
56
58
  require 'chef/knife/vault_delete'
57
59
  require 'chef/scan_access_control'
58
60
  require 'chef/file_access_control/unix'
@@ -827,6 +829,31 @@ retry
827
829
 
828
830
  return if nodeonly
829
831
 
832
+ vaults_to_clean.each { |vault|
833
+ MU::MommaCat.lock("vault-#{vault['vault']}", false, true)
834
+ MU.log "Purging unknown clients from #{vault['vault']} #{vault['item']}", MU::DEBUG
835
+ output = %x{#{@knife} data bag show "#{vault['vault']}" "#{vault['item']}_keys" --format json}
836
+ # This is an ugly workaround for --clean-unknown-clients, which in
837
+ # fact cleans known clients.
838
+ if output
839
+ begin
840
+ vault_cfg = JSON.parse(output)
841
+ if vault_cfg['clients']
842
+ searchstr = vault_cfg['clients'].map { |c| "name:"+c }.join(" OR ")
843
+ MU.log "Preserving client list for vault #{vault['vault']} #{vault['item']}", MU::DEBUG, details: vault_cfg['clients']
844
+ if !noop
845
+ ::Chef::Knife.run(['vault', 'rotate', 'keys', vault['vault'], vault['item'], "--clean-unknown-clients"])
846
+ ::Chef::Knife.run(['vault', 'update', vault['vault'], vault['item'], "--search", searchstr])
847
+ ::Chef::Knife.run(['vault', 'refresh', vault['vault'], vault['item']])
848
+ end
849
+ end
850
+ rescue JSON::ParserError => e
851
+ MU.log "Error parsing JSON from data bag #{vault['vault']} #{vault['item']}_keys, skipping vault client cleanse", MU::WARN
852
+ end
853
+ end
854
+ MU::MommaCat.unlock("vault-#{vault['vault']}")
855
+ }
856
+
830
857
  begin
831
858
  deleteSecret(vault: node) if !noop
832
859
  rescue MuNoSuchSecret
@@ -39,6 +39,7 @@ module MU
39
39
  end
40
40
 
41
41
  @@litters = {}
42
+ @@litters_loadtime = {}
42
43
  @@litter_semaphore = Mutex.new
43
44
 
44
45
  # Return a {MU::MommaCat} instance for an existing deploy. Use this instead
@@ -62,17 +63,32 @@ module MU
62
63
  @@litter_semaphore.synchronize {
63
64
  littercache = @@litters.dup
64
65
  }
66
+ if littercache[deploy_id] and @@litters_loadtime[deploy_id]
67
+ deploy_root = File.expand_path(MU.dataDir+"/deployments")
68
+ this_deploy_dir = deploy_root+"/"+deploy_id
69
+ if File.exist?("#{this_deploy_dir}/deployment.json")
70
+ lastmod = File.mtime("#{this_deploy_dir}/deployment.json")
71
+ if lastmod > @@litters_loadtime[deploy_id]
72
+ MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE
73
+ use_cache = false
74
+ end
75
+ end
76
+ end
65
77
  rescue ThreadError => e
66
78
  # already locked by a parent caller and this is a read op, so this is ok
67
79
  raise e if !e.message.match(/recursive locking/)
68
80
  littercache = @@litters.dup
69
81
  end
82
+
70
83
  if !use_cache or littercache[deploy_id].nil?
84
+ need_gc = !littercache[deploy_id].nil?
71
85
  newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
72
86
  # This, we have to synchronize, as it's a write
73
87
  @@litter_semaphore.synchronize {
74
- @@litters[deploy_id] ||= newlitter
88
+ @@litters[deploy_id] = newlitter
89
+ @@litters_loadtime[deploy_id] = Time.now
75
90
  }
91
+ GC.start if need_gc
76
92
  elsif set_context_to_me
77
93
  MU::MommaCat.setThreadContext(@@litters[deploy_id])
78
94
  end
@@ -80,6 +96,18 @@ module MU
80
96
  # MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
81
97
  end
82
98
 
99
+ # Update the in-memory cache of a given deploy. This is intended for use by
100
+ # {#save!}, primarily.
101
+ # @param deploy_id [String]
102
+ # @param litter [MU::MommaCat]
103
+ def self.updateLitter(deploy_id, litter)
104
+ return if litter.nil?
105
+ @@litter_semaphore.synchronize {
106
+ @@litters[deploy_id] = litter
107
+ @@litters_loadtime[deploy_id] = Time.now
108
+ }
109
+ end
110
+
83
111
  attr_reader :initializing
84
112
  attr_reader :public_key
85
113
  attr_reader :deploy_secret
@@ -633,8 +661,9 @@ module MU
633
661
  # @param max_length [Integer]: The maximum length of the resulting resource name.
634
662
  # @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated.
635
663
  # @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
664
+ # @param allowed_chars [Regexp]: A pattern of characters that are legal for this resource name, such as +/[a-zA-Z0-9-]/+
636
665
  # @return [String]: A full name string for this resource
637
- def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'])
666
+ def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], allowed_chars: nil)
638
667
  if name.nil?
639
668
  raise MuError, "Got no argument to MU::MommaCat.getResourceName"
640
669
  end
@@ -657,6 +686,19 @@ module MU
657
686
  basename = @appname.upcase + "-" + @environment.upcase + name.upcase
658
687
  end
659
688
 
689
+ subchar = if allowed_chars
690
+ if !"-".match(allowed_chars)
691
+ if "_".match(allowed_chars)
692
+ "_"
693
+ else
694
+ ""
695
+ end
696
+ else
697
+ "-"
698
+ end
699
+ end
700
+
701
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
660
702
  begin
661
703
  if (basename.length + reserved) > max_length
662
704
  MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG
@@ -669,6 +711,7 @@ module MU
669
711
  basename.slice!((max_length-(reserved+3))..basename.length)
670
712
  basename.sub!(/-$/, "")
671
713
  basename = basename + "-" + @seed.upcase
714
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
672
715
  else
673
716
  # If we have to strip anything, assume we've lost uniqueness and
674
717
  # will have to compensate with #genUniquenessString.
@@ -676,6 +719,7 @@ module MU
676
719
  reserved = 4
677
720
  basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
678
721
  basename = basename + "-" + @seed.upcase + "-" + name.upcase
722
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
679
723
  end
680
724
  end
681
725
  end while (basename.length + reserved) > max_length
@@ -702,6 +746,7 @@ module MU
702
746
  else
703
747
  muname = basename
704
748
  end
749
+ muname.gsub!(allowed_chars, subchar) if allowed_chars
705
750
 
706
751
  return muname
707
752
  end
@@ -901,7 +946,7 @@ module MU
901
946
  if MU.myCloud == "AWS"
902
947
  MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
903
948
  end
904
- MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
949
+ MU::MommaCat.getLitter(MU.deploy_id)
905
950
  MU::MommaCat.syncMonitoringConfig(false)
906
951
  MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})"
907
952
  FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt")
@@ -927,7 +972,7 @@ module MU
927
972
  MU.log "Creating #{ssh_dir}", MU::DEBUG
928
973
  Dir.mkdir(ssh_dir, 0700)
929
974
  if Process.uid == 0 and @mu_user != "mu"
930
- File.chown(Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir)
975
+ FileUtils.chown Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir
931
976
  end
932
977
  end
933
978
  if !File.exist?("#{ssh_dir}/#{@ssh_key_name}")
@@ -1102,23 +1147,28 @@ module MU
1102
1147
 
1103
1148
  # Iterate over all known deployments and look for instances that have been
1104
1149
  # terminated, but not yet cleaned up, then clean them up.
1105
- def self.cleanTerminatedInstances
1150
+ def self.cleanTerminatedInstances(debug = false)
1151
+ loglevel = debug ? MU::NOTICE : MU::DEBUG
1106
1152
  MU::MommaCat.lock("clean-terminated-instances", false, true)
1107
- MU.log "Checking for harvested instances in need of cleanup", MU::DEBUG
1153
+ MU.log "Checking for harvested instances in need of cleanup", loglevel
1108
1154
  parent_thread_id = Thread.current.object_id
1109
1155
  purged = 0
1156
+
1110
1157
  MU::MommaCat.listDeploys.each { |deploy_id|
1111
1158
  next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
1112
- MU.log "Checking for dead wood in #{deploy_id}", MU::DEBUG
1159
+ MU.log "Checking for dead wood in #{deploy_id}", loglevel
1113
1160
  need_reload = false
1114
1161
  @cleanup_threads << Thread.new {
1115
1162
  MU.dupGlobals(parent_thread_id)
1116
1163
  deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
1117
1164
  purged_this_deploy = 0
1165
+ MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys
1118
1166
  if deploy.kittens.has_key?("servers")
1167
+ MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id
1119
1168
  deploy.kittens["servers"].values.each { |nodeclasses|
1120
1169
  nodeclasses.each_pair { |nodeclass, servers|
1121
1170
  deletia = []
1171
+ MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys
1122
1172
  servers.each_pair { |mu_name, server|
1123
1173
  server.describe
1124
1174
  if !server.cloud_id
@@ -1145,15 +1195,16 @@ module MU
1145
1195
  servers.delete(mu_name)
1146
1196
  }
1147
1197
  if purged_this_deploy > 0
1148
- # XXX some kind of filter (obey sync_siblings on nodes' configs)
1149
- deploy.syncLitter(servers.keys)
1198
+ # XXX triggering_node needs to take more than one node name
1199
+ deploy.syncLitter(servers.keys, triggering_node: deletia.first)
1150
1200
  end
1151
1201
  }
1152
1202
  }
1153
1203
  end
1154
1204
  if need_reload
1205
+ MU.log "Saving modified deploy #{deploy_id}", loglevel
1155
1206
  deploy.save!
1156
- MU::MommaCat.getLitter(deploy_id, use_cache: false)
1207
+ MU::MommaCat.getLitter(deploy_id)
1157
1208
  end
1158
1209
  MU.purgeGlobals
1159
1210
  }
@@ -1161,6 +1212,8 @@ module MU
1161
1212
  @cleanup_threads.each { |t|
1162
1213
  t.join
1163
1214
  }
1215
+ MU.log "cleanTerminatedInstances threads complete", loglevel
1216
+ MU::MommaCat.unlock("clean-terminated-instances", true)
1164
1217
  @cleanup_threads = []
1165
1218
 
1166
1219
  if purged > 0
@@ -1168,8 +1221,9 @@ module MU
1168
1221
  MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
1169
1222
  end
1170
1223
  MU::MommaCat.syncMonitoringConfig
1224
+ GC.start
1171
1225
  end
1172
- MU::MommaCat.unlock("clean-terminated-instances", true)
1226
+ MU.log "cleanTerminatedInstances returning", loglevel
1173
1227
  end
1174
1228
 
1175
1229
  @@dummy_cache = {}
@@ -1890,22 +1944,51 @@ end
1890
1944
  }
1891
1945
  end
1892
1946
 
1947
+ # Clean an IP address out of ~/.ssh/known hosts
1948
+ # @param ip [String]: The IP to remove
1949
+ # @return [void]
1950
+ def self.removeIPFromSSHKnownHosts(ip)
1951
+ return if ip.nil?
1952
+ sshdir = "#{@myhome}/.ssh"
1953
+ knownhosts = "#{sshdir}/known_hosts"
1954
+
1955
+ if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
1956
+ MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
1957
+ if !@noop
1958
+ File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
1959
+ f.flock(File::LOCK_EX)
1960
+ newlines = Array.new
1961
+ delete_block = false
1962
+ f.readlines.each { |line|
1963
+ next if line.match(/^#{Regexp.quote(ip)} /)
1964
+ newlines << line
1965
+ }
1966
+ f.rewind
1967
+ f.truncate(0)
1968
+ f.puts(newlines)
1969
+ f.flush
1970
+ f.flock(File::LOCK_UN)
1971
+ }
1972
+ end
1973
+ end
1974
+ end
1975
+
1893
1976
  # Clean a node's entries out of ~/.ssh/config
1894
- # @param node [String]: The node's name
1977
+ # @param nodename [String]: The node's name
1895
1978
  # @return [void]
1896
- def self.removeHostFromSSHConfig(node)
1979
+ def self.removeHostFromSSHConfig(nodename)
1897
1980
  sshdir = "#{@myhome}/.ssh"
1898
1981
  sshconf = "#{sshdir}/config"
1899
1982
 
1900
- if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{node} /)
1901
- MU.log "Expunging old #{node} entry from #{sshconf}", MU::DEBUG
1983
+ if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /)
1984
+ MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
1902
1985
  if !@noop
1903
1986
  File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
1904
1987
  f.flock(File::LOCK_EX)
1905
1988
  newlines = Array.new
1906
1989
  delete_block = false
1907
1990
  f.readlines.each { |line|
1908
- if line.match(/^Host #{node}(\s|$)/)
1991
+ if line.match(/^Host #{nodename}(\s|$)/)
1909
1992
  delete_block = true
1910
1993
  elsif line.match(/^Host /)
1911
1994
  delete_block = false
@@ -1986,6 +2069,9 @@ end
1986
2069
  end
1987
2070
 
1988
2071
  MU::MommaCat.removeHostFromSSHConfig(node)
2072
+ if server and server.canonicalIP
2073
+ MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP)
2074
+ end
1989
2075
  # XXX add names paramater with useful stuff
1990
2076
  MU::MommaCat.addHostToSSHConfig(
1991
2077
  server,
@@ -2574,7 +2660,7 @@ MESSAGE_END
2574
2660
  update_servers = update_servers - skip
2575
2661
  end
2576
2662
 
2577
- return if update_servers.size < 1
2663
+ return if MU.inGem? || update_servers.size < 1
2578
2664
  threads = []
2579
2665
  parent_thread_id = Thread.current.object_id
2580
2666
  update_servers.each { |sibling|
@@ -2690,9 +2776,16 @@ MESSAGE_END
2690
2776
  Dir.chdir(MU.myRoot+"/modules")
2691
2777
 
2692
2778
  # XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
2693
- cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2779
+ if MU.inGem?
2780
+ cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2781
+ else
2782
+ cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2783
+ end
2784
+
2694
2785
  MU.log cmd, MU::NOTICE
2786
+
2695
2787
  output = %x{#{cmd}}
2788
+
2696
2789
  Dir.chdir(origdir)
2697
2790
 
2698
2791
  retries = 0
@@ -2782,6 +2875,7 @@ MESSAGE_END
2782
2875
  def save!(triggering_node = nil, force: false, origin: nil)
2783
2876
 
2784
2877
  return if @no_artifacts and !force
2878
+
2785
2879
  MU::MommaCat.deploy_struct_semaphore.synchronize {
2786
2880
  MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG
2787
2881
 
@@ -2834,6 +2928,7 @@ MESSAGE_END
2834
2928
  deploy.flock(File::LOCK_UN)
2835
2929
  deploy.close
2836
2930
  @need_deploy_flush = false
2931
+ MU::MommaCat.updateLitter(@deploy_id, self)
2837
2932
  end
2838
2933
 
2839
2934
  if !@original_config.nil? and @original_config.is_a?(Hash)