cloud-mu 3.0.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/Dockerfile +25 -0
  3. data/README.md +2 -2
  4. data/bin/mu-node-manage +1 -1
  5. data/chefignore +1 -0
  6. data/cloud-mu.gemspec +2 -2
  7. data/extras/clean-stock-amis +0 -0
  8. data/extras/generate-stock-images +0 -0
  9. data/extras/list-stock-amis +0 -0
  10. data/extras/vault_tools/export_vaults.sh +0 -0
  11. data/extras/vault_tools/recreate_vaults.sh +0 -0
  12. data/extras/vault_tools/test_vaults.sh +0 -0
  13. data/kitchen.yml +97 -0
  14. data/modules/mu.rb +14 -2
  15. data/modules/mu/cloud.rb +26 -0
  16. data/modules/mu/clouds/aws.rb +1 -2
  17. data/modules/mu/clouds/aws/container_cluster.rb +2 -1
  18. data/modules/mu/clouds/aws/database.rb +2 -2
  19. data/modules/mu/clouds/aws/loadbalancer.rb +1 -1
  20. data/modules/mu/clouds/aws/search_domain.rb +1 -1
  21. data/modules/mu/clouds/aws/server.rb +9 -1
  22. data/modules/mu/clouds/aws/vpc.rb +1 -2
  23. data/modules/mu/config.rb +83 -3
  24. data/modules/mu/config/bucket.yml +1 -1
  25. data/modules/mu/config/cache_cluster.yml +1 -9
  26. data/modules/mu/config/container_cluster.yml +1 -1
  27. data/modules/mu/config/database.yml +3 -3
  28. data/modules/mu/config/log.yml +8 -3
  29. data/modules/mu/config/msg_queue.yml +1 -1
  30. data/modules/mu/config/nosqldb.yml +1 -1
  31. data/modules/mu/config/notifier.yml +1 -1
  32. data/modules/mu/config/search_domain.yml +2 -2
  33. data/modules/mu/config/server.yml +23 -3
  34. data/modules/mu/config/server_pool.yml +5 -2
  35. data/modules/mu/config/storage_pool.yml +1 -1
  36. data/modules/mu/config/vpc.rb +6 -6
  37. data/modules/mu/config/vpc.yml +54 -3
  38. data/modules/mu/groomers/chef.rb +27 -0
  39. data/modules/mu/mommacat.rb +113 -18
  40. metadata +18 -22
  41. data/Berksfile.lock +0 -179
  42. data/bin/mu-azure-tests +0 -43
  43. data/cookbooks/mu-tools/files/default/Mu_CA.pem +0 -33
  44. data/modules/mu/kittens.rb +0 -20651
  45. data/modules/mu/mu.yaml.rb +0 -276
  46. data/modules/scratchpad.erb +0 -1
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: bucket
3
3
  acl: public-read
4
4
  versioning: true
@@ -1,17 +1,9 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  name: redis
3
3
  engine: redis
4
4
  creation_style: new
5
5
  size: cache.t2.medium
6
- name: memcache
7
- creation_style: new
8
- engine: memcached
9
- size: cache.t2.medium
10
6
  <% else %>
11
- name: redis
12
- engine: redis
13
- creation_style: new
14
- size: cache.t2.medium
15
7
  name: memcache
16
8
  creation_style: new
17
9
  engine: memcached
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  name: k8s
3
3
  flavor: EKS
4
4
  instance_type: t2.medium
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
 
3
3
  name: database-complex
4
4
  size: db.r4.large
@@ -12,7 +12,7 @@ backup_retention_period: 10
12
12
  cluster_node_count: 2
13
13
  create_cluster: true
14
14
  vpc:
15
- vpc_name: <%= vpc_name %>
15
+ name: <%= vpc_name %>
16
16
  create_read_replica: true
17
17
  master_user: Bob
18
18
  multi_az_on_create: true
@@ -21,7 +21,7 @@ multi_az_on_create: true
21
21
 
22
22
  name: database-simple
23
23
  vpc:
24
- vpc_name: <%= vpc_name %>
24
+ name: <%= vpc_name %>
25
25
  size: <%= db_size %>
26
26
  engine: mariadb
27
27
  storage: 5
@@ -1,6 +1,11 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= logs_name %>
3
- # TODO: BUILD OUT COMPLEX EXAMPLE
3
+ filters:
4
+ - name: myfilter
5
+ metric_name: LogMetrics/myfilter
6
+ namespace: ok
7
+ search_pattern: failed
8
+ value: yes
4
9
  <% else %>
5
10
  name: <%= logs_name %>
6
- <% end %>
11
+ <% end %>
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= queues_name %>
3
3
  visibility_timeout: "1 hour"
4
4
  failqueue:
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: complex
3
3
  stream: NEW_IMAGE
4
4
  read_capacity: 50
@@ -1,4 +1,4 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: notifier
3
3
  subscriptions:
4
4
  - endpoint: admin@example.com
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
 
3
3
  name: searchdomain-complex
4
4
  instance_type: t2.small.elasticsearch
@@ -17,7 +17,7 @@ advanced_options:
17
17
  # user_pool_id: "us-east-1_eSwWA1VYQ"
18
18
  slow_logs: <%= logs_name %>
19
19
  vpc:
20
- vpc_name: <%= vpc_name %>
20
+ name: <%= vpc_name %>
21
21
 
22
22
  <% else %> # IF NOT COMPLEX THEN ASSUME SIMPLE
23
23
 
@@ -1,11 +1,31 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: servercomplex
3
3
  size: <%= instance_type %>
4
4
  vpc:
5
- vpc_name: <%= vpc_name %>
5
+ name: <%= vpc_name %>
6
+ subnet_pref: public
7
+ platform: ubuntu
8
+ ssh_user: ubuntu
9
+ associate_public_ip: true
10
+ canned_iam_policies:
11
+ - AmazonDynamoDBReadOnlyAccess
12
+ - AmazonElastiCacheFullAccess
13
+ - AWSLambdaExecute
14
+ groomer: Ansible
15
+ run_list:
16
+ - geerlingguy.java
17
+ - geerlingguy.nginx
18
+ - sensu.sensu
19
+ tags:
20
+ - key: ThisIsATag
21
+ value: ThisIsAValue
22
+ src_dst_check: false
23
+ storage:
24
+ - device: /dev/xvdg
25
+ size: 50
6
26
  <% else %>
7
27
  name: serversimple
8
28
  size: <%= instance_type %>
9
29
  vpc:
10
- vpc_name: <%= vpc_name %>
30
+ name: <%= vpc_name %>
11
31
  <% end %>
@@ -1,8 +1,8 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= server_pools_name %>
3
3
  cloud: AWS
4
4
  vpc:
5
- vpc_name: <%= vpc_name %>
5
+ name: <%= vpc_name %>
6
6
  alarms:
7
7
  - comparison_operator: "GreaterThanThreshold"
8
8
  metric_name: "HTTPCode_Target_5XX_Count"
@@ -66,6 +66,9 @@ basis:
66
66
  name: <%= server_pools_name %>
67
67
  min_size: 1
68
68
  max_size: 1
69
+ vpc:
70
+ name: <%= vpc_name %>
71
+ subnet_pref: public
69
72
  basis:
70
73
  launch_config:
71
74
  name: <%= server_pools_name %>
@@ -1,4 +1,4 @@
1
- <% if $complexity == "complex" %>
1
+ <% if complexity == "complex" %>
2
2
  # XXX glue to other resources in test BoKs, if you want to test complexity
3
3
  name: efs
4
4
  mount_points:
@@ -390,16 +390,16 @@ module MU
390
390
  "description" => "The ID of a VPN, NAT, or Internet gateway attached to your VPC. #INTERNET will refer to this VPC's default internet gateway, if one exists. #NAT will refer to a this VPC's NAT gateway, and will implicitly create one if none exists. #DENY will ensure that the subnets associated with this route do *not* have a route outside of the VPC's local address space (primarily for Google Cloud, where we must explicitly disable egress to the internet)."
391
391
  },
392
392
  "nat_host_id" => {
393
- "type" => "string",
394
- "description" => "The instance id of a NAT host in this VPN."
393
+ "type" => "string",
394
+ "description" => "The instance id of a NAT host in this VPC."
395
395
  },
396
396
  "nat_host_name" => {
397
- "type" => "string",
398
- "description" => "The MU resource name or Name tag of a NAT host in this VPN."
397
+ "type" => "string",
398
+ "description" => "The MU resource name or Name tag of a NAT host in this VPC."
399
399
  },
400
400
  "interface" => {
401
- "type" => "string",
402
- "description" => "A network interface over which to route."
401
+ "type" => "string",
402
+ "description" => "A network interface over which to route."
403
403
  }
404
404
  }
405
405
  }
@@ -1,6 +1,57 @@
1
- <% if $complexity == 'complex' %>
1
+ <% if complexity == 'complex' %>
2
2
  name: <%= vpc_name %>
3
- # TODO: BUILD OUT COMPLEX EXAMPLE
3
+ create_nat_gateway: true
4
+ ip_block: 10.231.0.0/16
5
+ enable_traffic_logging: true
6
+ region: us-east-2
7
+ availability_zones:
8
+ - us-east-2a
9
+ - us-east-2c
10
+ - us-east-2e
11
+ route_tables:
12
+ - name: public
13
+ routes:
14
+ - destination_network: 0.0.0.0/0
15
+ gateway: "#INTERNET"
16
+ - name: private
17
+ routes:
18
+ - destination_network: 0.0.0.0/0
19
+ gateway: "#NAT"
20
+ subnets:
21
+ - name: Subnet0Internet
22
+ availability_zone: us-east-2a
23
+ ip_block: 10.0.0.0/19
24
+ route_table: internet
25
+ map_public_ips: true
26
+ create_nat_gateway: true
27
+ - name: Subnet0Private
28
+ availability_zone: us-east-2a
29
+ ip_block: 10.0.32.0/19
30
+ route_table: private
31
+ - name: Subnet1Internet
32
+ availability_zone: us-east-2c
33
+ ip_block: 10.0.64.0/19
34
+ route_table: internet
35
+ map_public_ips: true
36
+ - name: Subnet1Private
37
+ availability_zone: us-east-2c
38
+ ip_block: 10.0.96.0/19
39
+ route_table: private
40
+ - name: Subnet2Internet
41
+ availability_zone: us-east-2e
42
+ ip_block: 10.0.128.0/19
43
+ route_table: internet
44
+ map_public_ips: true
45
+ - name: Subnet2Private
46
+ availability_zone: us-east-2e
47
+ route_table: private
48
+ ip_block: 10.0.160.0/19
49
+ - name: NonRoutable1
50
+ availability_zone: us-east-2a
51
+ ip_block: 10.0.192.0/19
52
+ - name: NonRoutable2
53
+ availability_zone: us-east-2c
54
+ ip_block: 10.0.224.0/19
4
55
  <% else %>
5
56
  name: <%= vpc_name %>
6
- <% end %>
57
+ <% end %>
@@ -48,11 +48,13 @@ module MU
48
48
  require 'chef'
49
49
  require 'chef/api_client_v1'
50
50
  require 'chef/knife'
51
+ require 'chef/application/knife'
51
52
  require 'chef/knife/ssh'
52
53
  require 'chef/knife/bootstrap'
53
54
  require 'chef/knife/node_delete'
54
55
  require 'chef/knife/client_delete'
55
56
  require 'chef/knife/data_bag_delete'
57
+ require 'chef/knife/data_bag_show'
56
58
  require 'chef/knife/vault_delete'
57
59
  require 'chef/scan_access_control'
58
60
  require 'chef/file_access_control/unix'
@@ -827,6 +829,31 @@ retry
827
829
 
828
830
  return if nodeonly
829
831
 
832
+ vaults_to_clean.each { |vault|
833
+ MU::MommaCat.lock("vault-#{vault['vault']}", false, true)
834
+ MU.log "Purging unknown clients from #{vault['vault']} #{vault['item']}", MU::DEBUG
835
+ output = %x{#{@knife} data bag show "#{vault['vault']}" "#{vault['item']}_keys" --format json}
836
+ # This is an ugly workaround for --clean-unknown-clients, which in
837
+ # fact cleans known clients.
838
+ if output
839
+ begin
840
+ vault_cfg = JSON.parse(output)
841
+ if vault_cfg['clients']
842
+ searchstr = vault_cfg['clients'].map { |c| "name:"+c }.join(" OR ")
843
+ MU.log "Preserving client list for vault #{vault['vault']} #{vault['item']}", MU::DEBUG, details: vault_cfg['clients']
844
+ if !noop
845
+ ::Chef::Knife.run(['vault', 'rotate', 'keys', vault['vault'], vault['item'], "--clean-unknown-clients"])
846
+ ::Chef::Knife.run(['vault', 'update', vault['vault'], vault['item'], "--search", searchstr])
847
+ ::Chef::Knife.run(['vault', 'refresh', vault['vault'], vault['item']])
848
+ end
849
+ end
850
+ rescue JSON::ParserError => e
851
+ MU.log "Error parsing JSON from data bag #{vault['vault']} #{vault['item']}_keys, skipping vault client cleanse", MU::WARN
852
+ end
853
+ end
854
+ MU::MommaCat.unlock("vault-#{vault['vault']}")
855
+ }
856
+
830
857
  begin
831
858
  deleteSecret(vault: node) if !noop
832
859
  rescue MuNoSuchSecret
@@ -39,6 +39,7 @@ module MU
39
39
  end
40
40
 
41
41
  @@litters = {}
42
+ @@litters_loadtime = {}
42
43
  @@litter_semaphore = Mutex.new
43
44
 
44
45
  # Return a {MU::MommaCat} instance for an existing deploy. Use this instead
@@ -62,17 +63,32 @@ module MU
62
63
  @@litter_semaphore.synchronize {
63
64
  littercache = @@litters.dup
64
65
  }
66
+ if littercache[deploy_id] and @@litters_loadtime[deploy_id]
67
+ deploy_root = File.expand_path(MU.dataDir+"/deployments")
68
+ this_deploy_dir = deploy_root+"/"+deploy_id
69
+ if File.exist?("#{this_deploy_dir}/deployment.json")
70
+ lastmod = File.mtime("#{this_deploy_dir}/deployment.json")
71
+ if lastmod > @@litters_loadtime[deploy_id]
72
+ MU.log "Deployment metadata for #{deploy_id} was modified on disk, reload", MU::NOTICE
73
+ use_cache = false
74
+ end
75
+ end
76
+ end
65
77
  rescue ThreadError => e
66
78
  # already locked by a parent caller and this is a read op, so this is ok
67
79
  raise e if !e.message.match(/recursive locking/)
68
80
  littercache = @@litters.dup
69
81
  end
82
+
70
83
  if !use_cache or littercache[deploy_id].nil?
84
+ need_gc = !littercache[deploy_id].nil?
71
85
  newlitter = MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
72
86
  # This, we have to synchronize, as it's a write
73
87
  @@litter_semaphore.synchronize {
74
- @@litters[deploy_id] ||= newlitter
88
+ @@litters[deploy_id] = newlitter
89
+ @@litters_loadtime[deploy_id] = Time.now
75
90
  }
91
+ GC.start if need_gc
76
92
  elsif set_context_to_me
77
93
  MU::MommaCat.setThreadContext(@@litters[deploy_id])
78
94
  end
@@ -80,6 +96,18 @@ module MU
80
96
  # MU::MommaCat.new(deploy_id, set_context_to_me: set_context_to_me)
81
97
  end
82
98
 
99
+ # Update the in-memory cache of a given deploy. This is intended for use by
100
+ # {#save!}, primarily.
101
+ # @param deploy_id [String]
102
+ # @param litter [MU::MommaCat]
103
+ def self.updateLitter(deploy_id, litter)
104
+ return if litter.nil?
105
+ @@litter_semaphore.synchronize {
106
+ @@litters[deploy_id] = litter
107
+ @@litters_loadtime[deploy_id] = Time.now
108
+ }
109
+ end
110
+
83
111
  attr_reader :initializing
84
112
  attr_reader :public_key
85
113
  attr_reader :deploy_secret
@@ -633,8 +661,9 @@ module MU
633
661
  # @param max_length [Integer]: The maximum length of the resulting resource name.
634
662
  # @param need_unique_string [Boolean]: Whether to forcibly append a random three-character string to the name to ensure it's unique. Note that this behavior will be automatically invoked if the name must be truncated.
635
663
  # @param scrub_mu_isms [Boolean]: Don't bother with generating names specific to this deployment. Used to generate generic CloudFormation templates, amongst other purposes.
664
+ # @param allowed_chars [Regexp]: A pattern of characters that are legal for this resource name, such as +/[a-zA-Z0-9-]/+
636
665
  # @return [String]: A full name string for this resource
637
- def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'])
666
+ def getResourceName(name, max_length: 255, need_unique_string: false, use_unique_string: nil, reuse_unique_string: false, scrub_mu_isms: @original_config['scrub_mu_isms'], allowed_chars: nil)
638
667
  if name.nil?
639
668
  raise MuError, "Got no argument to MU::MommaCat.getResourceName"
640
669
  end
@@ -657,6 +686,19 @@ module MU
657
686
  basename = @appname.upcase + "-" + @environment.upcase + name.upcase
658
687
  end
659
688
 
689
+ subchar = if allowed_chars
690
+ if !"-".match(allowed_chars)
691
+ if "_".match(allowed_chars)
692
+ "_"
693
+ else
694
+ ""
695
+ end
696
+ else
697
+ "-"
698
+ end
699
+ end
700
+
701
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
660
702
  begin
661
703
  if (basename.length + reserved) > max_length
662
704
  MU.log "Stripping name down from #{basename}[#{basename.length.to_s}] (reserved: #{reserved.to_s}, max_length: #{max_length.to_s})", MU::DEBUG
@@ -669,6 +711,7 @@ module MU
669
711
  basename.slice!((max_length-(reserved+3))..basename.length)
670
712
  basename.sub!(/-$/, "")
671
713
  basename = basename + "-" + @seed.upcase
714
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
672
715
  else
673
716
  # If we have to strip anything, assume we've lost uniqueness and
674
717
  # will have to compensate with #genUniquenessString.
@@ -676,6 +719,7 @@ module MU
676
719
  reserved = 4
677
720
  basename.sub!(/-[^-]+-#{@seed.upcase}-#{Regexp.escape(name.upcase)}$/, "")
678
721
  basename = basename + "-" + @seed.upcase + "-" + name.upcase
722
+ basename.gsub!(allowed_chars, subchar) if allowed_chars
679
723
  end
680
724
  end
681
725
  end while (basename.length + reserved) > max_length
@@ -702,6 +746,7 @@ module MU
702
746
  else
703
747
  muname = basename
704
748
  end
749
+ muname.gsub!(allowed_chars, subchar) if allowed_chars
705
750
 
706
751
  return muname
707
752
  end
@@ -901,7 +946,7 @@ module MU
901
946
  if MU.myCloud == "AWS"
902
947
  MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
903
948
  end
904
- MU::MommaCat.getLitter(MU.deploy_id, use_cache: false)
949
+ MU::MommaCat.getLitter(MU.deploy_id)
905
950
  MU::MommaCat.syncMonitoringConfig(false)
906
951
  MU.log "Grooming complete for '#{name}' mu_name on \"#{MU.handle}\" (#{MU.deploy_id})"
907
952
  FileUtils.touch(MU.dataDir+"/deployments/#{MU.deploy_id}/#{name}_done.txt")
@@ -927,7 +972,7 @@ module MU
927
972
  MU.log "Creating #{ssh_dir}", MU::DEBUG
928
973
  Dir.mkdir(ssh_dir, 0700)
929
974
  if Process.uid == 0 and @mu_user != "mu"
930
- File.chown(Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir)
975
+ FileUtils.chown Etc.getpwnam(@mu_user).uid, Etc.getpwnam(@mu_user).gid, ssh_dir
931
976
  end
932
977
  end
933
978
  if !File.exist?("#{ssh_dir}/#{@ssh_key_name}")
@@ -1102,23 +1147,28 @@ module MU
1102
1147
 
1103
1148
  # Iterate over all known deployments and look for instances that have been
1104
1149
  # terminated, but not yet cleaned up, then clean them up.
1105
- def self.cleanTerminatedInstances
1150
+ def self.cleanTerminatedInstances(debug = false)
1151
+ loglevel = debug ? MU::NOTICE : MU::DEBUG
1106
1152
  MU::MommaCat.lock("clean-terminated-instances", false, true)
1107
- MU.log "Checking for harvested instances in need of cleanup", MU::DEBUG
1153
+ MU.log "Checking for harvested instances in need of cleanup", loglevel
1108
1154
  parent_thread_id = Thread.current.object_id
1109
1155
  purged = 0
1156
+
1110
1157
  MU::MommaCat.listDeploys.each { |deploy_id|
1111
1158
  next if File.exist?(deploy_dir(deploy_id)+"/.cleanup")
1112
- MU.log "Checking for dead wood in #{deploy_id}", MU::DEBUG
1159
+ MU.log "Checking for dead wood in #{deploy_id}", loglevel
1113
1160
  need_reload = false
1114
1161
  @cleanup_threads << Thread.new {
1115
1162
  MU.dupGlobals(parent_thread_id)
1116
1163
  deploy = MU::MommaCat.getLitter(deploy_id, set_context_to_me: true)
1117
1164
  purged_this_deploy = 0
1165
+ MU.log "#{deploy_id} has some kittens in it", loglevel, details: deploy.kittens.keys
1118
1166
  if deploy.kittens.has_key?("servers")
1167
+ MU.log "#{deploy_id} has some servers declared", loglevel, details: deploy.object_id
1119
1168
  deploy.kittens["servers"].values.each { |nodeclasses|
1120
1169
  nodeclasses.each_pair { |nodeclass, servers|
1121
1170
  deletia = []
1171
+ MU.log "Checking status of servers under '#{nodeclass}'", loglevel, details: servers.keys
1122
1172
  servers.each_pair { |mu_name, server|
1123
1173
  server.describe
1124
1174
  if !server.cloud_id
@@ -1145,15 +1195,16 @@ module MU
1145
1195
  servers.delete(mu_name)
1146
1196
  }
1147
1197
  if purged_this_deploy > 0
1148
- # XXX some kind of filter (obey sync_siblings on nodes' configs)
1149
- deploy.syncLitter(servers.keys)
1198
+ # XXX triggering_node needs to take more than one node name
1199
+ deploy.syncLitter(servers.keys, triggering_node: deletia.first)
1150
1200
  end
1151
1201
  }
1152
1202
  }
1153
1203
  end
1154
1204
  if need_reload
1205
+ MU.log "Saving modified deploy #{deploy_id}", loglevel
1155
1206
  deploy.save!
1156
- MU::MommaCat.getLitter(deploy_id, use_cache: false)
1207
+ MU::MommaCat.getLitter(deploy_id)
1157
1208
  end
1158
1209
  MU.purgeGlobals
1159
1210
  }
@@ -1161,6 +1212,8 @@ module MU
1161
1212
  @cleanup_threads.each { |t|
1162
1213
  t.join
1163
1214
  }
1215
+ MU.log "cleanTerminatedInstances threads complete", loglevel
1216
+ MU::MommaCat.unlock("clean-terminated-instances", true)
1164
1217
  @cleanup_threads = []
1165
1218
 
1166
1219
  if purged > 0
@@ -1168,8 +1221,9 @@ module MU
1168
1221
  MU::Cloud::AWS.openFirewallForClients # XXX add the other clouds, or abstract
1169
1222
  end
1170
1223
  MU::MommaCat.syncMonitoringConfig
1224
+ GC.start
1171
1225
  end
1172
- MU::MommaCat.unlock("clean-terminated-instances", true)
1226
+ MU.log "cleanTerminatedInstances returning", loglevel
1173
1227
  end
1174
1228
 
1175
1229
  @@dummy_cache = {}
@@ -1890,22 +1944,51 @@ end
1890
1944
  }
1891
1945
  end
1892
1946
 
1947
+ # Clean an IP address out of ~/.ssh/known hosts
1948
+ # @param ip [String]: The IP to remove
1949
+ # @return [void]
1950
+ def self.removeIPFromSSHKnownHosts(ip)
1951
+ return if ip.nil?
1952
+ sshdir = "#{@myhome}/.ssh"
1953
+ knownhosts = "#{sshdir}/known_hosts"
1954
+
1955
+ if File.exist?(knownhosts) and File.open(knownhosts).read.match(/^#{Regexp.quote(ip)} /)
1956
+ MU.log "Expunging old #{ip} entry from #{knownhosts}", MU::NOTICE
1957
+ if !@noop
1958
+ File.open(knownhosts, File::CREAT|File::RDWR, 0600) { |f|
1959
+ f.flock(File::LOCK_EX)
1960
+ newlines = Array.new
1961
+ delete_block = false
1962
+ f.readlines.each { |line|
1963
+ next if line.match(/^#{Regexp.quote(ip)} /)
1964
+ newlines << line
1965
+ }
1966
+ f.rewind
1967
+ f.truncate(0)
1968
+ f.puts(newlines)
1969
+ f.flush
1970
+ f.flock(File::LOCK_UN)
1971
+ }
1972
+ end
1973
+ end
1974
+ end
1975
+
1893
1976
  # Clean a node's entries out of ~/.ssh/config
1894
- # @param node [String]: The node's name
1977
+ # @param nodename [String]: The node's name
1895
1978
  # @return [void]
1896
- def self.removeHostFromSSHConfig(node)
1979
+ def self.removeHostFromSSHConfig(nodename)
1897
1980
  sshdir = "#{@myhome}/.ssh"
1898
1981
  sshconf = "#{sshdir}/config"
1899
1982
 
1900
- if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{node} /)
1901
- MU.log "Expunging old #{node} entry from #{sshconf}", MU::DEBUG
1983
+ if File.exist?(sshconf) and File.open(sshconf).read.match(/ #{nodename} /)
1984
+ MU.log "Expunging old #{nodename} entry from #{sshconf}", MU::DEBUG
1902
1985
  if !@noop
1903
1986
  File.open(sshconf, File::CREAT|File::RDWR, 0600) { |f|
1904
1987
  f.flock(File::LOCK_EX)
1905
1988
  newlines = Array.new
1906
1989
  delete_block = false
1907
1990
  f.readlines.each { |line|
1908
- if line.match(/^Host #{node}(\s|$)/)
1991
+ if line.match(/^Host #{nodename}(\s|$)/)
1909
1992
  delete_block = true
1910
1993
  elsif line.match(/^Host /)
1911
1994
  delete_block = false
@@ -1986,6 +2069,9 @@ end
1986
2069
  end
1987
2070
 
1988
2071
  MU::MommaCat.removeHostFromSSHConfig(node)
2072
+ if server and server.canonicalIP
2073
+ MU::MommaCat.removeIPFromSSHKnownHosts(server.canonicalIP)
2074
+ end
1989
2075
  # XXX add names paramater with useful stuff
1990
2076
  MU::MommaCat.addHostToSSHConfig(
1991
2077
  server,
@@ -2574,7 +2660,7 @@ MESSAGE_END
2574
2660
  update_servers = update_servers - skip
2575
2661
  end
2576
2662
 
2577
- return if update_servers.size < 1
2663
+ return if MU.inGem? || update_servers.size < 1
2578
2664
  threads = []
2579
2665
  parent_thread_id = Thread.current.object_id
2580
2666
  update_servers.each { |sibling|
@@ -2690,9 +2776,16 @@ MESSAGE_END
2690
2776
  Dir.chdir(MU.myRoot+"/modules")
2691
2777
 
2692
2778
  # XXX what's the safest way to find the 'bundle' executable in both gem and non-gem installs?
2693
- cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2779
+ if MU.inGem?
2780
+ cmd = %Q{thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2781
+ else
2782
+ cmd = %Q{bundle exec thin --threaded --daemonize --port #{MU.mommaCatPort} --pid #{daemonPidFile} --log #{daemonLogFile} --ssl --ssl-key-file #{MU.muCfg['ssl']['key']} --ssl-cert-file #{MU.muCfg['ssl']['cert']} --ssl-disable-verify --tag mu-momma-cat -R mommacat.ru start}
2783
+ end
2784
+
2694
2785
  MU.log cmd, MU::NOTICE
2786
+
2695
2787
  output = %x{#{cmd}}
2788
+
2696
2789
  Dir.chdir(origdir)
2697
2790
 
2698
2791
  retries = 0
@@ -2782,6 +2875,7 @@ MESSAGE_END
2782
2875
  def save!(triggering_node = nil, force: false, origin: nil)
2783
2876
 
2784
2877
  return if @no_artifacts and !force
2878
+
2785
2879
  MU::MommaCat.deploy_struct_semaphore.synchronize {
2786
2880
  MU.log "Saving deployment #{MU.deploy_id}", MU::DEBUG
2787
2881
 
@@ -2834,6 +2928,7 @@ MESSAGE_END
2834
2928
  deploy.flock(File::LOCK_UN)
2835
2929
  deploy.close
2836
2930
  @need_deploy_flush = false
2931
+ MU::MommaCat.updateLitter(@deploy_id, self)
2837
2932
  end
2838
2933
 
2839
2934
  if !@original_config.nil? and @original_config.is_a?(Hash)