elasticsearch-drain 0.0.4 → 0.1.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: b4a387835090abe799f17003384cf7981dd4c6c4
4
- data.tar.gz: ad7011c24bad8354389715c6b7a84aea56ed34d3
3
+ metadata.gz: 3099c4c4c099c9128357fb2dad397fbfd54bb0b2
4
+ data.tar.gz: 1e1c56d6bf0a91a9b6f30c86c6d14ecf36c1d341
5
5
  SHA512:
6
- metadata.gz: 58cef0753a1b0a874eb321d8b5ab731704661c5c3619451f23e3432a880924069a73e07155cbacc5c23cb721cd227e1fad78ff5d78dd7c56c50f546800c4b935
7
- data.tar.gz: 6e2e83b799067ef16c4865c29949007f6ff30d492a93dec8c0070d61b535d35fa3c1efc3c042649c34218089efcae1edb74fdcdcf8c6e9b6ee51a6a208b31542
6
+ metadata.gz: f8b791433e9e39697b59e9d8b3f45d60bc42c01e48c972ed372e379bb2fe6668d9f633fc2d108c35cb5a3f415cb15e455711b081a0481e678bdd9e0ecfb0aee5
7
+ data.tar.gz: 284eb015e4b3a5013308b14d029326b4e6f2452e4b27a985898554521d2d7fa7bc176ba6ae6551384d46a73eea747a4e840ec55e75bd2fd32f972be7bcab8d23
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- elasticsearch-drain (0.0.3)
4
+ elasticsearch-drain (0.0.1)
5
5
  aws-sdk (~> 2)
6
6
  elasticsearch (~> 1.0)
7
7
  thor (~> 0.19)
@@ -14,25 +14,25 @@ GEM
14
14
  ast (2.1.0)
15
15
  astrolabe (1.3.1)
16
16
  parser (~> 2.2)
17
- aws-sdk (2.2.0)
18
- aws-sdk-resources (= 2.2.0)
19
- aws-sdk-core (2.2.0)
17
+ aws-sdk (2.6.14)
18
+ aws-sdk-resources (= 2.6.14)
19
+ aws-sdk-core (2.6.14)
20
20
  jmespath (~> 1.0)
21
- aws-sdk-resources (2.2.0)
22
- aws-sdk-core (= 2.2.0)
21
+ aws-sdk-resources (2.6.14)
22
+ aws-sdk-core (= 2.6.14)
23
23
  coderay (1.1.0)
24
24
  crack (0.4.2)
25
25
  safe_yaml (~> 1.0.0)
26
26
  docile (1.1.5)
27
- elasticsearch (1.0.14)
28
- elasticsearch-api (= 1.0.14)
29
- elasticsearch-transport (= 1.0.14)
30
- elasticsearch-api (1.0.14)
27
+ elasticsearch (1.1.0)
28
+ elasticsearch-api (= 1.1.0)
29
+ elasticsearch-transport (= 1.1.0)
30
+ elasticsearch-api (1.1.0)
31
31
  multi_json
32
32
  elasticsearch-extensions (0.0.18)
33
33
  ansi
34
34
  ruby-prof
35
- elasticsearch-transport (1.0.14)
35
+ elasticsearch-transport (1.1.0)
36
36
  faraday
37
37
  multi_json
38
38
  faraday (0.9.2)
@@ -53,7 +53,7 @@ GEM
53
53
  guard-compat (~> 1.2)
54
54
  minitest (>= 3.0)
55
55
  hashdiff (0.2.2)
56
- jmespath (1.1.3)
56
+ jmespath (1.3.1)
57
57
  json (1.8.3)
58
58
  listen (3.0.3)
59
59
  rb-fsevent (>= 0.9.3)
@@ -62,7 +62,7 @@ GEM
62
62
  method_source (0.8.2)
63
63
  minitest (5.8.1)
64
64
  mixlib-shellout (2.2.3)
65
- multi_json (1.11.2)
65
+ multi_json (1.12.1)
66
66
  multipart-post (2.0.0)
67
67
  nenv (0.2.0)
68
68
  notiffany (0.0.8)
@@ -127,3 +127,6 @@ DEPENDENCIES
127
127
  vcr (~> 2.9)
128
128
  webmock (~> 1.21)
129
129
  yard (~> 0.8)
130
+
131
+ BUNDLED WITH
132
+ 1.13.2
@@ -38,12 +38,12 @@ module Elasticsearch
38
38
  @asg_client ||= AutoScaling.new(@asg_name, @region)
39
39
  end
40
40
 
41
- # Convience method to access {Elasticsearch::Drain::Nodes}
41
+ # Convenience method to access {Elasticsearch::Drain::Nodes}
42
42
  def nodes
43
- Nodes.new(client, asg)
43
+ @nodes ||= Nodes.new(client, asg)
44
44
  end
45
45
 
46
- # Convience method to access {Elasticsearch::Drain::Cluster#cluster}
46
+ # Convenience method to access {Elasticsearch::Drain::Cluster#cluster}
47
47
  #
48
48
  # @return [Elasticsearch::API::Cluster] Elasticsearch cluster client
49
49
  def cluster
@@ -52,7 +52,7 @@ module Elasticsearch
52
52
 
53
53
  def active_nodes_in_asg
54
54
  instances = asg.instances
55
- nodes.nodes_in_asg(reload: true, instances: instances)
55
+ nodes.filter_nodes(instances, true)
56
56
  end
57
57
 
58
58
  module Errors
@@ -8,20 +8,27 @@ module Elasticsearch
8
8
  attr_reader :asg
9
9
 
10
10
  # @attribute [r]
11
- # EC2 Client
12
- attr_reader :ec2_client
11
+ # AWS region
12
+ attr_reader :region
13
13
 
14
14
  def initialize(asg, region)
15
15
  @asg = asg
16
- @asg_client = Aws::AutoScaling::Client.new(region: region)
17
- @ec2_client = Aws::EC2::Client.new(region: region)
16
+ @region = region
18
17
  @instances = nil
19
18
  @instance_ids = nil
20
19
  end
21
20
 
21
+ def asg_client
22
+ Aws::AutoScaling::Client.new(region: region)
23
+ end
24
+
25
+ def ec2_client
26
+ Aws::EC2::Client.new(region: region)
27
+ end
28
+
22
29
  def find_instances_in_asg
23
30
  instances = []
24
- @asg_client.describe_auto_scaling_instances.each do |page|
31
+ asg_client.describe_auto_scaling_instances.each do |page|
25
32
  instances << page.auto_scaling_instances.map do |i|
26
33
  i.instance_id if i.auto_scaling_group_name == asg
27
34
  end
@@ -38,7 +45,7 @@ module Elasticsearch
38
45
  instances = []
39
46
  find_instances_in_asg if @instance_ids.nil?
40
47
  return [] if @instance_ids.empty?
41
- @ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
48
+ ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
42
49
  instances << page.reservations.map(&:instances)
43
50
  end
44
51
  instances.flatten!
@@ -49,7 +56,7 @@ module Elasticsearch
49
56
  #
50
57
  # @return [Struct] AutoScaling Group
51
58
  def describe_autoscaling_group
52
- groups = @asg_client.describe_auto_scaling_groups(
59
+ groups = asg_client.describe_auto_scaling_groups(
53
60
  auto_scaling_group_names: [asg]
54
61
  )
55
62
  groups.auto_scaling_groups.first
@@ -81,11 +88,11 @@ module Elasticsearch
81
88
  # @option [FixNum] count (0) The new MinSize of the AutoScalingGroup
82
89
  # @return [Struct] Empty response from the sdk
83
90
  def min_size=(count = 0)
84
- @asg_client.update_auto_scaling_group(
91
+ asg_client.update_auto_scaling_group(
85
92
  auto_scaling_group_name: asg,
86
93
  min_size: count
87
94
  )
88
- wait_until(0) do
95
+ wait_until(count) do
89
96
  min_size
90
97
  end
91
98
  end
@@ -108,7 +115,7 @@ module Elasticsearch
108
115
 
109
116
  def detach_instance(instance_id)
110
117
  current_desired_capacity = desired_capacity
111
- @asg_client.detach_instances(
118
+ asg_client.detach_instances(
112
119
  instance_ids: [instance_id],
113
120
  auto_scaling_group_name: asg,
114
121
  should_decrement_desired_capacity: true
@@ -2,27 +2,93 @@ require 'thor'
2
2
 
3
3
  module Elasticsearch
4
4
  class Drain
5
- class CLI < ::Thor
5
+ class CLI < ::Thor # rubocop:disable Metrics/ClassLength
6
6
  package_name :elasticsearch
7
7
 
8
8
  attr_reader :drainer
9
9
  attr_accessor :active_nodes
10
10
 
11
+ # rubocop:disable Metrics/LineLength
11
12
  desc 'asg', 'Drain all documents from all nodes in an EC2 AutoScaling Group'
12
13
  option :host, default: 'localhost:9200'
13
14
  option :asg, required: true
14
15
  option :region, required: true
16
+ option :nodes, type: :array, desc: 'A comma separated list of node IDs to drain. If specified, the --number option has no effect'
17
+ option :number, type: :numeric, desc: 'The number of nodes to drain'
18
+ option :continue, type: :boolean, default: true, desc: 'Whether to continue draining nodes once the first iteration of --number is complete'
19
+ # rubocop:enable Metrics/LineLength
15
20
  def asg # rubocop:disable Metrics/MethodLength
16
21
  @drainer = Elasticsearch::Drain.new(options[:host],
17
22
  options[:asg],
18
23
  options[:region])
24
+
19
25
  ensure_cluster_healthy
20
26
  @active_nodes = drainer.active_nodes_in_asg
27
+
28
+ # If :nodes are specified, :number has no effect
29
+ if options[:nodes]
30
+ say "Nodes #{options[:nodes].join(', ')} have been specified, the --number option has no effect"
31
+ number_to_drain = nil
32
+ currently_draining_nodes = nil
33
+ else
34
+ number_to_drain = options[:number]
35
+ currently_draining_nodes = drainer.cluster.currently_draining('_id')
36
+ end
37
+
38
+ # If a node or nodes are specified, only drain the requested node(s)
39
+ @active_nodes = active_nodes.find_all do |n|
40
+ instance_id = drainer.asg.instance(n.ipaddress).instance_id
41
+ options[:nodes].include?(instance_id)
42
+ end if options[:nodes]
43
+
21
44
  do_exit { say_status 'Complete', 'Nothing to do', :green } if active_nodes.empty?
22
- say_status 'Found Nodes', "AutoScalingGroup: #{instances}", :magenta
23
- ensure_cluster_healthy
24
- drain_nodes
25
- remove_nodes
45
+ say_status 'Found Nodes', "AutoScalingGroup: #{instances(active_nodes)}", :magenta
46
+
47
+ until active_nodes.empty?
48
+ ensure_cluster_healthy
49
+
50
+ nodes = active_nodes
51
+
52
+ # If there are nodes in cluster settings "transient.cluster.routing.allocation.exclude"
53
+ # test if those nodes are still in the ASG. If so, work on them first unless nodes are
54
+ # specified.
55
+ if currently_draining_nodes
56
+ nodes_to_drain = active_nodes.find_all { |n| currently_draining_nodes.split(',').include?(n.id) }
57
+
58
+ # If the list of nodes_to_drain isn't empty, we want to set nodes to the list of nodes
59
+ # we've already been working on.
60
+ unless nodes_to_drain.empty?
61
+ nodes = nodes_to_drain
62
+
63
+ say_status 'Active Nodes', "Resuming drain process on #{instances(nodes)}", :magenta
64
+ end
65
+
66
+ # We should only process currently_draining_nodes once
67
+ currently_draining_nodes = nil
68
+ end
69
+
70
+ # If we specify a number but DON'T specify nodes, sample the active_nodes.
71
+ if number_to_drain
72
+ nodes = nodes.sample(number_to_drain.to_i)
73
+ say_status 'Active Nodes', "Sampled #{number_to_drain} nodes and got #{instances(nodes)}", :magenta
74
+ end
75
+
76
+ @active_nodes = nodes unless options[:continue]
77
+
78
+ drain_nodes(nodes)
79
+ remove_nodes(nodes)
80
+
81
+ # Remove the drained nodes from the list of active_nodes
82
+ @active_nodes -= nodes
83
+
84
+ unless active_nodes.empty?
85
+ say_status 'Drain Nodes', "#{active_nodes.length} nodes remaining", :green
86
+
87
+ sleep_time = wait_sleep_time
88
+ say_status 'Waiting', "Sleeping for #{sleep_time} seconds before the next iteration", :green
89
+ sleep sleep_time
90
+ end
91
+ end
26
92
  say_status 'Complete', 'Draining nodes complete!', :green
27
93
  end
28
94
 
@@ -40,29 +106,54 @@ module Elasticsearch
40
106
  exit code
41
107
  end
42
108
 
43
- def instances
44
- instances = active_nodes.map(&:ipaddress)
109
+ def instances(nodes)
110
+ instances = nodes.map(&:ipaddress)
45
111
  instances.join(' ')
46
112
  end
47
113
 
48
- def drain_nodes
49
- drainer.asg.min_size = 0
50
- nodes_to_drain = active_nodes.map(&:id).join(',')
114
+ def adjusted_min_size(nodes)
115
+ min_size = drainer.asg.min_size
116
+ desired_capacity = drainer.asg.desired_capacity
117
+ desired_min_size = if (desired_capacity - nodes.length) >= min_size # Removing the nodes won't violate the min_size
118
+ # Reduce the asg min_size proportionally
119
+ (min_size - nodes.length) <= 0 ? 0 : (min_size - nodes.length)
120
+ else
121
+ # Removing the nodes will result in the min_size being violated
122
+ desired_capacity - nodes.length
123
+ end
124
+ say_status 'Debug', 'min_size = #{min_size}, desired_capacity = #{desired_capacity}, desired_min_size = #{desired_min_size}', :magenta
125
+ desired_min_size
126
+ end
127
+
128
+ def drain_nodes(nodes)
129
+ drainer.asg.min_size = adjusted_min_size(nodes)
130
+ nodes_to_drain = nodes.map(&:id).join(',')
51
131
  say_status 'Drain Nodes', "Draining nodes: #{nodes_to_drain}", :magenta
52
132
  drainer.cluster.drain_nodes(nodes_to_drain, '_id')
53
133
  end
54
134
 
55
- def remove_nodes # rubocop:disable Metrics/MethodLength
56
- while active_nodes.length > 0
57
- active_nodes.each do |instance|
58
- self.active_nodes = drainer.active_nodes_in_asg
135
+ def wait_sleep_time
136
+ ips = active_nodes.map(&:ipaddress)
137
+ bytes = drainer.nodes.filter_nodes(ips).map(&:bytes_stored)
138
+ sleep_time = 10
139
+ sleep_time = 30 if bytes.any? { |b| b >= 100_000 }
140
+ sleep_time = 60 if bytes.any? { |b| b >= 1_000_000 }
141
+ sleep_time = 120 if bytes.any? { |b| b >= 10_000_000_000 }
142
+ sleep_time
143
+ end
144
+
145
+ def remove_nodes(nodes) # rubocop:disable Metrics/MethodLength
146
+ while nodes.length > 0
147
+ sleep_time = wait_sleep_time
148
+ nodes.each do |instance|
149
+ instance = drainer.nodes.filter_nodes([instance.ipaddress], true).first
59
150
  if instance.bytes_stored > 0
60
151
  say_status 'Drain Status', "Node #{instance.ipaddress} has #{instance.bytes_stored} bytes to move", :blue
61
- sleep 2
152
+ sleep sleep_time
62
153
  else
63
154
  next unless remove_node(instance)
64
- self.active_nodes = drainer.active_nodes_in_asg
65
- break if active_nodes.length < 1
155
+ nodes.delete_if { |n| n.ipaddress == instance.ipaddress }
156
+ break if nodes.length < 1
66
157
  say_status 'Waiting', 'Sleeping for 1 minute before removing the next node', :green
67
158
  sleep 60
68
159
  end
@@ -71,12 +162,13 @@ module Elasticsearch
71
162
  end
72
163
 
73
164
  def remove_node(instance) # rubocop:disable Metrics/MethodLength
74
- instance_id = drainer.asg.instance(ipaddress).instance_id
165
+ instance_id = drainer.asg.instance(instance.ipaddress).instance_id
75
166
  instance.instance_id = instance_id
76
167
  say_status(
77
168
  'Removing Node',
78
169
  "Removing #{instance.ipaddress} from Elasticsearch cluster and #{drainer.asg.asg} AutoScalingGroup",
79
- :magenta)
170
+ :magenta
171
+ )
80
172
  sleep 5 unless instance.in_recovery?
81
173
  node = "#{instance.instance_id}(#{instance.ipaddress})"
82
174
  ensure_cluster_healthy
@@ -34,6 +34,11 @@ module Elasticsearch
34
34
  }
35
35
  )
36
36
  end
37
+
38
+ def currently_draining(exclude_by = '_ip')
39
+ settings = cluster.get_settings(:flat_settings => true)
40
+ settings.fetch('transient', {}).fetch("cluster.routing.allocation.exclude.#{exclude_by}", nil)
41
+ end
37
42
  end
38
43
  end
39
44
  end
@@ -99,7 +99,7 @@ module Elasticsearch
99
99
  @asg.ec2_client.wait_until(:instance_terminated,
100
100
  instance_ids: [instance_id]) do |w|
101
101
  w.max_attempts = 10
102
- w.delay = 30
102
+ w.delay = 60
103
103
  end
104
104
  end
105
105
  end
@@ -26,7 +26,7 @@ module Elasticsearch
26
26
  # Get list of nodes in the cluster
27
27
  #
28
28
  # @return [Array<OpenStruct>] Array of node objects
29
- def nodes(reload: false)
29
+ def nodes(reload = false)
30
30
  load if reload
31
31
  @info['nodes'].map do |node|
32
32
  Drain::Node.new(
@@ -38,8 +38,8 @@ module Elasticsearch
38
38
  end
39
39
  end
40
40
 
41
- def nodes_in_asg(reload: false, instances:)
42
- nodes(reload: false).find_all { |n| instances.include? n.ipaddress }
41
+ def filter_nodes(instances, reload = false)
42
+ nodes(reload).find_all { |n| instances.include? n.ipaddress }
43
43
  end
44
44
  end
45
45
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: elasticsearch-drain
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.4
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Andrew Thompson
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-11-23 00:00:00.000000000 Z
11
+ date: 2020-08-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
142
142
  version: '0'
143
143
  requirements: []
144
144
  rubyforge_project:
145
- rubygems_version: 2.4.4
145
+ rubygems_version: 2.4.3
146
146
  signing_key:
147
147
  specification_version: 4
148
148
  summary: Elasticsearch node replacement utility that tries to keep the cluster healthy