elasticsearch-drain 0.0.4 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b4a387835090abe799f17003384cf7981dd4c6c4
-  data.tar.gz: ad7011c24bad8354389715c6b7a84aea56ed34d3
+  metadata.gz: 3099c4c4c099c9128357fb2dad397fbfd54bb0b2
+  data.tar.gz: 1e1c56d6bf0a91a9b6f30c86c6d14ecf36c1d341
 SHA512:
-  metadata.gz: 58cef0753a1b0a874eb321d8b5ab731704661c5c3619451f23e3432a880924069a73e07155cbacc5c23cb721cd227e1fad78ff5d78dd7c56c50f546800c4b935
-  data.tar.gz: 6e2e83b799067ef16c4865c29949007f6ff30d492a93dec8c0070d61b535d35fa3c1efc3c042649c34218089efcae1edb74fdcdcf8c6e9b6ee51a6a208b31542
+  metadata.gz: f8b791433e9e39697b59e9d8b3f45d60bc42c01e48c972ed372e379bb2fe6668d9f633fc2d108c35cb5a3f415cb15e455711b081a0481e678bdd9e0ecfb0aee5
+  data.tar.gz: 284eb015e4b3a5013308b14d029326b4e6f2452e4b27a985898554521d2d7fa7bc176ba6ae6551384d46a73eea747a4e840ec55e75bd2fd32f972be7bcab8d23

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    elasticsearch-drain (0.0.3)
+    elasticsearch-drain (0.0.1)
       aws-sdk (~> 2)
       elasticsearch (~> 1.0)
       thor (~> 0.19)
@@ -14,25 +14,25 @@ GEM
     ast (2.1.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    aws-sdk (2.2.0)
-      aws-sdk-resources (= 2.2.0)
-    aws-sdk-core (2.2.0)
+    aws-sdk (2.6.14)
+      aws-sdk-resources (= 2.6.14)
+    aws-sdk-core (2.6.14)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.2.0)
-      aws-sdk-core (= 2.2.0)
+    aws-sdk-resources (2.6.14)
+      aws-sdk-core (= 2.6.14)
     coderay (1.1.0)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     docile (1.1.5)
-    elasticsearch (1.0.14)
-      elasticsearch-api (= 1.0.14)
-      elasticsearch-transport (= 1.0.14)
-    elasticsearch-api (1.0.14)
+    elasticsearch (1.1.0)
+      elasticsearch-api (= 1.1.0)
+      elasticsearch-transport (= 1.1.0)
+    elasticsearch-api (1.1.0)
       multi_json
     elasticsearch-extensions (0.0.18)
       ansi
       ruby-prof
-    elasticsearch-transport (1.0.14)
+    elasticsearch-transport (1.1.0)
       faraday
       multi_json
     faraday (0.9.2)
@@ -53,7 +53,7 @@ GEM
       guard-compat (~> 1.2)
       minitest (>= 3.0)
     hashdiff (0.2.2)
-    jmespath (1.1.3)
+    jmespath (1.3.1)
     json (1.8.3)
     listen (3.0.3)
       rb-fsevent (>= 0.9.3)
@@ -62,7 +62,7 @@ GEM
     method_source (0.8.2)
     minitest (5.8.1)
     mixlib-shellout (2.2.3)
-    multi_json (1.11.2)
+    multi_json (1.12.1)
     multipart-post (2.0.0)
     nenv (0.2.0)
     notiffany (0.0.8)
@@ -127,3 +127,6 @@ DEPENDENCIES
   vcr (~> 2.9)
   webmock (~> 1.21)
   yard (~> 0.8)
+
+BUNDLED WITH
+   1.13.2

data/lib/elasticsearch/drain.rb

@@ -38,12 +38,12 @@ module Elasticsearch
       @asg_client ||= AutoScaling.new(@asg_name, @region)
     end
 
-    # Convience method to access {Elasticsearch::Drain::Nodes}
+    # Convenience method to access {Elasticsearch::Drain::Nodes}
     def nodes
-      Nodes.new(client, asg)
+      @nodes ||= Nodes.new(client, asg)
     end
 
-    # Convience method to access {Elasticsearch::Drain::Cluster#cluster}
+    # Convenience method to access {Elasticsearch::Drain::Cluster#cluster}
     #
     # @return [Elasticsearch::API::Cluster] Elasticsearch cluster client
     def cluster
@@ -52,7 +52,7 @@ module Elasticsearch
 
     def active_nodes_in_asg
       instances = asg.instances
-      nodes.nodes_in_asg(reload: true, instances: instances)
+      nodes.filter_nodes(instances, true)
     end
 
     module Errors

data/lib/elasticsearch/drain/autoscaling.rb

@@ -8,20 +8,27 @@ module Elasticsearch
       attr_reader :asg
 
       # @attribute [r]
-      # EC2 Client
-      attr_reader :ec2_client
+      # AWS region
+      attr_reader :region
 
       def initialize(asg, region)
         @asg = asg
-        @asg_client = Aws::AutoScaling::Client.new(region: region)
-        @ec2_client = Aws::EC2::Client.new(region: region)
+        @region = region
         @instances = nil
         @instance_ids = nil
       end
 
+      def asg_client
+        Aws::AutoScaling::Client.new(region: region)
+      end
+
+      def ec2_client
+        Aws::EC2::Client.new(region: region)
+      end
+
       def find_instances_in_asg
         instances = []
-        @asg_client.describe_auto_scaling_instances.each do |page|
+        asg_client.describe_auto_scaling_instances.each do |page|
           instances << page.auto_scaling_instances.map do |i|
             i.instance_id if i.auto_scaling_group_name == asg
           end
@@ -38,7 +45,7 @@ module Elasticsearch
         instances = []
         find_instances_in_asg if @instance_ids.nil?
         return [] if @instance_ids.empty?
-        @ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
+        ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
           instances << page.reservations.map(&:instances)
         end
         instances.flatten!
@@ -49,7 +56,7 @@ module Elasticsearch
       #
       # @return [Struct] AutoScaling Group
       def describe_autoscaling_group
-        groups = @asg_client.describe_auto_scaling_groups(
+        groups = asg_client.describe_auto_scaling_groups(
           auto_scaling_group_names: [asg]
         )
         groups.auto_scaling_groups.first
@@ -81,11 +88,11 @@ module Elasticsearch
       # @option [FixNum] count (0) The new MinSize of the AutoScalingGroup
       # @return [Struct] Empty response from the sdk
       def min_size=(count = 0)
-        @asg_client.update_auto_scaling_group(
+        asg_client.update_auto_scaling_group(
           auto_scaling_group_name: asg,
           min_size: count
         )
-        wait_until(0) do
+        wait_until(count) do
           min_size
         end
       end
@@ -108,7 +115,7 @@ module Elasticsearch
 
       def detach_instance(instance_id)
         current_desired_capacity = desired_capacity
-        @asg_client.detach_instances(
+        asg_client.detach_instances(
           instance_ids: [instance_id],
           auto_scaling_group_name: asg,
           should_decrement_desired_capacity: true

data/lib/elasticsearch/drain/cli.rb

@@ -2,27 +2,93 @@ require 'thor'
 
 module Elasticsearch
   class Drain
-    class CLI < ::Thor
+    class CLI < ::Thor # rubocop:disable Metrics/ClassLength
       package_name :elasticsearch
 
       attr_reader :drainer
       attr_accessor :active_nodes
 
+      # rubocop:disable Metrics/LineLength
       desc 'asg', 'Drain all documents from all nodes in an EC2 AutoScaling Group'
       option :host, default: 'localhost:9200'
       option :asg, required: true
       option :region, required: true
+      option :nodes, type: :array, desc: 'A comma separated list of node IDs to drain. If specified, the --number option has no effect'
+      option :number, type: :numeric, desc: 'The number of nodes to drain'
+      option :continue, type: :boolean, default: true, desc: 'Whether to continue draining nodes once the first iteration of --number is complete'
+      # rubocop:enable Metrics/LineLength
       def asg # rubocop:disable Metrics/MethodLength
         @drainer = Elasticsearch::Drain.new(options[:host],
                                             options[:asg],
                                             options[:region])
+
         ensure_cluster_healthy
         @active_nodes = drainer.active_nodes_in_asg
+
+        # If :nodes are specified, :number has no effect
+        if options[:nodes]
+          say "Nodes #{options[:nodes].join(', ')} have been specified, the --number option has no effect"
+          number_to_drain = nil
+          currently_draining_nodes = nil
+        else
+          number_to_drain = options[:number]
+          currently_draining_nodes = drainer.cluster.currently_draining('_id')
+        end
+
+        # If a node or nodes are specified, only drain the requested node(s)
+        @active_nodes = active_nodes.find_all do |n|
+          instance_id = drainer.asg.instance(n.ipaddress).instance_id
+          options[:nodes].include?(instance_id)
+        end if options[:nodes]
+
         do_exit { say_status 'Complete', 'Nothing to do', :green } if active_nodes.empty?
-        say_status 'Found Nodes', "AutoScalingGroup: #{instances}", :magenta
-        ensure_cluster_healthy
-        drain_nodes
-        remove_nodes
+        say_status 'Found Nodes', "AutoScalingGroup: #{instances(active_nodes)}", :magenta
+
+        until active_nodes.empty?
+          ensure_cluster_healthy
+
+          nodes = active_nodes
+
+          # If there are nodes in cluster settings "transient.cluster.routing.allocation.exclude"
+          # test if those nodes are still in the ASG. If so, work on them first unless nodes are
+          # specified.
+          if currently_draining_nodes
+            nodes_to_drain = active_nodes.find_all { |n| currently_draining_nodes.split(',').include?(n.id) }
+
+            # If the list of nodes_to_drain isn't empty, we want to set nodes to the list of nodes
+            # we've already been working on.
+            unless nodes_to_drain.empty?
+              nodes = nodes_to_drain
+
+              say_status 'Active Nodes', "Resuming drain process on #{instances(nodes)}", :magenta
+            end
+
+            # We should only process currently_draining_nodes once
+            currently_draining_nodes = nil
+          end
+
+          # If we specify a number but DON'T specify nodes, sample the active_nodes.
+          if number_to_drain
+            nodes = nodes.sample(number_to_drain.to_i)
+            say_status 'Active Nodes', "Sampled #{number_to_drain} nodes and got #{instances(nodes)}", :magenta
+          end
+
+          @active_nodes = nodes unless options[:continue]
+
+          drain_nodes(nodes)
+          remove_nodes(nodes)
+
+          # Remove the drained nodes from the list of active_nodes
+          @active_nodes -= nodes
+
+          unless active_nodes.empty?
+            say_status 'Drain Nodes', "#{active_nodes.length} nodes remaining", :green
+
+            sleep_time = wait_sleep_time
+            say_status 'Waiting', "Sleeping for #{sleep_time} seconds before the next iteration", :green
+            sleep sleep_time
+          end
+        end
         say_status 'Complete', 'Draining nodes complete!', :green
       end
 
@@ -40,29 +106,54 @@ module Elasticsearch
           exit code
         end
 
-        def instances
-          instances = active_nodes.map(&:ipaddress)
+        def instances(nodes)
+          instances = nodes.map(&:ipaddress)
           instances.join(' ')
         end
 
-        def drain_nodes
-          drainer.asg.min_size = 0
-          nodes_to_drain = active_nodes.map(&:id).join(',')
+        def adjusted_min_size(nodes)
+          min_size = drainer.asg.min_size
+          desired_capacity = drainer.asg.desired_capacity
+          desired_min_size = if (desired_capacity - nodes.length) >= min_size # Removing the nodes won't violate the min_size
+                               # Reduce the asg min_size proportionally
+                               (min_size - nodes.length) <= 0 ? 0 : (min_size - nodes.length)
+                             else
+                               # Removing the nodes will result in the min_size being violated
+                               desired_capacity - nodes.length
+                             end
+          say_status 'Debug', 'min_size = #{min_size}, desired_capacity = #{desired_capacity}, desired_min_size = #{desired_min_size}', :magenta
+          desired_min_size
+        end
+
+        def drain_nodes(nodes)
+          drainer.asg.min_size = adjusted_min_size(nodes)
+          nodes_to_drain = nodes.map(&:id).join(',')
           say_status 'Drain Nodes', "Draining nodes: #{nodes_to_drain}", :magenta
           drainer.cluster.drain_nodes(nodes_to_drain, '_id')
         end
 
-        def remove_nodes # rubocop:disable Metrics/MethodLength
-          while active_nodes.length > 0
-            active_nodes.each do |instance|
-              self.active_nodes = drainer.active_nodes_in_asg
+        def wait_sleep_time
+          ips = active_nodes.map(&:ipaddress)
+          bytes = drainer.nodes.filter_nodes(ips).map(&:bytes_stored)
+          sleep_time = 10
+          sleep_time = 30 if bytes.any? { |b| b >= 100_000 }
+          sleep_time = 60 if bytes.any? { |b| b >= 1_000_000 }
+          sleep_time = 120 if bytes.any? { |b| b >= 10_000_000_000 }
+          sleep_time
+        end
+
+        def remove_nodes(nodes) # rubocop:disable Metrics/MethodLength
+          while nodes.length > 0
+            sleep_time = wait_sleep_time
+            nodes.each do |instance|
+              instance = drainer.nodes.filter_nodes([instance.ipaddress], true).first
               if instance.bytes_stored > 0
                 say_status 'Drain Status', "Node #{instance.ipaddress} has #{instance.bytes_stored} bytes to move", :blue
-                sleep 2
+                sleep sleep_time
               else
                 next unless remove_node(instance)
-                self.active_nodes = drainer.active_nodes_in_asg
-                break if active_nodes.length < 1
+                nodes.delete_if { |n| n.ipaddress == instance.ipaddress }
+                break if nodes.length < 1
                 say_status 'Waiting', 'Sleeping for 1 minute before removing the next node', :green
                 sleep 60
               end
@@ -71,12 +162,13 @@ module Elasticsearch
         end
 
         def remove_node(instance) # rubocop:disable Metrics/MethodLength
-          instance_id = drainer.asg.instance(ipaddress).instance_id
+          instance_id = drainer.asg.instance(instance.ipaddress).instance_id
           instance.instance_id = instance_id
           say_status(
             'Removing Node',
             "Removing #{instance.ipaddress} from Elasticsearch cluster and #{drainer.asg.asg} AutoScalingGroup",
-            :magenta)
+            :magenta
+          )
           sleep 5 unless instance.in_recovery?
           node = "#{instance.instance_id}(#{instance.ipaddress})"
           ensure_cluster_healthy

data/lib/elasticsearch/drain/cluster.rb

@@ -34,6 +34,11 @@ module Elasticsearch
           }
         )
       end
+
+      def currently_draining(exclude_by = '_ip')
+        settings = cluster.get_settings(:flat_settings => true)
+        settings.fetch('transient', {}).fetch("cluster.routing.allocation.exclude.#{exclude_by}", nil)
+      end
     end
   end
 end

data/lib/elasticsearch/drain/node.rb

@@ -99,7 +99,7 @@ module Elasticsearch
         @asg.ec2_client.wait_until(:instance_terminated,
                                    instance_ids: [instance_id]) do |w|
           w.max_attempts = 10
-          w.delay = 30
+          w.delay = 60
         end
       end
     end

data/lib/elasticsearch/drain/nodes.rb

@@ -26,7 +26,7 @@ module Elasticsearch
       # Get list of nodes in the cluster
       #
       # @return [Array<OpenStruct>] Array of node objects
-      def nodes(reload: false)
+      def nodes(reload = false)
         load if reload
         @info['nodes'].map do |node|
           Drain::Node.new(
@@ -38,8 +38,8 @@ module Elasticsearch
         end
       end
 
-      def nodes_in_asg(reload: false, instances:)
-        nodes(reload: false).find_all { |n| instances.include? n.ipaddress }
+      def filter_nodes(instances, reload = false)
+        nodes(reload).find_all { |n| instances.include? n.ipaddress }
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-drain
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.2
 platform: ruby
 authors:
 - Andrew Thompson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-11-23 00:00:00.000000000 Z
+date: 2020-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -142,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.4
+rubygems_version: 2.4.3
 signing_key: 
 specification_version: 4
 summary: Elasticsearch node replacement utility that tries to keep the cluster healthy