elasticsearch-drain 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5636da7177de03b2cc46e1f5f9c3382c542c25b1
-  data.tar.gz: ae9b51d1c2ace5e83497a6a92ba956523d713e01
+  metadata.gz: 087e3d64398481deae576f955f1b22d5cd0e5e36
+  data.tar.gz: cf6ad7098dc51e8881d63d739cf53cd46e137e9a
 SHA512:
-  metadata.gz: 7bbe3a3434ff69a13826251cb29561a6d95431294a7bc8590883480edbba598ff206c4d17d048a17a053e95fae4fb53f19ddf624599bf334215fa0b79ab34c1b
-  data.tar.gz: dfec04ba5e72334f4d75a597b512230f93670374d22afcbfc92f4b6e3a0683a2cf191394dba4072b84c68fefdf25f1def1417984d5a44f1f4d6a2616142ec346
+  metadata.gz: c75c5d28d018e42f29e39f91a8cd8c94bf9596efe90baf742e72d39986c4fa1c6ed246c23de8277cefbda1713c1b224dbb4178b4c8c8bf227aee7781ab8d3586
+  data.tar.gz: 37c5eac72feea75743dfabd03858fdee00b9e033353e69e0cd5163a857931c8276a5472b314c81effeb6038ecbe3e9e499f76f5b9c3e62084f754471b758cab2

data/Gemfile.lock

@@ -14,25 +14,25 @@ GEM
     ast (2.1.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    aws-sdk (2.4.4)
-      aws-sdk-resources (= 2.4.4)
-    aws-sdk-core (2.4.4)
+    aws-sdk (2.6.14)
+      aws-sdk-resources (= 2.6.14)
+    aws-sdk-core (2.6.14)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.4.4)
-      aws-sdk-core (= 2.4.4)
+    aws-sdk-resources (2.6.14)
+      aws-sdk-core (= 2.6.14)
     coderay (1.1.0)
     crack (0.4.2)
       safe_yaml (~> 1.0.0)
     docile (1.1.5)
-    elasticsearch (1.0.14)
-      elasticsearch-api (= 1.0.14)
-      elasticsearch-transport (= 1.0.14)
-    elasticsearch-api (1.0.14)
+    elasticsearch (1.1.0)
+      elasticsearch-api (= 1.1.0)
+      elasticsearch-transport (= 1.1.0)
+    elasticsearch-api (1.1.0)
       multi_json
     elasticsearch-extensions (0.0.18)
       ansi
       ruby-prof
-    elasticsearch-transport (1.0.14)
+    elasticsearch-transport (1.1.0)
       faraday
       multi_json
     faraday (0.9.2)
@@ -127,3 +127,6 @@ DEPENDENCIES
   vcr (~> 2.9)
   webmock (~> 1.21)
   yard (~> 0.8)
+
+BUNDLED WITH
+   1.13.2

data/lib/elasticsearch/drain.rb

@@ -38,12 +38,12 @@ module Elasticsearch
       @asg_client ||= AutoScaling.new(@asg_name, @region)
     end
 
-    # Convience method to access {Elasticsearch::Drain::Nodes}
+    # Convenience method to access {Elasticsearch::Drain::Nodes}
     def nodes
       @nodes ||= Nodes.new(client, asg)
     end
 
-    # Convience method to access {Elasticsearch::Drain::Cluster#cluster}
+    # Convenience method to access {Elasticsearch::Drain::Cluster#cluster}
     #
     # @return [Elasticsearch::API::Cluster] Elasticsearch cluster client
     def cluster

data/lib/elasticsearch/drain/autoscaling.rb

@@ -8,20 +8,27 @@ module Elasticsearch
       attr_reader :asg
 
       # @attribute [r]
-      # EC2 Client
-      attr_reader :ec2_client
+      # AWS region
+      attr_reader :region
 
       def initialize(asg, region)
         @asg = asg
-        @asg_client = Aws::AutoScaling::Client.new(region: region)
-        @ec2_client = Aws::EC2::Client.new(region: region)
+        @region = region
         @instances = nil
         @instance_ids = nil
       end
 
+      def asg_client
+        Aws::AutoScaling::Client.new(region: region)
+      end
+
+      def ec2_client
+        Aws::EC2::Client.new(region: region)
+      end
+
       def find_instances_in_asg
         instances = []
-        @asg_client.describe_auto_scaling_instances.each do |page|
+        asg_client.describe_auto_scaling_instances.each do |page|
           instances << page.auto_scaling_instances.map do |i|
             i.instance_id if i.auto_scaling_group_name == asg
           end
@@ -38,7 +45,7 @@ module Elasticsearch
         instances = []
         find_instances_in_asg if @instance_ids.nil?
         return [] if @instance_ids.empty?
-        @ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
+        ec2_client.describe_instances(instance_ids: @instance_ids).each do |page|
           instances << page.reservations.map(&:instances)
         end
         instances.flatten!
@@ -49,7 +56,7 @@ module Elasticsearch
       #
       # @return [Struct] AutoScaling Group
       def describe_autoscaling_group
-        groups = @asg_client.describe_auto_scaling_groups(
+        groups = asg_client.describe_auto_scaling_groups(
           auto_scaling_group_names: [asg]
         )
         groups.auto_scaling_groups.first
@@ -81,11 +88,11 @@ module Elasticsearch
       # @option [FixNum] count (0) The new MinSize of the AutoScalingGroup
       # @return [Struct] Empty response from the sdk
       def min_size=(count = 0)
-        @asg_client.update_auto_scaling_group(
+        asg_client.update_auto_scaling_group(
           auto_scaling_group_name: asg,
           min_size: count
         )
-        wait_until(0) do
+        wait_until(count) do
           min_size
         end
       end
@@ -108,7 +115,7 @@ module Elasticsearch
 
       def detach_instance(instance_id)
         current_desired_capacity = desired_capacity
-        @asg_client.detach_instances(
+        asg_client.detach_instances(
           instance_ids: [instance_id],
           auto_scaling_group_name: asg,
           should_decrement_desired_capacity: true

data/lib/elasticsearch/drain/cli.rb

@@ -2,35 +2,93 @@ require 'thor'
 
 module Elasticsearch
   class Drain
-    class CLI < ::Thor
+    class CLI < ::Thor # rubocop:disable Metrics/ClassLength
       package_name :elasticsearch
 
       attr_reader :drainer
       attr_accessor :active_nodes
 
+      # rubocop:disable Metrics/LineLength
       desc 'asg', 'Drain all documents from all nodes in an EC2 AutoScaling Group'
       option :host, default: 'localhost:9200'
       option :asg, required: true
       option :region, required: true
-      option :nodes
+      option :nodes, type: :array, desc: 'A comma separated list of node IDs to drain. If specified, the --number option has no effect'
+      option :number, type: :numeric, desc: 'The number of nodes to drain'
+      option :continue, type: :boolean, default: true, desc: 'Whether to continue draining nodes once the first iteration of --number is complete'
+      # rubocop:enable Metrics/LineLength
       def asg # rubocop:disable Metrics/MethodLength
         @drainer = Elasticsearch::Drain.new(options[:host],
                                             options[:asg],
                                             options[:region])
+
         ensure_cluster_healthy
         @active_nodes = drainer.active_nodes_in_asg
 
+        # If :nodes are specified, :number has no effect
+        if options[:nodes]
+          say "Nodes #{options[:nodes].join(', ')} have been specified, the --number option has no effect"
+          number_to_drain = nil
+          currently_draining_nodes = nil
+        else
+          number_to_drain = options[:number]
+          currently_draining_nodes = drainer.cluster.currently_draining('_id')
+        end
+
         # If a node or nodes are specified, only drain the requested node(s)
         @active_nodes = active_nodes.find_all do |n|
           instance_id = drainer.asg.instance(n.ipaddress).instance_id
-          options[:nodes].split(',').include?(instance_id)
+          options[:nodes].include?(instance_id)
         end if options[:nodes]
 
         do_exit { say_status 'Complete', 'Nothing to do', :green } if active_nodes.empty?
-        say_status 'Found Nodes', "AutoScalingGroup: #{instances}", :magenta
-        ensure_cluster_healthy
-        drain_nodes
-        remove_nodes
+        say_status 'Found Nodes', "AutoScalingGroup: #{instances(active_nodes)}", :magenta
+
+        until active_nodes.empty?
+          ensure_cluster_healthy
+
+          nodes = active_nodes
+
+          # If there are nodes in cluster settings "transient.cluster.routing.allocation.exclude"
+          # test if those nodes are still in the ASG. If so, work on them first unless nodes are
+          # specified.
+          if currently_draining_nodes
+            nodes_to_drain = active_nodes.find_all { |n| currently_draining_nodes.split(',').include?(n.id) }
+
+            # If the list of nodes_to_drain isn't empty, we want to set nodes to the list of nodes
+            # we've already been working on.
+            unless nodes_to_drain.empty?
+              nodes = nodes_to_drain
+
+              say_status 'Active Nodes', "Resuming drain process on #{instances(nodes)}", :magenta
+            end
+
+            # We should only process currently_draining_nodes once
+            currently_draining_nodes = nil
+          end
+
+          # If we specify a number but DON'T specify nodes, sample the active_nodes.
+          if number_to_drain
+            nodes = nodes.sample(number_to_drain.to_i)
+            say_status 'Active Nodes', "Sampled #{number_to_drain} nodes and got #{instances(nodes)}", :magenta
+          end
+
+          @active_nodes = nodes unless options[:continue]
+
+          drain_nodes(nodes)
+          remove_nodes(nodes)
+
+          # Remove the drained nodes from the list of active_nodes
+          @active_nodes -= nodes
+
+          unless active_nodes.empty?
+            say_status 'Drain Nodes', "#{active_nodes.length} nodes remaining", :green
+
+            sleep_time = wait_sleep_time
+            say_status 'Waiting', "Sleeping for #{sleep_time} seconds before the next iteration", :green
+            sleep sleep_time
+          end
+        end
         say_status 'Complete', 'Draining nodes complete!', :green
       end
 
@@ -48,42 +106,53 @@ module Elasticsearch
           exit code
         end
 
-        def instances
-          instances = active_nodes.map(&:ipaddress)
+        def instances(nodes)
+          instances = nodes.map(&:ipaddress)
           instances.join(' ')
         end
 
-        def adjusted_min_size
+        def adjusted_min_size(nodes)
           min_size = drainer.asg.min_size
           desired_capacity = drainer.asg.desired_capacity
-          if (desired_capacity - active_nodes.length) >= min_size # Removing the active_nodes won't violate the min_size
-            # Reduce the asg min_size proportionally
-            desired_min_size = (min_size - active_nodes.length) <= 0 ? 0 : (min_size - active_nodes.length)
-          else
-            # Removing the active_nodes will result in the min_size being violated
-            desired_min_size = desired_capacity - active_nodes.length
-          end
+          desired_min_size = if (desired_capacity - nodes.length) >= min_size # Removing the nodes won't violate the min_size
+                               # Reduce the asg min_size proportionally
+                               (min_size - nodes.length) <= 0 ? 0 : (min_size - nodes.length)
+                             else
+                               # Removing the nodes will result in the min_size being violated
+                               desired_capacity - nodes.length
+                             end
           desired_min_size
         end
 
-        def drain_nodes
-          drainer.asg.min_size = adjusted_min_size
-          nodes_to_drain = active_nodes.map(&:id).join(',')
+        def drain_nodes(nodes)
+          drainer.asg.min_size = adjusted_min_size(nodes)
+          nodes_to_drain = nodes.map(&:id).join(',')
           say_status 'Drain Nodes', "Draining nodes: #{nodes_to_drain}", :magenta
           drainer.cluster.drain_nodes(nodes_to_drain, '_id')
         end
 
-        def remove_nodes # rubocop:disable Metrics/MethodLength
-          while active_nodes.length > 0
-            active_nodes.each do |instance|
-              instance = drainer.nodes.filter_nodes([instance], true).first
+        def wait_sleep_time
+          ips = active_nodes.map(&:ipaddress)
+          bytes = drainer.nodes.filter_nodes(ips).map(&:bytes_stored)
+          sleep_time = 10
+          sleep_time = 30 if bytes.any? { |b| b >= 100_000 }
+          sleep_time = 60 if bytes.any? { |b| b >= 1_000_000 }
+          sleep_time = 120 if bytes.any? { |b| b >= 10_000_000_000 }
+          sleep_time
+        end
+
+        def remove_nodes(nodes) # rubocop:disable Metrics/MethodLength
+          while nodes.length > 0
+            sleep_time = wait_sleep_time
+            nodes.each do |instance|
+              instance = drainer.nodes.filter_nodes([instance.ipaddress], true).first
               if instance.bytes_stored > 0
                 say_status 'Drain Status', "Node #{instance.ipaddress} has #{instance.bytes_stored} bytes to move", :blue
-                sleep 2
+                sleep sleep_time
               else
                 next unless remove_node(instance)
-                active_nodes.delete_if { |n| n.ipaddress == instance.ipaddress }
-                break if active_nodes.length < 1
+                nodes.delete_if { |n| n.ipaddress == instance.ipaddress }
+                break if nodes.length < 1
                 say_status 'Waiting', 'Sleeping for 1 minute before removing the next node', :green
                 sleep 60
               end
@@ -97,7 +166,8 @@ module Elasticsearch
           say_status(
             'Removing Node',
             "Removing #{instance.ipaddress} from Elasticsearch cluster and #{drainer.asg.asg} AutoScalingGroup",
-            :magenta)
+            :magenta
+          )
           sleep 5 unless instance.in_recovery?
           node = "#{instance.instance_id}(#{instance.ipaddress})"
           ensure_cluster_healthy

data/lib/elasticsearch/drain/cluster.rb

@@ -34,6 +34,11 @@ module Elasticsearch
           }
         )
       end
+
+      def currently_draining(exclude_by = '_ip')
+        settings = cluster.get_settings(:flat_settings => true)
+        settings.fetch('transient', {}).fetch("cluster.routing.allocation.exclude.#{exclude_by}", nil)
+      end
     end
   end
 end

data/lib/elasticsearch/drain/node.rb

@@ -99,7 +99,7 @@ module Elasticsearch
         @asg.ec2_client.wait_until(:instance_terminated,
                                    instance_ids: [instance_id]) do |w|
           w.max_attempts = 10
-          w.delay = 30
+          w.delay = 60
         end
       end
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-drain
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Andrew Thompson
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-09 00:00:00.000000000 Z
+date: 2018-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -158,4 +158,3 @@ test_files:
 - test/elasticsearch/drain/test_nodes.rb
 - test/elasticsearch/test_drain.rb
 - test/test_helper.rb
-has_rdoc: