elasticsearch-manager 0.1.1 → 0.1.2.pre

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    N2FlYzM2ZDk2Njg4M2I5NTJkOGQ4M2ZjMGM3ZjA2MjFiMjk3MmY3ZQ==
-  data.tar.gz: !binary |-
-    OGVjMGNiNjUxZjhjZGNmNThmYWZlZDZiOWVkMzYyY2M1MzI3YWZiOQ==
+SHA1:
+  metadata.gz: 4e33a67d294a013df625637e54a74b3768103eff
+  data.tar.gz: bacfd5645a75323de255fcdb4e7eed862d611ef5
 SHA512:
-  metadata.gz: !binary |-
-    Yzk1YTMwYjBmYmRkZTVmYjU0ZmYwNjg1NzhhMTRiNDk3NjczMDdjMjVkNDNm
-    Mjk2NGYyOTM5YmNjOGZjMmNhNTkwNmQ0M2NlMzBhMjRjMzk0MGUxODUwZmVj
-    NDViNjU2N2ZjNDZlNGRlNzFmYTgzODNhYWUzNjY1MDE4MTg2MWI=
-  data.tar.gz: !binary |-
-    Y2Y1YTg4Mjg0NjhjN2ZhZjZiYzZiMmU2Zjk3MjhiNmNhM2VkZWIwOWE3NmUw
-    ODhiNDA0ZjI2NWEyZDhkNGFmZTAzODJiYTY0YzZjZWE2NzJlNjE3MGVkZjVj
-    Mjg0ZmRhODI5ZmU1NjU4N2NhMzFjZmVlNDk3MjNjYzA2YzYwNTA=
+  metadata.gz: f481ee93f5117ad7ca76d2986061bb93fb8a8c4767246e2b80814d806a7a44b519d52103cafa6fbc5957c07ca8289832a8a44d1b27809b60a6f71c75b9366702
+  data.tar.gz: 7cdca00339dd4c242afbbb4c1ec78d9782c86649a0dcdee31630d70deda76be98b9ad791cbd355f84cef71f25d90c7bd364007d8ea7d8abb3f4eb2bef0e155cc

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    elasticsearch-manager (0.1.1)
+    elasticsearch-manager (0.1.2)
       colorize (~> 0.7)
       highline (~> 1.7)
       json (~> 1.8)

data/bin/elasticsearch-manager

@@ -19,8 +19,12 @@ opt_parser = OptionParser.new do |opts|
 elasticsearch-manager [options] <command>
   
 Available commands:
-  rolling-restart    -- Restart elasticsearch across the entire cluster, one node at a time.
+  rolling-restart    -- Restart elasticsearch across the entire cluster, one node at a time
+  list-nodes         -- List IPs of nodes in the cluster
+  shard-state        -- Print current shard states
   status             -- Check the current state of the cluster (green/yellow/red)
+  diable-routing     -- Disable shard routing allocation
+  enable-routing     -- Enable shard routing allocation
 
 EOC
 
@@ -51,8 +55,16 @@ cmd = ARGV[0]
 case cmd
 when 'rolling-restart'
   ec = CMD.rolling_restart(options)
+when 'list-nodes'
+  ec = CMD.list_nodes(options)
+when 'shard-state'
+  ec = CMD.shard_states(options)
 when 'status'
   ec = CMD.status(options)
+when 'disable-routing'
+  ec = CMD.disable_routing(options)
+when 'enable-routing'
+  ec = CMD.enable_routing(options)
 else
   puts "Unknown command: #{cmd}\n" 
   puts opt_parser

data/lib/elasticsearch/client/elasticsearch.rb

@@ -29,6 +29,16 @@ module Elasticsearch
         get('/_cluster/settings')
       end
 
+      def node_concurrent_recoveries(num_recoveries = 2)
+        data = {
+          'transient' => {
+            'cluster.routing.allocation.node_concurrent_recoveries' => num_recoveries
+          }
+        }.to_json
+
+        put('/_cluster/settings', data)
+      end
+
       def routing(disable = true)
         data = {
           'transient' => {

data/lib/elasticsearch/manager/cmd.rb

@@ -1,8 +1,7 @@
 require 'colorize'
 require 'net/ssh'
 
-require 'elasticsearch/manager/manager'
-require 'elasticsearch/manager/rollingrestart'
+require 'elasticsearch/manager'
 
 
 module Elasticsearch
@@ -17,8 +16,9 @@ module Elasticsearch
           print_cluster_status(manager, 'The cluster is currently unstable! Not proceeding with rolling-restart')
           return 2
         end
-        puts "Discovering cluster members...\n"
+        print "Discovering cluster members..." if opts[:verbose]
         manager.cluster_members!
+        print "\rDiscovering cluster members... Done!\n" if opts[:verbose]
         timeout = opts[:timeout] || 600
         sleep_interval = opts[:sleep_interval] || 30
         begin
@@ -31,6 +31,51 @@ module Elasticsearch
         return 0
       end
   
+      def self.list_nodes(opts)
+        manager = _manager(opts)
+        print "Discovering cluster members..." if opts[:verbose]
+        manager.cluster_members!
+        print "\rDiscovering cluster members... Done!\n" if opts[:verbose]
+        manager.list_node_ips
+        return 0
+      end
+
+      def self.disable_routing(opts)
+        manager = _manager(opts)
+        print "Disabling shard routing allocation..."
+        msg = if manager.disable_routing
+                "disabled!".colorize(:green)
+              else
+                "error, unable to disable shard routing allocation!".colorize(:red)
+              end
+        print "\rDisabling shard routing allocation... #{msg}\n"
+        return 0
+      end
+
+      def self.enable_routing(opts)
+        manager = _manager(opts)
+        print "Enabling shard routing allocation..."
+        msg = if manager.enable_routing
+                "enabled!".colorize(:green)
+              else
+                "error, unable to enable shard routing allocation!".colorize(:red)
+              end
+        print "\rEnabling shard routing allocation... #{msg}\n"
+        return 0
+      end
+
+      def self.shard_states(opts)
+        manager = _manager(opts)
+        print "Discovering cluster members..." if opts[:verbose]
+        manager.cluster_members!
+        print "\rDiscovering cluster members... Done!\n" if opts[:verbose]
+        puts "UNASSIGNED: #{manager.state.count_unassigned_shards}"
+        manager.nodes.each do |node|
+          puts "#{node.ip}:\tSTARTED: #{node.count_started_shards}\t|\tINITIALIZING: #{node.count_initializing_shards}\t|\tRELOCATING: #{node.count_relocating_shards}"
+        end
+        return 0
+      end
+
       def self.status(opts)
         manager = _manager(opts)
         status = manager.cluster_status

data/lib/elasticsearch/manager/manager.rb

@@ -6,10 +6,11 @@ module Elasticsearch
     class ESManager
       include Elasticsearch::Model
 
-      attr_accessor :leader, :members, :nodes
+      attr_accessor :leader, :members, :nodes, :state
 
       def initialize(cluster_host = 'localhost', port = 9200)
         @client = Elasticsearch::Client::ESClient.new(cluster_host, port)
+        @state = nil
         @leader = nil
         @nodes = nil
         @members = nil
@@ -30,8 +31,9 @@ module Elasticsearch
       end
 
       def cluster_members!
-        state = cluster_state
+        @state = cluster_state
         @nodes = state.nodes
+        @nodes.sort! { |a,b| a.id <=> b.id }
         @leader = @nodes.select { |n| n.master }[0].ip
         @members = @nodes.map { |n| n.ip }
       end
@@ -55,6 +57,12 @@ module Elasticsearch
         ret = @client.routing(false)
         ret['transient']['cluster']['routing']['allocation']['enable'] == 'all'
       end
+
+      def set_concurrent_recoveries(num_recoveries)
+        ret = @client.node_concurrent_recoveries(num_recoveries)
+        # Elasticache seems to return integer settings as strings when setting them...
+        ret['transient']['cluster']['routing']['allocation']['node_concurrent_recoveries'] == num_recoveries.to_s
+      end
     end
   end
 end

data/lib/elasticsearch/manager/nodes.rb

@@ -0,0 +1,15 @@
+module Elasticsearch
+  module Manager
+
+    class ESManager
+      def list_node_ips
+        puts "#{@leader} -- master"
+        @members.each do |m|
+          unless m == @leader
+            puts m
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/manager/rollingrestart.rb

@@ -29,8 +29,10 @@ module Elasticsearch
 
       def restart_node(node_ip, timeout, sleep_interval)
           puts "\nRestarting Elasticsearch on node: #{node_ip}"
+          # Pull the current node's state
+          n = @state.nodes.select { |n| n.ip == node_ip }[0]
+
           raise "Could not disable shard routing prior to restarting node: #{node_ip}".colorize(:red) unless disable_routing
-          
           Net::SSH.start(node_ip, ENV['USER']) do |ssh|
             ssh.exec 'sudo service elasticsearch restart'
           end
@@ -43,6 +45,10 @@ module Elasticsearch
             raise NodeAvailableTimeout, "Node did not become available after waiting #{timeout} seconds...".colorize(:red)
           end
 
+          # Make sure the cluster is willing to concurrently recover as many
+          # shards per node as this node happens to have.
+          raise "Could not update node_concurrent_recoveries prior to restarting node: #{node_ip}".colorize(:red) unless set_concurrent_recoveries(n.count_started_shards + 1)
+
           raise "Could not re-enable shard routing following restart of node: #{node_ip}".colorize(:red) unless enable_routing
 
           begin

data/lib/elasticsearch/manager/version.rb

@@ -1,5 +1,5 @@
 module Elasticsearch
   module Manager
-    VERSION = "0.1.1"
+    VERSION = "0.1.2.pre"
   end
 end

data/lib/elasticsearch/manager.rb

@@ -1,4 +1,5 @@
 require_relative 'manager/manager'
+require_relative 'manager/nodes'
 require_relative 'manager/rollingrestart'
 require_relative 'manager/version'
 

data/lib/elasticsearch/model/node.rb

@@ -34,6 +34,19 @@ module Elasticsearch
           end
         end
       end
+
+      def count_initializing_shards
+        shards.select { |s| s.state.upcase == 'INITIALIZING' }.length
+      end
+
+      def count_relocating_shards
+        shards.select { |s| s.state.upcase == 'RELOCATING' }.length
+      end
+
+      def count_started_shards
+        shards.select { |s| s.state.upcase == 'STARTED' }.length
+      end
+
       extend Representer
     end
   end

data/lib/elasticsearch/model/routing_nodes.rb

@@ -13,11 +13,11 @@ module Elasticsearch
         property :master_node
         nested :routing_nodes do
           property :unassigned, setter: (lambda do |v,args|
-            self.unassigned = v.map {|s| Elasticsearch::Manager::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(s) }
+            self.unassigned = v.map {|s| Elasticsearch::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(s) }
           end)
           property :nodes, setter: (lambda do |v,args|
             self.nodes = v.map do |id, shards|
-              shards.each { |shard| Elasticsearch::Manager::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(shard) }
+              shards.each { |shard| Elasticsearch::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(shard) }
             end.flatten
           end)
         end

data/lib/elasticsearch/model/state.rb

@@ -2,22 +2,45 @@ require 'ostruct'
 require 'representable/json'
 
 module Elasticsearch
-  class ClusterState < OpenStruct
-    module Representer
-      include Representable::JSON
-      include Representable::Hash
-      include Representable::Hash::AllowSymbols
-  
-      property :cluster_name
-      property :master_node
-      property :nodes, setter: (lambda do |v,args|
-        self.nodes = v.map do |id,node|
-          n = Elasticsearch::Model::Node.new.extend(Elasticsearch::Model::Node::Representer).from_hash(node)
-          n.id, n.master, n.ip = id, id == self.master_node, n.transport_address[/\d+\.\d+\.\d+\.\d+/]
-          n
-        end
-      end)
+  module Model
+    class ClusterState < OpenStruct
+      module Representer
+        include Representable::JSON
+        include Representable::Hash
+        include Representable::Hash::AllowSymbols
+
+        property :cluster_name
+        property :master_node
+        property :nodes, setter: (lambda do |v,args|
+          self.nodes = v.map do |id,node|
+            n = Elasticsearch::Model::Node.new.extend(Elasticsearch::Model::Node::Representer).from_hash(node)
+            n.id, n.master, n.ip = id, id == self.master_node, n.transport_address[/\d+\.\d+\.\d+\.\d+/]
+            n
+          end
+        end)
+        property :routing_nodes, setter: (lambda do |v,args|
+          v.each do |status,routes|
+            if status == 'nodes'
+              routes.each do |id,shards|
+                n = self.nodes.select { |n| n.id == id }[0]
+                s = shards.map do |shard|
+                  Elasticsearch::Model::Shard.new.extend(Elasticsearch::Model::Shard::Representer).from_hash(shard)
+                end
+                n.shards = s
+              end
+            elsif status == 'unassigned'
+              self.unassigned_shards = routes.map do |shard|
+                Elasticsearch::Model::Shard.new.extend(Elasticsearch::Model::Shard::Representer).from_hash(shard)
+              end
+            end
+          end
+        end)
+      end
+
+      def count_unassigned_shards
+        unassigned_shards.length
+      end
+      extend Representer
     end
-    extend Representer
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticsearch-manager
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2.pre
 platform: ruby
 authors:
 - Brian Oldfield
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-08 00:00:00.000000000 Z
+date: 2015-05-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -203,6 +203,7 @@ files:
 - lib/elasticsearch/manager/cmd.rb
 - lib/elasticsearch/manager/errors.rb
 - lib/elasticsearch/manager/manager.rb
+- lib/elasticsearch/manager/nodes.rb
 - lib/elasticsearch/manager/rollingrestart.rb
 - lib/elasticsearch/manager/version.rb
 - lib/elasticsearch/model.rb
@@ -235,14 +236,14 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.4.6