elasticsearch-manager 0.1.0

data/lib/elasticsearch/client.rb

@@ -0,0 +1,8 @@
+require_relative 'client/base'
+require_relative 'client/elasticsearch'
+
+
+module Elasticsearch
+  module Client
+  end
+end

data/lib/elasticsearch/client/base.rb

@@ -0,0 +1,71 @@
+require 'logger'
+require 'rest-client'
+
+module Elasticsearch
+  module Client
+    class Base
+      def initialize(host = 'localhost', port = 9200, logger = Logger.new(STDOUT))
+        @host = host
+        @port = port
+        @logger = logger
+      end
+
+      def get(uri, params = nil)
+        raw = _get(uri, params)
+        if !raw.headers[:content_type].nil? && raw.headers[:content_type][/json/]
+          JSON.parse(raw)
+        else
+          raw
+        end
+      end
+
+      def put(uri, body, params = nil)
+        raw = _put(uri, body, params)
+        if !raw.headers[:content_type].nil? && raw.headers[:content_type][/json/]
+          JSON.parse(raw)
+        else
+          raw
+        end
+      end
+
+      def _get(uri, params = nil)
+        url = _build_url(uri)
+        opts = _prep_opts(params)
+
+        begin
+          return RestClient.get url, opts
+        rescue Exception => e
+          @logger.error(e.message)
+          raise e
+          raise IOError.new "Unable to complete get request: #{e}"
+        end
+      end
+
+      def _put(uri, body, params = nil)
+        url = _build_url(uri)
+        opts = _prep_opts(params)
+
+        begin
+          return RestClient.put url, body, opts
+        rescue Exception => e
+          @logger.error(e.message)
+          raise IOError.new "Unable to complete put request: #{e}"
+        end
+      end
+
+      def _prep_opts(params)
+        unless params.nil?
+                 {:params => params}
+               else
+                 {}
+               end
+      end
+
+      protected
+
+      def _build_url(uri)
+        "http://#{@host}:#{@port}#{uri}"
+      end
+    end
+  end
+end

data/lib/elasticsearch/client/elasticsearch.rb

@@ -0,0 +1,43 @@
+require 'json'
+
+
+module Elasticsearch
+  module Client
+    class ESClient < Base
+      def status
+        data = get('/_cluster/health')
+        data['status']
+      end
+
+      def green?
+        status == 'green'
+      end
+
+      def health
+        get('/_cluster/health')
+      end
+
+      def nodes
+        get('/_nodes')
+      end
+
+      def state
+        get('/_cluster/state')
+      end
+
+      def settings
+        get('/_cluster/settings')
+      end
+
+      def routing(disable = true)
+        data = {
+          'transient' => {
+            'cluster.routing.allocation.enable' => disable ? 'none' : 'all'
+          }
+        }.to_json
+
+        put('/_cluster/settings', data)
+      end
+    end
+  end
+end

data/lib/elasticsearch/manager.rb

@@ -0,0 +1,8 @@
+require_relative 'manager/manager'
+require_relative 'manager/rollingrestart'
+require_relative 'manager/version'
+
+module Elasticsearch
+  module Manager
+  end
+end

data/lib/elasticsearch/manager/cmd.rb

@@ -0,0 +1,62 @@
+require 'colorize'
+require 'net/ssh'
+
+require 'elasticsearch/manager/manager'
+require 'elasticsearch/manager/rollingrestart'
+
+
+module Elasticsearch
+  module Manager
+    module CMD
+      include Elasticsearch::Manager
+
+      def self.rolling_restart(opts)
+        manager = _manager(opts)
+        # Check that the cluster is stable?
+        unless manager.cluster_stable?
+          print_cluster_stable(manager)
+          return 2
+        end
+        puts "Discovering cluster members...\n"
+        manager.cluster_members!
+        timeout = opts[:timeout] || 600
+        sleep_interval = opts[:sleep_interval] || 30
+        begin
+          manager.rolling_restart(timeout, sleep_interval)
+        rescue Exception => e
+          puts e
+          return 2
+        end
+        puts 'Rolling restart complete.'
+        return 0
+      end
+  
+      def self.status(opts)
+        manager = _manager(opts)
+        status = manager.cluster_status
+        puts "The Elasticsearch cluster is currently: #{status.colorize(status.to_sym)}"
+        return 0
+      end
+
+      protected
+      def self._manager(opts)
+        ESManager.new(opts[:hostname], opts[:port])
+      end
+
+      def self.print_cluster_stable(manager)
+        health = manager.cluster_health
+        puts 'The cluster is currently unstable! Not proceeding with rolling-restart'
+        puts "\tCluster status: #{health.status.colorize(health.status.to_sym)}"
+
+        relocating = health.relocating_shards == 0 ? :green : :red
+        puts "\tRelocating shards: #{health.relocating_shards.to_s.colorize(relocating)}"
+
+        initializing = health.initializing_shards == 0 ? :green : :red
+        puts "\tInitializing shards: #{health.relocating_shards.to_s.colorize(relocating)}"
+
+        unassigned = health.unassigned_shards == 0 ? :green : :red
+        puts "\tUnassigned shards: #{health.relocating_shards.to_s.colorize(relocating)}"
+      end
+    end
+  end
+end

data/lib/elasticsearch/manager/errors.rb

@@ -0,0 +1,12 @@
+module Elasticsearch
+  module Manager
+    class StabalizationTimeout < StandardError
+    end
+
+    class NodeAvailableTimeout < StandardError
+    end
+
+    class UserRequestedStop < StandardError
+    end
+  end
+end

data/lib/elasticsearch/manager/manager.rb

@@ -0,0 +1,60 @@
+require 'elasticsearch/client'
+require 'elasticsearch/model'
+
+module Elasticsearch
+  module Manager
+    class ESManager
+      include Elasticsearch::Model
+
+      attr_accessor :leader, :members, :nodes
+
+      def initialize(cluster_host = 'localhost', port = 9200)
+        @client = Elasticsearch::Client::ESClient.new(cluster_host, port)
+        @leader = nil
+        @nodes = nil
+        @members = nil
+      end
+
+      def cluster_green?
+        @client.green?
+      end
+
+      def cluster_status
+        @client.status
+      end
+
+      def cluster_stable?
+        health = cluster_health
+        moving = [health.relocating_shards, health.initializing_shards, health.unassigned_shards]
+        cluster_green? && moving.all? { |x| x == 0 } 
+      end
+
+      def cluster_members!
+        state = cluster_state
+        @nodes = state.nodes
+        @leader = @nodes.select { |n| n.master }[0].ip
+        @members = @nodes.map { |n| n.ip }
+      end
+
+      def cluster_health
+        health = @client.health
+        Health.new.extend(Health::Representer).from_hash(health)
+      end
+
+      def cluster_state
+        state = @client.state
+        ClusterState.new.extend(ClusterState::Representer).from_hash(state)
+      end
+
+      def disable_routing
+        ret = @client.routing(true)
+        ret['transient']['cluster']['routing']['allocation']['enable'] == 'none'
+      end
+
+      def enable_routing
+        ret = @client.routing(false)
+        ret['transient']['cluster']['routing']['allocation']['enable'] == 'all'
+      end
+    end
+  end
+end

data/lib/elasticsearch/manager/rollingrestart.rb

@@ -0,0 +1,79 @@
+require 'colorize'
+require 'net/ssh'
+require 'timeout'
+require 'highline'
+
+require 'elasticsearch/manager/errors'
+require 'elasticsearch/manager/manager'
+
+
+module Elasticsearch
+  module Manager
+
+    class ESManager
+      def rolling_restart(timeout = 600, sleep_interval = 30)
+        highline = HighLine.new
+        @members.each do |m|
+          restart_node(m, timeout, sleep_interval) unless m == @leader
+          if m != @members[-1]
+            unless highline.agree('Continue with rolling restart of cluster? (y/n) ')
+              raise UserRequestedStop, "Stopping rolling restart at user request!".colorize(:red)
+            end
+          end
+        end
+        unless highline.agree("\nRestarting current cluster master, continue? (y/n) ")
+          raise UserRequestedStop, "Stopping rolling restart at user request before restarting master node!".colorize(:red)
+        end
+        restart_node(@leader, timeout, sleep_interval)
+      end
+
+      def restart_node(node_ip, timeout, sleep_interval)
+          puts "\nRestarting Elasticsearch on node: #{node_ip}"
+          raise "Could not disable shard routing prior to restarting node: #{node_ip}".colorize(:red) unless disable_routing
+          
+          Net::SSH.start(node_ip, ENV['USER']) do |ssh|
+            ssh.exec 'sudo service elasticsearch restart'
+          end
+          puts "Elasticsearch restarted on node: #{node_ip}"
+
+          begin
+            wait_for_node_available(node_ip, timeout, sleep_interval)
+            puts "Node back up!".colorize(:green)
+          rescue Timeout::Error
+            raise NodeAvailableTimeout, "Node did not become available after waiting #{timeout} seconds...".colorize(:red)
+          end
+
+          raise "Could not re-enable shard routing following restart of node: #{node_ip}".colorize(:red) unless enable_routing
+
+          begin
+            wait_for_stable(timeout, sleep_interval)
+            puts "Cluster stabilized!".colorize(:green)
+          rescue Timeout::Error
+            raise StabalizationTimeout, "Cluster not re-stabilize after waiting #{timeout} seconds...".colorize(:red)
+          end
+      end 
+
+      protected
+
+      def wait_for_stable(timeout = 600, sleep_interval = 30)
+        Timeout.timeout(timeout) do
+          while !cluster_stable?
+            puts "Waiting for cluster to stabilize...".colorize(:yellow)
+            sleep(sleep_interval)
+          end
+        end
+      end
+
+      def wait_for_node_available(node_ip, timeout = 600, sleep_interval = 30)
+        Timeout.timeout(timeout) do
+          state = cluster_state
+          while !state.nodes.map { |n| n.ip }.include?(node_ip)
+            puts "Waiting for node to become available...".colorize(:yellow)
+            sleep(sleep_interval)
+            state = cluster_state
+          end
+        end
+      end
+    end
+  end
+end

data/lib/elasticsearch/manager/version.rb

@@ -0,0 +1,5 @@
+module Elasticsearch
+  module Manager
+    VERSION = "0.1.0"
+  end
+end

data/lib/elasticsearch/model.rb

@@ -0,0 +1,5 @@
+require_relative 'model/health'
+require_relative 'model/node'
+require_relative 'model/routing_nodes'
+require_relative 'model/state'
+require_relative 'model/shard'

data/lib/elasticsearch/model/health.rb

@@ -0,0 +1,26 @@
+require 'ostruct'
+require 'representable/json'
+
+module Elasticsearch
+  module Model
+    class Health < OpenStruct
+      module Representer
+        include Representable::JSON
+        include Representable::Hash
+        include Representable::Hash::AllowSymbols
+  
+        property :cluster_name
+        property :status
+        property :timed_out
+        property :number_of_nodes
+        property :number_of_data_nodes
+        property :active_primary_shards
+        property :active_shards
+        property :relocating_shards
+        property :initializing_shards
+        property :unassigned_shards
+      end
+      extend Representer
+    end
+  end
+end

data/lib/elasticsearch/model/node.rb

@@ -0,0 +1,40 @@
+require 'ostruct'
+require 'representable/json'
+
+module Elasticsearch
+  module Model
+    class Node < OpenStruct
+      module Representer
+        include Representable::JSON
+        include Representable::Hash
+        include Representable::Hash::AllowSymbols
+  
+        property :name
+        property :transport_address
+        property :host
+        property :ip
+        property :version
+        property :build
+        property :http_address
+  
+        nested :settings do
+          nested :gateway do
+            property :gateway_expected_nodes, as: :expected_nodes
+          end
+  
+          nested :http do
+            property :http_port, as: :port
+          end
+  
+          nested :path do
+            property :data_path, as: :data
+            property :home_path, as: :home
+            property :logs_path, as: :logs
+            property :conf_path, as: :conf
+          end
+        end
+      end
+      extend Representer
+    end
+  end
+end

data/lib/elasticsearch/model/routing_nodes.rb

@@ -0,0 +1,28 @@
+require 'ostruct'
+require 'representable/json'
+
+module Elasticsearch
+  module Model
+    class RoutingNodes < OpenStruct
+      module Representer
+        include Representable::JSON
+        include Representable::Hash
+        include Representable::Hash::AllowSymbols
+  
+        property :cluster_name
+        property :master_node
+        nested :routing_nodes do
+          property :unassigned, setter: (lambda do |v,args|
+            self.unassigned = v.map {|s| Elasticsearch::Manager::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(s) }
+          end)
+          property :nodes, setter: (lambda do |v,args|
+            self.nodes = v.map do |id, shards|
+              shards.each { |shard| Elasticsearch::Manager::Model::Shard.new.extend(Elastiman::Model::Shard::Representer).from_hash(shard) }
+            end.flatten
+          end)
+        end
+      end
+      extend Representer
+    end
+  end
+end