elasticsearch-transport 0.0.2

data/lib/elasticsearch/transport/client.rb

@@ -0,0 +1,123 @@
+module Elasticsearch
+  module Transport
+
+    # Handles communication with an Elasticsearch cluster.
+    #
+    # See {file:README.md README} for usage and code examples.
+    #
+    class Client
+      DEFAULT_TRANSPORT_CLASS  = Transport::HTTP::Faraday
+
+      DEFAULT_LOGGER = lambda do
+        require 'logger'
+        logger = Logger.new(STDERR)
+        logger.progname = 'elasticsearch'
+        logger.formatter = proc { |severity, datetime, progname, msg| "#{datetime}: #{msg}\n" }
+        logger
+      end
+
+      DEFAULT_TRACER = lambda do
+        require 'logger'
+        logger = Logger.new(STDERR)
+        logger.progname = 'elasticsearch.tracer'
+        logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+        logger
+      end
+
+      # Returns the transport object.
+      #
+      # @see Elasticsearch::Transport::Transport::Base
+      # @see Elasticsearch::Transport::Transport::HTTP::Faraday
+      #
+      attr_accessor :transport
+
+      # Create a client connected to an Elasticsearch cluster.
+      #
+      # @option arguments [String,Array] :hosts Single host passed as a String or Hash, or multiple hosts
+      #                                         passed as an Array; `host` or `url` keys are also valid
+      #
+      # @option arguments [Boolean] :log   Use the default logger (disabled by default)
+      #
+      # @option arguments [Boolean] :trace Use the default tracer (disabled by default)
+      #
+      # @option arguments [Object] :logger An instance of a Logger-compatible object
+      #
+      # @option arguments [Object] :tracer An instance of a Logger-compatible object
+      #
+      # @option arguments [Number] :resurrect_after After how many seconds a dead connection should be tried again
+      #
+      # @option arguments [Boolean,Number] :reload_connections Reload connections after X requests (false by default)
+      #
+      # @option arguments [Boolean] :randomize_hosts   Shuffle connections on initialization and reload (false by default)
+      #
+      # @option arguments [Integer] :sniffer_timeout   Timeout for reloading connections in seconds (1 by default)
+      #
+      # @option arguments [Boolean,Number] :retry_on_failure   Retry X times when request fails before raising and
+      #                                                        exception (false by default)
+      #
+      # @option arguments [Boolean] :reload_on_failure Reload connections after failure (false by default)
+      #
+      # @option arguments [Constant] :transport_class  A specific transport class to use, will be initialized by
+      #                                                the client and passed hosts and all arguments
+      #
+      # @option arguments [Object] :transport A specific transport instance
+      #
+      # @option arguments [Constant] :serializer_class A specific serializer class to use, will be initialized by
+      #                                               the transport and passed the transport instance
+      #
+      # @option arguments [Constant] :selector An instance of selector strategy implemented with
+      #                                        {Elasticsearch::Transport::Transport::Connections::Selector::Base}.
+      #
+      def initialize(arguments={})
+        transport_class  = arguments[:transport_class] || DEFAULT_TRANSPORT_CLASS
+        hosts            = arguments[:hosts] || arguments[:host] || arguments[:url]
+
+        arguments[:logger] ||= arguments[:log]   ? DEFAULT_LOGGER.call() : nil
+        arguments[:tracer] ||= arguments[:trace] ? DEFAULT_TRACER.call() : nil
+        arguments[:reload_connections] ||= false
+        arguments[:retry_on_failure]   ||= false
+        arguments[:reload_on_failure]  ||= false
+        arguments[:randomize_hosts]    ||= false
+
+        @transport = arguments[:transport] || \
+                     transport_class.new(:hosts => __extract_hosts(hosts, arguments), :options => arguments)
+      end
+
+      # Performs a request through delegation to {#transport}.
+      #
+      def perform_request(method, path, params={}, body=nil)
+        transport.perform_request method, path, params, body
+      end
+
+      # Normalizes and returns hosts configuration.
+      #
+      # Arrayifies the `hosts_config` argument and extracts `host` and `port` info from strings.
+      # Performs shuffling when the `randomize_hosts` option is set.
+      #
+      # @return [Array<Hash>]
+      # @raise  [ArgumentError]
+      #
+      # @api private
+      #
+      def __extract_hosts(hosts_config=nil, options={})
+        hosts_config = hosts_config.nil? ? ['localhost'] : Array(hosts_config)
+
+        hosts = hosts_config.map do |host|
+          case host
+          when String
+            # TODO: Handle protocol?
+            host, port = host.split(':')
+            { :host => host, :port => port }
+          when Hash
+            host
+          else
+            raise ArgumentError, "Please pass host as a String or Hash, #{host.class} given."
+          end
+        end
+
+        hosts.shuffle! if options[:randomize_hosts]
+        hosts
+      end
+    end
+  end
+end

data/lib/elasticsearch/transport/extensions/test_cluster.rb

@@ -0,0 +1,163 @@
+require 'ansi/code'
+
+module Elasticsearch
+
+  # A convenience Ruby class for starting and stopping a separate testing cluster,
+  # to not depend on -- and not mess up -- <localhost:9200>.
+  #
+  module TestCluster
+    require 'timeout'
+    require 'net/http'
+    require 'uri'
+
+    @@number_of_nodes = 2
+    @@pids            = []
+
+    # Start a cluster
+    #
+    # Starts the desired number of nodes in test-suitable configuration (memory store, no persistence, etc).
+    #
+    # @option arguments [String]  :command      Elasticsearch command (default: `elasticsearch`).
+    # @option arguments [Integer] :count        Number of desired nodes (default: 2).
+    # @option arguments [String]  :cluster_name Cluster name (default: `elasticsearch-ruby-test`).
+    # @option arguments [String]  :port         Starting port number; will be auto-incremented (default: 9250).
+    #
+    # You can also use environment variables to set these options.
+    #
+    def start(arguments={})
+      arguments[:command] = ENV['TEST_CLUSTER_COMMAND'] || 'elasticsearch'
+
+      unless system "which #{arguments[:command]} > /dev/null 2>&1"
+        STDERR.puts ANSI.red("[ERROR] Elasticsearch can't be started, is it installed? Run: $ which elasticsearch"), ''
+        abort
+      end
+
+      @@number_of_nodes = arguments[:count] if arguments[:count]
+
+      arguments[:port]         = (ENV['TEST_CLUSTER_PORT'] || 9250).to_i
+      arguments[:cluster_name] = ENV['TEST_CLUSTER_NAME'] || 'elasticsearch-ruby-test'
+      arguments[:node_name]    = 'node'
+
+      if running? :on => arguments[:port], :as => arguments[:cluster_name]
+        print ANSI.red("Elasticsearch cluster already running")
+        __wait_for_green(arguments[:port])
+        exit(0)
+      end
+
+      print ANSI.faint("Starting ") + ANSI.ansi(@@number_of_nodes.to_s, :bold, :faint) + ANSI.faint(" Elasticsearch nodes")
+
+      @@number_of_nodes.times do |n|
+        n += 1
+        pidfile = File.expand_path("tmp/elasticsearch-#{n}.pid", Dir.pwd)
+        pid = Process.spawn <<-COMMAND
+          #{arguments[:command]} \
+            -D es.foreground=yes \
+            -D es.cluster.name=#{arguments[:cluster_name]} \
+            -D es.node.name=#{arguments[:node_name]}-#{n} \
+            -D es.http.port=#{arguments[:port].to_i + (n-1)} \
+            -D es.gateway.type=none \
+            -D es.index.store.type=memory \
+            -D es.network.host=0.0.0.0 \
+            -D es.discovery.zen.ping.multicast.enabled=true \
+            -D es.pidfile=#{pidfile} \
+            > /dev/null 2>&1
+        COMMAND
+        Process.detach pid
+      end
+
+      __wait_for_green(arguments[:port])
+    end
+
+    # Stop the cluster.
+    #
+    # Gets the PID numbers from pidfiles in `$CWD/tmp` and stops any matching nodes.
+    #
+    def stop
+      pids     = __get_pids
+      pidfiles = __get_pidfiles
+
+      unless pids.empty?
+        print "Stopping Elasticsearch nodes... "
+        pids.each_with_index do |pid, i|
+          begin
+            print ANSI.green("stopped PID #{pid}. ") if Process.kill 'KILL', pid
+          rescue Exception => e
+            print ANSI.red("[#{e.class}] PID #{pid} not found. ")
+          end
+          File.delete pidfiles[i] if pidfiles[i] && File.exists?(pidfiles[i])
+        end
+        puts
+      end
+    end
+
+    # Returns true when a specific test node is running.
+    #
+    # @option arguments [Integer] :on The port on which the node is running.
+    # @option arguments [String]  :as The cluster name.
+    #
+    def running?(arguments={})
+      port         = arguments[:on] || 9250
+      cluster_name = arguments[:as] || 'elasticsearch-ruby-test'
+
+      if cluster_health = Timeout::timeout(0.25) { __get_cluster_health(port) } rescue nil
+        return cluster_health['cluster_name']    == cluster_name && \
+               cluster_health['number_of_nodes'] == @@number_of_nodes
+      end
+      return false
+    end
+
+    # Blocks the process and waits for the cluster to be in a "green" state.
+    # Prints information about the cluster on STDOUT.
+    #
+    def __wait_for_green(port=9250)
+      uri = URI("http://localhost:#{port}/_cluster/health")
+
+      Timeout::timeout(30) do
+        loop do
+          response = Net::HTTP.get(uri) rescue nil
+          if response
+            pids = __get_pids
+
+            json = MultiJson.load(response)
+            if json['status'] == 'green' && json['number_of_nodes'].to_i == @@number_of_nodes
+              puts '',
+                   ANSI.faint('-'*80),
+                   ANSI.faint(
+                    'Cluster: '.ljust(20)         + json['cluster_name'].to_s    + "\n" +
+                    'Status: '.ljust(20)          + json['status'].to_s          + "\n" +
+                    'Number of nodes: '.ljust(20) + json['number_of_nodes'].to_s + "\n" +
+                    'PIDs'.ljust(20)              + pids.inspect
+                  ),
+                  ANSI.faint('-'*80)
+              break
+            end
+          end
+          print ANSI.faint('.')
+          sleep 1
+        end
+      end
+    end
+
+    # Tries to load cluster health information
+    #
+    def __get_cluster_health(port=9250)
+      uri = URI("http://localhost:#{port}/_cluster/health")
+      if response = Net::HTTP.get(uri) rescue nil
+        return MultiJson.load(response)
+      end
+    end
+
+    # Returns a collection of PID numbers from pidfiles.
+    def __get_pids
+      __get_pidfiles.map { |pidfile| File.read(pidfile).to_i }.uniq
+    end
+
+    # Returns a collection of files with PID information.
+    #
+    def __get_pidfiles
+      Dir[File.expand_path('tmp/elasticsearch-*.pid', Dir.pwd)]
+    end
+
+    extend self
+  end
+end

data/lib/elasticsearch/transport/transport/base.rb

@@ -0,0 +1,236 @@
+module Elasticsearch
+  module Transport
+    module Transport
+
+      # @abstract Module with common functionality for transport implementations.
+      #
+      module Base
+        DEFAULT_PORT             = 9200
+        DEFAULT_PROTOCOL         = 'http'
+        DEFAULT_RELOAD_AFTER     = 10_000 # Requests
+        DEFAULT_RESURRECT_AFTER  = 60     # Seconds
+        DEFAULT_MAX_TRIES        = 3      # Requests
+        DEFAULT_SERIALIZER_CLASS = Serializer::MultiJson
+
+        attr_reader   :hosts, :options, :connections, :counter, :last_request_at, :protocol
+        attr_accessor :serializer, :sniffer, :logger, :tracer, :reload_after, :resurrect_after, :max_tries
+
+        # Creates a new transport object.
+        #
+        # @param arguments [Hash] Settings and options for the transport
+        # @param block     [Proc] Lambda or Proc which can be evaluated in the context of the "session" object
+        #
+        # @option arguments [Array] :hosts   An Array of normalized hosts information
+        # @option arguments [Array] :options A Hash with options (usually passed by {Client})
+        #
+        # @see Client#initialize
+        #
+        def initialize(arguments={}, &block)
+          @hosts       = arguments[:hosts]   || []
+          @options     = arguments[:options] || {}
+          @block       = block
+          @connections = __build_connections
+
+          @serializer  = options[:serializer] || ( options[:serializer_class] ? options[:serializer_class].new(self) : DEFAULT_SERIALIZER_CLASS.new(self) )
+          @protocol    = options[:protocol] || DEFAULT_PROTOCOL
+
+          @logger      = options[:logger]
+          @tracer      = options[:tracer]
+
+          @sniffer     = options[:sniffer_class] ? options[:sniffer_class].new(self) : Sniffer.new(self)
+          @counter     = 0
+          @last_request_at = Time.now
+          @reload_after    = options[:reload_connections].is_a?(Fixnum) ? options[:reload_connections] : DEFAULT_RELOAD_AFTER
+          @resurrect_after = options[:resurrect_after] || DEFAULT_RESURRECT_AFTER
+          @max_tries       = options[:retry_on_failure].is_a?(Fixnum)   ? options[:retry_on_failure]   : DEFAULT_MAX_TRIES
+        end
+
+        # Returns a connection from the connection pool by delegating to {Connections::Collection#get_connection}.
+        #
+        # Resurrects dead connection if the `resurrect_after` timeout has passed.
+        # Increments the counter and performs connection reloading if the `reload_connections` option is set.
+        #
+        # @return [Connections::Connection]
+        # @see    Connections::Collection#get_connection
+        #
+        def get_connection(options={})
+          resurrect_dead_connections! if Time.now > @last_request_at + @resurrect_after
+
+          connection = connections.get_connection(options)
+          @counter  += 1
+
+          reload_connections!         if @options[:reload_connections] && counter % reload_after == 0
+          connection
+        end
+
+        # Reloads and replaces the connection collection based on cluster information.
+        #
+        # @see Sniffer#hosts
+        #
+        def reload_connections!
+          hosts = sniffer.hosts
+          __rebuild_connections :hosts => hosts, :options => options
+          self
+        rescue SnifferTimeoutError
+          logger.error "[SnifferTimeoutError] Timeout when reloading connections." if logger
+          self
+        end
+
+        # Tries to "resurrect" all eligible dead connections.
+        #
+        # @see Connections::Connection#resurrect!
+        #
+        def resurrect_dead_connections!
+          connections.dead.each { |c| c.resurrect! }
+        end
+
+        # Replaces the connections collection.
+        #
+        # @api private
+        #
+        def __rebuild_connections(arguments={})
+          @hosts       = arguments[:hosts]    || []
+          @options     = arguments[:options]  || {}
+          @connections = __build_connections
+        end
+
+        # Log request and response information.
+        #
+        # @api private
+        #
+        def __log(method, path, params, body, url, response, json, took, duration)
+          logger.info  "#{method.to_s.upcase} #{url} " +
+                       "[status:#{response.status}, request:#{sprintf('%.3fs', duration)}, query:#{took}]"
+          logger.debug "> #{__convert_to_json(body)}" if body
+          logger.debug "< #{response.body}"
+        end
+
+        # Log failed request.
+        #
+        # @api private
+        def __log_failed(response)
+          logger.fatal "[#{response.status}] #{response.body}"
+        end
+
+        # Trace the request in the `curl` format.
+        #
+        # @api private
+        def __trace(method, path, params, body, url, response, json, took, duration)
+          trace_url  = "http://localhost:9200/#{path}?pretty" +
+                       ( params.empty? ? '' : "&#{::Faraday::Utils::ParamsHash[params].to_query}" )
+          trace_body = body ? " -d '#{__convert_to_json(body, :pretty => true)}'" : ''
+          tracer.info  "curl -X #{method.to_s.upcase} '#{trace_url}'#{trace_body}\n"
+          tracer.debug "# #{Time.now.iso8601} [#{response.status}] (#{format('%.3f', duration)}s)\n#"
+          tracer.debug json ? serializer.dump(json, :pretty => true).gsub(/^/, '# ').sub(/\}$/, "\n# }")+"\n" : "# #{response.body}\n"
+        end
+
+        # Raise error specific for the HTTP response status or a generic server error
+        #
+        # @api private
+        def __raise_transport_error(response)
+          error = ERRORS[response.status] || ServerError
+          raise error.new "[#{response.status}] #{response.body}"
+        end
+
+        # Converts any non-String object to JSON
+        #
+        # @api private
+        def __convert_to_json(o=nil, options={})
+          o = o.is_a?(String) ? o : serializer.dump(o, options)
+        end
+
+        # Performs a request to Elasticsearch, while handling logging, tracing, marking dead connections,
+        # retrying the request and reloading the connections.
+        #
+        # @abstract The transport implementation has to implement this method either in full,
+        #           or by invoking this method with a block. See {HTTP::Faraday#perform_request} for an example.
+        #
+        # @param method [String] Request method
+        # @param path   [String] The API endpoint
+        # @param params [Hash]   Request parameters (will be serialized by {Connections::Connection#full_url})
+        # @param body   [Hash]   Request body (will be serialized by the {#serializer})
+        # @param block  [Proc]   Code block to evaluate, passed from the implementation
+        #
+        # @return [Response]
+        # @raise  [NoMethodError] If no block is passed
+        # @raise  [ServerError]   If request failed on server
+        # @raise  [Error]         If no connection is available
+        #
+        def perform_request(method, path, params={}, body=nil, &block)
+          raise NoMethodError, "Implement this method in your transport class" unless block_given?
+          start = Time.now if logger || tracer
+          tries = 0
+
+          begin
+            tries     += 1
+            connection = get_connection or raise Error.new("Cannot get new connection from pool.")
+            url        = connection.full_url(path, params)
+            response   = block.call(connection, url)
+
+            connection.healthy! if connection.failures > 0
+
+          rescue *host_unreachable_exceptions => e
+            logger.error "[#{e.class}] #{e.message} #{connection.host.inspect}" if logger
+
+            connection.dead!
+
+            if @options[:reload_on_failure] and tries < connections.all.size
+              logger.warn "[#{e.class}] Reloading connections (attempt #{tries} of #{connections.size})" if logger
+              reload_connections! and retry
+            end
+
+            if @options[:retry_on_failure]
+              logger.warn "[#{e.class}] Attempt #{tries} connecting to #{connection.host.inspect}" if logger
+              if tries < max_tries
+                retry
+              else
+                logger.fatal "[#{e.class}] Cannot connect to #{connection.host.inspect} after #{tries} tries" if logger
+                raise e
+              end
+            else
+              raise e
+            end
+
+          rescue Exception => e
+            logger.fatal "[#{e.class}] #{e.message} (#{connection.host.inspect})" if logger
+            raise e
+          end
+
+          json     = serializer.load(response.body) if response.body.to_s =~ /^\{/
+          took     = (json['took'] ? sprintf('%.3fs', json['took']/1000.0) : 'n/a') rescue 'n/a' if logger || tracer
+          duration = Time.now-start if logger || tracer
+
+          __log   method, path, params, body, url, response, json, took, duration if logger
+          __trace method, path, params, body, url, response, json, took, duration if tracer
+
+          if response.status.to_i >= 300
+            __log_failed response if logger
+            __raise_transport_error response
+          else
+            Response.new response.status, json || response.body, response.headers
+          end
+        ensure
+          @last_request_at = Time.now
+        end
+
+        # @abstract Returns an Array of connection errors specific to the transport implementation.
+        #           See {HTTP::Faraday#host_unreachable_exceptions} for an example.
+        #
+        # @return [Array]
+        #
+        def host_unreachable_exceptions
+          [Errno::ECONNREFUSED]
+        end
+
+        # @abstract A transport implementation must implement this method.
+        #           See {HTTP::Faraday#__build_connections} for an example.
+        #
+        # @return [Connections::Collection]
+        # @api    private
+        def __build_connections
+          raise NoMethodError, "Implement this method in your class"
+        end
+      end
+    end
+  end
+end