logstash-output-elasticsearch 3.0.2-java → 4.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7b8baebc24fbd14637d41ef87a5a0eeff7daf94f
-  data.tar.gz: e1ebb6428b9f00175f1cd816dc19d0c0ed4c36d9
+  metadata.gz: 1463814b1c058872439a7cff351407b2c2a6d442
+  data.tar.gz: 28b8279a5cf3bb64e2003bc7e5272989918a2eb7
 SHA512:
-  metadata.gz: ccb0eec4af4e4dcadcf39f3db0ee986799887f202627722c3eb88677d5a1c7996e4338ce60ad123ebbeca44d90cc235d23a95885eda47e6ac0b3c5115853d053
-  data.tar.gz: 534769a76b11249b73a91622928271aa9a7c8f1ab2ebde6d50bc117e0d0046356655f4652825110c7a88ab57ae75e8c896ea17f51ecadc200f8ed57dddd8c8ed
+  metadata.gz: 7d7ac9a08e59b1698121a9aaf13b195a032961a3df092c7a58ed183611869efe95803928e5dce25fe8b8e9a0394a1eae7e15cb4b2fa4b7b44a0bee1362321bdb
+  data.tar.gz: a97ecd3ea4a6d391b7b7e77d38956d9c7c8089ca5406513301a2e46e5f1e12fb82edd4f0a0dce048a473e903965012246c8b99978fb207eae384fb35a4887238

data/CHANGELOG.md

@@ -1,12 +1,25 @@
+## 4.1.0
+- breaking,config: Removed obsolete config `host` and `port`. Please use the `hosts` config with the `[host:port]` syntax.
+- breaking,config: Removed obsolete config `index_type`. Please use `document_type` instead.
+- breaking,config: Set config `max_retries` and `retry_max_items` as obsolete
+
+## 4.0.0
+ - Make this plugin threadsafe. Workers no longer needed or supported
+ - Add pool_max and pool_max_per_route options
+
 ## 3.0.2
-  - Fix issues where URI based paths in 'hosts' would not function correctly
+ - Fix issues where URI based paths in 'hosts' would not function correctly
+
 ## 3.0.1
-  - Republish all the gems under jruby.
+ - Republish all the gems under jruby.
+
 ## 3.0.0
-  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+ - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+
 ## 2.7.0
  - Add `pipeline` configuration option for setting an ingest pipeline to run upon indexing
 
+
 ## 2.6.2
  - Fix bug where update index actions would not work with events with 'data' field
 

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
+
 gemspec

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -1,5 +1,4 @@
 require "logstash/outputs/elasticsearch/template_manager"
-require "logstash/outputs/elasticsearch/buffer"
 
 module LogStash; module Outputs; class ElasticSearch;
   module Common
@@ -13,16 +12,11 @@ module LogStash; module Outputs; class ElasticSearch;
       setup_hosts # properly sets @hosts
       build_client
       install_template
-      setup_buffer_and_handler
       check_action_validity
 
       @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
     end
 
-    def receive(event)
-      @buffer << event_action_tuple(event)
-    end
-
     # Receive an array of events and immediately attempt to index them (no buffering)
     def multi_receive(events)
       events.each_slice(@flush_size) do |slice|
@@ -37,10 +31,6 @@ module LogStash; module Outputs; class ElasticSearch;
       [action, params, event]
     end
 
-    def flush
-      @buffer.flush
-    end
-
     def setup_hosts
       @hosts = Array(@hosts)
       if @hosts.empty?
@@ -53,12 +43,6 @@ module LogStash; module Outputs; class ElasticSearch;
       TemplateManager.install_template(self)
     end
 
-    def setup_buffer_and_handler
-      @buffer = ::LogStash::Outputs::ElasticSearch::Buffer.new(@logger, @flush_size, @idle_flush_time) do |actions|
-        retrying_submit(actions)
-      end
-    end
-
     def check_action_validity
       raise LogStash::ConfigurationError, "No action specified!" unless @action
 
@@ -75,33 +59,55 @@ module LogStash; module Outputs; class ElasticSearch;
       VALID_HTTP_ACTIONS
     end
 
-    def retrying_submit(actions)
+    def retrying_submit(actions)      
       # Initially we submit the full list of actions
       submit_actions = actions
 
+      sleep_interval = @retry_initial_interval
+
       while submit_actions && submit_actions.length > 0
-        return if !submit_actions || submit_actions.empty? # If everything's a success we move along
+        
         # We retry with whatever is didn't succeed
         begin
           submit_actions = submit(submit_actions)
+          if submit_actions && submit_actions.size > 0
+            @logger.error("Retrying individual actions")
+            submit_actions.each {|action| @logger.error("Action", action) }
+          end
         rescue => e
-          @logger.warn("Encountered an unexpected error submitting a bulk request! Will retry.",
-                       :message => e.message,
+          @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",
+                       :error_message => e.message,
                        :class => e.class.name,
                        :backtrace => e.backtrace)
         end
 
-        sleep @retry_max_interval if submit_actions && submit_actions.length > 0
+        # Everything was a success!
+        break if !submit_actions || submit_actions.empty?
+
+        # If we're retrying the action sleep for the recommended interval
+        # Double the interval for the next time through to achieve exponential backoff
+        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+        sleep_interval = next_sleep_interval(sleep_interval)
       end
     end
 
-    def submit(actions)
-      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash]}
+    def sleep_for_interval(sleep_interval)
+      Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+      next_sleep_interval(sleep_interval)
+    end
 
-      bulk_response = safe_bulk(es_actions,actions)
+    def next_sleep_interval(current_interval)
+      doubled = current_interval * 2
+      doubled > @retry_max_interval ? @retry_max_interval : doubled
+    end
 
-      # If there are no errors, we're done here!
-      return unless bulk_response["errors"]
+    def submit(actions)
+      bulk_response = safe_bulk(actions)
+      
+      # If the response is nil that means we were in a retry loop
+      # and aborted since we're shutting down
+      # If it did return and there are no errors we're good as well
+      return if bulk_response.nil? || !bulk_response["errors"]
 
       actions_to_retry = []
       bulk_response["items"].each_with_index do |response,idx|
@@ -168,38 +174,64 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     # Rescue retryable errors during bulk submission
-    def safe_bulk(es_actions,actions)
-      @client.bulk(es_actions)
-    rescue Manticore::SocketException, Manticore::SocketTimeout => e
-      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
-      # and let the user sort it out from there
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
-          " but Elasticsearch appears to be unreachable or down!",
-        :error_message => e.message,
-        :class => e.class.name,
-        :client_config => @client.client_options,
-      )
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-
-      # We retry until there are no errors! Errors should all go to the retry queue
-      sleep @retry_max_interval
-      retry unless @stopping.true?
-    rescue => e
-      # For all other errors print out full connection issues
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
-          " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
-          "the configuration provided?",
-        :error_message => e.message,
-        :error_class => e.class.name,
-        :backtrace => e.backtrace,
-        :client_config => @client.client_options,
-      )
-
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-
-      raise e
+    def safe_bulk(actions)
+      sleep_interval = @retry_initial_interval
+      begin
+        es_actions = actions.map {|action_type, params, event| [action_type, params, event.to_hash]}
+        response = @client.bulk(es_actions)
+        response
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::HostUnreachableError => e
+        # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
+        # and let the user sort it out from there
+        @logger.error(
+          "Attempted to send a bulk request to elasticsearch'"+
+            " but Elasticsearch appears to be unreachable or down!",
+          :error_message => e.message,
+          :class => e.class.name,
+          :will_retry_in_seconds => sleep_interval
+        )
+        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+        # We retry until there are no errors! Errors should all go to the retry queue
+        sleep_interval = sleep_for_interval(sleep_interval)
+        retry unless @stopping.true?
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::NoConnectionAvailableError => e
+        @logger.error(
+          "Attempted to send a bulk request to elasticsearch, but no there are no living connections in the connection pool. Perhaps Elasticsearch is unreachable or down?",
+          :error_message => e.message,
+          :class => e.class.name,
+          :will_retry_in_seconds => sleep_interval
+        )
+        Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+        sleep_interval = next_sleep_interval(sleep_interval)
+        retry unless @stopping.true?
+      rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
+        if RETRYABLE_CODES.include?(e.response_code)
+          log_hash = {:code => e.response_code, :url => e.url}
+          log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
+          @logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
+
+          sleep_interval = sleep_for_interval(sleep_interval)
+          retry unless @stopping.true?
+        else
+          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.code)
+        end
+      rescue => e
+        # Stuff that should never happen
+        # For all other errors print out full connection issues
+        @logger.error(
+          "An unknown error occurred sending a bulk request to Elasticsearch. We will retry indefinitely",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+        )
+
+        @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
+
+        # We retry until there are no errors! Errors should all go to the retry queue
+        sleep_interval = sleep_for_interval(sleep_interval)        
+        retry unless @stopping.true?
+      end
     end
   end
 end; end; end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -6,16 +6,10 @@ module LogStash; module Outputs; class ElasticSearch
       # delete old data or only search specific date ranges.
       # Indexes may not contain uppercase characters.
       # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
-      # LS uses Joda to format the index pattern from event timestamp. 
-      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. 
+      # LS uses Joda to format the index pattern from event timestamp.
+      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
       mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
 
-      # The index type to write events to. Generally you should try to write only
-      # similar events to the same 'type'. String expansion `%{foo}` works here.
-      #
-      # Deprecated in favor of `docoument_type` field.
-      mod.config :index_type, :validate => :string, :obsolete => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
-
       # The document type to write events to. Generally you should try to write only
       # similar events to the same 'type'. String expansion `%{foo}` works here.
       # Unless you set 'document_type', the event 'type' will be used if it exists
@@ -81,28 +75,12 @@ module LogStash; module Outputs; class ElasticSearch
       # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
       mod.config :hosts, :validate => :array, :default => ["127.0.0.1"]
 
-      mod.config :host, :obsolete => "Please use the 'hosts' setting instead. You can specify multiple entries separated by comma in 'host:port' format."
-
-      # The port setting is obsolete.  Please use the 'hosts' setting instead.
-      # Hosts entries can be in "host:port" format.
-      mod.config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
-
       # This plugin uses the bulk index API for improved indexing performance.
-      # In Logstashes >= 2.2 this setting defines the maximum sized bulk request Logstash will make
+      # This setting defines the maximum sized bulk request Logstash will make
       # You you may want to increase this to be in line with your pipeline's batch size.
       # If you specify a number larger than the batch size of your pipeline it will have no effect,
       # save for the case where a filter increases the size of an inflight batch by outputting
       # events.
-      #
-      # In Logstashes <= 2.1 this plugin uses its own internal buffer of events.
-      # This config option sets that size. In these older logstashes this size may
-      # have a significant impact on heap usage, whereas in 2.2+ it will never increase it.
-      # To make efficient bulk API calls, we will buffer a certain number of
-      # events before flushing that out to Elasticsearch. This setting
-      # controls how many events will be buffered before sending a batch
-      # of events. Increasing the `flush_size` has an effect on Logstash's heap size.
-      # Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
-      # or have increased the `flush_size` to a higher value.
       mod.config :flush_size, :validate => :number, :default => 500
 
       # The amount of time since last flush before a flush is forced.
@@ -124,8 +102,8 @@ module LogStash; module Outputs; class ElasticSearch
       # Create a new document with source if `document_id` doesn't exist in Elasticsearch
       mod.config :doc_as_upsert, :validate => :boolean, :default => false
 
-      # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
-      mod.config :max_retries, :validate => :number, :default => 3
+      #Obsolete since 4.1.0
+      mod.config :max_retries, :obsolete => "This setting no longer does anything. Please remove it from your config"
 
       # Set script name for scripted update mode
       mod.config :script, :validate => :string, :default => ""
@@ -145,12 +123,14 @@ module LogStash; module Outputs; class ElasticSearch
       # if enabled, script is in charge of creating non-existent document (scripted update)
       mod.config :scripted_upsert, :validate => :boolean, :default => false
 
-      # Set max interval between bulk retries.
-      mod.config :retry_max_interval, :validate => :number, :default => 2
+      # Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+      mod.config :retry_initial_interval, :validate => :number, :default => 2
+
+      # Set max interval in seconds between bulk retries.
+      mod.config :retry_max_interval, :validate => :number, :default => 64
 
-      # DEPRECATED This setting no longer does anything. If you need to change the number of retries in flight
-      # try increasing the total number of workers to better handle this.
-      mod.config :retry_max_items, :validate => :number, :default => 500, :deprecated => true
+      #Obsolete since 4.1.0
+      mod.config :retry_max_items, :obsolete => "This setting no longer does anything. Please remove it from your config"
 
       # The number of times Elasticsearch should internally retry an update/upserted document
       # See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -0,0 +1,63 @@
+require 'manticore'
+
+module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+  class ManticoreAdapter
+    attr_reader :manticore, :logger
+
+    def initialize(logger, options={})
+      @logger = logger
+      @options = options || {}
+      @options[:ssl] = @options[:ssl] || {}
+
+      # We manage our own retries directly, so let's disable them here
+      @options[:automatic_retries] = 0
+      # We definitely don't need cookies
+      @options[:cookies] = false
+
+      @request_options = @options[:headers] ? {:headers => @options[:headers]} : {}
+      @manticore = ::Manticore::Client.new(@options)
+    end
+
+    def client
+      @manticore
+    end
+
+    # Performs the request by invoking {Transport::Base#perform_request} with a block.
+    #
+    # @return [Response]
+    # @see    Transport::Base#perform_request
+    #
+    def perform_request(url, method, path, params={}, body=nil)
+      
+      
+      params = (params || {}).merge @request_options
+      params[:body] = body if body
+      url_and_path = (url + path).to_s # Convert URI object to string
+
+
+      resp = @manticore.send(method.downcase, url_and_path, params)
+
+      # Manticore returns lazy responses by default
+      # We want to block for our usage, this will wait for the repsonse
+      # to finish
+      resp.call
+
+      # 404s are excluded because they are valid codes in the case of
+      # template installation. We might need a better story around this later
+      # but for our current purposes this is correct
+      if resp.code < 200 || resp.code > 299 && resp.code != 404
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, url_and_path, body)
+      end
+
+      resp
+    end
+
+    def close
+      @manticore.close
+    end
+
+    def host_unreachable_exceptions
+      [::Manticore::Timeout,::Manticore::SocketException, ::Manticore::ClientProtocolException, ::Manticore::ResolutionFailure, Manticore::SocketTimeout]
+    end
+  end
+end; end; end; end

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -0,0 +1,378 @@
+module LogStash; module Outputs; class ElasticSearch; class HttpClient;
+  class Pool
+    class NoConnectionAvailableError < Error; end
+    class BadResponseCodeError < Error
+      attr_reader :url, :response_code, :body
+
+      def initialize(response_code, url, body)
+        @response_code = response_code
+        @url = url
+        @body = body
+      end
+
+      def message
+        "Got response code '#{response_code}' contact Elasticsrearch at URL '#{@url}'"
+      end
+    end
+    class HostUnreachableError < Error;
+      attr_reader :original_error, :url
+
+      def initialize(original_error, url)
+        @original_error = original_error
+        @url = url
+      end
+
+      def message
+        "Elasticsearch Unreachable: [#{@url}][#{original_error.class}] #{original_error.message}"
+      end
+    end
+
+    attr_reader :logger, :adapter, :sniffing, :sniffer_delay, :resurrect_delay, :auth, :healthcheck_path
+
+    DEFAULT_OPTIONS = {
+      :healthcheck_path => '/'.freeze,
+      :scheme => 'http',
+      :resurrect_delay => 5,
+      :auth => nil, # Can be set to {:user => 'user', :password => 'pass'}
+      :sniffing => false,
+      :sniffer_delay => 10,
+    }.freeze
+
+    def initialize(logger, adapter, initial_urls=[], options={})
+      @logger = logger
+      @adapter = adapter
+
+      DEFAULT_OPTIONS.merge(options).tap do |merged|
+        @healthcheck_path = merged[:healthcheck_path]
+        @scheme = merged[:scheme]
+        @resurrect_delay = merged[:resurrect_delay]
+        @auth = merged[:auth]
+        @sniffing = merged[:sniffing]
+        @sniffer_delay = merged[:sniffer_delay]
+      end
+
+      # Override the scheme if one is explicitly set in urls
+      if initial_urls.any? {|u| u.scheme == 'https'} && @scheme == 'http'
+        raise ArgumentError, "HTTP was set as scheme, but an HTTPS URL was passed in!"
+      end
+
+      # Used for all concurrent operations in this class
+      @state_mutex = Mutex.new
+
+      # Holds metadata about all URLs
+      @url_info = {}
+      @stopping = false
+
+      update_urls(initial_urls)
+      start_resurrectionist
+      start_sniffer if @sniffing
+    end
+
+    def close
+      @state_mutex.synchronize { @stopping = true }
+
+      logger.debug  "Stopping sniffer"
+      stop_sniffer
+
+      logger.debug  "Stopping resurrectionist"
+      stop_resurrectionist
+
+      logger.debug  "Waiting for in use manticore connections"
+      wait_for_in_use_connections
+
+      logger.debug("Closing adapter #{@adapter}")
+      @adapter.close
+    end
+
+    def wait_for_in_use_connections
+      until in_use_connections.empty?
+        logger.info "Blocked on shutdown to in use connections #{@state_mutex.synchronize {@url_info}}"
+        sleep 1
+      end
+    end
+
+    def in_use_connections
+      @state_mutex.synchronize { @url_info.values.select {|v| v[:in_use] > 0 } }
+    end
+
+    def alive_urls_count
+      @state_mutex.synchronize { @url_info.values.select {|v| !v[:dead] }.count }
+    end
+
+    def url_info
+      @state_mutex.synchronize { @url_info }
+    end
+
+    def urls
+      url_info.keys
+    end
+
+    def until_stopped(task_name, delay)
+      last_done = Time.now
+      until @state_mutex.synchronize { @stopping }
+        begin
+          now = Time.now
+          if (now - last_done) >= delay
+            last_done = now
+            yield
+          end
+          sleep 1
+        rescue => e
+          logger.warn(
+            "Error while performing #{task_name}",
+            :error_message => e.message,
+            :class => e.class.name,
+            :backtrace => e.backtrace
+            )
+        end
+      end
+    end
+
+    def start_sniffer
+      @sniffer = Thread.new do
+        until_stopped("sniffing", sniffer_delay) do
+          begin
+            sniff!
+          rescue NoConnectionAvailableError => e
+            @state_mutex.synchronize { # Synchronize around @url_info
+              logger.warn("Elasticsearch output attempted to sniff for new connections but cannot. No living connections are detected. Pool contains the following current URLs", :url_info => @url_info) }
+          end
+        end
+      end
+    end
+
+    # Sniffs the cluster then updates the internal URLs
+    def sniff!
+      update_urls(check_sniff)
+    end
+
+    ES1_SNIFF_RE_URL  = /\[([^\/]*)?\/?([^:]*):([0-9]+)\]/
+    ES2_SNIFF_RE_URL  = /([^\/]*)?\/?([^:]*):([0-9]+)/
+    # Sniffs and returns the results. Does not update internal URLs!
+    def check_sniff
+      url, resp = perform_request(:get, '_nodes')
+      parsed = LogStash::Json.load(resp.body)
+      parsed['nodes'].map do |id,info|
+        # TODO Make sure this works with shield. Does that listed
+        # stuff as 'https_address?'
+        addr_str = info['http_address'].to_s
+        next unless addr_str # Skip hosts with HTTP disabled
+
+
+        # Only connect to nodes that serve data
+        # this will skip connecting to client, tribe, and master only nodes
+        # Note that if 'attributes' is NOT set, then that's just a regular node
+        # with master + data + client enabled, so we allow that
+        attributes = info['attributes']
+        next if attributes && attributes['data'] == 'false'
+
+        matches = addr_str.match(ES1_SNIFF_RE_URL) || addr_str.match(ES2_SNIFF_RE_URL)
+        if matches
+          host = matches[1].empty? ? matches[2] : matches[1]
+          port = matches[3]
+          URI.parse("#{@scheme}://#{host}:#{port}")
+        end
+      end.compact
+    end
+
+    def stop_sniffer
+      @sniffer.join if @sniffer
+    end
+
+    def sniffer_alive?
+      @sniffer ? @sniffer.alive? : nil
+    end
+
+    def start_resurrectionist
+      @resurrectionist = Thread.new do
+        until_stopped("resurrection", @resurrect_delay) do
+          resurrect_dead!
+        end
+      end
+    end
+
+    def resurrect_dead!
+      # Try to keep locking granularity low such that we don't affect IO...
+      @state_mutex.synchronize { @url_info.select {|url,meta| meta[:dead] } }.each do |url,meta|
+        begin
+          @logger.info("Checking url #{url} with path #{@healthcheck_path} to see if node resurrected")
+          perform_request_to_url(url, "HEAD", @healthcheck_path)
+          # If no exception was raised it must have succeeded!
+          logger.warn("Resurrected connection to dead ES instance at #{url}")
+          @state_mutex.synchronize { meta[:dead] = false }
+        rescue HostUnreachableError => e
+          logger.debug("Attempted to resurrect connection to dead ES instance at #{url}, got an error [#{e.class}] #{e.message}")
+        end
+      end
+    end
+
+    def stop_resurrectionist
+      @resurrectionist.join
+    end
+
+    def resurrectionist_alive?
+      @resurrectionist.alive?
+    end
+
+    def perform_request(method, path, params={}, body=nil)
+      with_connection do |url|
+        resp = perform_request_to_url(url, method, path, params, body)
+        [url, resp]
+      end
+    end
+
+    [:get, :put, :post, :delete, :patch, :head].each do |method|
+      define_method(method) do |path, params={}, body=nil|
+        perform_request(method, path, params, body)
+      end
+    end
+
+    def perform_request_to_url(url, method, path, params={}, body=nil)
+      res = @adapter.perform_request(url, method, path, params, body)
+    rescue *@adapter.host_unreachable_exceptions => e
+      raise HostUnreachableError.new(e, url), "Could not reach host #{e.class}: #{e.message}"
+    end
+
+    def normalize_url(uri)
+      raise ArgumentError, "Only URI objects may be passed in!" unless uri.is_a?(URI)
+      uri = uri.clone
+
+      # Set credentials if need be
+      if @auth && !uri.user
+        uri.user ||= @auth[:user]
+        uri.password ||= @auth[:password]
+      end
+
+      uri.scheme = @scheme
+
+      uri
+    end
+
+    def update_urls(new_urls)
+      # Normalize URLs
+      new_urls = new_urls.map(&method(:normalize_url))
+
+      # Used for logging nicely
+      state_changes = {:removed => [], :added => []}
+      @state_mutex.synchronize do
+        # Add new connections
+        new_urls.each do |url|
+          # URI objects don't have real hash equality! So, since this isn't perf sensitive we do a linear scan
+          unless @url_info.keys.include?(url)
+            state_changes[:added] << url.to_s
+            add_url(url)
+          end
+        end
+
+        # Delete connections not in the new list
+        @url_info.each do |url,_|
+          unless new_urls.include?(url)
+            state_changes[:removed] << url.to_s
+            remove_url(url)
+          end
+        end
+      end
+
+      if state_changes[:removed].size > 0 || state_changes[:added].size > 0
+        logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
+      end
+    end
+
+    def size
+      @state_mutex.synchronize { @url_info.size }
+    end
+
+    def add_url(url)
+      @url_info[url] ||= empty_url_meta
+    end
+
+    def remove_url(url)
+      @url_info.delete(url)
+    end
+
+    def empty_url_meta
+      {
+        :in_use => 0,
+        :dead => false
+      }
+    end
+
+    def with_connection
+      url, url_meta = get_connection
+
+      # Custom error class used here so that users may retry attempts if they receive this error
+      # should they choose to
+      raise NoConnectionAvailableError, "No Available connections" unless url
+      yield url
+    rescue HostUnreachableError => e
+      # Mark the connection as dead here since this is likely not transient
+      mark_dead(url, e)
+      raise e
+    rescue BadResponseCodeError => e
+      # These aren't discarded from the pool because these are often very transient
+      # errors
+      raise e
+    rescue => e
+      logger.warn("UNEXPECTED POOL ERROR", :e => e)
+      raise e
+    ensure
+      return_connection(url)
+    end
+
+    def mark_dead(url, error)
+      @state_mutex.synchronize do
+        meta = @url_info[url]
+        # In case a sniff happened removing the metadata just before there's nothing to mark
+        # This is an extreme edge case, but it can happen!
+        return unless meta 
+        logger.warn("Marking url as dead. Last error: [#{error.class}] #{error.message}",
+                    :url => url, :error_message => error.message, :error_class => error.class.name)
+        meta[:dead] = true
+        meta[:last_error] = error
+        meta[:last_errored_at] = Time.now
+      end
+    end
+
+    def url_meta(url)
+      @state_mutex.synchronize do
+        @url_info[url]
+      end
+    end
+
+    def get_connection
+      @state_mutex.synchronize do
+        # The goal here is to pick a random connection from the least-in-use connections
+        # We want some randomness so that we don't hit the same node over and over, but
+        # we also want more 'fair' behavior in the event of high concurrency
+        eligible_set = nil
+        lowest_value_seen = nil
+        @url_info.each do |url,meta|
+          meta_in_use = meta[:in_use]
+          next if meta[:dead]
+
+          if lowest_value_seen.nil? || meta_in_use < lowest_value_seen
+            lowest_value_seen = meta_in_use
+            eligible_set = [[url, meta]]
+          elsif lowest_value_seen == meta_in_use
+            eligible_set << [url, meta]
+          end
+        end
+
+        return nil if eligible_set.nil?
+
+        pick, pick_meta = eligible_set.sample
+        pick_meta[:in_use] += 1
+
+        [pick, pick_meta]
+      end
+    end
+
+    def return_connection(url)
+      @state_mutex.synchronize do
+        if @url_info[url] # Guard against the condition where the connection has already been deleted
+          @url_info[url][:in_use] -= 1
+        end
+      end
+    end
+  end
+end; end; end; end;