logstash-output-elasticsearch 2.0.0.beta6-java → 2.0.0.pre.beta2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c3914887728f304938a7f0414212daff422424a8
-  data.tar.gz: e627c4d8f3722e30f9ff4c32298792aa14adeeeb
+  metadata.gz: 1eddba06256936b077c451667043e068411ad626
+  data.tar.gz: 0c2720a59a0fced2bf4dd2eb40c780ee9fb9a458
 SHA512:
-  metadata.gz: 4e5facd0d14d905fa723901d58b952b5547a35fdd81bc1848b2552aaeb8ae0f559ef1059b2e916bc838517b3e1d21227a089a316cb2e5ee545089934c992386d
-  data.tar.gz: e788b461bdc1e24fceebec5b72a0bc16f8f1f4603cd12b1c0e0af708beccb74f202a7a99c54b16f3554072bdfba7bbeccca9294be9bf408ff4765967883ddd24
+  metadata.gz: 016018991bc202ddfaccb630b7e87b8a4951c7de1fad7f86190846ccd21a7a077317b91fc7308f99086ed91013ab21c90c66850fb44b54156a52fd7431bde34e
+  data.tar.gz: dbf3a51b89577d185845eecc985ae57cb9624878f47ac52f203bcc3bcc28c334bd464f20af0b4be1f4cbe82e79ea4e82a476e58e25ead16b4129ff6b310b627f

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+Gemfile.lock
+.bundle
+.idea
+*~

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/elasticsearch.rb

@@ -10,32 +10,44 @@ require "thread" # for safe queueing
 require "uri" # for escaping user input
 require "logstash/outputs/elasticsearch/http_client"
 
-# This plugin is the recommended method of storing logs in Elasticsearch.
-# If you plan on using the Kibana web interface, you'll want to use this output.
+# This output lets you store logs in Elasticsearch and is the most recommended
+# output for Logstash. If you plan on using the Kibana web interface, you'll
+# want to use this output.
 #
-# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
-# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
-# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-# to upgrade Logstash in lock-step. For those wishing to use the node or transport protocols please see the 'elasticsearch_java' gem.
+# This output only speaks the HTTP, which is the preferred protocol for interacting with Elasticsearch. By default
+# Elasticsearch exposes HTTP on port 9200.
+#
+# We strongly encourage the use of HTTP over the node protocol. It is just as
+# fast and far easier to administer. For those wishing to use the java protocol please see the 'elasticsearch_java' gem.
 #
 # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
 #
 # ==== Retry Policy
 #
-# This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
-# either partial or total failures. Events are retried if they fail due to either a network error or the status codes
-# 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
+# By default all bulk requests to ES are synchronous. Not all events in the bulk requests
+# always make it successfully. For example, there could be events which are not formatted
+# correctly for the index they are targeting (type mismatch in mapping). So that we minimize loss of 
+# events, we have a specific retry policy in place. We retry all events which fail to be reached by 
+# Elasticsearch for network related issues. We retry specific events which exhibit errors under a separate 
+# policy described below. Events of this nature are ones which experience ES error codes described as 
+# retryable errors.
+#
+# *Retryable Errors:*
 #
-# The retry policy's logic can be described as follows:
+# - 429, Too Many Requests (RFC6585)
+# - 503, The server is currently unable to handle the request due to a temporary overloading or maintenance of the server.
+# 
+# Here are the rules of what is retried when:
 #
-# - Block and retry all events in the bulk response that experience transient network exceptions until
+# - Block and retry all events in bulk response that experiences transient network exceptions until
 #   a successful submission is received by Elasticsearch.
-# - Retry the subset of sent events which resulted in ES errors of a retryable nature.
-# - Events which returned retryable error codes will be pushed onto a separate queue for
-#   retrying events. Events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries).
-#   The size of this queue is capped by the value set in :retry_max_items.
-# - Events from the retry queue are submitted again when the queue reaches its max size or when
-#   the max interval time is reached. The max interval time is configurable via :retry_max_interval.
+# - Retry subset of sent events which resulted in ES errors of a retryable nature which can be found 
+#   in RETRYABLE_CODES
+# - For events which returned retryable error codes, they will be pushed onto a separate queue for 
+#   retrying events. events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries). The size of 
+#   this queue is capped by the value set in :retry_max_items.
+# - Events from the retry queue are submitted again either when the queue reaches its max size or when
+#   the max interval time is reached, which is set in :retry_max_interval.
 # - Events which are not retryable or have reached their max retry count are logged to stderr.
 class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   attr_reader :client
@@ -237,9 +249,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     client_settings[:path] = "/#{@path}/".gsub(/\/+/, "/") # Normalize slashes
     @logger.debug? && @logger.debug("Normalizing http path", :path => @path, :normalized => client_settings[:path])
 
-    if @hosts.nil? || @hosts.empty?
+    if @host.nil?
       @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
-      @hosts = ["localhost"]
+      @host = ["localhost"]
     end
 
     client_settings.merge! setup_ssl()
@@ -254,7 +266,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     common_options.merge! update_options if @action == 'update'
 
     @client = LogStash::Outputs::Elasticsearch::HttpClient.new(
-      common_options.merge(:hosts => @hosts, :port => @port)
+      common_options.merge(:hosts => @hosts)
     )
 
     if @manage_template
@@ -348,25 +360,19 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
       bulk_response = @client.bulk(es_actions)
 
-      next unless bulk_response["errors"]
-
-      actions_to_retry = []
-
-      bulk_response["items"].each_with_index do |resp,idx|
-        action_type, action_props = resp.first
-
-        status = action_props["status"]
-        action = actions[idx]
-
-        if RETRYABLE_CODES.include?(status)
-          @logger.warn "retrying failed action with response code: #{status}"
-          actions_to_retry << action
-        elsif not SUCCESS_CODES.include?(status)
-          @logger.warn "Failed action. ", status: status, action: action, response: resp
+      if bulk_response["errors"]
+        actions_with_responses = actions.zip(bulk_response['statuses'])
+        actions_to_retry = []
+        actions_with_responses.each do |action, resp_code|
+          if RETRYABLE_CODES.include?(resp_code)
+            @logger.warn "retrying failed action with response code: #{resp_code}"
+            actions_to_retry << action
+          elsif not SUCCESS_CODES.include?(resp_code)
+            @logger.warn "failed action with response of #{resp_code}, dropping action: #{action}"
+          end
         end
+        retry_push(actions_to_retry) unless actions_to_retry.empty?
       end
-
-      retry_push(actions_to_retry) unless actions_to_retry.empty?
     end
   end
 
@@ -408,6 +414,10 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   def teardown
     @client.stop_sniffing!
 
+    if @cacert # remove temporary jks store created from the cacert
+      File.delete(@truststore)
+    end
+
     @retry_teardown_requested.make_true
     # First, make sure retry_timer_thread is stopped
     # to ensure we do not signal a retry based on 
@@ -451,15 +461,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     if @cacert && @truststore
       raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
     end
-
     ssl_options = {}
-
-    if @cacert
-      ssl_options[:ca_file] = @cacert
+    if @cacert then
+      @truststore, ssl_options[:truststore_password] = generate_jks @cacert
     elsif @truststore
       ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
     end
-
     ssl_options[:truststore] = @truststore if @truststore
     if @keystore
       ssl_options[:keystore] = @keystore
@@ -486,6 +493,29 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     }
   end
 
+  private
+  def generate_jks cert_path
+
+    require 'securerandom'
+    require 'tempfile'
+    require 'java'
+    import java.io.FileInputStream
+    import java.io.FileOutputStream
+    import java.security.KeyStore
+    import java.security.cert.CertificateFactory
+
+    jks = java.io.File.createTempFile("cert", ".jks")
+
+    ks = KeyStore.getInstance "JKS"
+    ks.load nil, nil
+    cf = CertificateFactory.getInstance "X.509"
+    cert = cf.generateCertificate FileInputStream.new(cert_path)
+    ks.setCertificateEntry "cacert", cert
+    pwd = SecureRandom.urlsafe_base64(9)
+    ks.store FileOutputStream.new(jks), pwd.to_java.toCharArray
+    [jks.path, pwd]
+  end
+
   private
   # in charge of submitting any actions in @retry_queue that need to be 
   # retried

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -50,7 +50,9 @@ module LogStash::Outputs::Elasticsearch
         end
       end.flatten
 
-      @client.bulk(:body => bulk_body)
+      bulk_response = @client.bulk(:body => bulk_body)
+
+      self.class.normalize_bulk_response(bulk_response)
     end
 
     def start_sniffing!
@@ -84,8 +86,7 @@ module LogStash::Outputs::Elasticsearch
       client_settings = options[:client_settings] || {}
 
       uris = hosts.map do |host|
-        proto = client_settings[:ssl] ? "https"  : "http"
-        "#{proto}://#{host}:#{port}#{client_settings[:path]}"
+        "http://#{host}:#{port}#{client_settings[:path]}"
       end
 
       @client_options = {
@@ -104,11 +105,27 @@ module LogStash::Outputs::Elasticsearch
         @client_options[:headers] = { "Authorization" => "Basic #{token}" }
       end
 
-      @logger.debug? && @logger.debug("Elasticsearch HTTP client options", client_options)
-
       Elasticsearch::Client.new(client_options)
     end
 
+    def self.normalize_bulk_response(bulk_response)
+      if bulk_response["errors"]
+        # The structure of the response from the REST Bulk API is follows:
+        # {"took"=>74, "errors"=>true, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+        #                                                    "_type"=>"logs",
+        #                                                    "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+        #                                                    "_version"=>1,
+        #                                                    "status"=>400,
+        #                                                    "error"=>"MapperParsingException[failed to parse]..."}}]}
+        # where each `item` is a hash of {OPTYPE => Hash[]}. calling first, will retrieve
+        # this hash as a single array with two elements, where the value is the second element (i.first[1])
+        # then the status of that item is retrieved.
+        {"errors" => true, "statuses" => bulk_response["items"].map { |i| i.first[1]['status'] }}
+      else
+        {"errors" => false}
+      end
+    end
+
     def template_exists?(name)
       @client.indices.get_template(:name => name)
       return true

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.0.0.beta6'
+  s.version         = '2.0.0-beta2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = `git ls-files`.split($\)
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
@@ -21,7 +21,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency 'concurrent-ruby'
-  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.13', '~> 1.0']
+  s.add_runtime_dependency 'elasticsearch', ['>= 1.0.10', '~> 1.0']
   s.add_runtime_dependency 'stud', ['>= 0.0.17', '~> 0.0']
   s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core", '>= 1.4.0', '< 2.0.0'

data/spec/integration/outputs/retry_spec.rb

@@ -11,18 +11,8 @@ describe "failures in bulk class expected behavior", :integration => true do
   let(:max_retries) { 3 }
 
   def mock_actions_with_response(*resp)
-    expanded_responses = resp.map do |resp|
-      items = resp["statuses"] && resp["statuses"].map do |status|
-        {"create" => {"status" => status, "error" => "Error for #{status}"}}
-      end
-
-      {
-        "errors" => resp["errors"],
-        "items" => items
-      }
-    end
-
-    allow_any_instance_of(LogStash::Outputs::Elasticsearch::HttpClient).to receive(:bulk).and_return(*expanded_responses)
+    LogStash::Outputs::Elasticsearch::HttpClient
+      .any_instance.stub(:bulk).and_return(*resp)
   end
 
   subject! do
@@ -32,7 +22,7 @@ describe "failures in bulk class expected behavior", :integration => true do
       "template_overwrite" => true,
       "hosts" => get_host(),
       "port" => get_port(),
-      "retry_max_items" => 2,
+      "retry_max_items" => 10,
       "retry_max_interval" => 1,
       "max_retries" => max_retries
     }
@@ -88,7 +78,7 @@ describe "failures in bulk class expected behavior", :integration => true do
     subject.receive(event1)
     subject.receive(event2)
     subject.buffer_flush(:final => true)
-    sleep(10)
+    sleep(3)
   end
 
   it "should retry actions with response status of 429" do

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -3,6 +3,27 @@ require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
 describe LogStash::Outputs::Elasticsearch::HttpClient do
+  context "successful" do
+    it "should map correctly" do
+      bulk_response = {"took"=>74, "errors"=>false, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                                                          "_type"=>"logs", "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+                                                                          "_version"=>1, "status"=>201}}]} 
+      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      bulk_response = {"took"=>71, "errors"=>true,
+                       "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                             "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
+                                             "error"=>"MapperParsingException[failed to parse]..."}}]}
+      actual = LogStash::Outputs::Elasticsearch::HttpClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> true, "statuses"=> [400]}
+    end
+  end
+
   describe "sniffing" do
     let(:base_options) { {:hosts => ["127.0.0.1"] }}
     let(:client) { LogStash::Outputs::Elasticsearch::HttpClient.new(base_options.merge(client_opts)) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.0.0.beta6
+  version: 2.0.0.pre.beta2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-09-15 00:00:00.000000000 Z
+date: 2015-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.13
+        version: 1.0.10
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -40,7 +40,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.13
+        version: 1.0.10
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
@@ -174,14 +174,15 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".gitignore"
 - CHANGELOG.md
 - CONTRIBUTORS
 - Gemfile
 - LICENSE
 - NOTICE.TXT
 - README.md
+- Rakefile
 - lib/logstash/outputs/elasticsearch.rb
-- lib/logstash/outputs/elasticsearch.rb.orig
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - logstash-output-elasticsearch.gemspec
@@ -236,4 +237,3 @@ test_files:
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
-has_rdoc:

data/lib/logstash/outputs/elasticsearch.rb.orig

@@ -1,555 +0,0 @@
-# encoding: utf-8
-require "logstash/namespace"
-require "logstash/environment"
-require "logstash/outputs/base"
-require "logstash/json"
-require "concurrent"
-require "stud/buffer"
-require "socket" # for Socket.gethostname
-require "thread" # for safe queueing
-require "uri" # for escaping user input
-require "logstash/outputs/elasticsearch/http_client"
-
-# This plugin is the recommended method of storing logs in Elasticsearch.
-# If you plan on using the Kibana web interface, you'll want to use this output.
-#
-# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
-# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
-# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-# to upgrade Logstash in lock-step. For those wishing to use the node or transport protocols please see the 'elasticsearch_java' gem.
-#
-# You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
-#
-# ==== Retry Policy
-#
-# This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
-# either partial or total failures. Events are retried if they fail due to either a network error or the status codes
-# 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
-#
-# The retry policy's logic can be described as follows:
-#
-# - Block and retry all events in the bulk response that experience transient network exceptions until
-#   a successful submission is received by Elasticsearch.
-# - Retry the subset of sent events which resulted in ES errors of a retryable nature.
-# - Events which returned retryable error codes will be pushed onto a separate queue for
-#   retrying events. Events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries).
-#   The size of this queue is capped by the value set in :retry_max_items.
-# - Events from the retry queue are submitted again when the queue reaches its max size or when
-#   the max interval time is reached. The max interval time is configurable via :retry_max_interval.
-# - Events which are not retryable or have reached their max retry count are logged to stderr.
-class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
-  attr_reader :client
-
-  include Stud::Buffer
-  RETRYABLE_CODES = [409, 429, 503]
-  SUCCESS_CODES = [200, 201]
-
-  config_name "elasticsearch"
-
-  # The index to write events to. This can be dynamic using the `%{foo}` syntax.
-  # The default value will partition your indices by day so you can more easily
-  # delete old data or only search specific date ranges.
-  # Indexes may not contain uppercase characters.
-  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
-  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
-
-  # The index type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # 
-  # Deprecated in favor of `document_type` field.
-  config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
-
-  # The document type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # Unless you set 'document_type', the event 'type' will be used if it exists 
-  # otherwise the document type will be assigned the value of 'logs'
-  config :document_type, :validate => :string
-
-  # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
-  # a default mapping template for Elasticsearch will be applied, if you do not
-  # already have one set to match the index pattern defined (default of
-  # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
-  # the template will be applied to all indices starting with `logstash-*`
-  #
-  # If you have dynamic templating (e.g. creating indices based on field names)
-  # then you should set `manage_template` to false and use the REST API to upload
-  # your templates manually.
-  config :manage_template, :validate => :boolean, :default => true
-
-  # This configuration option defines how the template is named inside Elasticsearch.
-  # Note that if you have used the template management features and subsequently
-  # change this, you will need to prune the old template manually, e.g.
-  #
-  # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
-  #
-  # where `OldTemplateName` is whatever the former setting was.
-  config :template_name, :validate => :string, :default => "logstash"
-
-  # You can set the path to your own template here, if you so desire.
-  # If not set, the included template will be used.
-  config :template, :validate => :path
-
-  # Overwrite the current template with whatever is configured
-  # in the `template` and `template_name` directives.
-  config :template_overwrite, :validate => :boolean, :default => false
-
-  # The document ID for the index. Useful for overwriting existing entries in
-  # Elasticsearch with the same ID.
-  config :document_id, :validate => :string
-
-  # A routing override to be applied to all processed events.
-  # This can be dynamic using the `%{foo}` syntax.
-  config :routing, :validate => :string
-
-  # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `host` parameter.
-  # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
-  #     `"127.0.0.1"`
-  #     `["127.0.0.1:9200","127.0.0.2:9200"]`
-  # It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `host` list
-  # to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes.
-
-  config :hosts, :validate => :array
-
-  # You can set the remote port as part of the host, or explicitly here as well
-  config :port, :validate => :string, :default => 9200
-
-  # This plugin uses the bulk index api for improved indexing performance.
-  # To make efficient bulk api calls, we will buffer a certain number of
-  # events before flushing that out to Elasticsearch. This setting
-  # controls how many events will be buffered before sending a batch
-  # of events.
-  config :flush_size, :validate => :number, :default => 500
-
-  # The amount of time since last flush before a flush is forced.
-  #
-  # This setting helps ensure slow event rates don't get stuck in Logstash.
-  # For example, if your `flush_size` is 100, and you have received 10 events,
-  # and it has been more than `idle_flush_time` seconds since the last flush,
-  # Logstash will flush those 10 events automatically.
-  #
-  # This helps keep both fast and slow log streams moving along in
-  # near-real-time.
-  config :idle_flush_time, :validate => :number, :default => 1
-
-  # The Elasticsearch action to perform. Valid actions are: `index`, `delete`.
-  #
-  # Use of this setting *REQUIRES* you also configure the `document_id` setting
-  # because `delete` actions all require a document id.
-  #
-  # What does each action do?
-  #
-  # - index: indexes a document (an event from Logstash).
-  # - delete: deletes a document by id
-  # - create: indexes a document, fails if a document by that id already exists in the index.
-  # - update: updates a document by id
-  # following action is not supported by HTTP protocol
-  #
-  # For more details on actions, check out the http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
-  config :action, :validate => %w(index delete create update), :default => "index"
-
-  # Username and password (only valid when protocol is HTTP; this setting works with HTTP or HTTPS auth)
-  config :user, :validate => :string
-  config :password, :validate => :password
-
-  # HTTP Path at which the Elasticsearch server lives. Use this if you must run ES behind a proxy that remaps
-  # the root path for the Elasticsearch HTTP API lives. This option is ignored for non-HTTP transports.
-  config :path, :validate => :string, :default => "/"
-
-  # SSL Configurations (only valid when protocol is HTTP)
-  #
-  # Enable SSL
-  config :ssl, :validate => :boolean, :default => false
-
-  # Validate the server's certificate
-  # Disabling this severely compromises security
-  # For more information read https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
-  config :ssl_certificate_verification, :validate => :boolean, :default => true
-
-  # The .cer or .pem file to validate the server's certificate
-  config :cacert, :validate => :path
-
-  # The JKS truststore to validate the server's certificate
-  # Use either `:truststore` or `:cacert`
-  config :truststore, :validate => :path
-
-  # Set the truststore password
-  config :truststore_password, :validate => :password
-
-  # The keystore used to present a certificate to the server
-  # It can be either .jks or .p12
-  config :keystore, :validate => :path
-
-  # Set the truststore password
-  config :keystore_password, :validate => :password
-
-  # Enable cluster sniffing
-  # Asks host for the list of all cluster nodes and adds them to the hosts list
-  # Will return ALL nodes with HTTP enabled (including master nodes!). If you use
-  # this with master nodes, you probably want to disable HTTP on them by setting
-  # `http.enabled` to false in their elasticsearch.yml.
-  config :sniffing, :validate => :boolean, :default => false
-
-  # How long to wait, in seconds, between sniffing attempts
-  config :sniffing_delay, :validate => :number, :default => 30
-
-  # Set max retry for each event
-  config :max_retries, :validate => :number, :default => 3
-
-  # Set retry policy for events that failed to send
-  config :retry_max_items, :validate => :number, :default => 5000
-
-  # Set max interval between bulk retries
-  config :retry_max_interval, :validate => :number, :default => 5
-
-  # Set the address of a forward HTTP proxy. Must be used with the 'http' protocol
-  # Can be either a string, such as 'http://localhost:123' or a hash in the form
-  # {host: 'proxy.org' port: 80 scheme: 'http'}
-  # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
-  config :proxy
-
-  # Enable doc_as_upsert for update mode
-  # create a new document with source if document_id doesn't exists
-  config :doc_as_upsert, :validate => :boolean, :default => false
-
-  # Set upsert content for update mode
-  # create a new document with this parameter as json string if document_id doesn't exists
-  config :upsert, :validate => :string, :default => ""
-
-  public
-  def register
-    @hosts = Array(@hosts)
-    # retry-specific variables
-    @retry_flush_mutex = Mutex.new
-    @retry_teardown_requested = Concurrent::AtomicBoolean.new(false)
-    # needs flushing when interval
-    @retry_queue_needs_flushing = ConditionVariable.new
-    @retry_queue_not_full = ConditionVariable.new
-    @retry_queue = Queue.new
-    @submit_mutex = Mutex.new
-
-    client_settings = {}
-    common_options = {
-      :client_settings => client_settings,
-      :sniffing => @sniffing,
-      :sniffing_delay => @sniffing_delay
-    }
-
-    client_settings[:path] = "/#{@path}/".gsub(/\/+/, "/") # Normalize slashes
-    @logger.debug? && @logger.debug("Normalizing http path", :path => @path, :normalized => client_settings[:path])
-
-    if @hosts.nil? || @hosts.empty?
-      @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
-      @hosts = ["localhost"]
-    end
-
-    client_settings.merge! setup_ssl()
-    client_settings.merge! setup_proxy()
-    common_options.merge! setup_basic_auth()
-
-    # Update API setup
-    update_options = {
-      :upsert => @upsert,
-      :doc_as_upsert => @doc_as_upsert
-    }
-    common_options.merge! update_options if @action == 'update'
-
-    @client = LogStash::Outputs::Elasticsearch::HttpClient.new(
-      common_options.merge(:hosts => @hosts, :port => @port)
-    )
-
-    if @manage_template
-      begin
-        @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
-        @client.template_install(@template_name, get_template, @template_overwrite)
-      rescue => e
-        @logger.error("Failed to install template: #{e.message}")
-      end
-    end
-
-    @logger.info("New Elasticsearch output", :hosts => @hosts, :port => @port)
-
-    @client_idx = 0
-
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
-
-    @retry_timer_thread = Thread.new do
-      loop do
-        sleep(@retry_max_interval)
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-      end
-    end
-
-    @retry_thread = Thread.new do
-      while @retry_teardown_requested.false?
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.wait(@retry_flush_mutex) }
-        retry_flush
-      end
-    end
-  end # def register
-
-  public
-  def get_template
-    if @template.nil?
-      @template = ::File.expand_path('elasticsearch/elasticsearch-template.json', ::File.dirname(__FILE__))
-      if !File.exists?(@template)
-        raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
-      end
-    end
-    template_json = IO.read(@template).gsub(/\n/,'')
-    template = LogStash::Json.load(template_json)
-    @logger.info("Using mapping template", :template => template)
-    return template
-  end # def get_template
-
-  public
-  def receive(event)
-    return unless output?(event)
-
-    # block until we have not maxed out our 
-    # retry queue. This is applying back-pressure
-    # to slow down the receive-rate
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
-    }
-
-    event['@metadata']['retry_count'] = 0
-
-    # Set the 'type' value for the index.
-    type = if @document_type
-             event.sprintf(@document_type)
-           elsif @index_type # deprecated
-             event.sprintf(@index_type)
-           else
-             event["type"] || "logs"
-           end
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      :_type => type,
-      :_routing => @routing ? event.sprintf(@routing) : nil
-    }
-    
-    params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
-
-    buffer_receive([event.sprintf(@action), params, event])
-  end # def receive
-
-  public
-  # The submit method can be called from both the
-  # Stud::Buffer flush thread and from our own retry thread.
-  def submit(actions)
-    @submit_mutex.synchronize do
-      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash] }
-
-      bulk_response = @client.bulk(es_actions)
-
-<<<<<<< c2b39d4ff5a5ced59a9e7229fc135cf2a966bdc4
-      if bulk_response["errors"]
-        actions_to_retry = []
-
-        bulk_response['items'].each_with_index do |item,idx|
-          action = es_actions[idx]
-          action_type, props = item.first # These are all hashes with one value, so we destructure them here
-
-          status = props['status']
-          error = props['error']
-
-          if RETRYABLE_CODES.include?(status)
-            @logger.warn "retrying failed action with response code: #{status}"
-            actions_to_retry << action
-          elsif not SUCCESS_CODES.include?(status)
-            @logger.warn "failed action", status: status, error: error, action: action
-          end
-        end
-
-        retry_push(actions_to_retry) unless actions_to_retry.empty?
-=======
-      next unless bulk_response["errors"]
-
-      actions_to_retry = []
-
-      bulk_response["items"].each_with_index do |resp,idx|
-        action_type, action_props = resp.first
-
-        status = action_props["status"]
-        action = es_actions[idx]
-
-        if RETRYABLE_CODES.include?(status)
-          @logger.warn "retrying failed action with response code: #{status}"
-          actions_to_retry << action
-        elsif not SUCCESS_CODES.include?(status)
-          @logger.warn "failed action with response of #{status}, dropping action: #{action}"
-        end
->>>>>>> WIP for better retry errors
-      end
-
-      retry_push(actions_to_retry) unless actions_to_retry.empty?
-    end
-  end
-
-  # When there are exceptions raised upon submission, we raise an exception so that
-  # Stud::Buffer will retry to flush
-  public
-  def flush(actions, teardown = false)
-    begin
-      submit(actions)
-    rescue Manticore::SocketException => e
-      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
-      # and let the user sort it out from there
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
-          " but Elasticsearch appears to be unreachable or down!",
-        :client_config => @client.client_options,
-        :error_message => e.message
-      )
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-    rescue => e
-      # For all other errors print out full connection issues
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
-            " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
-          "the configuration provided?",
-        :client_config => @client.client_options,
-        :error_message => e.message,
-        :error_class => e.class.name,
-        :backtrace => e.backtrace
-      )
-
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-
-      raise e
-    end
-  end # def flush
-
-  public
-  def teardown
-    @client.stop_sniffing!
-
-    @retry_teardown_requested.make_true
-    # First, make sure retry_timer_thread is stopped
-    # to ensure we do not signal a retry based on 
-    # the retry interval.
-    Thread.kill(@retry_timer_thread)
-    @retry_timer_thread.join
-    # Signal flushing in the case that #retry_flush is in 
-    # the process of waiting for a signal.
-    @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-    # Now, #retry_flush is ensured to not be in a state of 
-    # waiting and can be safely joined into the main thread
-    # for further final execution of an in-process remaining call.
-    @retry_thread.join
-
-    # execute any final actions along with a proceeding retry for any 
-    # final actions that did not succeed.
-    buffer_flush(:final => true)
-    retry_flush
-  end
-
-  private
-  def setup_proxy
-    return {} unless @proxy
-
-    # Symbolize keys
-    proxy = if @proxy.is_a?(Hash)
-              Hash[@proxy.map {|k,v| [k.to_sym, v]}]
-            elsif @proxy.is_a?(String)
-              @proxy
-            else
-              raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{@proxy}''!"
-            end
-
-    return {:proxy => proxy}
-  end
-
-  private
-  def setup_ssl
-    return {} unless @ssl
-
-    if @cacert && @truststore
-      raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
-    end
-
-    ssl_options = {}
-
-    if @cacert
-      ssl_options[:ca_file] = @cacert
-    elsif @truststore
-      ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
-    end
-
-    ssl_options[:truststore] = @truststore if @truststore
-    if @keystore
-      ssl_options[:keystore] = @keystore
-      ssl_options[:keystore_password] = @keystore_password.value if @keystore_password
-    end
-    if @ssl_certificate_verification == false
-      @logger.warn [
-        "** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
-        "** WARNING ** You have enabled encryption but DISABLED certificate verification.",
-        "** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
-      ].join("\n")
-      ssl_options[:verify] = false
-    end
-    { ssl: ssl_options }
-  end
-
-  private
-  def setup_basic_auth
-    return {} unless @user && @password
-
-    {
-      :user => ::URI.escape(@user, "@:"),
-      :password => ::URI.escape(@password.value, "@:")
-    }
-  end
-
-  private
-  # in charge of submitting any actions in @retry_queue that need to be 
-  # retried
-  #
-  # This method is not called concurrently. It is only called by @retry_thread
-  # and once that thread is ended during the teardown process, a final call 
-  # to this method is done upon teardown in the main thread.
-  def retry_flush()
-    unless @retry_queue.empty?
-      buffer = @retry_queue.size.times.map do
-        next_action, next_doc, next_event = @retry_queue.pop
-        next_event['@metadata']['retry_count'] += 1
-
-        if next_event['@metadata']['retry_count'] > @max_retries
-          @logger.error "too many attempts at sending event. dropping: #{next_event}"
-          nil
-        else
-          [next_action, next_doc, next_event]
-        end
-      end.compact
-
-      submit(buffer) unless buffer.empty?
-    end
-
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.signal if @retry_queue.size < @retry_max_items
-    }
-  end
-
-  private
-  def retry_push(actions)
-    Array(actions).each{|action| @retry_queue << action}
-    @retry_flush_mutex.synchronize {
-      @retry_queue_needs_flushing.signal if @retry_queue.size >= @retry_max_items
-    }
-  end
-
-  @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
-
-  @@plugins.each do |plugin|
-    name = plugin.name.split('-')[-1]
-    require "logstash/outputs/elasticsearch/#{name}"
-  end
-
-end # class LogStash::Outputs::Elasticsearch