logstash-output-amazon_es 2.0.1-java → 6.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fe28932780893f008536b1c504907e747d1c5505
-  data.tar.gz: 3af463b9a92973d567b43e079aee1761ff8bc9be
+SHA256:
+  metadata.gz: 3c010485bbf51026b39f574b1bf1f532f81dde611519c9048e44f5b3559edaca
+  data.tar.gz: eb34d6b24ff01e2856012ce75dc2fd92f641f8e926c191f447461dcdf97114a9
 SHA512:
-  metadata.gz: fe91a7d8491a64030589e5767bdf23f7cc19098688d41626cc46aa7b56451ff7109eb2acc40a41587ff1ddb2dccce16e96e592012d911160b86967340e011bce
-  data.tar.gz: 9bd5f852a9e5e09770b3159b751f8e7ad66e3cc6b8b52030e0a082463b903af6559de106522ef568e3edbabb70309968187483024225f8ef010c72974cd6bf62
+  metadata.gz: 79f7ca9c8188d8a9ffa6a68e0e882d005fc8f47137734a1d741c2df0fad92b830ae6415cbde3b4cc0c91989ff7e5e2f8d8694914f086c5e3f4adaa0e520054e7
+  data.tar.gz: 83f261d5982333ea68bfd161f8eed25388b98f8e1f03f6512eacabb6b4782987f374c4342bb3715efbfad01f414ef518b7dc9cc0be438a96cfbea6d855fc5336

data/CONTRIBUTORS

@@ -0,0 +1,12 @@
+The following is a list of people who have contributed ideas, code, bug
+reports, or in general have helped logstash along its way.
+
+Contributors:
+* Frank Xu (qinyaox)
+* Qingyu Zhou (zhoqingy)
+* Ankit Malpani(malpani)
+
+Note: If you've sent us patches, bug reports, or otherwise contributed to
+Logstash, and you aren't on the list above and want to be, please let us know
+and we'll make sure you're here. Contributions from folks like you are what make
+open source awesome.

data/Gemfile

@@ -1,3 +1,11 @@
 source 'https://rubygems.org'
 
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/LICENSE

@@ -1,202 +1,13 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
+Copyright (c) 2012-2018 Elasticsearch <http://www.elastic.co>
 
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
 
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
+    http://www.apache.org/licenses/LICENSE-2.0
 
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -4,21 +4,6 @@ This is a plugin for [Logstash](https://github.com/elastic/logstash).
 
 It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
 
-# Setting Up
-
-## Installation
-One command installation
-`bin/plugin install logstash-output-amazon_es`
-
-While we are in the process of getting this plugin fully integrated within logstash to make installation simpler,
-if above does not work, or you would like to patch code here is a workaround to install this plugin within your logstash:
-
-1. Check out/clone this code from github
-2. Build plugin using - `gem build logstash-output-amazon_es.gemspec` ( this works with jruby and rubygem versions > 1.9)
-3. Install plugin using `<logstash-home>/bin/plugin install logstash-output-amazon_es-0.2.0-java.gem` (or the non java variant)
-4. For 2.3 support, please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-1.0-java.gem' 
-5. For 5.2 support, please use '<logstash-home>/bin/logstash-plugin install logstash-output-amazon_es-2.0.0-java.gem' 
-
 ## Configuration for Amazon Elasticsearch Output plugin
 
 To run the Logstash output Amazon Elasticsearch plugin simply add a configuration following the below documentation.
@@ -29,12 +14,12 @@ An example configuration:
 	    amazon_es {
 	        hosts => ["foo.us-east-1.es.amazonaws.com"]
 	        region => "us-east-1"
-			# aws_access_key_id, aws_secret_access_key optional if instance profile is configured
-			aws_access_key_id => 'ACCESS_KEY'
-			aws_secret_access_key => 'SECRET_KEY'
-			index => "production-logs-%{+YYYY.MM.dd}"
-		}
-	}
+	        # aws_access_key_id, aws_secret_access_key optional if instance profile is configured
+	        aws_access_key_id => 'ACCESS_KEY'
+	        aws_secret_access_key => 'SECRET_KEY'
+	        index => "production-logs-%{+YYYY.MM.dd}"
+            }
+        }
 
 * Required Parameters
 	* hosts (array of string) - Amazon Elasticsearch domain endpoint. eg ["foo.us-east-1.es.amazonaws.com"]
@@ -42,27 +27,15 @@ An example configuration:
 
 * Optional Parameters
 	* Credential parameters
-		* aws_access_key_id, :validate => :string - Optional AWS Access key
+	    * aws_access_key_id, :validate => :string - Optional AWS Access key
 		* aws_secret_access_key, :validate => :string - Optional AWS Secret Key  
-		   The credential resolution logic can be described as follows:
-		   - User passed aws_access_key_id and aws_secret_access_key in aes configuration
-		   - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
+		    The credential resolution logic can be described as follows:
+		    - User passed aws_access_key_id and aws_secret_access_key in aes configuration
+		    - Environment Variables - AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY
 		     (RECOMMENDED since they are recognized by all the AWS SDKs and CLI except for .NET),
 		     or AWS_ACCESS_KEY and AWS_SECRET_KEY (only recognized by Java SDK)
-		   - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
-		   - Instance profile credentials delivered through the Amazon EC2 metadata service
-	* Retry Parameters
-		* max_retries (number, default => 3) - Set max retry for each event
-		* retry_max_items (number, default => 5000) - Set retry queue size for events that failed to send
-		* retry_max_interval (number, default => 5) - Set max interval between bulk retries
-	* index (string - all lowercase, default => "logstash-%{+YYYY.MM.dd}") - Elasticsearch index to write events into
-	* flush_size (number , default => 500) - This setting controls how many events will be buffered before sending a batch of events in bulk API
-	* idle_flush_time (number, default => 1) - The amount of time in seconds since last flush before a flush is forced.
-		This setting helps ensure slow event rates don't get stuck in Logstash.
-		For example, if your `flush_size` is 100, and you have received 10 events,
-		and it has been more than `idle_flush_time` seconds since the last flush,
-		Logstash will flush those 10 events automatically.
-		This helps keep both fast and slow log streams moving along in near-real-time.
+		    - Credential profiles file at the default location (~/.aws/credentials) shared by all AWS SDKs and the AWS CLI
+		    - Instance profile credentials delivered through the Amazon EC2 metadata service
 	* template (path) - You can set the path to your own template here, if you so desire. If not set, the included template will be used.
 	* template_name (string, default => "logstash") - defines how the template is named inside Elasticsearch
 	* port (string, default 443) - Amazon Elasticsearch Service listens on port 443 - https (default) and 80 - http. Tweak this for custom proxy.
@@ -81,7 +54,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 
 ## Developing
 
-### 1. Plugin Development and Testing
+### 1. Plugin Developement and Testing
 
 #### Code
 - To get started, you'll need JRuby with the Bundler gem installed.
@@ -107,17 +80,6 @@ bundle install
 bundle exec rspec
 ```
 
-- Run integration tests
-
-Dependencies: [Docker](http://docker.com)
-
-Before the test suite is run, we will load and run an
-Elasticsearch instance within a docker container. This container
-will be cleaned up when suite has finished.
-
-```sh
-bundle exec rspec --tag integration
-```
 
 ### 2. Running your unpublished Plugin in Logstash
 
@@ -129,7 +91,12 @@ gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
 ```
 - Install plugin
 ```sh
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
 bin/plugin install --no-verify
+
 ```
 - Run Logstash with your plugin
 ```sh
@@ -138,6 +105,15 @@ bin/logstash -e 'filter {awesome {}}'
 At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
 
 #### 2.2 Run in an installed Logstash
+Before build your gemfile, please make sure use JRuby. Here is how you can know your local ruby version:
+```sh
+rvm list
+```
+Please make sure you current using jruby. Here is how you can change to jruby
+```sh
+rvm jruby
+```
+
 
 You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
 
@@ -147,7 +123,12 @@ gem build logstash-filter-awesome.gemspec
 ```
 - Install the plugin from the Logstash home
 ```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+# Logstash 2.3 and higher
+bin/logstash-plugin install --no-verify
+
+# Prior to Logstash 2.3
+bin/plugin install --no-verify
+
 ```
 - Start Logstash and proceed to test the plugin
 
@@ -155,19 +136,7 @@ bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
 
 All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
 
-Programming is not a required skill. Whatever you've seen about open source and maintainers or community members saying "send patches or die" - you will not see that here.
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
 
 It is more important to the community that you are able to contribute.
 
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elastic/logstash/blob/master/CONTRIBUTING.md) file.
-
-## Building the Logstash output plugin with Docker
-
-**Prerequisites:**
-
-- [Docker Engine](https://www.docker.com/products/docker-engine) >= 1.9.1
-- [Docker Compose](https://docs.docker.com/compose/) >= 1.6.0
-
-		docker-compose up
-
-This will result in a newly created binary inside the host-mounted volume `${PWD}` named `logstash-output-amazon_es-<VERSION>-java.gem`. Where `<VERSION>` is defined as value of `s.version` in [logstash-output-amazon_es.gemspec](logstash-output-amazon_es.gemspec) file.

data/lib/logstash/outputs/amazon_es.rb

@@ -2,117 +2,130 @@
 require "logstash/namespace"
 require "logstash/environment"
 require "logstash/outputs/base"
-require "logstash/outputs/amazon_es/http_client"
 require "logstash/json"
 require "concurrent"
 require "stud/buffer"
-require "socket"
-require "thread"
-require "uri"
-
-# This output plugin emits data to Amazon Elasticsearch with support for signing requests using AWS V4 Signatures
+require "socket" # for Socket.gethostname
+require "thread" # for safe queueing
+require "uri" # for escaping user input
+require "forwardable"
+
+# .Compatibility Note
+# [NOTE]
+# ================================================================================
+# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
+# called `http.content_type.required`. If this option is set to `true`, and you
+# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
+# plugin to version 6.2.5 or higher.
+# 
+# ================================================================================
+#
+# This plugin is the recommended method of storing logs in Elasticsearch.
+# If you plan on using the Kibana web interface, you'll want to use this output.
+#
+# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
+# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
+# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
+# to upgrade Logstash in lock-step. 
+# 
+# You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
+#
+# ==== Template management for Elasticsearch 5.x
+# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
+# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
+# behavior.
 #
+# ** Users installing ES 5.x and LS 5.x **
+# This change will not affect you and you will continue to use the ES defaults.
 #
-# The configuration and experience is similar to logstash-output-elasticsearch plugin and we have added Signature V4 support for the same
-# Some of the default configurations like connection timeouts have been tuned for optimal performance with Amazon Elasticsearch
+# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
+# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
+# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
+# the new template is installed.
 #
 # ==== Retry Policy
 #
-# This plugin uses the same retry policy as logstash-output-elasticsearch, It uses bulk API to optimize its
-# imports into Elasticsearch.. These requests may experience either partial or total failures.
-# Events are retried if they fail due to either a network error or the status codes
-# 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
+# The retry policy has changed significantly in the 2.2.0 release.
+# This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
+# either partial or total failures.
 #
-# The retry policy's logic can be described as follows:
+# The following errors are retried infinitely:
 #
-# - Block and retry all events in the bulk response that experience transient network exceptions until
-#   a successful submission is received by Elasticsearch.
-# - Retry the subset of sent events which resulted in ES errors of a retryable nature.
-# - Events which returned retryable error codes will be pushed onto a separate queue for
-#   retrying events. Events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries).
-#   The size of this queue is capped by the value set in :retry_max_items.
-# - Events from the retry queue are submitted again when the queue reaches its max size or when
-#   the max interval time is reached. The max interval time is configurable via :retry_max_interval.
-# - Events which are not retryable or have reached their max retry count are logged to stderr.
-
-class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
-  attr_reader :client
-
-  include Stud::Buffer
-  RETRYABLE_CODES = [409, 429, 503]
-  SUCCESS_CODES = [200, 201]
-
-  config_name "amazon_es"
-
-  # The index to write events to. This can be dynamic using the `%{foo}` syntax.
-  # The default value will partition your indices by day so you can more easily
-  # delete old data or only search specific date ranges.
-  # Indexes may not contain uppercase characters.
-  # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
-  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
-
-  # The index type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  #
-  # Deprecated in favor of `document_type` field.
-  config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
-
-  # The document type to write events to. Generally you should try to write only
-  # similar events to the same 'type'. String expansion `%{foo}` works here.
-  # Unless you set 'document_type', the event 'type' will be used if it exists
-  # otherwise the document type will be assigned the value of 'logs'
-  config :document_type, :validate => :string
-
-  # Starting in Logstash 1.3 (unless you set option `manage_template` to false)
-  # a default mapping template for Elasticsearch will be applied, if you do not
-  # already have one set to match the index pattern defined (default of
-  # `logstash-%{+YYYY.MM.dd}`), minus any variables.  For example, in this case
-  # the template will be applied to all indices starting with `logstash-*`
-  #
-  # If you have dynamic templating (e.g. creating indices based on field names)
-  # then you should set `manage_template` to false and use the REST API to upload
-  # your templates manually.
-  config :manage_template, :validate => :boolean, :default => true
-
-  # This configuration option defines how the template is named inside Elasticsearch.
-  # Note that if you have used the template management features and subsequently
-  # change this, you will need to prune the old template manually, e.g.
-  #
-  # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
-  #
-  # where `OldTemplateName` is whatever the former setting was.
-  config :template_name, :validate => :string, :default => "logstash"
+# - Network errors (inability to connect)
+# - 429 (Too many requests) and
+# - 503 (Service unavailable) errors
+#
+# NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
+# It is more performant for Elasticsearch to retry these exceptions than this plugin.
+#
+# ==== Batch Sizes ====
+# This plugin attempts to send batches of events as a single request. However, if
+# a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
+#
+# ==== DNS Caching
+#
+# This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
+# a global setting for the JVM.
+#
+# As an example, to set your DNS TTL to 1 second you would set
+# the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
+#
+# Keep in mind that a connection with keepalive enabled will
+# not reevaluate its DNS value while the keepalive is in effect.
+#
+# ==== HTTP Compression
+#
+# This plugin supports request and response compression. Response compression is enabled by default and 
+# for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for 
+# it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in 
+# Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
+#
+# For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 
+# setting in their Logstash config file.
+#
+class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
+  declare_threadsafe!
 
-  # You can set the path to your own template here, if you so desire.
-  # If not set, the included template will be used.
-  config :template, :validate => :path
+  require "logstash/outputs/amazon_es/http_client"
+  require "logstash/outputs/amazon_es/http_client_builder"
+  require "logstash/outputs/amazon_es/common_configs"
+  require "logstash/outputs/amazon_es/common"
 
-  # Overwrite the current template with whatever is configured
-  # in the `template` and `template_name` directives.
-  config :template_overwrite, :validate => :boolean, :default => false
+  # Protocol agnostic (i.e. non-http, non-java specific) configs go here
+  include(LogStash::Outputs::ElasticSearch::CommonConfigs)
 
-  # The document ID for the index. Useful for overwriting existing entries in
-  # Elasticsearch with the same ID.
-  config :document_id, :validate => :string
+  # Protocol agnostic methods
+  include(LogStash::Outputs::ElasticSearch::Common)
 
-  # A routing override to be applied to all processed events.
-  # This can be dynamic using the `%{foo}` syntax.
-  config :routing, :validate => :string
+  config_name "amazon_es"
 
+  # The Elasticsearch action to perform. Valid actions are:
+  #
+  # - index: indexes a document (an event from Logstash).
+  # - delete: deletes a document by id (An id is required for this action)
+  # - create: indexes a document, fails if a document by that id already exists in the index.
+  # - update: updates a document by id. Update has a special case where you can upsert -- update a
+  #   document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
+  #   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
+  # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
+  #   would use the foo field for the action
+  #
+  # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
+  config :action, :validate => :string, :default => "index"
 
-# Set the endpoint of your Amazon Elasticsearch domain. This will always be array of size 1
-#     ["foo.us-east-1.es.amazonaws.com"]
-  config :hosts, :validate => :array
+  # Username to authenticate to a secure Elasticsearch cluster
+  config :user, :validate => :string
+  # Password to authenticate to a secure Elasticsearch cluster
+  config :password, :validate => :password
 
   # You can set the remote port as part of the host, or explicitly here as well
-  config :port, :validate => :string, :default => 443
+  config :port, :validate => :number, :default => 443
 
   # Sets the protocol thats used to connect to elastisearch
   config :protocol, :validate => :string, :default => "https"
 
   #Signing specific details
   config :region, :validate => :string, :default => "us-east-1"
-
   # Credential resolution logic works as follows:
   #
   # - User passed aws_access_key_id and aws_secret_access_key in aes configuration
@@ -124,344 +137,126 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
   config :aws_access_key_id, :validate => :string
   config :aws_secret_access_key, :validate => :string
 
-
-  # This plugin uses the bulk index api for improved indexing performance.
-  # To make efficient bulk api calls, we will buffer a certain number of
-  # events before flushing that out to Elasticsearch. This setting
-  # controls how many events will be buffered before sending a batch
-  # of events.
-  config :flush_size, :validate => :number, :default => 500
-
-  # The amount of time since last flush before a flush is forced.
-  #
-  # This setting helps ensure slow event rates don't get stuck in Logstash.
-  # For example, if your `flush_size` is 100, and you have received 10 events,
-  # and it has been more than `idle_flush_time` seconds since the last flush,
-  # Logstash will flush those 10 events automatically.
-  #
-  # This helps keep both fast and slow log streams moving along in
-  # near-real-time.
-  config :idle_flush_time, :validate => :number, :default => 1
-
-  # The Elasticsearch action to perform. Valid actions are: `index`, `delete`.
-  #
-  # Use of this setting *REQUIRES* you also configure the `document_id` setting
-  # because `delete` actions all require a document id.
-  #
-  # What does each action do?
-  #
-  # - index: indexes a document (an event from Logstash).
-  # - delete: deletes a document by id
-  # - create: indexes a document, fails if a document by that id already exists in the index.
-  # - update: updates a document by id
-  # following action is not supported by HTTP protocol
-  #
-  # For more details on actions, check out the http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
-  config :action, :validate => %w(index delete create update), :default => "index"
-
-  # Username and password (only valid when protocol is HTTP; this setting works with HTTP or HTTPS auth)
-  config :user, :validate => :string
-  config :password, :validate => :password
-
-  # HTTP Path at which the Elasticsearch server lives. Use this if you must run ES behind a proxy that remaps
+  # HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
   # the root path for the Elasticsearch HTTP API lives.
-  config :path, :validate => :string, :default => "/"
-
-  # Set max retry for each event
-  config :max_retries, :validate => :number, :default => 3
-
-  # Set retry policy for events that failed to send
-  config :retry_max_items, :validate => :number, :default => 5000
-
-  # Set max interval between bulk retries
-  config :retry_max_interval, :validate => :number, :default => 5
-
-  # Set the address of a forward HTTP proxy. Must be used with the 'http' protocol
-  # Can be either a string, such as 'http://localhost:123' or a hash in the form
-  # {host: 'proxy.org' port: 80 scheme: 'http'}
-  # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
-  config :proxy
-
-  # Enable doc_as_upsert for update mode
-  # create a new document with source if document_id doesn't exists
-  config :doc_as_upsert, :validate => :boolean, :default => false
-
-  # Set upsert content for update mode
-  # create a new document with this parameter as json string if document_id doesn't exists
-  config :upsert, :validate => :string, :default => ""
-
-  public
-  def register
-    @hosts = Array(@hosts)
-    # retry-specific variables
-    @retry_flush_mutex = Mutex.new
-    @retry_teardown_requested = Concurrent::AtomicBoolean.new(false)
-    # needs flushing when interval
-    @retry_queue_needs_flushing = ConditionVariable.new
-    @retry_queue_not_full = ConditionVariable.new
-    @retry_queue = Queue.new
-    @submit_mutex = Mutex.new
-
-    client_settings = {}
-    common_options = {
-      :client_settings => client_settings
-    }
-
-    client_settings[:path] = "/#{@path}/".gsub(/\/+/, "/") # Normalize slashes
-    @logger.debug? && @logger.debug("Normalizing http path", :path => @path, :normalized => client_settings[:path])
-
-    if @hosts.nil? || @hosts.empty?
-      @logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
-      @hosts = ["localhost"]
-    end
-
-    client_settings.merge! setup_proxy()
-    common_options.merge! setup_basic_auth()
-
-    # Update API setup
-    update_options = {
-      :upsert => @upsert,
-      :doc_as_upsert => @doc_as_upsert
-    }
-    common_options.merge! update_options if @action == 'update'
-
-    @client = LogStash::Outputs::AES::HttpClient.new(
-      common_options.merge(:hosts => @hosts, :port => @port, :region => @region, :aws_access_key_id => @aws_access_key_id, :aws_secret_access_key => @aws_secret_access_key,:protocol => @protocol)
-    )
-
-    if @manage_template
-      begin
-        @logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
-        @client.template_install(@template_name, get_template, @template_overwrite)
-      rescue => e
-        @logger.error("Failed to install template: #{e.message}")
-      end
-    end
-
-    @logger.info("New Elasticsearch output", :hosts => @hosts, :port => @port)
-
-    @client_idx = 0
-
-    buffer_initialize(
-      :max_items => @flush_size,
-      :max_interval => @idle_flush_time,
-      :logger => @logger
-    )
-
-    @retry_timer_thread = Thread.new do
-      loop do
-        sleep(@retry_max_interval)
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-      end
-    end
-
-    @retry_thread = Thread.new do
-      while @retry_teardown_requested.false?
-        @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.wait(@retry_flush_mutex) }
-        retry_flush
-      end
-    end
-  end # def register
-
-  public
-  def get_template
-    if @template.nil?
-      @template = ::File.expand_path('amazon_es/elasticsearch-template.json', ::File.dirname(__FILE__))
-      if !File.exists?(@template)
-        raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
-      end
-    end
-    template_json = IO.read(@template).gsub(/\n/,'')
-    template = LogStash::Json.load(template_json)
-    @logger.info("Using mapping template", :template => template)
-    return template
-  end # def get_template
-
-  public
-  def receive(event)
-    return unless output?(event)
-
-    # block until we have not maxed out our
-    # retry queue. This is applying back-pressure
-    # to slow down the receive-rate
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
-    }
-
-    event.set('[@metadata][retry_count]', 0)
-
-    # Set the 'type' value for the index.
-    type = if @document_type
-             event.sprintf(@document_type)
-           elsif @index_type # deprecated
-             event.sprintf(@index_type)
-           else
-             event.get('type') || 'logs'
-           end
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      :_type => type,
-      :_routing => @routing ? event.sprintf(@routing) : nil
-    }
-
-    params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
-
-    buffer_receive([event.sprintf(@action), params, event])
-  end # def receive
-
-  public
-  # The submit method can be called from both the
-  # Stud::Buffer flush thread and from our own retry thread.
-  def submit(actions)
-    @submit_mutex.synchronize do
-      es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash] }
-
-      bulk_response = @client.bulk(es_actions)
-
-      if bulk_response["errors"] && bulk_response["items"]
-        actions_to_retry = []
-
-        bulk_response['items'].each_with_index do |item,idx|
-          action = es_actions[idx]
-          action_type, props = item.first # These are all hashes with one value, so we destructure them here
-
-          status = props['status']
-          error = props['error']
-
-          if RETRYABLE_CODES.include?(status)
-            @logger.warn "retrying failed action with response code: #{status}"
-            actions_to_retry << action
-          elsif not SUCCESS_CODES.include?(status)
-            @logger.warn "failed action", status: status, error: error, action: action
-          end
-        end
-
-        retry_push(actions_to_retry) unless actions_to_retry.empty?
-      end
-    end
-  end
-
-  # When there are exceptions raised upon submission, we raise an exception so that
-  # Stud::Buffer will retry to flush
-  public
-  def flush(actions, teardown = false)
-    begin
-      submit(actions)
-    rescue Manticore::SocketException => e
-      # If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
-      # and let the user sort it out from there
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
-          " but Elasticsearch appears to be unreachable or down!",
-        :client_config => @client.client_options,
-        :error_message => e.message
-      )
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-    rescue => e
-      # For all other errors print out full connection issues
-      @logger.error(
-        "Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
-            " but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
-          "the configuration provided?",
-        :client_config => @client.client_options,
-        :error_message => e.message,
-        :error_class => e.class.name,
-        :backtrace => e.backtrace
-      )
-
-      @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
-
-      raise e
-    end
-  end # def flush
-
-  public
-  def teardown
-
-    @retry_teardown_requested.make_true
-    # First, make sure retry_timer_thread is stopped
-    # to ensure we do not signal a retry based on
-    # the retry interval.
-    Thread.kill(@retry_timer_thread)
-    @retry_timer_thread.join
-    # Signal flushing in the case that #retry_flush is in
-    # the process of waiting for a signal.
-    @retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
-    # Now, #retry_flush is ensured to not be in a state of
-    # waiting and can be safely joined into the main thread
-    # for further final execution of an in-process remaining call.
-    @retry_thread.join
-
-    # execute any final actions along with a proceeding retry for any
-    # final actions that did not succeed.
-    buffer_flush(:final => true)
-    retry_flush
-  end
-
-  private
-  def setup_proxy
-    return {} unless @proxy
-
-    # Symbolize keys
-    proxy = if @proxy.is_a?(Hash)
-              Hash[@proxy.map {|k,v| [k.to_sym, v]}]
-            elsif @proxy.is_a?(String)
-              @proxy
-            else
-              raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{@proxy}''!"
-            end
-
-    return {:proxy => proxy}
-  end
-
-  private
-  def setup_basic_auth
-    return {} unless @user && @password
-
-    {
-      :user => ::URI.escape(@user, "@:"),
-      :password => ::URI.escape(@password.value, "@:")
-    }
-  end
-
-
-  private
-  # in charge of submitting any actions in @retry_queue that need to be
-  # retried
-  #
-  # This method is not called concurrently. It is only called by @retry_thread
-  # and once that thread is ended during the teardown process, a final call
-  # to this method is done upon teardown in the main thread.
-  def retry_flush()
-    unless @retry_queue.empty?
-      buffer = @retry_queue.size.times.map do
-        next_action, next_doc, next_event = @retry_queue.pop
-        next_event.set('[@metadata][retry_count]', next_event.get('[@metadata][retry_count]') + 1)
-
-        if next_event.get('[@metadata][retry_count]') > @max_retries
-          @logger.error "too many attempts at sending event. dropping: #{next_event}"
-          nil
-        else
-          [next_action, next_doc, next_event]
-        end
-      end.compact
-
-      submit(buffer) unless buffer.empty?
-    end
-
-    @retry_flush_mutex.synchronize {
-      @retry_queue_not_full.signal if @retry_queue.size < @retry_max_items
-    }
+  # Note that if you use paths as components of URLs in the 'hosts' field you may
+  # not also set this field. That will raise an error at startup
+  config :path, :validate => :string
+
+  # HTTP Path to perform the _bulk requests to
+  # this defaults to a concatenation of the path parameter and "_bulk"
+  config :bulk_path, :validate => :string
+
+  # Pass a set of key value pairs as the URL query string. This query string is added
+  # to every host listed in the 'hosts' configuration. If the 'hosts' list contains
+  # urls that already have query strings, the one specified here will be appended.
+  config :parameters, :validate => :hash
+
+  # Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
+  # is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+  # If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+  config :ssl, :validate => :boolean
+
+  # Option to validate the server's certificate. Disabling this severely compromises security.
+  # For more information on disabling certificate verification please read
+  # https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+  config :ssl_certificate_verification, :validate => :boolean, :default => true
+
+  # The .cer or .pem file to validate the server's certificate
+  config :cacert, :validate => :path
+
+  # The JKS truststore to validate the server's certificate.
+  # Use either `:truststore` or `:cacert`
+  config :truststore, :validate => :path
+
+  # Set the truststore password
+  config :truststore_password, :validate => :password
+
+  # The keystore used to present a certificate to the server.
+  # It can be either .jks or .p12
+  config :keystore, :validate => :path
+
+  # Set the keystore password
+  config :keystore_password, :validate => :password
+
+  # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
+  # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
+  # this with master nodes, you probably want to disable HTTP on them by setting
+  # `http.enabled` to false in their amazon_es.yml. You can either use the `sniffing` option or
+  # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
+  config :sniffing, :validate => :boolean, :default => false
+
+  # How long to wait, in seconds, between sniffing attempts
+  config :sniffing_delay, :validate => :number, :default => 5
+
+  # HTTP Path to be used for the sniffing requests
+  # the default value is computed by concatenating the path value and "_nodes/http"
+  # if sniffing_path is set it will be used as an absolute path
+  # do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
+  config :sniffing_path, :validate => :string
+
+  # Set the address of a forward HTTP proxy.
+  # This used to accept hashes as arguments but now only accepts
+  # arguments of the URI type to prevent leaking credentials.
+  config :proxy, :validate => :uri
+
+  # Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
+  # a timeout occurs, the request will be retried.
+  config :timeout, :validate => :number, :default => 60
+
+  # Set the Elasticsearch errors in the whitelist that you don't want to log.
+  # A useful example is when you want to skip all 409 errors
+  # which are `document_already_exists_exception`.
+  config :failure_type_logging_whitelist, :validate => :array, :default => []
+
+  # While the output tries to reuse connections efficiently we have a maximum.
+  # This sets the maximum number of open connections the output will create.
+  # Setting this too low may mean frequently closing / opening connections
+  # which is bad.
+  config :pool_max, :validate => :number, :default => 1000
+
+  # While the output tries to reuse connections efficiently we have a maximum per endpoint.
+  # This sets the maximum number of open connections per endpoint the output will create.
+  # Setting this too low may mean frequently closing / opening connections
+  # which is bad.
+  config :pool_max_per_route, :validate => :number, :default => 100
+
+  # HTTP Path where a HEAD request is sent when a backend is marked down
+  # the request is sent in the background to see if it has come back again
+  # before it is once again eligible to service requests.
+  # If you have custom firewall rules you may need to change this
+  config :healthcheck_path, :validate => :string
+
+  # How frequently, in seconds, to wait between resurrection attempts.
+  # Resurrection is the process by which backend endpoints marked 'down' are checked
+  # to see if they have come back to life
+  config :resurrect_delay, :validate => :number, :default => 5
+
+  # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+  # You may want to set this lower, if you get connection errors regularly
+  # Quoting the Apache commons docs (this client is based Apache Commmons):
+  # 'Defines period of inactivity in milliseconds after which persistent connections must
+  # be re-validated prior to being leased to the consumer. Non-positive value passed to
+  # this method disables connection validation. This check helps detect connections that
+  # have become stale (half-closed) while kept inactive in the pool.'
+  # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+  config :validate_after_inactivity, :validate => :number, :default => 10000
+
+  # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
+  config :http_compression, :validate => :boolean, :default => false
+
+  # Custom Headers to send on each request to amazon_es nodes
+  config :custom_headers, :validate => :hash, :default => {}
+
+  def build_client
+    params["metric"] = metric
+    @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end
 
-  private
-  def retry_push(actions)
-    Array(actions).each{|action| @retry_queue << action}
-    @retry_flush_mutex.synchronize {
-      @retry_queue_needs_flushing.signal if @retry_queue.size >= @retry_max_items
-    }
+  def close
+    @stopping.make_true
+    stop_template_installer
+    @client.close if @client
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-amazon_es-/ }
@@ -471,4 +266,4 @@ class LogStash::Outputs::AmazonES < LogStash::Outputs::Base
     require "logstash/outputs/amazon_es/#{name}"
   end
 
-end
+end # class LogStash::Outputs::Elasticsearch