logstash-output-elasticsearch 7.3.1-java → 7.3.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 84e98668e085b0b9f4514e8bf574f502f5e3084e
-  data.tar.gz: ce5720c504d1c225cf4deade46096f0eba7cff76
+  metadata.gz: 42a74efb39e22b7169dc0ff278c364b1b6b8f282
+  data.tar.gz: 703b6a84b574bd8db99dd8dabb1fa16c77b34eb3
 SHA512:
-  metadata.gz: 17fed98200315e4c6182a91cda4ff0762a6dc11af70048cb037ed87ea48e62a1835b35e0ce170800e76b9174c4a0382024c41e22ca632a97cffedc1378bc90e9
-  data.tar.gz: 8433b4438c50c0d7ae0d52404f706917ae5ce581423a32f8abd32425340158798374e1c6ebd8c703fb36a9713a2131be07a67098a5e7f5bc75c5f11b8fa68c46
+  metadata.gz: f0bc0d68d637ee45d5b2a4243f03f72ce367dc34c81f4366b16c46ee9da3034d7d42799d90b518663ee894eb5bf59110a9530d8a35ab259bab047335ac3200f4
+  data.tar.gz: 5d6d9edc56c353d3ea1350a54ff1142d0af0242c587d54a6a26b646b9fd54bfef939f8925d346e0e25e0d78c0ab487df6419c899c6b7e2ed410e21233099c27e

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.3.2
+ - Fix error where a 429 would cause this output to crash
+ - Wait for all inflight requests to complete before stopping
+
 ## 7.3.1
  - Fix the backwards compatibility layer used for detecting DLQ capabilities in logstash core
 

data/Gemfile

@@ -1,4 +1,10 @@
 source 'https://rubygems.org'
 
-
 gemspec
+
+logstash_path = "../../logstash"
+
+if Dir.exist?(logstash_path) && ENV["LOGSTASH_SOURCE"] == 1
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,679 @@
+:plugin: elasticsearch
+:type: output
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Elasticsearch
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+.Compatibility Note
+[NOTE]
+================================================================================
+Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
+called `http.content_type.required`. If this option is set to `true`, and you
+are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
+plugin to version 6.2.5 or higher.
+
+================================================================================
+
+This plugin is the recommended method of storing logs in Elasticsearch.
+If you plan on using the Kibana web interface, you'll want to use this output.
+
+This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
+We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
+yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
+to upgrade Logstash in lock-step. 
+
+You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
+
+==== Template management for Elasticsearch 5.x
+Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
+Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
+behavior.
+
+** Users installing ES 5.x and LS 5.x **
+This change will not affect you and you will continue to use the ES defaults.
+
+** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
+LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
+`.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
+the new template is installed.
+
+==== Retry Policy
+
+The retry policy has changed significantly in the 2.2.0 release.
+This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
+either partial or total failures.
+
+The following errors are retried infinitely:
+
+- Network errors (inability to connect)
+- 429 (Too many requests) and
+- 503 (Service unavailable) errors
+
+NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
+It is more performant for Elasticsearch to retry these exceptions than this plugin.
+
+==== Batch Sizes ====
+This plugin attempts to send batches of events as a single request. However, if
+a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
+
+==== DNS Caching
+
+This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
+a global setting for the JVM.
+
+As an example, to set your DNS TTL to 1 second you would set
+the `LS_JAVA_OPTS` environment variable to `-Dnetworkaddress.cache.ttl=1`.
+
+Keep in mind that a connection with keepalive enabled will
+not reevaluate its DNS value while the keepalive is in effect.
+
+==== HTTP Compression
+
+This plugin supports request and response compression. Response compression is enabled by default and 
+for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for 
+it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in 
+Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
+
+For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 
+setting in their Logstash config file.
+
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Elasticsearch Output Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-action>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-bulk_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-cacert>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
+| <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-http_compression>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-index>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-keystore>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-keystore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-manage_template>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-parameters>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-parent>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-pipeline>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-pool_max>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-pool_max_per_route>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-proxy>> |<<uri,uri>>|No
+| <<plugins-{type}s-{plugin}-resurrect_delay>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_initial_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_max_interval>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-retry_on_conflict>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-routing>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-script>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-script_lang>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-script_type>> |<<string,string>>, one of `["inline", "indexed", "file"]`|No
+| <<plugins-{type}s-{plugin}-script_var_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-scripted_upsert>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-sniffing>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-sniffing_delay>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-sniffing_path>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-template>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-template_name>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-template_overwrite>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-timeout>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-truststore>> |a valid filesystem path|No
+| <<plugins-{type}s-{plugin}-truststore_password>> |<<password,password>>|No
+| <<plugins-{type}s-{plugin}-upsert>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-user>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-validate_after_inactivity>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-version>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-version_type>> |<<string,string>>, one of `["internal", "external", "external_gt", "external_gte", "force"]`|No
+|=======================================================================
+
+Also see <<plugins-{type}s-common-options>> for a list of options supported by all
+output plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-action"]
+===== `action` 
+
+  * Value type is <<string,string>>
+  * Default value is `"index"`
+
+Protocol agnostic (i.e. non-http, non-java specific) configs go here
+Protocol agnostic methods
+The Elasticsearch action to perform. Valid actions are:
+
+- index: indexes a document (an event from Logstash).
+- delete: deletes a document by id (An id is required for this action)
+- create: indexes a document, fails if a document by that id already exists in the index.
+- update: updates a document by id. Update has a special case where you can upsert -- update a
+  document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
+  in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
+- A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
+  would use the foo field for the action
+
+For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
+
+[id="plugins-{type}s-{plugin}-bulk_path"]
+===== `bulk_path` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+HTTP Path to perform the _bulk requests to
+this defaults to a concatenation of the path parameter and "_bulk"
+
+[id="plugins-{type}s-{plugin}-cacert"]
+===== `cacert` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The .cer or .pem file to validate the server's certificate
+
+[id="plugins-{type}s-{plugin}-doc_as_upsert"]
+===== `doc_as_upsert` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Enable `doc_as_upsert` for update mode.
+Create a new document with source if `document_id` doesn't exist in Elasticsearch
+
+[id="plugins-{type}s-{plugin}-document_id"]
+===== `document_id` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The document ID for the index. Useful for overwriting existing entries in
+Elasticsearch with the same ID.
+
+[id="plugins-{type}s-{plugin}-document_type"]
+===== `document_type` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The document type to write events to. Generally you should try to write only
+similar events to the same 'type'. String expansion `%{foo}` works here.
+Unless you set 'document_type', the event 'type' will be used if it exists
+otherwise the document type will be assigned the value of 'logs'
+
+[id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
+===== `failure_type_logging_whitelist` 
+
+  * Value type is <<array,array>>
+  * Default value is `[]`
+
+Set the Elasticsearch errors in the whitelist that you don't want to log.
+A useful example is when you want to skip all 409 errors
+which are `document_already_exists_exception`.
+
+[id="plugins-{type}s-{plugin}-flush_size"]
+===== `flush_size`  (DEPRECATED)
+
+  * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
+  * Value type is <<number,number>>
+  * There is no default value for this setting.
+
+
+
+[id="plugins-{type}s-{plugin}-healthcheck_path"]
+===== `healthcheck_path` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+HTTP Path where a HEAD request is sent when a backend is marked down
+the request is sent in the background to see if it has come back again
+before it is once again eligible to service requests.
+If you have custom firewall rules you may need to change this
+
+[id="plugins-{type}s-{plugin}-hosts"]
+===== `hosts` 
+
+  * Value type is <<uri,uri>>
+  * Default value is `[//127.0.0.1]`
+
+Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
+Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+    `"127.0.0.1"`
+    `["127.0.0.1:9200","127.0.0.2:9200"]`
+    `["http://127.0.0.1"]`
+    `["https://127.0.0.1:9200"]`
+    `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
+to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
+
+Any special characters present in the URLs here MUST be URL escaped! This means `#` should be put in as `%23` for instance.
+
+[id="plugins-{type}s-{plugin}-http_compression"]
+===== `http_compression` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
+
+[id="plugins-{type}s-{plugin}-idle_flush_time"]
+===== `idle_flush_time`  (DEPRECATED)
+
+  * DEPRECATED WARNING: This configuration item is deprecated and may not be available in future versions.
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+
+
+[id="plugins-{type}s-{plugin}-index"]
+===== `index` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash-%{+YYYY.MM.dd}"`
+
+The index to write events to. This can be dynamic using the `%{foo}` syntax.
+The default value will partition your indices by day so you can more easily
+delete old data or only search specific date ranges.
+Indexes may not contain uppercase characters.
+For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+LS uses Joda to format the index pattern from event timestamp.
+Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
+
+[id="plugins-{type}s-{plugin}-keystore"]
+===== `keystore` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The keystore used to present a certificate to the server.
+It can be either .jks or .p12
+
+[id="plugins-{type}s-{plugin}-keystore_password"]
+===== `keystore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the keystore password
+
+[id="plugins-{type}s-{plugin}-manage_template"]
+===== `manage_template` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+From Logstash 1.3 onwards, a template is applied to Elasticsearch during
+Logstash's startup if one with the name `template_name` does not already exist.
+By default, the contents of this template is the default template for
+`logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
+`logstash-*`.  Should you require support for other index names, or would like
+to change the mappings in the template in general, a custom template can be
+specified by setting `template` to the path of a template file.
+
+Setting `manage_template` to false disables this feature.  If you require more
+control over template creation, (e.g. creating indices dynamically based on
+field names) you should set `manage_template` to false and use the REST
+API to apply your templates manually.
+
+[id="plugins-{type}s-{plugin}-parameters"]
+===== `parameters` 
+
+  * Value type is <<hash,hash>>
+  * There is no default value for this setting.
+
+Pass a set of key value pairs as the URL query string. This query string is added
+to every host listed in the 'hosts' configuration. If the 'hosts' list contains
+urls that already have query strings, the one specified here will be appended.
+
+[id="plugins-{type}s-{plugin}-parent"]
+===== `parent` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+For child documents, ID of the associated parent.
+This can be dynamic using the `%{foo}` syntax.
+
+[id="plugins-{type}s-{plugin}-password"]
+===== `password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Password to authenticate to a secure Elasticsearch cluster
+
+[id="plugins-{type}s-{plugin}-path"]
+===== `path` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+HTTP Path at which the Elasticsearch server lives. Use this if you must run Elasticsearch behind a proxy that remaps
+the root path for the Elasticsearch HTTP API lives.
+Note that if you use paths as components of URLs in the 'hosts' field you may
+not also set this field. That will raise an error at startup
+
+[id="plugins-{type}s-{plugin}-pipeline"]
+===== `pipeline` 
+
+  * Value type is <<string,string>>
+  * Default value is `nil`
+
+Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
+here like `pipeline => "%{INGEST_PIPELINE}"`
+
+[id="plugins-{type}s-{plugin}-pool_max"]
+===== `pool_max` 
+
+  * Value type is <<number,number>>
+  * Default value is `1000`
+
+While the output tries to reuse connections efficiently we have a maximum.
+This sets the maximum number of open connections the output will create.
+Setting this too low may mean frequently closing / opening connections
+which is bad.
+
+[id="plugins-{type}s-{plugin}-pool_max_per_route"]
+===== `pool_max_per_route` 
+
+  * Value type is <<number,number>>
+  * Default value is `100`
+
+While the output tries to reuse connections efficiently we have a maximum per endpoint.
+This sets the maximum number of open connections per endpoint the output will create.
+Setting this too low may mean frequently closing / opening connections
+which is bad.
+
+[id="plugins-{type}s-{plugin}-proxy"]
+===== `proxy` 
+
+  * Value type is <<uri,uri>>
+  * There is no default value for this setting.
+
+Set the address of a forward HTTP proxy.
+This used to accept hashes as arguments but now only accepts
+arguments of the URI type to prevent leaking credentials.
+
+[id="plugins-{type}s-{plugin}-resurrect_delay"]
+===== `resurrect_delay` 
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+How frequently, in seconds, to wait between resurrection attempts.
+Resurrection is the process by which backend endpoints marked 'down' are checked
+to see if they have come back to life
+
+[id="plugins-{type}s-{plugin}-retry_initial_interval"]
+===== `retry_initial_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `2`
+
+Set initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+
+[id="plugins-{type}s-{plugin}-retry_max_interval"]
+===== `retry_max_interval` 
+
+  * Value type is <<number,number>>
+  * Default value is `64`
+
+Set max interval in seconds between bulk retries.
+
+[id="plugins-{type}s-{plugin}-retry_on_conflict"]
+===== `retry_on_conflict` 
+
+  * Value type is <<number,number>>
+  * Default value is `1`
+
+The number of times Elasticsearch should internally retry an update/upserted document
+See the https://www.elastic.co/guide/en/elasticsearch/guide/current/partial-updates.html[partial updates]
+for more info
+
+[id="plugins-{type}s-{plugin}-routing"]
+===== `routing` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+A routing override to be applied to all processed events.
+This can be dynamic using the `%{foo}` syntax.
+
+[id="plugins-{type}s-{plugin}-script"]
+===== `script` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Set script name for scripted update mode
+
+[id="plugins-{type}s-{plugin}-script_lang"]
+===== `script_lang` 
+
+  * Value type is <<string,string>>
+  * Default value is `"painless"`
+
+Set the language of the used script. If not set, this defaults to painless in ES 5.0
+
+[id="plugins-{type}s-{plugin}-script_type"]
+===== `script_type` 
+
+  * Value can be any of: `inline`, `indexed`, `file`
+  * Default value is `["inline"]`
+
+Define the type of script referenced by "script" variable
+ inline : "script" contains inline script
+ indexed : "script" contains the name of script directly indexed in elasticsearch
+ file    : "script" contains the name of script stored in elasticseach's config directory
+
+[id="plugins-{type}s-{plugin}-script_var_name"]
+===== `script_var_name` 
+
+  * Value type is <<string,string>>
+  * Default value is `"event"`
+
+Set variable name passed to script (scripted update)
+
+[id="plugins-{type}s-{plugin}-scripted_upsert"]
+===== `scripted_upsert` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+if enabled, script is in charge of creating non-existent document (scripted update)
+
+[id="plugins-{type}s-{plugin}-sniffing"]
+===== `sniffing` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.
+Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
+this with master nodes, you probably want to disable HTTP on them by setting
+`http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
+manually enter multiple Elasticsearch hosts using the `hosts` parameter.
+
+[id="plugins-{type}s-{plugin}-sniffing_delay"]
+===== `sniffing_delay` 
+
+  * Value type is <<number,number>>
+  * Default value is `5`
+
+How long to wait, in seconds, between sniffing attempts
+
+[id="plugins-{type}s-{plugin}-sniffing_path"]
+===== `sniffing_path` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+HTTP Path to be used for the sniffing requests
+the default value is computed by concatenating the path value and "_nodes/http"
+if sniffing_path is set it will be used as an absolute path
+do not use full URL here, only paths, e.g. "/sniff/_nodes/http"
+
+[id="plugins-{type}s-{plugin}-ssl"]
+===== `ssl` 
+
+  * Value type is <<boolean,boolean>>
+  * There is no default value for this setting.
+
+Enable SSL/TLS secured communication to Elasticsearch cluster. Leaving this unspecified will use whatever scheme
+is specified in the URLs listed in 'hosts'. If no explicit protocol is specified plain HTTP will be used.
+If SSL is explicitly disabled here the plugin will refuse to start if an HTTPS URL is given in 'hosts'
+
+[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
+===== `ssl_certificate_verification` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `true`
+
+Option to validate the server's certificate. Disabling this severely compromises security.
+For more information on disabling certificate verification please read
+https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf
+
+[id="plugins-{type}s-{plugin}-template"]
+===== `template` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+You can set the path to your own template here, if you so desire.
+If not set, the included template will be used.
+
+[id="plugins-{type}s-{plugin}-template_name"]
+===== `template_name` 
+
+  * Value type is <<string,string>>
+  * Default value is `"logstash"`
+
+This configuration option defines how the template is named inside Elasticsearch.
+Note that if you have used the template management features and subsequently
+change this, you will need to prune the old template manually, e.g.
+
+`curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
+
+where `OldTemplateName` is whatever the former setting was.
+
+[id="plugins-{type}s-{plugin}-template_overwrite"]
+===== `template_overwrite` 
+
+  * Value type is <<boolean,boolean>>
+  * Default value is `false`
+
+The template_overwrite option will always overwrite the indicated template
+in Elasticsearch with either the one indicated by template or the included one.
+This option is set to false by default. If you always want to stay up to date
+with the template provided by Logstash, this option could be very useful to you.
+Likewise, if you have your own template file managed by puppet, for example, and
+you wanted to be able to update it regularly, this option could help there as well.
+
+Please note that if you are using your own customized version of the Logstash
+template (logstash), setting this to true will make Logstash to overwrite
+the "logstash" template (i.e. removing all customized settings)
+
+[id="plugins-{type}s-{plugin}-timeout"]
+===== `timeout` 
+
+  * Value type is <<number,number>>
+  * Default value is `60`
+
+Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
+a timeout occurs, the request will be retried.
+
+[id="plugins-{type}s-{plugin}-truststore"]
+===== `truststore` 
+
+  * Value type is <<path,path>>
+  * There is no default value for this setting.
+
+The JKS truststore to validate the server's certificate.
+Use either `:truststore` or `:cacert`
+
+[id="plugins-{type}s-{plugin}-truststore_password"]
+===== `truststore_password` 
+
+  * Value type is <<password,password>>
+  * There is no default value for this setting.
+
+Set the truststore password
+
+[id="plugins-{type}s-{plugin}-upsert"]
+===== `upsert` 
+
+  * Value type is <<string,string>>
+  * Default value is `""`
+
+Set upsert content for update mode.s
+Create a new document with this parameter as json string if `document_id` doesn't exists
+
+[id="plugins-{type}s-{plugin}-user"]
+===== `user` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+Username to authenticate to a secure Elasticsearch cluster
+
+[id="plugins-{type}s-{plugin}-validate_after_inactivity"]
+===== `validate_after_inactivity` 
+
+  * Value type is <<number,number>>
+  * Default value is `10000`
+
+How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
+You may want to set this lower, if you get connection errors regularly
+Quoting the Apache commons docs (this client is based Apache Commmons):
+'Defines period of inactivity in milliseconds after which persistent connections must
+be re-validated prior to being leased to the consumer. Non-positive value passed to
+this method disables connection validation. This check helps detect connections that
+have become stale (half-closed) while kept inactive in the pool.'
+See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
+
+[id="plugins-{type}s-{plugin}-version"]
+===== `version` 
+
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The version to use for indexing. Use sprintf syntax like `%{my_version}` to use a field value here.
+See https://www.elastic.co/blog/elasticsearch-versioning-support.
+
+[id="plugins-{type}s-{plugin}-version_type"]
+===== `version_type` 
+
+  * Value can be any of: `internal`, `external`, `external_gt`, `external_gte`, `force`
+  * There is no default value for this setting.
+
+The version_type to use for indexing.
+See https://www.elastic.co/blog/elasticsearch-versioning-support.
+See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-index_.html#_version_types
+
+
+
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -4,6 +4,11 @@ module LogStash; module Outputs; class ElasticSearch;
   module Common
     attr_reader :client, :hosts
 
+    # These are codes for temporary recoverable conditions
+    # 429 just means that ES has too much traffic ATM
+    # 503 means it , or a proxy is temporarily unavailable
+    RETRYABLE_CODES = [429, 503]
+
     DLQ_CODES = [400, 404]
     SUCCESS_CODES = [200, 201]
     CONFLICT_CODE = 409
@@ -252,16 +257,16 @@ module LogStash; module Outputs; class ElasticSearch;
 
           # We treat 429s as a special case because these really aren't errors, but
           # rather just ES telling us to back off a bit, which we do.
-          # The other retryable codes are 502 and 503, which are true errors
+          # The other retryable code is 503, which are true errors
           # Even though we retry the user should be made aware of these
           if e.response_code == 429
-            @logger.debug(message, log_hash)
+            logger.debug(message, log_hash)
           else
-            @logger.error(message, log_hash)
+            logger.error(message, log_hash)
           end
 
           sleep_interval = sleep_for_interval(sleep_interval)
-          retry unless @stopping.true?
+          retry
         else
           log_hash = {:code => e.response_code, 
                       :response_body => e.response_body}

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.3.1'
+  s.version         = '7.3.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -202,8 +202,41 @@ describe "outputs/elasticsearch" do
 
     end
 
-  end
+    context "429 errors" do
+      let(:event) { ::LogStash::Event.new("foo" => "bar") }
+      let(:error) do 
+        ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
+          429, double("url").as_null_object, double("request body"), double("response body")
+        )
+      end
+      let(:logger) { double("logger").as_null_object }
+
+      before(:each) do
+        i = 0
+        bulk_param =  [["index", anything, event.to_hash]]
+
+        allow(eso).to receive(:logger).and_return(logger)
 
+        # Fail the first time bulk is called, succeed the next time
+        allow(eso.client).to receive(:bulk).with(bulk_param) do
+          i += 1
+          if i == 1
+            raise error
+          end
+        end
+        eso.multi_receive([event])
+      end
+
+      it "should retry the 429 till it goes away" do
+        expect(eso.client).to have_received(:bulk).twice
+      end
+
+      it "should log a debug message" do
+        expect(eso.logger).to have_received(:debug).with(/Encountered a retryable error/i, anything)
+      end
+    end
+  end
+  
   context "with timeout set" do
     let(:listener) { Flores::Random.tcp_listener }
     let(:port) { listener[2] }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.3.1
+  version: 7.3.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-04-27 00:00:00.000000000 Z
+date: 2017-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -208,6 +208,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/outputs/elasticsearch.rb
 - lib/logstash/outputs/elasticsearch/common.rb
 - lib/logstash/outputs/elasticsearch/common_configs.rb