logstash-output-elasticsearch 10.2.2-java → 10.2.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cee8d8a64a98ebc0e90746c1e84323fbe842ae0afdbf072e530b50cbb4b27950
-  data.tar.gz: 2b24860b48ef18b5c28cfc9b62990ecc90bd202269b309b681181e0c18b9bae6
+  metadata.gz: 6be61b215a17faa3cec976576510d0f153be3a343162fa921e77398659b764a7
+  data.tar.gz: 35ab3d0b82ada32bd091091b2f1759681b976356837301c35bf0d14016cc6fa9
 SHA512:
-  metadata.gz: 07c5415be2991fdfed79ecbd27027272f9a59a72797b6a1efc0de021f9ae876e8949374a307da9663c3c36c6603ed8a8e4aedcf0009076354d0fe079572e2330
-  data.tar.gz: 47e5ca4b87727558c5c99a2e7134de3f7fec886b46f007f4af287e4ba91ee1e9fa71d578428da2149e42d2dce37ae0dd79f84684fb824f50004ed23b13345e1b
+  metadata.gz: 134a73bd275a3f1c50d453fc9d4ee3f7b17a54293099d0b0c539e8cb87f2fc66993f2ad237b62ea789580a18a0ddeb785bade259ca24305d0211cea18f477e52
+  data.tar.gz: 23f201b5a03917d066f558f34d3b14d7bb6218cc8ef090ec3d26766f4aaab318d430256463a61cc00828fc60a3f24de290411a0015dbd689dd93a9b8add4bf87

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 10.2.3
+  - Opened type removal logic for extension. This allows X-Pack Elasticsearch output to continue using types for special case `/_monitoring` bulk endpoint, enabling a fix for LogStash #11312. [#900](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/900)
+
 ## 10.2.2
-  - Fixed 8.x type removal compatibility issue [#892](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/892) 
+  - Fixed 8.x type removal compatibility issue [#892](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/892)
 
 ## 10.2.1
   - Fixed wording and corrected option in documentation [#881](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/881) [#883](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/883)
@@ -168,14 +171,14 @@
 
 ## 6.2.3
 - Fixed a bug introduced in 6.2.2 where passwords needing escapes were not actually sent to ES properly
-  encoded. 
+  encoded.
 
 ## 6.2.2
 - Fixed a bug that forced users to URL encode the `password` option.
   If you are currently manually escaping your passwords upgrading to this version
   will break authentication. You should unescape your password if you have implemented
   this workaround as it will otherwise be doubly encoded.
-  URL escaping is STILL required for passwords inline with URLs in the `hosts` option. 
+  URL escaping is STILL required for passwords inline with URLs in the `hosts` option.
 
 ## 6.2.1
 - When an HTTP error is encountered, log the response body instead of the request.
@@ -189,7 +192,7 @@
 
 ## 6.0.0
 - Proxies requiring auth now always work when a URL is specified
-- It is no longer possible to specify a proxy as a hash due to security reasons 
+- It is no longer possible to specify a proxy as a hash due to security reasons
 - Fix URL normalization logic to correctly apply all settings to sniffed hosts
 - Proxies requiring auth now always work when a URL is specified
 - Switch internals to new LogStash::Util::SafeURI type for more defensive approach to logging credentials

data/docs/index.asciidoc

@@ -64,6 +64,58 @@ LS will not force upgrade the template, if `logstash` template already exists. T
 `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
 the new template is installed.
 
+==== Writing to different indices: best practices
+
+[NOTE]
+================================================================================
+You cannot use dynamic variable substitution when `ilm_enabled` is `true` and 
+when using `ilm_rollover_pattern`.
+
+================================================================================
+
+If you're sending events to the same Elasticsearch cluster but you're targeting different indices you can:
+
+ * use different Elasticsearch outputs, each one with a different value for the `index` parameter
+ * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter
+
+Each Elasticsearch output is a new client connected to the cluster:
+
+ * it has to initialize the client and connect to Elasticsearch (restart time is longer if you have more clients)
+ * it has an associated connection pool
+
+In order to minimize the number of open connections to Elasticsearch, maximize the bulk size and reduce the number of "small" bulk requests (which could easily fill up the queue), it is usually more efficient to have a single Elasticsearch output.
+
+Example:
+[source,ruby]
+    output {
+      elasticsearch {
+        index => "%{[some_field][sub_field]}-%{+YYYY.MM.dd}"
+      }
+    }
+
+**What to do in case there is no field in the event containing the destination index prefix?**
+
+You can use the `mutate` filter and conditionals to add a `[@metadata]` field (see https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html#metadata) to set
+the destination index for each event. The `[@metadata]` fields will not be sent to Elasticsearch.
+
+Example:
+[source,ruby]
+    filter {
+      if [log_type] in [ "test", "staging" ] {
+        mutate { add_field => { "[@metadata][target_index]" => "test-%{+YYYY.MM}" } }
+      } else if [log_type] == "production" {
+        mutate { add_field => { "[@metadata][target_index]" => "prod-%{+YYYY.MM.dd}" } }
+      } else {
+        mutate { add_field => { "[@metadata][target_index]" => "unknown-%{+YYYY}" } }
+      }
+    }
+    output {
+      elasticsearch {
+        index => "%{[@metadata][target_index]}"
+      }
+    }
+
+
 ==== Retry Policy
 
 The retry policy has changed significantly in the 8.1.1 release.
@@ -423,6 +475,8 @@ NOTE: If both `index` and `ilm_rollover_alias` are specified, `ilm_rollover_alia
 
 NOTE: Updating the rollover alias will require the index template to be rewritten
 
+NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as `index` does.
+
 [id="plugins-{type}s-{plugin}-index"]
 ===== `index` 
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -62,9 +62,12 @@ module LogStash; module Outputs; class ElasticSearch;
       !!maximum_seen_major_version
     end
 
+    def use_event_type?(client)
+      client.maximum_seen_major_version < 8
+    end
+
     # Convert the event into a 3-tuple of action, params, and event
     def event_action_tuple(event)
-
       action = event.sprintf(@action)
 
       params = {
@@ -73,7 +76,7 @@ module LogStash; module Outputs; class ElasticSearch;
         routing_field_name => @routing ? event.sprintf(@routing) : nil
       }
 
-      params[:_type] = get_event_type(event) if client.maximum_seen_major_version < 8
+      params[:_type] = get_event_type(event) if use_event_type?(client)
 
       if @pipeline
         params[:pipeline] = event.sprintf(@pipeline)

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.2.2'
+  s.version         = '10.2.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 10.2.2
+  version: 10.2.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-11-15 00:00:00.000000000 Z
+date: 2019-11-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement