logstash-output-elasticsearch 10.5.0-java → 10.7.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 71b5bb8d2a918b37ee594c9bcbe11ec872dbfb2ee30094a6b2ede8ea4550fb46
-  data.tar.gz: fc4c5cba1d3cc99cfbdf17f3658723d97889e7d0c824dc430986f5f64ea652e0
+  metadata.gz: 311303e9bfc0369329af5ec4da290d92e428bddee2e022fb1f22967fb1f1d6d9
+  data.tar.gz: c88b0816b49bd5f5ab232156340f946b674e70a62b86f8a1c4c25ed985840c6e
 SHA512:
-  metadata.gz: 036130a59c31c2634db172b37a918fdcb29ca93d608f1cfb80a54a5f0e3620b0766acec221f0c2ef0c65e7f764085ac67f16d5263fea9b72d5b5256f8ac7be98
-  data.tar.gz: 0b27c6975420f0390d945bc7bdabd9fd5afabe12d8a5a20a4ccb6cc2d2ea5da2946942c76b8fc67de5bee97063b5771770a74e89a7f40f846df1979c798e9499
+  metadata.gz: 7d9910d4f28a864ac45450cd2be179268d8b83c91f8dab57328e6c772b06a39d7d8aa3fa7c34cddcd45a185bc994da2f2b46ee3b29bf2dc219a3189b6e995b30
+  data.tar.gz: de41059508b3f06466446d3a183d4959cdaaacffb00492d917acc57824b4864a694c5834c8e210d8d277ae186c146d6bf3c6b9ab0c0cd1ab73bc7f95a6f4e871

data/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 10.7.0
+ - Changed: don't set the pipeline parameter if the value resolves to an empty string [#962](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/962)
+
+## 10.6.2
+ - [DOC] Added clarifying info on http compression settings and behaviors [#943](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/943)
+ - [DOC] Fixed entry for ilm_policy default value[#956](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/956)
+
+## 10.6.1
+ - Fixed an issue introduced in 10.6.0 that broke Logstash Core's monitoring feature when this plugin is run in Logstash 7.7-7.8. [#953](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/953)
+
+## 10.6.0
+ - Added `ecs_compatiblity` mode, for managing ECS-compatable templates [#952](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/952)
+
+## 10.5.1
+  - [DOC] Removed outdated compatibility notices, reworked cloud notice, and fixed formatting for `hosts` examples [#938](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/938)
+
 ## 10.5.0
   - Added api_key support [#934](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934)
 

data/docs/index.asciidoc

@@ -21,6 +21,21 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
+If you plan to use the Kibana web interface to analyze data transformed by
+Logstash, use the Elasticsearch output plugin to get your data into
+Elasticsearch. 
+
+This output only speaks the HTTP protocol as it is the preferred protocol for
+interacting with Elasticsearch. In previous versions it was possible to
+communicate with Elasticsearch through the transport protocol, which is now
+reserved for internal cluster communication between nodes
+https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-transport.html[communication between nodes].
+Using the https://www.elastic.co/guide/en/elasticsearch/reference/current/java-clients.html[transport protocol] 
+to communicate with the cluster has been deprecated in Elasticsearch 7.0.0 and
+will be removed in 8.0.0
+
+You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
+
 .Compatibility Note
 [NOTE]
 ================================================================================
@@ -36,47 +51,26 @@ ensure your template uses the `_doc` document-type before
 connecting to Elasticsearch 7.x.
 ================================================================================
 
-.Compatibility Note
-[NOTE]
-================================================================================
-Starting with Elasticsearch 5.3, there's an {ref}/modules-http.html[HTTP setting]
-called `http.content_type.required`. If this option is set to `true`, and you
-are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
-plugin to version 6.2.5 or higher.
-
-================================================================================
+===== Hosted {es} Service on Elastic Cloud
 
-If you plan to use the Kibana web
-interface, use the Elasticsearch output plugin to get your log data into
-Elasticsearch. 
-
-TIP: You can run Elasticsearch on your own hardware, or use our
+You can run Elasticsearch on your own hardware, or use our
 https://www.elastic.co/cloud/elasticsearch-service[hosted {es} Service] on
-Elastic Cloud. The Elasticsearch Service is available on both AWS and GCP.
-{ess-trial}[Try the {es} Service for free].
-
-This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
-We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
-yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-to upgrade Logstash in lock-step. 
+Elastic Cloud. The Elasticsearch Service is available on AWS, Google Cloud
+Platform, and Microsoft Azure. {ess-trial}[Try the {es} Service for free].
 
-You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
+==== Compatibility with the Elastic Common Schema (ECS)
 
-==== Template management for Elasticsearch 5.x
+This plugin will persist events to Elasticsearch in the shape produced by
+your pipeline, and _cannot_ be used to re-shape the event structure into a
+shape that complies with ECS. To produce events that fully comply with ECS,
+you will need to populate ECS-defined fields throughout your pipeline
+definition.
 
-Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
-Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
-behavior.
-
-**Users installing ES 5.x and LS 5.x**
-
-This change will not affect you and you will continue to use the ES defaults.
-
-**Users upgrading from LS 2.x to LS 5.x with ES 5.x**
-
-LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
-`.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
-the new template is installed.
+However, the Elasticsearch Index Templates it manages can be configured to
+be ECS-compatible by setting <<plugins-{type}s-{plugin}-ecs_compatibility>>.
+By having an ECS-compatible template in place, we can ensure that Elasticsearch
+is prepared to create and index fields in a way that is compatible with ECS,
+and will correctly reject events with fields that conflict and cannot be coerced.
 
 ==== Writing to different indices: best practices
 
@@ -87,7 +81,7 @@ when using `ilm_rollover_alias`.
 
 ================================================================================
 
-If you're sending events to the same Elasticsearch cluster but you're targeting different indices you can:
+If you're sending events to the same Elasticsearch cluster, but you're targeting different indices you can:
 
  * use different Elasticsearch outputs, each one with a different value for the `index` parameter
  * use one Elasticsearch output and use the dynamic variable substitution for the `index` parameter
@@ -224,13 +218,17 @@ not reevaluate its DNS value while the keepalive is in effect.
 
 ==== HTTP Compression
 
-This plugin supports request and response compression. Response compression is enabled by default and 
-for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for 
-it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[in 
-Elasticsearch] to take advantage of response compression when using this plugin
+This plugin supports request and response compression. Response compression is
+enabled by default for HTTP and for Elasticsearch versions 5.0 and later. 
 
-For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 
-setting in their Logstash config file.
+You don't have to set any configs in Elasticsearch for it to send back a
+compressed response. For versions before 5.0, or if HTTPS is enabled,
+`http.compression` must be set to `true`
+https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[in
+Elasticsearch] to take advantage of response compression when using this plugin.
+
+For requests compression, regardless of the Elasticsearch version, enable the
+`http_compression` setting in the Logstash config file.
 
 ==== Authentication
 
@@ -254,6 +252,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-ecs_compatibility>> | <<string,string>>|No
 | <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-healthcheck_path>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-hosts>> |<<uri,uri>>|No
@@ -413,6 +412,25 @@ If you don't set a value for this option:
 - for elasticsearch clusters 6.x: the value of 'doc' will be used;
 - for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
 
+[id="plugins-{type}s-{plugin}-ecs_compatibility"]
+===== `ecs_compatibility`
+
+* Value type is <<string,string>>
+* Supported values are:
+** `disabled`: does not provide ECS-compatible templates
+** `v1`: provides defaults that are compatible with v1 of the Elastic Common Schema
+* Default value depends on which version of Logstash is running:
+** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
+** Otherwise, the default value is `disabled`.
+
+Controls this plugin's compatibility with the https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema (ECS)],
+including the installation of ECS-compatible index templates.
+The value of this setting affects the _default_ values of:
+
+* <<plugins-{type}s-{plugin}-index>>
+* <<plugins-{type}s-{plugin}-template_name>>
+* <<plugins-{type}s-{plugin}-ilm_rollover_alias>>
+
 [id="plugins-{type}s-{plugin}-failure_type_logging_whitelist"]
 ===== `failure_type_logging_whitelist` 
 
@@ -453,11 +471,15 @@ If you have custom firewall rules you may need to change this
 
 Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
 Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
+
+Examples:
+
     `"127.0.0.1"`
     `["127.0.0.1:9200","127.0.0.2:9200"]`
     `["http://127.0.0.1"]`
     `["https://127.0.0.1:9200"]`
     `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
+
 It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
 to prevent LS from sending bulk requests to the master nodes.  So this parameter should only reference either data or client nodes in Elasticsearch.
 
@@ -505,7 +527,7 @@ NOTE: The pattern must finish with a dash and a number that will be automaticall
 ===== `ilm_policy`
 
   * Value type is <<string,string>>
-  * Default value is `logstash`
+  * Default value is `logstash-policy`
 
 Modify this setting to use a custom Index Lifecycle Management policy, rather than the default. If this value is not set, the default policy will
 be automatically installed into Elasticsearch
@@ -516,7 +538,9 @@ NOTE: If this setting is specified, the policy must already exist in Elasticsear
 ===== `ilm_rollover_alias`
 
   * Value type is <<string,string>>
-  * Default value is `logstash`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `logstash`
+  ** ECS Compatibility enabled: `ecs-logstash`
 
 The rollover alias is the alias where indices managed using Index Lifecycle Management will be written to.
 
@@ -530,7 +554,9 @@ NOTE: `ilm_rollover_alias` does NOT support dynamic variable substitution as `in
 ===== `index` 
 
   * Value type is <<string,string>>
-  * Default value is `"logstash-%{+yyyy.MM.dd}"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `"logstash-%{+yyyy.MM.dd}"`
+  ** ECS Compatibility enabled: `"ecs-logstash-%{+yyyy.MM.dd}"`
 
 The index to write events to. This can be dynamic using the `%{foo}` syntax.
 The default value will partition your indices by day so you can more easily
@@ -564,7 +590,8 @@ Set the keystore password
   * Default value is `true`
 
 From Logstash 1.3 onwards, a template is applied to Elasticsearch during
-Logstash's startup if one with the name `template_name` does not already exist.
+Logstash's startup if one with the name <<plugins-{type}s-{plugin}-template_name>>
+does not already exist.
 By default, the contents of this template is the default template for
 `logstash-%{+YYYY.MM.dd}` which always matches indices based on the pattern
 `logstash-*`.  Should you require support for other index names, or would like
@@ -620,8 +647,8 @@ not also set this field. That will raise an error at startup
   * Value type is <<string,string>>
   * Default value is `nil`
 
-Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration
-here like `pipeline => "%{INGEST_PIPELINE}"`
+Set which ingest pipeline you wish to execute for an event. You can also use event dependent configuration here
+like `pipeline => "%{[@metadata][pipeline]}"`. The pipeline parameter won't be set if the value resolves to empty string ("").
 
 [id="plugins-{type}s-{plugin}-pool_max"]
 ===== `pool_max` 
@@ -815,7 +842,10 @@ If not set, the included template will be used.
 ===== `template_name` 
 
   * Value type is <<string,string>>
-  * Default value is `"logstash"`
+  * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
+  ** ECS Compatibility disabled: `logstash`
+  ** ECS Compatibility enabled: `ecs-logstash`
+
 
 This configuration option defines how the template is named inside Elasticsearch.
 Note that if you have used the template management features and subsequently

data/lib/logstash/outputs/elasticsearch.rb

@@ -92,6 +92,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/outputs/elasticsearch/common"
   require "logstash/outputs/elasticsearch/ilm"
 
+  require 'logstash/plugin_mixins/ecs_compatibility_support'
+
   # Protocol agnostic (i.e. non-http, non-java specific) configs go here
   include(LogStash::Outputs::ElasticSearch::CommonConfigs)
 
@@ -101,6 +103,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Methods for ILM support
   include(LogStash::Outputs::ElasticSearch::Ilm)
 
+  # ecs_compatibility option, provided by Logstash core or the support adapter.
+  include(LogStash::PluginMixins::ECSCompatibilitySupport)
+
   config_name "elasticsearch"
 
   # The Elasticsearch action to perform. Valid actions are:
@@ -242,6 +247,34 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Custom Headers to send on each request to elasticsearch nodes
   config :custom_headers, :validate => :hash, :default => {}
 
+  def initialize(*params)
+    super
+    setup_ecs_compatibility_related_defaults
+  end
+
+  def setup_ecs_compatibility_related_defaults
+    case ecs_compatibility
+    when :disabled
+      @default_index = "logstash-%{+yyyy.MM.dd}"
+      @default_ilm_rollover_alias = "logstash"
+      @default_template_name = 'logstash'
+    when :v1
+      @default_index = "ecs-logstash-%{+yyyy.MM.dd}"
+      @default_ilm_rollover_alias = "ecs-logstash"
+      @default_template_name = 'ecs-logstash'
+    else
+      fail("unsupported ECS Compatibility `#{ecs_compatibility}`")
+    end
+
+    @index ||= default_index
+    @ilm_rollover_alias ||= default_ilm_rollover_alias
+    @template_name ||= default_template_name
+  end
+
+  attr_reader :default_index
+  attr_reader :default_ilm_rollover_alias
+  attr_reader :default_template_name
+
   # @override to handle proxy => '' as if none was set
   def config_init(params)
     proxy = params['proxy']

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -60,8 +60,15 @@ module LogStash; module Outputs; class ElasticSearch;
       !!maximum_seen_major_version
     end
 
-    def use_event_type?(client)
-      client.maximum_seen_major_version < 8
+    ##
+    # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
+    #          where a `client` argument is both required and ignored. In later versions of
+    #          Logstash Core it is optional and ignored, but to make it optional here would
+    #          allow us to accidentally break compatibility with Logstashes where it was required.
+    # @param noop_required_client [nil]: required `nil` for legacy reasons.
+    # @return [Boolean]
+    def use_event_type?(noop_required_client)
+      maximum_seen_major_version < 8
     end
 
     # Convert the event into a 3-tuple of action, params, and event
@@ -74,10 +81,16 @@ module LogStash; module Outputs; class ElasticSearch;
         routing_field_name => @routing ? event.sprintf(@routing) : nil
       }
 
-      params[:_type] = get_event_type(event) if use_event_type?(client)
+      params[:_type] = get_event_type(event) if use_event_type?(nil)
 
       if @pipeline
-        params[:pipeline] = event.sprintf(@pipeline)
+        value = event.sprintf(@pipeline)
+        # convention: empty string equates to not using a pipeline
+        # this is useful when using a field reference in the pipeline setting, e.g.
+        #      elasticsearch {
+        #        pipeline => "%{[@metadata][pipeline]}"
+        #      }
+        params[:pipeline] = value unless value.empty?
       end
 
       if @parent
@@ -347,11 +360,11 @@ module LogStash; module Outputs; class ElasticSearch;
       type = if @document_type
                event.sprintf(@document_type)
              else
-               if client.maximum_seen_major_version < 6
+               if maximum_seen_major_version < 6
                  event.get("type") || DEFAULT_EVENT_TYPE_ES6
-               elsif client.maximum_seen_major_version == 6
+               elsif maximum_seen_major_version == 6
                  DEFAULT_EVENT_TYPE_ES6
-               elsif client.maximum_seen_major_version == 7
+               elsif maximum_seen_major_version == 7
                  DEFAULT_EVENT_TYPE_ES7
                else
                  nil
@@ -436,7 +449,7 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def default_index?(index)
-      @index == LogStash::Outputs::ElasticSearch::CommonConfigs::DEFAULT_INDEX_NAME
+      @index == @default_index
     end
 
     def dlq_enabled?

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -17,7 +17,7 @@ module LogStash; module Outputs; class ElasticSearch
       # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
       # LS uses Joda to format the index pattern from event timestamp.
       # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here].
-      mod.config :index, :validate => :string, :default => DEFAULT_INDEX_NAME
+      mod.config :index, :validate => :string
 
       mod.config :document_type, 
         :validate => :string, 
@@ -44,7 +44,7 @@ module LogStash; module Outputs; class ElasticSearch
       # `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
       #
       # where `OldTemplateName` is whatever the former setting was.
-      mod.config :template_name, :validate => :string, :default => "logstash"
+      mod.config :template_name, :validate => :string
 
       # You can set the path to your own template here, if you so desire.
       # If not set, the included template will be used.
@@ -153,7 +153,7 @@ module LogStash; module Outputs; class ElasticSearch
       mod.config :ilm_enabled, :validate => [true, false, 'true', 'false', 'auto'], :default => 'auto'
 
       # Rollover alias used for indexing data. If rollover alias doesn't exist, Logstash will create it and map it to the relevant index
-      mod.config :ilm_rollover_alias, :validate => :string, :default => DEFAULT_ROLLOVER_ALIAS
+      mod.config :ilm_rollover_alias, :validate => :string
 
       # appends “{now/d}-000001” by default for new index creation, subsequent rollover indices will increment based on this pattern i.e. “000002”
       # {now/d} is date math, and will insert the appropriate value automatically.

data/lib/logstash/outputs/elasticsearch/ilm.rb

@@ -12,7 +12,7 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     def default_rollover_alias?(rollover_alias)
-      rollover_alias == LogStash::Outputs::ElasticSearch::DEFAULT_ROLLOVER_ALIAS
+      rollover_alias == default_ilm_rollover_alias
     end
 
     def ilm_alias_set?

data/lib/logstash/outputs/elasticsearch/template_manager.rb

@@ -3,14 +3,15 @@ module LogStash; module Outputs; class ElasticSearch
     # To be mixed into the elasticsearch plugin base
     def self.install_template(plugin)
       return unless plugin.manage_template
-      if plugin.template.nil?
-        plugin.logger.info("Using default mapping template")
-      else
+      if plugin.template
         plugin.logger.info("Using mapping template from", :path => plugin.template)
+        template = read_template_file(plugin.template)
+      else
+        plugin.logger.info("Using a default mapping template", :es_version => plugin.maximum_seen_major_version,
+                                                               :ecs_compatibility => plugin.ecs_compatibility)
+        template = load_default_template(plugin.maximum_seen_major_version, plugin.ecs_compatibility)
       end
 
-
-      template = get_template(plugin.template, plugin.maximum_seen_major_version)
       add_ilm_settings_to_template(plugin, template) if plugin.ilm_in_use?
       plugin.logger.info("Attempting to install template", :manage_template => template)
       install(plugin.client, template_name(plugin), template, plugin.template_overwrite)
@@ -19,9 +20,11 @@ module LogStash; module Outputs; class ElasticSearch
     end
 
     private
-    def self.get_template(path, es_major_version)
-      template_path = path || default_template_path(es_major_version)
+    def self.load_default_template(es_major_version, ecs_compatibility)
+      template_path = default_template_path(es_major_version, ecs_compatibility)
       read_template_file(template_path)
+    rescue => e
+      fail "Failed to load default template for Elasticsearch v#{es_major_version} with ECS #{ecs_compatibility}; caused by: #{e.inspect}"
     end
 
     def self.install(client, template_name, template, template_overwrite)
@@ -46,9 +49,9 @@ module LogStash; module Outputs; class ElasticSearch
       plugin.ilm_in_use? && !plugin.original_params.key?('template_name') ? plugin.ilm_rollover_alias : plugin.template_name
     end
 
-    def self.default_template_path(es_major_version)
+    def self.default_template_path(es_major_version, ecs_compatibility=:disabled)
       template_version = es_major_version == 1 ? 2 : es_major_version
-      default_template_name = "elasticsearch-template-es#{template_version}x.json"
+      default_template_name = "templates/ecs-#{ecs_compatibility}/elasticsearch-#{template_version}x.json"
       ::File.expand_path(default_template_name, ::File.dirname(__FILE__))
     end