logstash-output-elasticsearch 10.8.6-java → 11.0.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69557d21ffe4079cabafcf86949f41d85cb6781f8898cebdc54b354117333b6b
-  data.tar.gz: a65b40a961335837f9ccff55472c0aeef033c5248cdcc579ffa98c6560fa377c
+  metadata.gz: dd0e368ed484aa214da94fcdc6978f919b3139cf90f8db462aba17f9c1e86670
+  data.tar.gz: 67b475fdd703d50d7bbb806adebd46c3b6657ead3019e76c7517e3c2428335be
 SHA512:
-  metadata.gz: e8be38c81c89f8dca5dad83c79106180967cb5ed6806ed4a0ce97db1296a15bd8a462da80ef4a663807648164ac410d3d57fc46b2412ef497b1f9d0a4d7b57c6
-  data.tar.gz: 1843e98054e65374fe4b72c5938b0d808fecca799294783893d75c955977ec0d34020cd688ec7a16e4e22bf8c6c2b9e53343e9bfd4dd0cbccee10e601d0b2e0f
+  metadata.gz: f1e85fea62c9173d0ffdbf739487d8bcbbcfb41f304893e37333d22a8f42b0f527506d926c9f983480ef1c76eee43869ded32b84df98e19fdd999b9d5a26baa9
+  data.tar.gz: 0d1df95d541fa6e1d1161763051a3b4dc8fed10f62d878e7b4aa556d00c17bf6daf44a2fdc0c356c14c9c5431457112da601942dcf34dbf5386b05235933146e

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 11.0.0
+ - Feat: Data stream support [#988](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/988)
+ - Refactor: reviewed logging format + restored ES (initial) setup error logging
+ - Feat: always check ES license [#1005](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1005)
+
+   Since Elasticsearch no longer provides an OSS artifact the plugin will no longer skip the license check on OSS Logstash. 
+
 ## 10.8.6
  - Fixed an issue where a single over-size event being rejected by Elasticsearch would cause the entire entire batch to be retried indefinitely. The oversize event will still be retried on its own and logging has been improved to include payload sizes in this situation [#972](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/972)
  - Fixed an issue with `http_compression => true` where a well-compressed payload could fit under our outbound 20MB limit but expand beyond Elasticsearch's 100MB limit, causing bulk failures. Bulk grouping is now determined entirely by the decompressed payload size [#823](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/823)

data/docs/index.asciidoc

@@ -21,17 +21,9 @@ include::{include_path}/plugin_header.asciidoc[]
 
 ==== Description
 
-If you plan to use the Kibana web interface to analyze data transformed by
-Logstash, use the Elasticsearch output plugin to get your data into
-Elasticsearch.
-
-This output only speaks the HTTP protocol as it is the preferred protocol for
-interacting with Elasticsearch. In previous versions it was possible to
-communicate with Elasticsearch through the transport protocol, which is now
-reserved for internal cluster communication between nodes
-{ref}/modules-transport.html[communication between nodes].
-Using the transport protocol to communicate with the cluster has been deprecated
-in Elasticsearch 7.0.0 and will be removed in 8.0.0
+Elasticsearch provides near real-time search and analytics for all types of
+data. The Elasticsearch output plugin can store both time series datasets (such
+as logs, events, and metrics) and non-time series data in Elasticsearch.
 
 You can https://www.elastic.co/elasticsearch/[learn more about Elasticsearch] on
 the website landing page or in the {ref}[Elasticsearch documentation].
@@ -70,6 +62,59 @@ By having an ECS-compatible template in place, we can ensure that Elasticsearch
 is prepared to create and index fields in a way that is compatible with ECS,
 and will correctly reject events with fields that conflict and cannot be coerced.
 
+[id="plugins-{type}s-{plugin}-data-streams"]
+==== Data streams
+
+The {es} output plugin can store both time series datasets (such
+as logs, events, and metrics) and non-time series data in Elasticsearch.
+
+The data stream options are recommended for indexing time series datasets (such
+as logs, metrics, and events) into {es}:
+
+* <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>
+* <<plugins-{type}s-{plugin}-data_stream_auto_routing>> 
+* <<plugins-{type}s-{plugin}-data_stream_dataset>> 
+* <<plugins-{type}s-{plugin}-data_stream_namespace>>
+* <<plugins-{type}s-{plugin}-data_stream_sync_fields>>
+* <<plugins-{type}s-{plugin}-data_stream_type>> 
+
+[id="plugins-{type}s-{plugin}-ds-examples"]
+===== Data stream configuration examples
+
+**Example: Basic default configuration**
+
+[source,sh]
+-----
+output {
+    elasticsearch {
+        hosts => "hostname"
+        data_stream => "true"
+    }
+}
+-----
+
+This example shows the minimal settings for processing data streams. Events
+with `data_stream.*`` fields are routed to the appropriate data streams. If the
+fields are missing, routing defaults to `logs-generic-logstash`.
+
+**Example: Customize data stream name**
+
+[source,sh]
+-----
+output {
+    elasticsearch {
+        hosts => "hostname"
+        data_stream => "true"
+        data_stream_type => "metrics"
+        data_stream_dataset => "foo"
+        data_stream_namespace => "bar"
+    }
+}
+-----
+
+
+
+
 ==== Writing to different indices: best practices
 
 [NOTE]
@@ -274,6 +319,12 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
+| <<plugins-{type}s-{plugin}-data_stream>> |<<string,string>>, one of `["true", "false", "auto"]`|No
+| <<plugins-{type}s-{plugin}-data_stream_auto_routing>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-data_stream_dataset>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-data_stream_namespace>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
@@ -335,23 +386,20 @@ output plugins.
 ===== `action`
 
   * Value type is <<string,string>>
-  * Default value is `"index"`
+  * Default value is `create` for data streams, and `index` for non-time series data. 
 
-Protocol agnostic (i.e. non-http, non-java specific) configs go here
-Protocol agnostic methods
 The Elasticsearch action to perform. Valid actions are:
 
-- index: indexes a document (an event from Logstash).
-- delete: deletes a document by id (An id is required for this action)
-- create: indexes a document, fails if a document by that id already exists in the index.
-- update: updates a document by id. Update has a special case where you can upsert -- update a
+- `index`: indexes a document (an event from Logstash).
+- `delete`: deletes a document by id (An id is required for this action)
+- `create`: indexes a document, fails if a document by that id already exists in the index.
+- `update`: updates a document by id. Update has a special case where you can upsert -- update a
   document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported
   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
 - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   would use the foo field for the action
 
-For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch
-bulk API documentation].
+For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation].
 
 [id="plugins-{type}s-{plugin}-api_key"]
 ===== `api_key`
@@ -405,6 +453,69 @@ Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used.
 For more details, check out the
 {logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation].
 
+[id="plugins-{type}s-{plugin}-data_stream"]
+===== `data_stream`
+
+* Value can be any of: `true`, `false` and `auto`
+* Default is `false` in Logstash 7.x and `auto` starting in Logstash 8.0.
+
+Defines whether data will be indexed into an Elasticsearch data stream.
+The other `data_stream_*` settings will be used only if this setting is enabled.
+
+Logstash handles the output as a data stream when the supplied configuration
+is compatible with data streams and this value is set to `auto`.
+
+[id="plugins-{type}s-{plugin}-data_stream_auto_routing"]
+===== `data_stream_auto_routing`
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`.
+
+Automatically routes events by deriving the data stream name using specific event
+fields with the `%{[data_stream][type]}-%{[data_stream][dataset]}-%{[data_stream][namespace]}` format.
+
+If enabled, the `data_stream.*` event fields will take precedence over the
+`data_stream_type`, `data_stream_dataset`, and `data_stream_namespace` settings,
+but will fall back to them if any of the fields are missing from the event.
+
+[id="plugins-{type}s-{plugin}-data_stream_dataset"]
+===== `data_stream_dataset`
+
+* Value type is <<string,string>>
+* Default value is `generic`.
+
+The data stream dataset used to construct the data stream at index time.
+
+[id="plugins-{type}s-{plugin}-data_stream_namespace"]
+===== `data_stream_namespace`
+
+* Value type is <<string,string>>
+* Default value is `default`.
+
+The data stream namespace used to construct the data stream at index time.
+
+[id="plugins-{type}s-{plugin}-data_stream_sync_fields"]
+===== `data_stream_sync_fields`
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`
+
+Automatically adds and syncs the `data_stream.*` event fields if they are missing from the
+event. This ensures that fields match the name of the data stream that is receiving events.
+
+NOTE: If existing `data_stream.*` event fields do not match the data stream name
+and `data_stream_auto_routing` is disabled, the event fields will be
+overwritten with a warning.
+
+[id="plugins-{type}s-{plugin}-data_stream_type"]
+===== `data_stream_type`
+
+* Value type is <<string,string>>
+* Default value is `logs`.
+
+The data stream type used to construct the data stream at index time.
+Currently, only `logs` and `metrics`are supported.
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
 
@@ -457,8 +568,7 @@ If you don't set a value for this option:
 ** When Logstash provides a `pipeline.ecs_compatibility` setting, its value is used as the default
 ** Otherwise, the default value is `disabled`.
 
-Controls this plugin's compatibility with the
-https://www.elastic.co/guide/en/ecs/current/index.html[Elastic Common Schema
+Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema
 (ECS)], including the installation of ECS-compatible index templates. The value
 of this setting affects the _default_ values of:
 

data/lib/logstash/outputs/elasticsearch.rb

@@ -3,8 +3,8 @@ require "logstash/namespace"
 require "logstash/environment"
 require "logstash/outputs/base"
 require "logstash/json"
-require "concurrent"
-require "stud/buffer"
+require "concurrent/atomic/atomic_boolean"
+require "stud/interval"
 require "socket" # for Socket.gethostname
 require "thread" # for safe queueing
 require "uri" # for escaping user input
@@ -92,6 +92,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   require "logstash/plugin_mixins/elasticsearch/api_configs"
   require "logstash/plugin_mixins/elasticsearch/common"
   require "logstash/outputs/elasticsearch/ilm"
+  require "logstash/outputs/elasticsearch/data_stream_support"
   require 'logstash/plugin_mixins/ecs_compatibility_support'
 
   # Protocol agnostic methods
@@ -106,6 +107,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Generic/API config options that any document indexer output needs
   include(LogStash::PluginMixins::ElasticSearch::APIConfigs)
 
+  # DS support
+  include(LogStash::Outputs::ElasticSearch::DataStreamSupport)
+
   DEFAULT_POLICY = "logstash-policy"
 
   config_name "elasticsearch"
@@ -122,7 +126,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   #   would use the foo field for the action
   #
   # For more details on actions, check out the http://www.elastic.co/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
-  config :action, :validate => :string, :default => "index"
+  config :action, :validate => :string # :default => "index" unless data_stream
 
   # The index to write events to. This can be dynamic using the `%{foo}` syntax.
   # The default value will partition your indices by day so you can more easily
@@ -247,6 +251,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # ILM policy to use, if undefined the default policy will be used.
   config :ilm_policy, :validate => :string, :default => DEFAULT_POLICY
 
+  attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
   attr_reader :default_template_name
@@ -257,26 +262,53 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   def register
-    @template_installed = Concurrent::AtomicBoolean.new(false)
+    @after_successful_connection_done = Concurrent::AtomicBoolean.new(false)
     @stopping = Concurrent::AtomicBoolean.new(false)
-    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
     check_action_validity
 
+    @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
+
     # the license_checking behaviour in the Pool class is externalized in the LogStash::ElasticSearchOutputLicenseChecker
     # class defined in license_check.rb. This license checking is specific to the elasticsearch output here and passed
     # to build_client down to the Pool class.
-    build_client(LicenseChecker.new(@logger))
+    @client = build_client(LicenseChecker.new(@logger))
+
+    @after_successful_connection_thread = after_successful_connection do
+      begin
+        finish_register
+        true # thread.value
+      rescue => e
+        # we do not want to halt the thread with an exception as that has consequences for LS
+        e # thread.value
+      ensure
+        @after_successful_connection_done.make_true
+      end
+    end
 
-    @template_installer = setup_after_successful_connection do
-      discover_cluster_uuid
-      install_template
-      setup_ilm if ilm_in_use?
+    # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
+    @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
+
+    if data_stream_config?
+      @event_mapper = -> (e) { data_stream_event_action_tuple(e) }
+      @event_target = -> (e) { data_stream_name(e) }
+      @index = "#{data_stream_type}-#{data_stream_dataset}-#{data_stream_namespace}".freeze # default name
+    else
+      @event_mapper = -> (e) { event_action_tuple(e) }
+      @event_target = -> (e) { e.sprintf(@index) }
     end
+
     @bulk_request_metrics = metric.namespace(:bulk_requests)
     @document_level_metrics = metric.namespace(:documents)
-    @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
+  end
+
+  # @override post-register when ES connection established
+  def finish_register
+    assert_es_version_supports_data_streams if data_stream_config?
+    discover_cluster_uuid
+    install_template
+    setup_ilm if ilm_in_use?
+    super
   end
 
   # @override to handle proxy => '' as if none was set
@@ -297,46 +329,47 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   # Receive an array of events and immediately attempt to index them (no buffering)
   def multi_receive(events)
-    until @template_installed.true?
-      sleep 1
+    wait_for_successful_connection if @after_successful_connection_done
+    retrying_submit map_events(events)
+  end
+
+  def map_events(events)
+    events.map(&@event_mapper)
+  end
+
+  def wait_for_successful_connection
+    after_successful_connection_done = @after_successful_connection_done
+    return unless after_successful_connection_done
+    stoppable_sleep 1 until after_successful_connection_done.true?
+
+    status = @after_successful_connection_thread && @after_successful_connection_thread.value
+    if status.is_a?(Exception) # check if thread 'halted' with an error
+      # keep logging that something isn't right (from every #multi_receive)
+      @logger.error "Elasticsearch setup did not complete normally, please review previously logged errors",
+                    message: status.message, exception: status.class
+    else
+      @after_successful_connection_done = nil # do not execute __method__ again if all went well
     end
-    retrying_submit(events.map {|e| event_action_tuple(e)})
   end
+  private :wait_for_successful_connection
 
   def close
     @stopping.make_true if @stopping
-    stop_template_installer
+    stop_after_successful_connection_thread
     @client.close if @client
   end
 
-  # not private because used by ILM specs
-  def stop_template_installer
-    @template_installer.join unless @template_installer.nil?
+  private
+
+  def stop_after_successful_connection_thread
+    @after_successful_connection_thread.join unless @after_successful_connection_thread.nil?
   end
 
-  # not private for elasticsearch_spec.rb
-  # Convert the event into a 3-tuple of action, params, and event
+  # Convert the event into a 3-tuple of action, params and event hash
   def event_action_tuple(event)
-    action = event.sprintf(@action)
-
-    params = {
-      :_id => @document_id ? event.sprintf(@document_id) : nil,
-      :_index => event.sprintf(@index),
-      routing_field_name => @routing ? event.sprintf(@routing) : nil
-    }
-
+    params = common_event_params(event)
     params[:_type] = get_event_type(event) if use_event_type?(nil)
 
-    if @pipeline
-      value = event.sprintf(@pipeline)
-      # convention: empty string equates to not using a pipeline
-      # this is useful when using a field reference in the pipeline setting, e.g.
-      #      elasticsearch {
-      #        pipeline => "%{[@metadata][pipeline]}"
-      #      }
-      params[:pipeline] = value unless value.empty?
-    end
-
     if @parent
       if @join_field
         join_value = event.get(@join_field)
@@ -348,26 +381,40 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
       end
     end
 
+    action = event.sprintf(@action || 'index')
+
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
       params[:_script] = event.sprintf(@script) if @script != ""
       params[retry_on_conflict_action_name] = @retry_on_conflict
     end
 
-    if @version
-      params[:version] = event.sprintf(@version)
-    end
-
-    if @version_type
-      params[:version_type] = event.sprintf(@version_type)
-    end
+    params[:version] = event.sprintf(@version) if @version
+    params[:version_type] = event.sprintf(@version_type) if @version_type
 
-    [action, params, event]
+    [action, params, event.to_hash]
   end
 
-  # not private for elasticsearch_spec.rb
-  def retry_on_conflict_action_name
-    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+  # @return Hash (initial) parameters for given event
+  # @private shared event params factory between index and data_stream mode
+  def common_event_params(event)
+    params = {
+        :_id => @document_id ? event.sprintf(@document_id) : nil,
+        :_index => @event_target.call(event),
+        routing_field_name => @routing ? event.sprintf(@routing) : nil
+    }
+
+    if @pipeline
+      value = event.sprintf(@pipeline)
+      # convention: empty string equates to not using a pipeline
+      # this is useful when using a field reference in the pipeline setting, e.g.
+      #      elasticsearch {
+      #        pipeline => "%{[@metadata][pipeline]}"
+      #      }
+      params[:pipeline] = value unless value.empty?
+    end
+
+    params
   end
 
   @@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
@@ -377,38 +424,47 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     require "logstash/outputs/elasticsearch/#{name}"
   end
 
-  private
+  def retry_on_conflict_action_name
+    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+  end
 
   def routing_field_name
     maximum_seen_major_version >= 6 ? :routing : :_routing
   end
 
   # Determine the correct value for the 'type' field for the given event
-  DEFAULT_EVENT_TYPE_ES6="doc".freeze
-  DEFAULT_EVENT_TYPE_ES7="_doc".freeze
+  DEFAULT_EVENT_TYPE_ES6 = "doc".freeze
+  DEFAULT_EVENT_TYPE_ES7 = "_doc".freeze
+
   def get_event_type(event)
     # Set the 'type' value for the index.
     type = if @document_type
              event.sprintf(@document_type)
            else
-             if maximum_seen_major_version < 6
-               event.get("type") || DEFAULT_EVENT_TYPE_ES6
-             elsif maximum_seen_major_version == 6
+             major_version = maximum_seen_major_version
+             if major_version < 6
+               es5_event_type(event)
+             elsif major_version == 6
                DEFAULT_EVENT_TYPE_ES6
-             elsif maximum_seen_major_version == 7
+             elsif major_version == 7
                DEFAULT_EVENT_TYPE_ES7
              else
                nil
              end
            end
 
-    if !(type.is_a?(String) || type.is_a?(Numeric))
-      @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
-    end
-
     type.to_s
   end
 
+  def es5_event_type(event)
+    type = event.get('type')
+    return DEFAULT_EVENT_TYPE_ES6 unless type
+    if !type.is_a?(String) && !type.is_a?(Numeric)
+      @logger.warn("Bad event type (non-string/integer type value set)", :type_class => type.class, :type_value => type, :event => event.to_hash)
+    end
+    type
+  end
+
   ##
   # WARNING: This method is overridden in a subclass in Logstash Core 7.7-7.8's monitoring,
   #          where a `client` argument is both required and ignored. In later versions of
@@ -424,7 +480,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   def install_template
     TemplateManager.install_template(self)
-    @template_installed.make_true
+  rescue => e
+    @logger.error("Failed to install template", message: e.message, exception: e.class, backtrace: e.backtrace)
   end
 
   def setup_ecs_compatibility_related_defaults
@@ -447,13 +504,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   end
 
   # To be overidden by the -java version
-  VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
+  VALID_HTTP_ACTIONS = ["index", "delete", "create", "update"]
   def valid_actions
     VALID_HTTP_ACTIONS
   end
 
   def check_action_validity
-    raise LogStash::ConfigurationError, "No action specified!" unless @action
+    return if @action.nil? # not set
+    raise LogStash::ConfigurationError, "No action specified!" if @action.empty?
 
     # If we're using string interpolation, we're good!
     return if @action =~ /%{.+}/