logstash-output-elasticsearch 10.6.0-java → 10.8.0-java

data/lib/logstash/{outputs → plugin_mixins}/elasticsearch/common.rb

@@ -1,7 +1,10 @@
 require "logstash/outputs/elasticsearch/template_manager"
 
-module LogStash; module Outputs; class ElasticSearch;
+module LogStash; module PluginMixins; module ElasticSearch
   module Common
+
+    # This module defines common methods that can be reused by alternate elasticsearch output plugins such as the elasticsearch_data_streams output.
+
     attr_reader :client, :hosts
 
     # These codes apply to documents, not at the request level
@@ -9,103 +12,26 @@ module LogStash; module Outputs; class ElasticSearch;
     DOC_SUCCESS_CODES = [200, 201]
     DOC_CONFLICT_CODE = 409
 
-    # When you use external versioning, you are communicating that you want
-    # to ignore conflicts. More obviously, since an external version is a
-    # constant part of the incoming document, we should not retry, as retrying
-    # will never succeed.
-    VERSION_TYPES_PERMITTING_CONFLICT = ["external", "external_gt", "external_gte"]
-
-    def register
-      @template_installed = Concurrent::AtomicBoolean.new(false)
-      @stopping = Concurrent::AtomicBoolean.new(false)
-      # To support BWC, we check if DLQ exists in core (< 5.4). If it doesn't, we use nil to resort to previous behavior.
-      @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
-      build_client
-      setup_after_successful_connection
-      check_action_validity
-      @bulk_request_metrics = metric.namespace(:bulk_requests)
-      @document_level_metrics = metric.namespace(:documents)
-      @logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized).map(&:to_s))
-    end
-
-    # Receive an array of events and immediately attempt to index them (no buffering)
-    def multi_receive(events)
-      until @template_installed.true?
-        sleep 1
-      end
-      retrying_submit(events.map {|e| event_action_tuple(e)})
-    end
-
-    def setup_after_successful_connection
-      @template_installer ||= Thread.new do
-        sleep_interval = @retry_initial_interval
-        until successful_connection? || @stopping.true?
-          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
-          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
-          sleep_interval = next_sleep_interval(sleep_interval)
-        end
-        if successful_connection?
-          discover_cluster_uuid
-          install_template
-          setup_ilm if ilm_in_use?
-        end
-      end
-    end
-
-    def stop_template_installer
-      @template_installer.join unless @template_installer.nil?
-    end
-
-    def successful_connection?
-      !!maximum_seen_major_version
-    end
-
-    def use_event_type?
-      maximum_seen_major_version < 8
-    end
-
-    # Convert the event into a 3-tuple of action, params, and event
-    def event_action_tuple(event)
-      action = event.sprintf(@action)
-
-      params = {
-        :_id => @document_id ? event.sprintf(@document_id) : nil,
-        :_index => event.sprintf(@index),
-        routing_field_name => @routing ? event.sprintf(@routing) : nil
-      }
-
-      params[:_type] = get_event_type(event) if use_event_type?
-
-      if @pipeline
-        params[:pipeline] = event.sprintf(@pipeline)
-      end
-
-      if @parent
-        if @join_field
-          join_value = event.get(@join_field)
-          parent_value = event.sprintf(@parent)
-          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
-          params[routing_field_name] = event.sprintf(@parent)
-        else
-          params[:parent] = event.sprintf(@parent)
-        end
-      end
-
-      if action == 'update'
-        params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
-        params[:_script] = event.sprintf(@script) if @script != ""
-        params[retry_on_conflict_action_name] = @retry_on_conflict
-      end
-
-      if @version
-        params[:version] = event.sprintf(@version)
-      end
-
-      if @version_type
-        params[:version_type] = event.sprintf(@version_type)
+    # Perform some ES options validations and Build the HttpClient.
+    # Note that this methods may sets the @user, @password, @hosts and @client ivars as a side effect.
+    # @param license_checker [#appropriate_license?] An optional license checker that will be used by the Pool class.
+    # @return [HttpClient] the new http client
+    def build_client(license_checker = nil)
+      params["license_checker"] = license_checker
+
+      # the following 3 options validation & setup methods are called inside build_client
+      # because they must be executed prior to building the client and logstash
+      # monitoring and management rely on directly calling build_client
+      # see https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/934#pullrequestreview-396203307
+      validate_authentication
+      fill_hosts_from_cloud_id
+      setup_hosts
+
+      params["metric"] = metric
+      if @proxy.eql?('')
+        @logger.warn "Supplied proxy setting (proxy => '') has no effect"
       end
-
-      [action, params, event]
+      @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
     end
 
     def validate_authentication
@@ -140,7 +66,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def hosts_default?(hosts)
       # NOTE: would be nice if pipeline allowed us a clean way to detect a config default :
-      hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(CommonConfigs::DEFAULT_HOST)
+      hosts.is_a?(Array) && hosts.size == 1 && hosts.first.equal?(LogStash::PluginMixins::ElasticSearch::APIConfigs::DEFAULT_HOST)
     end
     private :hosts_default?
 
@@ -193,17 +119,23 @@ module LogStash; module Outputs; class ElasticSearch;
       client.maximum_seen_major_version
     end
 
-    def routing_field_name
-      maximum_seen_major_version >= 6 ? :routing : :_routing
-    end
-
-    def retry_on_conflict_action_name
-      maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
+    def successful_connection?
+      !!maximum_seen_major_version
     end
 
-    def install_template
-      TemplateManager.install_template(self)
-      @template_installed.make_true
+    # launch a thread that waits for an initial successful connection to the ES cluster to call the given block
+    # @param block [Proc] the block to execute upon initial successful connection
+    # @return [Thread] the successful connection wait thread
+    def setup_after_successful_connection(&block)
+      Thread.new do
+        sleep_interval = @retry_initial_interval
+        until successful_connection? || @stopping.true?
+          @logger.debug("Waiting for connectivity to Elasticsearch cluster. Retrying in #{sleep_interval}s")
+          Stud.stoppable_sleep(sleep_interval) { @stopping.true? }
+          sleep_interval = next_sleep_interval(sleep_interval)
+        end
+        block.call if successful_connection?
+      end
     end
 
     def discover_cluster_uuid
@@ -215,22 +147,6 @@ module LogStash; module Outputs; class ElasticSearch;
       # @logger.error("Unable to retrieve elasticsearch cluster uuid", error => e.message)
     end
 
-    def check_action_validity
-      raise LogStash::ConfigurationError, "No action specified!" unless @action
-
-      # If we're using string interpolation, we're good!
-      return if @action =~ /%{.+}/
-      return if valid_actions.include?(@action)
-
-      raise LogStash::ConfigurationError, "Action '#{@action}' is invalid! Pick one of #{valid_actions} or use a sprintf style statement"
-    end
-
-    # To be overidden by the -java version
-    VALID_HTTP_ACTIONS=["index", "delete", "create", "update"]
-    def valid_actions
-      VALID_HTTP_ACTIONS
-    end
-
     def retrying_submit(actions)
       # Initially we submit the full list of actions
       submit_actions = actions
@@ -339,32 +255,6 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
-    # Determine the correct value for the 'type' field for the given event
-    DEFAULT_EVENT_TYPE_ES6="doc".freeze
-    DEFAULT_EVENT_TYPE_ES7="_doc".freeze
-    def get_event_type(event)
-      # Set the 'type' value for the index.
-      type = if @document_type
-               event.sprintf(@document_type)
-             else
-               if maximum_seen_major_version < 6
-                 event.get("type") || DEFAULT_EVENT_TYPE_ES6
-               elsif maximum_seen_major_version == 6
-                 DEFAULT_EVENT_TYPE_ES6
-               elsif maximum_seen_major_version == 7
-                 DEFAULT_EVENT_TYPE_ES7
-               else
-                 nil
-               end
-             end
-
-      if !(type.is_a?(String) || type.is_a?(Numeric))
-        @logger.warn("Bad event type! Non-string/integer type value set!", :type_class => type.class, :type_value => type.to_s, :event => event)
-      end
-
-      type.to_s
-    end
-
     # Rescue retryable errors during bulk submission
     def safe_bulk(actions)
       sleep_interval = @retry_initial_interval
@@ -435,10 +325,6 @@ module LogStash; module Outputs; class ElasticSearch;
       end
     end
 
-    def default_index?(index)
-      @index == @default_index
-    end
-
     def dlq_enabled?
       # TODO there should be a better way to query if DLQ is enabled
       # See more in: https://github.com/elastic/logstash/issues/8064
@@ -446,4 +332,4 @@ module LogStash; module Outputs; class ElasticSearch;
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
     end
   end
-end end end
+end; end; end

data/lib/logstash/plugin_mixins/elasticsearch/noop_license_checker.rb

@@ -0,0 +1,9 @@
+module LogStash; module PluginMixins; module ElasticSearch
+  class NoopLicenseChecker
+    INSTANCE = self.new
+
+    def appropriate_license?(pool, url)
+      true
+    end
+  end
+end; end; end

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.6.0'
+  s.version         = '10.8.0'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/es_spec_helper.rb

@@ -49,19 +49,10 @@ module ESHelper
     Time.now.strftime("%Y.%m.%d")
   end
 
-
-  def default_mapping_from_mappings(mappings)
-    if ESHelper.es_version_satisfies?(">=7")
-      mappings
-    else
-      mappings["_default_"]
-    end
-  end
-
   def field_properties_from_template(template_name, field)
-    mappings = @es.indices.get_template(name: template_name)[template_name]["mappings"]
-    mapping = default_mapping_from_mappings(mappings)
-    mapping["properties"][field]["properties"]
+    template = get_template(@es, template_name)
+    mappings = get_template_mappings(template)
+    mappings["properties"][field]["properties"]
   end
 
   def routing_field_name
@@ -105,6 +96,7 @@ module ESHelper
 
   def clean(client)
     client.indices.delete_template(:name => "*")
+    client.indices.delete_index_template(:name => "logstash*") rescue nil
     # This can fail if there are no indexes, ignore failure.
     client.indices.delete(:index => "*") rescue nil
     clean_ilm(client) if supports_ilm?(client)
@@ -182,6 +174,34 @@ module ESHelper
     }
   }
   end
+
+  def get_template(client, name)
+    if ESHelper.es_version_satisfies?(">=8")
+      t = client.indices.get_index_template(name: name)
+      t['index_templates'][0]['index_template']
+    else
+      t = client.indices.get_template(name: name)
+      t[name]
+    end
+  end
+
+  def get_template_settings(template)
+    if ESHelper.es_version_satisfies?(">=8")
+      template['template']['settings']
+    else
+      template['settings']
+    end
+  end
+
+  def get_template_mappings(template)
+    if ESHelper.es_version_satisfies?(">=8")
+      template['template']['mappings']
+    elsif ESHelper.es_version_satisfies?(">=7")
+      template['mappings']
+    else
+      template['mappings']["_default_"]
+    end
+  end
 end
 
 RSpec.configure do |config|

data/spec/fixtures/template-with-policy-es8x.json

@@ -0,0 +1,50 @@
+{
+  "index_patterns" : "overwrite-*",
+  "version" : 80001,
+  "template" : {
+    "settings" : {
+      "index.refresh_interval" : "1s",
+      "number_of_shards": 1
+    },
+    "mappings" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword", "ignore_above": 256 }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date" },
+        "@version": { "type": "keyword" },
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  },
+  "priority": 200,
+  "_meta" : {
+    "description": "index template for logstash-output-elasticsearch"
+  }
+}

data/spec/integration/outputs/ilm_spec.rb

@@ -8,7 +8,7 @@ shared_examples_for 'an ILM enabled Logstash' do
     let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
-      put_policy(@es,ilm_policy_name, policy)
+      put_policy(@es, ilm_policy_name, policy)
       subject.register
 
       subject.multi_receive([
@@ -108,9 +108,11 @@ shared_examples_for 'an ILM disabled Logstash' do
   it 'should not write the ILM settings into the template' do
     subject.register
     sleep(1)
-    expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
+
+    template = get_template(@es, "logstash")
+    expect(template).to have_index_pattern("logstash-*")
     if ESHelper.es_version_satisfies?(">= 2")
-      expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']).to be_nil
+      expect(get_template_settings(template)['index']['lifecycle']).to be_nil
     end
   end
 
@@ -152,16 +154,17 @@ shared_examples_for 'an ILM disabled Logstash' do
   end
 
   context 'with a custom template name' do
-    let (:template_name) { "custom_template_name" }
+    let (:template_name) { "logstash_custom_template_name" }
     let (:settings) { super.merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
       sleep(1)
 
-      expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("logstash-*")
+      template = get_template(@es, template_name)
+      expect(template).to have_index_pattern("logstash-*")
       if ESHelper.es_version_satisfies?(">= 2")
-        expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']).to be_nil
+        expect(get_template_settings(template)['index']['lifecycle']).to be_nil
       end
     end
   end
@@ -387,16 +390,20 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("logstash-*")
-          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['name']).to eq("logstash-policy")
-          expect(@es.indices.get_template(name: "logstash")["logstash"]["settings"]['index']['lifecycle']['rollover_alias']).to eq("logstash")
+
+          template = get_template(@es, "logstash")
+          expect(template).to have_index_pattern("logstash-*")
+          expect(get_template_settings(template)['index']['lifecycle']['name']).to eq("logstash-policy")
+          expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq("logstash")
         end
 
         it_behaves_like 'an ILM enabled Logstash'
       end
 
       context 'with a set index and a custom index pattern' do
-        if ESHelper.es_version_satisfies?(">= 7.0")
+        if ESHelper.es_version_satisfies?(">= 8.0")
+          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+        elsif ESHelper.es_version_satisfies?(">= 7.0")
           let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
         else
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
@@ -408,13 +415,15 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should not overwrite the index patterns' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: "logstash")["logstash"]).to have_index_pattern("overwrite-*")
+
+          template = get_template(@es, "logstash")
+          expect(template).to have_index_pattern("overwrite-*")
         end
       end
 
 
       context 'with a custom template' do
-        let (:ilm_rollover_alias) { "the_cat_in_the_hat" }
+        let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
         let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
@@ -422,7 +431,9 @@ if ESHelper.es_version_satisfies?(">= 6.6")
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
 
-        if ESHelper.es_version_satisfies?(">= 7.0")
+        if ESHelper.es_version_satisfies?(">= 8.0")
+          let (:template) { "spec/fixtures/template-with-policy-es8x.json" }
+        elsif ESHelper.es_version_satisfies?(">= 7.0")
           let (:template) { "spec/fixtures/template-with-policy-es7x.json" }
         else
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
@@ -460,13 +471,15 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         it 'should write the ILM settings into the template' do
           subject.register
           sleep(1)
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]).to have_index_pattern("#{ilm_rollover_alias}-*")
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
-          expect(@es.indices.get_template(name: ilm_rollover_alias)[ilm_rollover_alias]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
+
+          template = get_template(@es, ilm_rollover_alias)
+          expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
+          expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
+          expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
         end
 
         context 'with a different template_name' do
-          let (:template_name) { "custom_template_name" }
+          let (:template_name) { "logstash_custom_template_name" }
           let (:settings) { super.merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
@@ -474,9 +487,10 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           it 'should write the ILM settings into the template' do
             subject.register
             sleep(1)
-            expect(@es.indices.get_template(name: template_name)[template_name]).to have_index_pattern("#{ilm_rollover_alias}-*")
-            expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['name']).to eq(ilm_policy_name)
-            expect(@es.indices.get_template(name: template_name)[template_name]["settings"]['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
+            template = get_template(@es, template_name)
+            expect(template).to have_index_pattern("#{ilm_rollover_alias}-*")
+            expect(get_template_settings(template)['index']['lifecycle']['name']).to eq(ilm_policy_name)
+            expect(get_template_settings(template)['index']['lifecycle']['rollover_alias']).to eq(ilm_rollover_alias)
           end
         end