logstash-output-elasticsearch 11.8.0-java → 11.9.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 28dfa1d2a757a4cb14aa7972f13c13f78975a52db0cac8d39425fa6bb34a462c
-  data.tar.gz: 7a9b9e05f59e2d9024f8faf7d6d5faa58aeb123cb77fd4f89b88826787d5d4f5
+  metadata.gz: 4dbc44fb5e78b53368adb97d43b613c4a3c843e1a1308aa6d008cb105ee4301c
+  data.tar.gz: 9ed115a974e20bda89d63d6b2bf37dc5c7c42f8f40b46badbad072db0a698683
 SHA512:
-  metadata.gz: 8ee227e4dcb1f1af9cb7d1baf11cabb1e4f1a8ab87781494ca71f2d3a9ad57e6635819827c00090f597f79d33d8724f1a2e5e4047f506a0fde144c0dc5d4d71a
-  data.tar.gz: 6ad8e717ade5b09f46cf4bab5d9749c782cf654dd3e69f79010a230237cd251b7fd034b18f42ed09ffb46aa729c8596eae62a3cb41a8b33c36a7c23e7084adcf
+  metadata.gz: 9df0d61b80c78cd992b46428b24836db44094d636384940e976a0a09a58245bec59def6031d8426a24d41bdb0644ac49d4e8b558ef32c684554fbf03104a148c
+  data.tar.gz: 8c63cfaa83e3acad5d864dbf2d170fcf0e9111f31e8053fc2ce2a8a4d8af696355fda7b3e7152dc0764bce8e36c3afdd432c5a344f97a2b9b4b6c6c17e6a52d9

data/CHANGELOG.md

@@ -1,3 +1,12 @@
+## 11.9.1
+ - Fixes a possible infinite-retry-loop that could occur when this plugin is configured with an `action` whose value contains a [sprintf-style placeholder][] that fails to be resolved for an individual event. Events in this state will be routed to the pipeline's [dead letter queue][DLQ] if it is available, or will be logged-and-dropped so that the remaining events in the batch can be processed [#1080](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1080)
+ 
+[sprintf-style placeholder]: https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html#sprintf
+[DLQ]: https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
+
+## 11.9.0
+ - Feature: force unresolved dynamic index names to be sent into DLQ. This feature could be explicitly disabled using `dlq_on_failed_indexname_interpolation` setting [#1084](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1084)
+
 ## 11.8.0
  - Feature: Adds a new `dlq_custom_codes` option to customize DLQ codes [#1067](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1067)
  

data/README.md

@@ -19,7 +19,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 
 ## Developing
 
-### 1. Plugin Developement and Testing
+### 1. Plugin Development and Testing
 
 #### Code
 - To get started, you'll need JRuby with the Bundler gem installed.

data/docs/index.asciidoc

@@ -320,6 +320,7 @@ This plugin supports the following configuration options plus the
 | <<plugins-{type}s-{plugin}-data_stream_sync_fields>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-data_stream_type>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-dlq_custom_codes>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-doc_as_upsert>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-document_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-document_type>> |<<string,string>>|No
@@ -394,7 +395,7 @@ The Elasticsearch action to perform. Valid actions are:
   document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported
   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
 - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
-  would use the foo field for the action
+  would use the foo field for the action. If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}, and instead either will be sent to the pipeline's <<dead-letter-queues,dead letter queue (DLQ)>> if it is enabled, or will be logged and dropped.
 
 For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation].
 
@@ -533,6 +534,14 @@ This list is an addition to the ordinary error codes considered for this feature
 It's considered a configuration error to re-use the same predefined codes for success, DLQ or conflict.
 The option accepts a list of natural numbers corresponding to HTTP errors codes.
 
+[id="plugins-{type}s-{plugin}-dlq_on_failed_indexname_interpolation"]
+===== `dlq_on_failed_indexname_interpolation`
+
+* Value type is <<boolean,boolean>>
+* Default value is `true`.
+
+If enabled, failed index name interpolation events go into dead letter queue.
+
 [id="plugins-{type}s-{plugin}-doc_as_upsert"]
 ===== `doc_as_upsert`
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -120,6 +120,7 @@ module LogStash; module Outputs; class ElasticSearch;
       else
         stream_writer = body_stream
       end
+
       bulk_responses = []
       batch_actions = []
       bulk_actions.each_with_index do |action, index|
@@ -142,13 +143,16 @@ module LogStash; module Outputs; class ElasticSearch;
         stream_writer.write(as_json)
         batch_actions << action
       end
+
       stream_writer.close if http_compression
+
       logger.debug("Sending final bulk request for batch.",
                    :action_count => batch_actions.size,
                    :payload_size => stream_writer.pos,
                    :content_length => body_stream.size,
                    :batch_offset => (actions.size - batch_actions.size))
       bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
+
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end

data/lib/logstash/outputs/elasticsearch.rb

@@ -261,6 +261,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # The option accepts a list of natural numbers corresponding to HTTP errors codes.
   config :dlq_custom_codes, :validate => :number, :list => true, :default => []
 
+  # if enabled, failed index name interpolation events go into dead letter queue.
+  config :dlq_on_failed_indexname_interpolation, :validate => :boolean, :default => true
+
   attr_reader :client
   attr_reader :default_index
   attr_reader :default_ilm_rollover_alias
@@ -362,11 +365,48 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Receive an array of events and immediately attempt to index them (no buffering)
   def multi_receive(events)
     wait_for_successful_connection if @after_successful_connection_done
-    retrying_submit map_events(events)
+    events_mapped = safe_interpolation_map_events(events)
+    retrying_submit(events_mapped.successful_events)
+    unless events_mapped.event_mapping_errors.empty?
+      handle_event_mapping_errors(events_mapped.event_mapping_errors)
+    end
+  end
+
+  # @param: Arrays of FailedEventMapping
+  private
+  def handle_event_mapping_errors(event_mapping_errors)
+    # if DQL is enabled, log the events to provide issue insights to users.
+    if @dlq_writer
+      @logger.warn("Events could not be indexed and routing to DLQ, count: #{event_mapping_errors.size}")
+    end
+
+    event_mapping_errors.each do |event_mapping_error|
+      detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
+      handle_dlq_status(event_mapping_error.event, :warn, detailed_message)
+    end
+    @document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
   end
 
+  MapEventsResult = Struct.new(:successful_events, :event_mapping_errors)
+  FailedEventMapping = Struct.new(:event, :message)
+
+  private
+  def safe_interpolation_map_events(events)
+    successful_events = [] # list of LogStash::Outputs::ElasticSearch::EventActionTuple
+    event_mapping_errors = [] # list of FailedEventMapping
+    events.each do |event|
+      begin
+        successful_events << @event_mapper.call(event)
+      rescue EventMappingError => ie
+        event_mapping_errors << FailedEventMapping.new(event, ie.message)
+       end
+    end
+    MapEventsResult.new(successful_events, event_mapping_errors)
+  end
+
+  public
   def map_events(events)
-    events.map(&@event_mapper)
+    safe_interpolation_map_events(events).successful_events
   end
 
   def wait_for_successful_connection
@@ -414,6 +454,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     end
 
     action = event.sprintf(@action || 'index')
+    raise UnsupportedActionError, action unless VALID_HTTP_ACTIONS.include?(action)
 
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
@@ -441,12 +482,32 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   end
 
+  class EventMappingError < ArgumentError
+    def initialize(msg = nil)
+      super
+    end
+  end
+
+  class IndexInterpolationError < EventMappingError
+    def initialize(bad_formatted_index)
+      super("Badly formatted index, after interpolation still contains placeholder: [#{bad_formatted_index}]")
+    end
+  end
+
+  class UnsupportedActionError < EventMappingError
+    def initialize(bad_action)
+      super("Elasticsearch doesn't support [#{bad_action}] action")
+    end
+  end
+
   # @return Hash (initial) parameters for given event
   # @private shared event params factory between index and data_stream mode
   def common_event_params(event)
+    sprintf_index = @event_target.call(event)
+    raise IndexInterpolationError, sprintf_index if sprintf_index.match(/%{.*?}/) && dlq_on_failed_indexname_interpolation
     params = {
         :_id => @document_id ? event.sprintf(@document_id) : nil,
-        :_index => @event_target.call(event),
+        :_index => sprintf_index,
         routing_field_name => @routing ? event.sprintf(@routing) : nil
     }
 

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -206,19 +206,23 @@ module LogStash; module PluginMixins; module ElasticSearch
       doubled > @retry_max_interval ? @retry_max_interval : doubled
     end
 
-    def handle_dlq_status(message, action, status, response)
+    def handle_dlq_response(message, action, status, response)
+      _, action_params = action.event, [action[0], action[1], action[2]]
+      
+      # TODO: Change this to send a map with { :status => status, :action => action } in the future
+      detailed_message = "#{message} status: #{status}, action: #{action_params}, response: #{response}"
+      
+      log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
+      
+      handle_dlq_status(action.event, log_level, detailed_message)
+    end
+
+    def handle_dlq_status(event, log_level, message)
       # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
-        event, action = action.event, [action[0], action[1], action[2]]
-        # TODO: Change this to send a map with { :status => status, :action => action } in the future
-        @dlq_writer.write(event, "#{message} status: #{status}, action: #{action}, response: #{response}")
+        @dlq_writer.write(event, "#{message}")
       else
-        if dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception'
-          level = :error
-        else
-          level = :warn
-        end
-        @logger.send level, message, status: status, action: action, response: response
+        @logger.send log_level, message
       end
     end
 
@@ -255,7 +259,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         status = action_props["status"]
         error  = action_props["error"]
         action = actions[idx]
-        action_params = action[1]
 
         # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
         # - For 409, we log and drop. there is nothing we can do
@@ -269,7 +272,7 @@ module LogStash; module PluginMixins; module ElasticSearch
           @logger.warn "Failed action", status: status, action: action, response: response if log_failure_type?(error)
           next
         elsif @dlq_codes.include?(status)
-          handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
+          handle_dlq_response("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next
         else

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.8.0'
+  s.version         = '11.9.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'webmock'
+  s.add_development_dependency 'rspec-collection_matchers'
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/integration/outputs/index_spec.rb

@@ -1,5 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
+require 'cgi'
 
 describe "TARGET_BULK_BYTES", :integration => true do
   let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
@@ -45,16 +46,57 @@ describe "TARGET_BULK_BYTES", :integration => true do
   end
 end
 
-describe "indexing" do
+def curl_and_get_json_response(url, method: :get, retrieve_err_payload: false); require 'open3'
+  cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
+  begin
+    out, err, status = Open3.capture3(cmd)
+  rescue Errno::ENOENT
+    fail "curl not available, make sure curl binary is installed and available on $PATH"
+  end
+
+  if status.success?
+    http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+
+    if http_status.strip[0].to_i > 2
+      error = (LogStash::Json.load(out)['error']) rescue nil
+      if error
+        if retrieve_err_payload
+          return error
+        else
+          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
+        end
+      else
+        warn out
+        fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
+      end
+    end
+
+    LogStash::Json.load(out)
+  else
+    warn out
+    fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+  end
+end
+
+describe "indexing with sprintf resolution", :integration => true do
   let(:message) { "Hello from #{__FILE__}" }
   let(:event) { LogStash::Event.new("message" => message, "type" => type) }
-  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let (:index) { "%{[index_name]}_dynamic" }
   let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
-  let(:event_count) { 1 + rand(2) }
-  let(:config) { "not implemented" }
+  let(:event_count) { 1 }
+  let(:user) { "simpleuser" }
+  let(:password) { "abc123" }
+  let(:config) do
+    {
+      "hosts" => [ get_host_port ],
+      "user" => user,
+      "password" => password,
+      "index" => index
+    }
+  end
   let(:events) { event_count.times.map { event }.to_a }
   subject { LogStash::Outputs::ElasticSearch.new(config) }
-  
+
   let(:es_url) { "http://#{get_host_port}" }
   let(:index_url) { "#{es_url}/#{index}" }
 
@@ -63,33 +105,65 @@ describe "indexing" do
   let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
   let(:es_admin_pass) { 'elastic' }
 
-  def curl_and_get_json_response(url, method: :get); require 'open3'
-    cmd = "curl -s -v --show-error #{curl_opts} -X #{method.to_s.upcase} -k #{url}"
-    begin
-      out, err, status = Open3.capture3(cmd)
-    rescue Errno::ENOENT
-      fail "curl not available, make sure curl binary is installed and available on $PATH"
-    end
+  let(:initial_events) { [] }
 
-    if status.success?
-      http_status = err.match(/< HTTP\/1.1 (\d+)/)[1] || '0' # < HTTP/1.1 200 OK\r\n
+  let(:do_register) { true }
 
-      if http_status.strip[0].to_i > 2
-        error = (LogStash::Json.load(out)['error']) rescue nil
-        if error
-          fail "#{cmd.inspect} received an error: #{http_status}\n\n#{error.inspect}"
-        else
-          warn out
-          fail "#{cmd.inspect} unexpected response: #{http_status}\n\n#{err}"
-        end
-      end
+  before do
+    subject.register if do_register
+    subject.multi_receive(initial_events) if initial_events
+  end
 
-      LogStash::Json.load(out)
-    else
-      warn out
-      fail "#{cmd.inspect} process failed: #{status}\n\n#{err}"
+  after do
+    subject.do_close
+  end
+
+  let(:event) { LogStash::Event.new("message" => message, "type" => type, "index_name" => "test") }
+
+  it "should index successfully when field is resolved" do
+    expected_index_name = "test_dynamic"
+    subject.multi_receive(events)
+
+#     curl_and_get_json_response "#{es_url}/_refresh", method: :post
+
+    result = curl_and_get_json_response "#{es_url}/#{expected_index_name}"
+
+    expect(result[expected_index_name]).not_to be(nil)
+  end
+
+  context "when dynamic field doesn't resolve the index_name" do
+    let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+    let(:dlq_writer) { double('DLQ writer') }
+    before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
+
+    it "should doesn't create an index name with unresolved placeholders" do
+      expect(dlq_writer).to receive(:write).once.with(event, a_string_including("Badly formatted index, after interpolation still contains placeholder"))
+      subject.multi_receive(events)
+
+      escaped_index_name = CGI.escape("%{[index_name]}_dynamic")
+      result = curl_and_get_json_response "#{es_url}/#{escaped_index_name}", retrieve_err_payload: true
+      expect(result["root_cause"].first()["type"]).to eq("index_not_found_exception")
     end
   end
+end
+
+describe "indexing" do
+  let(:message) { "Hello from #{__FILE__}" }
+  let(:event) { LogStash::Event.new("message" => message, "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { ESHelper.es_version_satisfies?("< 7") ? "doc" : "_doc" }
+  let(:event_count) { 1 + rand(2) }
+  let(:config) { "not implemented" }
+  let(:events) { event_count.times.map { event }.to_a }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+  
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) { "#{es_url}/#{index}" }
+
+  let(:curl_opts) { nil }
+
+  let(:es_admin) { 'admin' } # default user added in ES -> 8.x requires auth credentials for /_refresh etc
+  let(:es_admin_pass) { 'elastic' }
 
   let(:initial_events) { [] }
 

data/spec/integration/outputs/unsupported_actions_spec.rb

@@ -0,0 +1,75 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Unsupported actions testing...", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  INDEX = "logstash-unsupported-actions-rejected"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => INDEX,
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "%{action_field}",
+      "document_id" => "%{doc_id}",
+      "ecs_compatibility" => "disabled"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    # index single doc for update purpose
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "2",
+      :body => { :message => 'Test to doc indexing', :counter => 1 }
+    )
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "3",
+      :body => { :message => 'Test to doc deletion', :counter => 2 }
+    )
+    @es.indices.refresh
+  end
+
+  context "multiple actions include unsupported action" do
+    let(:events) {[
+      LogStash::Event.new("action_field" => "index", "doc_id" => 1, "message"=> "hello"),
+      LogStash::Event.new("action_field" => "update", "doc_id" => 2, "message"=> "hi"),
+      LogStash::Event.new("action_field" => "delete", "doc_id" => 3),
+      LogStash::Event.new("action_field" => "unsupported_action", "doc_id" => 4, "message"=> "world!")
+    ]}
+
+    it "should reject unsupported doc" do
+      subject = get_es_output
+      subject.register
+      subject.multi_receive(events)
+
+      index_or_update = proc do |event|
+        action = event.get("action_field")
+        action.eql?("index") || action.eql?("update")
+      end
+
+      indexed_events = events.select { |event| index_or_update.call(event) }
+      rejected_events = events.select { |event| !index_or_update.call(event) }
+
+      indexed_events.each do |event|
+        response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)
+        expect(response['_source']['message']).to eq(event.get("message"))
+      end
+
+      rejected_events.each do |event|
+        expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -266,6 +266,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
             end
           end
         end
+
        end
      end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -3,8 +3,8 @@ require "base64"
 require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
-
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+require 'rspec/collection_matchers'
 
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
@@ -347,7 +347,7 @@ describe LogStash::Outputs::ElasticSearch do
                               }
                     },
                     # NOTE: this is an artificial success (usually everything fails with a 500) but even if some doc where
-                    # to succeed due the unexpected reponse items we can not clearly identify which actions to retry ...
+                    # to succeed due the unexpected response items we can not clearly identify which actions to retry ...
                     {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>201}},
                     {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>500,
                                "error"=>{"type" => "illegal_state_exception",
@@ -381,6 +381,104 @@ describe LogStash::Outputs::ElasticSearch do
         subject.multi_receive(events)
       end
     end
+
+    context "unsupported actions" do
+      let(:options) { super().merge("index" => "logstash", "action" => "%{action_field}") }
+
+      context "with multiple valid actions with one trailing invalid action" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 3, "message"=> "bye"),
+          LogStash::Event.new("action_field" => "unsupported_action", "id" => 4, "message"=> "world!")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action] action")
+          end
+        end
+      end
+
+      context "with one leading invalid action followed by multiple valid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "unsupported_action", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "index", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action] action")
+          end
+        end
+      end
+
+      context "with batch of multiple invalid actions and no valid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "unsupported_action1", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "unsupported_action2", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "unsupported_action4", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have(:no).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(4).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to include "Elasticsearch doesn't support"
+          end
+        end
+      end
+
+      context "with batch of intermixed valid and invalid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "unsupported_action2", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(2).items
+          expect(event_result.event_mapping_errors).to have_exactly(2).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to include "Elasticsearch doesn't support"
+          end
+        end
+      end
+
+      context "with batch of exactly one action that is invalid" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported action" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action3] action")
+          end
+        end
+      end
+    end
   end
 
   context '413 errors' do
@@ -768,17 +866,18 @@ describe LogStash::Outputs::ElasticSearch do
 
   context 'handling elasticsearch document-level status meant for the DLQ' do
     let(:options) { { "manage_template" => false } }
+    let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
+      let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
       context 'resorting to previous behaviour of logging the error' do
         context 'getting an invalid_index_name_exception' do
           it 'should log at ERROR level' do
             subject.instance_variable_set(:@logger, double("logger").as_null_object)
             mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
-            subject.handle_dlq_status("Could not index event to Elasticsearch.",
-              [:action, :params, :event], :some_status, mock_response)
+            subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
           end
         end
 
@@ -786,10 +885,9 @@ describe LogStash::Outputs::ElasticSearch do
           it 'should log at WARN level' do
             logger = double("logger").as_null_object
             subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
             mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
-            subject.handle_dlq_status("Could not index event to Elasticsearch.",
-              [:action, :params, :event], :some_status, mock_response)
+            subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
           end
         end
 
@@ -797,11 +895,10 @@ describe LogStash::Outputs::ElasticSearch do
           it 'should not fail, but just log a warning' do
             logger = double("logger").as_null_object
             subject.instance_variable_set(:@logger, logger)
-            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            expect(logger).to receive(:warn).with(a_string_including "Could not index event to Elasticsearch. status: some_status, action: [:action, :params, {")
             mock_response = { 'index' => {} }
             expect do
-              subject.handle_dlq_status("Could not index event to Elasticsearch.",
-                [:action, :params, :event], :some_status, mock_response)
+              subject.handle_dlq_response("Could not index event to Elasticsearch.", action, :some_status, mock_response)
             end.to_not raise_error
           end
         end
@@ -821,7 +918,7 @@ describe LogStash::Outputs::ElasticSearch do
         expect(dlq_writer).to receive(:write).once.with(event, /Could not index/)
         mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
         action = LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, event)
-        subject.handle_dlq_status("Could not index event to Elasticsearch.", action, 404, mock_response)
+        subject.handle_dlq_response("Could not index event to Elasticsearch.", action, 404, mock_response)
       end
     end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.8.0
+  version: 11.9.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-15 00:00:00.000000000 Z
+date: 2022-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -196,6 +196,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-collection_matchers
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -288,6 +302,7 @@ files:
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/unsupported_actions_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
@@ -373,6 +388,7 @@ test_files:
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/unsupported_actions_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb