logstash-output-elasticsearch 11.9.0-java → 11.9.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ad30f3f3af7faaebf3409ac23375e99531def7d30940330f0f617e007f341b1d
-  data.tar.gz: d06a9954211995b266d08cfbc6586164275d5d269ef5716ba18d4f7e3ca4cf5c
+  metadata.gz: 4dbc44fb5e78b53368adb97d43b613c4a3c843e1a1308aa6d008cb105ee4301c
+  data.tar.gz: 9ed115a974e20bda89d63d6b2bf37dc5c7c42f8f40b46badbad072db0a698683
 SHA512:
-  metadata.gz: ad84072b2dcc3d3a567b18cc89bbd3390e93303e952188053a8f61cbefbddb0a8bf46453fb8df0b94cecaf29304452b85fbe7a822bb9ece0763c4278b514ff91
-  data.tar.gz: 348d53c674b7b2730ce918403e705ddbda5f9745ca2564e922d3762bff9de7a455ff871c269669dc1e74d5e0c5f89c7997221eca4c851a073b1540b2a8bbdc15
+  metadata.gz: 9df0d61b80c78cd992b46428b24836db44094d636384940e976a0a09a58245bec59def6031d8426a24d41bdb0644ac49d4e8b558ef32c684554fbf03104a148c
+  data.tar.gz: 8c63cfaa83e3acad5d864dbf2d170fcf0e9111f31e8053fc2ce2a8a4d8af696355fda7b3e7152dc0764bce8e36c3afdd432c5a344f97a2b9b4b6c6c17e6a52d9

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 11.9.1
+ - Fixes a possible infinite-retry-loop that could occur when this plugin is configured with an `action` whose value contains a [sprintf-style placeholder][] that fails to be resolved for an individual event. Events in this state will be routed to the pipeline's [dead letter queue][DLQ] if it is available, or will be logged-and-dropped so that the remaining events in the batch can be processed [#1080](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1080)
+ 
+[sprintf-style placeholder]: https://www.elastic.co/guide/en/logstash/current/event-dependent-configuration.html#sprintf
+[DLQ]: https://www.elastic.co/guide/en/logstash/current/dead-letter-queues.html
+
 ## 11.9.0
  - Feature: force unresolved dynamic index names to be sent into DLQ. This feature could be explicitly disabled using `dlq_on_failed_indexname_interpolation` setting [#1084](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1084)
 

data/README.md

@@ -19,7 +19,7 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 
 ## Developing
 
-### 1. Plugin Developement and Testing
+### 1. Plugin Development and Testing
 
 #### Code
 - To get started, you'll need JRuby with the Bundler gem installed.

data/docs/index.asciidoc

@@ -395,7 +395,7 @@ The Elasticsearch action to perform. Valid actions are:
   document if not already present. See the `doc_as_upsert` option. NOTE: This does not work and is not supported
   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
 - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
-  would use the foo field for the action
+  would use the foo field for the action. If resolved action is not in [`index`, `delete`, `create`, `update`], the event will not be sent to {es}, and instead either will be sent to the pipeline's <<dead-letter-queues,dead letter queue (DLQ)>> if it is enabled, or will be logged and dropped.
 
 For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bulk API documentation].
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -120,6 +120,7 @@ module LogStash; module Outputs; class ElasticSearch;
       else
         stream_writer = body_stream
       end
+
       bulk_responses = []
       batch_actions = []
       bulk_actions.each_with_index do |action, index|
@@ -142,13 +143,16 @@ module LogStash; module Outputs; class ElasticSearch;
         stream_writer.write(as_json)
         batch_actions << action
       end
+
       stream_writer.close if http_compression
+
       logger.debug("Sending final bulk request for batch.",
                    :action_count => batch_actions.size,
                    :payload_size => stream_writer.pos,
                    :content_length => body_stream.size,
                    :batch_offset => (actions.size - batch_actions.size))
       bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
+
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end

data/lib/logstash/outputs/elasticsearch.rb

@@ -367,36 +367,41 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     wait_for_successful_connection if @after_successful_connection_done
     events_mapped = safe_interpolation_map_events(events)
     retrying_submit(events_mapped.successful_events)
-    unless events_mapped.failed_events.empty?
-      @logger.error("Can't map some events, needs to be handled by DLQ #{events_mapped.failed_events}")
-      send_failed_resolutions_to_dlq(events_mapped.failed_events)
+    unless events_mapped.event_mapping_errors.empty?
+      handle_event_mapping_errors(events_mapped.event_mapping_errors)
     end
   end
 
-  # @param: Arrays of EventActionTuple
+  # @param: Arrays of FailedEventMapping
   private
-  def send_failed_resolutions_to_dlq(failed_action_tuples)
-    failed_action_tuples.each do |action|
-      handle_dlq_status(action, "warn", "Could not resolve dynamic index")
+  def handle_event_mapping_errors(event_mapping_errors)
+    # if DQL is enabled, log the events to provide issue insights to users.
+    if @dlq_writer
+      @logger.warn("Events could not be indexed and routing to DLQ, count: #{event_mapping_errors.size}")
     end
+
+    event_mapping_errors.each do |event_mapping_error|
+      detailed_message = "#{event_mapping_error.message}; event: `#{event_mapping_error.event.to_hash_with_metadata}`"
+      handle_dlq_status(event_mapping_error.event, :warn, detailed_message)
+    end
+    @document_level_metrics.increment(:non_retryable_failures, event_mapping_errors.size)
   end
 
-  MapEventsResult = Struct.new(:successful_events, :failed_events)
+  MapEventsResult = Struct.new(:successful_events, :event_mapping_errors)
+  FailedEventMapping = Struct.new(:event, :message)
 
   private
   def safe_interpolation_map_events(events)
     successful_events = [] # list of LogStash::Outputs::ElasticSearch::EventActionTuple
-    failed_events = [] # list of LogStash::Event
+    event_mapping_errors = [] # list of FailedEventMapping
     events.each do |event|
-       begin
+      begin
         successful_events << @event_mapper.call(event)
-       rescue IndexInterpolationError, e
-         action = event.sprintf(@action || 'index')
-         event_action_tuple = EventActionTuple.new(action, [], event)
-         failed_events << event_action_tuple
+      rescue EventMappingError => ie
+        event_mapping_errors << FailedEventMapping.new(event, ie.message)
        end
     end
-    MapEventsResult.new(successful_events, failed_events)
+    MapEventsResult.new(successful_events, event_mapping_errors)
   end
 
   public
@@ -449,6 +454,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
     end
 
     action = event.sprintf(@action || 'index')
+    raise UnsupportedActionError, action unless VALID_HTTP_ACTIONS.include?(action)
 
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
@@ -476,13 +482,21 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   end
 
-  class IndexInterpolationError < ArgumentError
-    attr_reader :bad_formatted_index
+  class EventMappingError < ArgumentError
+    def initialize(msg = nil)
+      super
+    end
+  end
 
+  class IndexInterpolationError < EventMappingError
     def initialize(bad_formatted_index)
       super("Badly formatted index, after interpolation still contains placeholder: [#{bad_formatted_index}]")
+    end
+  end
 
-      @bad_formatted_index = bad_formatted_index
+  class UnsupportedActionError < EventMappingError
+    def initialize(bad_action)
+      super("Elasticsearch doesn't support [#{bad_action}] action")
     end
   end
 

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -214,13 +214,13 @@ module LogStash; module PluginMixins; module ElasticSearch
       
       log_level = dig_value(response, 'index', 'error', 'type') == 'invalid_index_name_exception' ? :error : :warn
       
-      handle_dlq_status(action, log_level, detailed_message)
+      handle_dlq_status(action.event, log_level, detailed_message)
     end
 
-    def handle_dlq_status(action, log_level, message)
+    def handle_dlq_status(event, log_level, message)
       # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
       if @dlq_writer
-        @dlq_writer.write(action.event, "#{message}")
+        @dlq_writer.write(event, "#{message}")
       else
         @logger.send log_level, message
       end
@@ -259,7 +259,6 @@ module LogStash; module PluginMixins; module ElasticSearch
         status = action_props["status"]
         error  = action_props["error"]
         action = actions[idx]
-        action_params = action[1]
 
         # Retry logic: If it is success, we move on. If it is a failure, we have 3 paths:
         # - For 409, we log and drop. there is nothing we can do

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '11.9.0'
+  s.version         = '11.9.1'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -33,6 +33,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'cabin', ['~> 0.6']
   s.add_development_dependency 'webrick'
   s.add_development_dependency 'webmock'
+  s.add_development_dependency 'rspec-collection_matchers'
   # Still used in some specs, we should remove this ASAP
   s.add_development_dependency 'elasticsearch'
 end

data/spec/integration/outputs/index_spec.rb

@@ -137,7 +137,7 @@ describe "indexing with sprintf resolution", :integration => true do
     before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
 
     it "should doesn't create an index name with unresolved placeholders" do
-      expect(dlq_writer).to receive(:write).once.with(event, /Could not resolve dynamic index/)
+      expect(dlq_writer).to receive(:write).once.with(event, a_string_including("Badly formatted index, after interpolation still contains placeholder"))
       subject.multi_receive(events)
 
       escaped_index_name = CGI.escape("%{[index_name]}_dynamic")

data/spec/integration/outputs/unsupported_actions_spec.rb

@@ -0,0 +1,75 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Unsupported actions testing...", :integration => true do
+  require "logstash/outputs/elasticsearch"
+
+  INDEX = "logstash-unsupported-actions-rejected"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => INDEX,
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "%{action_field}",
+      "document_id" => "%{doc_id}",
+      "ecs_compatibility" => "disabled"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    # index single doc for update purpose
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "2",
+      :body => { :message => 'Test to doc indexing', :counter => 1 }
+    )
+    @es.index(
+      :index => INDEX,
+      :type => doc_type,
+      :id => "3",
+      :body => { :message => 'Test to doc deletion', :counter => 2 }
+    )
+    @es.indices.refresh
+  end
+
+  context "multiple actions include unsupported action" do
+    let(:events) {[
+      LogStash::Event.new("action_field" => "index", "doc_id" => 1, "message"=> "hello"),
+      LogStash::Event.new("action_field" => "update", "doc_id" => 2, "message"=> "hi"),
+      LogStash::Event.new("action_field" => "delete", "doc_id" => 3),
+      LogStash::Event.new("action_field" => "unsupported_action", "doc_id" => 4, "message"=> "world!")
+    ]}
+
+    it "should reject unsupported doc" do
+      subject = get_es_output
+      subject.register
+      subject.multi_receive(events)
+
+      index_or_update = proc do |event|
+        action = event.get("action_field")
+        action.eql?("index") || action.eql?("update")
+      end
+
+      indexed_events = events.select { |event| index_or_update.call(event) }
+      rejected_events = events.select { |event| !index_or_update.call(event) }
+
+      indexed_events.each do |event|
+        response = @es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)
+        expect(response['_source']['message']).to eq(event.get("message"))
+      end
+
+      rejected_events.each do |event|
+        expect {@es.get(:index => INDEX, :type => doc_type, :id => event.get("doc_id"), :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -266,6 +266,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
             end
           end
         end
+
        end
      end
   end

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -3,8 +3,8 @@ require "base64"
 require "flores/random"
 require 'concurrent/atomic/count_down_latch'
 require "logstash/outputs/elasticsearch"
-
 require 'logstash/plugin_mixins/ecs_compatibility_support/spec_helper'
+require 'rspec/collection_matchers'
 
 describe LogStash::Outputs::ElasticSearch do
   subject(:elasticsearch_output_instance) { described_class.new(options) }
@@ -347,7 +347,7 @@ describe LogStash::Outputs::ElasticSearch do
                               }
                     },
                     # NOTE: this is an artificial success (usually everything fails with a 500) but even if some doc where
-                    # to succeed due the unexpected reponse items we can not clearly identify which actions to retry ...
+                    # to succeed due the unexpected response items we can not clearly identify which actions to retry ...
                     {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>201}},
                     {"index"=>{"_index"=>"bar2", "_type"=>"_doc", "_id"=>nil, "status"=>500,
                                "error"=>{"type" => "illegal_state_exception",
@@ -381,6 +381,104 @@ describe LogStash::Outputs::ElasticSearch do
         subject.multi_receive(events)
       end
     end
+
+    context "unsupported actions" do
+      let(:options) { super().merge("index" => "logstash", "action" => "%{action_field}") }
+
+      context "with multiple valid actions with one trailing invalid action" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 3, "message"=> "bye"),
+          LogStash::Event.new("action_field" => "unsupported_action", "id" => 4, "message"=> "world!")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action] action")
+          end
+        end
+      end
+
+      context "with one leading invalid action followed by multiple valid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "unsupported_action", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "index", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action] action")
+          end
+        end
+      end
+
+      context "with batch of multiple invalid actions and no valid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "unsupported_action1", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "unsupported_action2", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "unsupported_action4", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have(:no).items
+          event_result.successful_events.each do |action, _|
+            expect(action).to_not eql("unsupported_action")
+          end
+          expect(event_result.event_mapping_errors).to have_exactly(4).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to include "Elasticsearch doesn't support"
+          end
+        end
+      end
+
+      context "with batch of intermixed valid and invalid actions" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "unsupported_action2", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported actions" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(2).items
+          expect(event_result.event_mapping_errors).to have_exactly(2).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to include "Elasticsearch doesn't support"
+          end
+        end
+      end
+
+      context "with batch of exactly one action that is invalid" do
+        let(:events) {[
+          LogStash::Event.new("action_field" => "index", "id" => 1, "message"=> "world!"),
+          LogStash::Event.new("action_field" => "index", "id" => 2, "message"=> "hello"),
+          LogStash::Event.new("action_field" => "unsupported_action3", "id" => 3, "message"=> "hi"),
+          LogStash::Event.new("action_field" => "index", "id" => 4, "message"=> "bye")
+        ]}
+        it "rejects unsupported action" do
+          event_result = subject.send(:safe_interpolation_map_events, events)
+          expect(event_result.successful_events).to have_exactly(3).items
+          expect(event_result.event_mapping_errors).to have_exactly(1).items
+          event_result.event_mapping_errors.each do |event_mapping_error|
+            expect(event_mapping_error.message).to eql("Elasticsearch doesn't support [unsupported_action3] action")
+          end
+        end
+      end
+    end
   end
 
   context '413 errors' do
@@ -772,6 +870,7 @@ describe LogStash::Outputs::ElasticSearch do
 
     context 'when @dlq_writer is nil' do
       before { subject.instance_variable_set '@dlq_writer', nil }
+      let(:action) { LogStash::Outputs::ElasticSearch::EventActionTuple.new(:action, :params, LogStash::Event.new("foo" => "bar")) }
 
       context 'resorting to previous behaviour of logging the error' do
         context 'getting an invalid_index_name_exception' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 11.9.0
+  version: 11.9.1
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-09-20 00:00:00.000000000 Z
+date: 2022-09-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -196,6 +196,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-collection_matchers
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -288,6 +302,7 @@ files:
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/unsupported_actions_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb
@@ -373,6 +388,7 @@ test_files:
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
 - spec/integration/outputs/templates_spec.rb
+- spec/integration/outputs/unsupported_actions_spec.rb
 - spec/integration/outputs/update_spec.rb
 - spec/spec_helper.rb
 - spec/support/elasticsearch/api/actions/delete_ilm_policy.rb