logstash-output-elasticsearch 9.1.3-java → 9.1.4-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c039da495adec68b5bb57018b408430dede45c5389744d0d8b043d6210a60d08
-  data.tar.gz: e2a28d5dee5519a151fd501d61f41b1bf182dcd2f262a0b2588b6be7272c2067
+  metadata.gz: a8917f99f05252cd3a9386839e0dbdbcc5196660620a60f15b3505979f76dcde
+  data.tar.gz: f0eff97eb0951baa55a73db99131784ee9ddfd6498eab5b6d0bbf616582e728c
 SHA512:
-  metadata.gz: 4fa67d4e34cbf2d0c7de02cece71c2ea1e938a820a839878d9d07535590da657b71069312d843ef37e6e444f24149d165f61189abff7f0d4d59e1e0273d06cde
-  data.tar.gz: 908fed926ba7ccdecbd77f6e1493a811214824756e396f287ca9e82b5dd280c60b79c51299c4963bf0e1168561404a3d567a63f5f930915658a062e914289cc1
+  metadata.gz: 1832b2292c90c7c63ac4390a615abbd4215351f5949ec33d1e9a51e450af9066a09294cdb89b9bb24a9eed77d9546860eb929c80d6c515acb507b7c6bbab4397
+  data.tar.gz: 6bee0365e4003c390e098901ae6ec0f62dec19123b7cdef184bed92eecb9c1d54cf5bfc0130a202f0f8cae72c5ae1fbb3eb89cfdd7731ebba0da7afc816eff50

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 9.1.4
+  - Log an error -- not a warning -- when ES raises an invalid\_index\_name\_exception.
+
 ## 9.1.3
   - Improve plugin behavior when Elasticsearch is down on startup #758
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -213,14 +213,7 @@ module LogStash; module Outputs; class ElasticSearch;
           @logger.warn "Failed action.", status: status, action: action, response: response if !failure_type_logging_whitelist.include?(failure["type"])
           next
         elsif DOC_DLQ_CODES.include?(status)
-          action_event = action[2]
-          # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
-          if @dlq_writer
-            # TODO: Change this to send a map with { :status => status, :action => action } in the future
-            @dlq_writer.write(action_event, "Could not index event to Elasticsearch. status: #{status}, action: #{action}, response: #{response}")
-          else
-            @logger.warn "Could not index event to Elasticsearch.", status: status, action: action, response: response
-          end
+          handle_dlq_status("Could not index event to Elasticsearch.", action, status, response)
           @document_level_metrics.increment(:non_retryable_failures)
           next
         else
@@ -234,6 +227,22 @@ module LogStash; module Outputs; class ElasticSearch;
       actions_to_retry
     end
 
+    def handle_dlq_status(message, action, status, response)
+      # To support bwc, we check if DLQ exists. otherwise we log and drop event (previous behavior)
+      if @dlq_writer
+        # TODO: Change this to send a map with { :status => status, :action => action } in the future
+        @dlq_writer.write(action[2], "#{message} status: #{status}, action: #{action}, response: #{response}")
+      else
+        error_type = response.fetch('index', {}).fetch('error', {})['type']
+        if 'invalid_index_name_exception' == error_type
+          level = :error
+        else
+          level = :warn
+        end
+        @logger.send level, message, status: status, action: action, response: response
+      end
+    end
+
     # Determine the correct value for the 'type' field for the given event
     DEFAULT_EVENT_TYPE_ES6="doc".freeze
     DEFAULT_EVENT_TYPE_ES7="_doc".freeze

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '9.1.3'
+  s.version         = '9.1.4'
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/es_spec_helper.rb

@@ -2,6 +2,12 @@ require "logstash/devutils/rspec/spec_helper"
 require 'manticore'
 require 'elasticsearch'
 
+# by default exclude secure_integration tests unless requested
+# normal integration specs are already excluded by devutils' spec helper
+RSpec.configure do |config|
+   config.filter_run_excluding config.exclusion_filter.add(:secure_integration => true)
+end
+
 module ESHelper
   def get_host_port
     "127.0.0.1:9200"
@@ -17,6 +23,10 @@ module ESHelper
 
   def self.es_version_satisfies?(*requirement)
     es_version = RSpec.configuration.filter[:es_version] || ENV['ES_VERSION']
+    if es_version.nil?
+      puts "Info: ES_VERSION environment or 'es_version' tag wasn't set. Returning false to all `es_version_satisfies?` call."
+      return false
+    end
     es_release_version = Gem::Version.new(es_version).release
     Gem::Requirement.new(requirement).satisfied_by?(es_release_version)
   end

data/spec/integration/outputs/index_spec.rb

@@ -129,7 +129,7 @@ describe "indexing" do
     it_behaves_like("an indexer")
   end
 
-  describe "a secured indexer", :elasticsearch_secure => true do
+  describe "a secured indexer", :secure_integration => true do
     let(:user) { "simpleuser" }
     let(:password) { "abc123" }
     let(:cacert) { "spec/fixtures/test_certs/test.crt" }

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -17,7 +17,7 @@ describe LogStash::Outputs::ElasticSearch do
       allow(subject.client.pool).to receive(:start_resurrectionist)
       allow(subject.client.pool).to receive(:start_sniffer)
       allow(subject.client.pool).to receive(:healthcheck!)
-      allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
+      allow(subject.client).to receive(:maximum_seen_major_version).at_least(:once).and_return(maximum_seen_major_version)
       subject.client.pool.adapter.manticore.respond_with(:body => "{}")
     end
   end
@@ -358,6 +358,7 @@ describe LogStash::Outputs::ElasticSearch do
       let(:options) { super.merge("action" => "index") }
 
       it "should not set the retry_on_conflict parameter when creating an event_action_tuple" do
+        allow(subject.client).to receive(:maximum_seen_major_version).and_return(maximum_seen_major_version)
         action, params, event_data = subject.event_action_tuple(event)
         expect(params).not_to include({:_retry_on_conflict => num_retries})
       end
@@ -498,4 +499,60 @@ describe LogStash::Outputs::ElasticSearch do
       end
     end
   end
+
+  context 'handling elasticsearch document-level status meant for the DLQ' do
+    let(:options) { { "manage_template" => false } }
+    let(:logger)  { subject.instance_variable_get(:@logger) }
+
+    context 'when @dlq_writer is nil' do
+      before { subject.instance_variable_set '@dlq_writer', nil }
+
+      context 'resorting to previous behaviour of logging the error' do
+        context 'getting an invalid_index_name_exception' do
+          it 'should log at ERROR level' do
+            expect(logger).to receive(:error).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => { 'error' => { 'type' => 'invalid_index_name_exception' } } }
+            subject.handle_dlq_status("Could not index event to Elasticsearch.",
+              [:action, :params, :event], :some_status, mock_response)
+          end
+        end
+
+        context 'when getting any other exception' do
+          it 'should log at WARN level' do
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
+            subject.handle_dlq_status("Could not index event to Elasticsearch.",
+              [:action, :params, :event], :some_status, mock_response)
+          end
+        end
+
+        context 'when the response does not include [error]' do
+          it 'should not fail, but just log a warning' do
+            expect(logger).to receive(:warn).with(/Could not index/, hash_including(:status, :action, :response))
+            mock_response = { 'index' => {} }
+            expect do
+              subject.handle_dlq_status("Could not index event to Elasticsearch.",
+                [:action, :params, :event], :some_status, mock_response)
+            end.to_not raise_error
+          end
+        end
+      end
+    end
+
+    # DLQ writer always nil, no matter what I try here. So mocking it all the way
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      before { subject.instance_variable_set('@dlq_writer', dlq_writer) }
+
+      # Note: This is not quite the desired behaviour.
+      # We should still log when sending to the DLQ.
+      # This shall be solved by another issue, however: logstash-output-elasticsearch#772
+      it 'should send the event to the DLQ instead, and not log' do
+        expect(dlq_writer).to receive(:write).with(:event, /Could not index/)
+        mock_response = { 'index' => { 'error' => { 'type' => 'illegal_argument_exception' } } }
+        subject.handle_dlq_status("Could not index event to Elasticsearch.",
+          [:action, :params, :event], :some_status, mock_response)
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 9.1.3
+  version: 9.1.4
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-05-15 00:00:00.000000000 Z
+date: 2018-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement