logstash-output-elasticsearch 4.1.1-java → 4.1.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0063a188b7bd2e7bc419042bca990a04b7680b7a
-  data.tar.gz: 0a029a7275a7eba0e201457275f4490074ed92f1
+  metadata.gz: d34b7350d64567ba86294e3497d0d37216df93b0
+  data.tar.gz: 681dbbb922820129b82c483b2d108da395191f5a
 SHA512:
-  metadata.gz: 873f0c289ed656c300e0e2df2efee4cdb8f10efc7b6f5fd558715fc02ce02327d39be37434582c2704d2379a02d8c50b221cf8434a1025ee85e6a507b0f73629
-  data.tar.gz: 04a9a3a18264487cfda5328fecdb01737dcb5dac27b2c08e6db2f5de17ba56cbee1d3cb81739850cccf4142de5f5d2d7c9c9e65a593c70254d6833a155cb5786
+  metadata.gz: a3bb076147070c2f3e95da532f5203647e3b40ab3f25af3887cd36a2844634de890319d711bd9edc8167dcdb6aa996242e49ea5f50bf76512c608437087d1fc6
+  data.tar.gz: 50072bff0779e28344c7e7a5b686d53eac151b68403a6f6061b7d99b8964f10879c215578c0492488aa61ff78043b8c6676d72572bbd9f78f8af0fd7c4aeed5f

data/CHANGELOG.md

@@ -1,5 +1,8 @@
-## 4.1.1
+## 4.1.2
+
+- Added a configuration called failure_type_logging_whitelist which takes a list of strings, that are error types from elasticsearch, so we prevent logging WARN if elasticsearch fails with that action. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/423
 
+## 4.1.1
 - Fix bug where setting credentials would cause fatal errors. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/441
 
 ## 4.1.0

data/CONTRIBUTORS

@@ -25,6 +25,7 @@ Contributors:
 * Tal Levy (talevy)
 * Tom Hodder (tolland)
 * jimmyjones2
+* Gabriel Moskovicz (gmoskovicz)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/outputs/elasticsearch.rb

@@ -67,7 +67,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # - delete: deletes a document by id (An id is required for this action)
   # - create: indexes a document, fails if a document by that id already exists in the index.
   # - update: updates a document by id. Update has a special case where you can upsert -- update a
-  #   document if not already present. See the `upsert` option
+  #   document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
+  #   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
@@ -132,6 +133,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # a timeout occurs, the request will be retried.
   config :timeout, :validate => :number
 
+  # Set the Elasticsearch errors in the whitelist that you don't want to log. 
+  # A useful example is when you want to skip all 409 errors
+  # which are `document_already_exists_exception`.
+  config :failure_type_logging_whitelist, :validate => :array, :default => []
+
   # While the output tries to reuse connections efficiently we have a maximum.
   # This sets the maximum number of open connections the output will create.
   # Setting this too low may mean frequently closing / opening connections

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -112,17 +112,18 @@ module LogStash; module Outputs; class ElasticSearch;
       actions_to_retry = []
       bulk_response["items"].each_with_index do |response,idx|
         action_type, action_props = response.first
+
         status = action_props["status"]
-        error  = action_props["error"]
+        failure  = action_props["error"]
         action = actions[idx]
 
         if SUCCESS_CODES.include?(status)
           next
         elsif RETRYABLE_CODES.include?(status)
-          @logger.info "retrying failed action with response code: #{status} (#{error})"
+          @logger.info "retrying failed action with response code: #{status} (#{failure})"
           actions_to_retry << action
-        else
-          @logger.warn "Failed action. ", status: status, action: action, response: response
+        elsif !failure_type_logging_whitelist.include?(failure["type"])
+          @logger.warn "Failed action.", status: status, action: action, response: response
         end
       end
 
@@ -214,7 +215,7 @@ module LogStash; module Outputs; class ElasticSearch;
           sleep_interval = sleep_for_interval(sleep_interval)
           retry unless @stopping.true?
         else
-          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.code)
+          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code)
         end
       rescue => e
         # Stuff that should never happen

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '4.1.1'
+  s.version         = '4.1.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/routing_spec.rb

@@ -12,7 +12,7 @@ shared_examples "a routing indexer" do
     before do
       subject.register
       event_count.times do
-        subject.multi_receive([LogStash::Event.new("message" => "Hello World!", "type" => type)])
+        subject.multi_receive([LogStash::Event.new("message" => "test", "type" => type)])
       end
     end
 
@@ -26,6 +26,7 @@ shared_examples "a routing indexer" do
       # Wait until all events are available.
       Stud::try(10.times) do
         data = ""
+
         response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
         response.read_body { |chunk| data << chunk }
         result = LogStash::Json.load(data)

data/spec/integration/outputs/templates_spec.rb

@@ -75,9 +75,7 @@ describe "index template expected behavior", :integration => true do
   end
 
   it "make [geoip][location] a geo_point" do
-    results = @es.search(:body => { "query" => { "bool" => { "must" => { "match_all" => {} }, "filter" => { "geo_distance" => { "distance" => "1000km", "geoip.location" => { "lat" => 0.5, "lon" => 0.5 } } } } } })
-    insist { results["hits"]["total"] } == 1
-    insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
+    expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
   end
 
   it "aggregate .raw results correctly " do

data/spec/integration/outputs/update_spec.rb

@@ -1,6 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "Update actions", :integration => true do
+describe "Update actions", :integration => true, :version_2x_plus => true do
   require "logstash/outputs/elasticsearch"
 
   def get_es_output( options={} )
@@ -102,7 +102,7 @@ describe "Update actions", :integration => true do
         'script_type' => 'inline'
       })
       subject.register
-      subject.multi_receive([LogStash::Event.new("count" => 3 )])
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
       insist { r["_source"]["counter"] } == 4
     end

data/spec/unit/outputs/error_whitelist_spec.rb

@@ -0,0 +1,52 @@
+require "logstash/outputs/elasticsearch"
+require_relative "../../../spec/es_spec_helper"
+
+describe "whitelisting error types in expected behavior" do
+  let(:template) { '{"template" : "not important, will be updated by :index"}' }
+  let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
+  let(:action1) { ["index", {:_id=>1, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
+  let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
+
+  subject { LogStash::Outputs::ElasticSearch.new(settings) }
+
+  before :each do
+    allow(subject.logger).to receive(:warn)
+
+    subject.register
+
+    allow(subject.client).to receive(:bulk).and_return(
+      {
+        "errors" => true,
+        "items" => [{
+          "create" => {
+            "status" => 409, 
+            "error" => {
+              "type" => "document_already_exists_exception",
+              "reason" => "[shard] document already exists"
+            }
+          }
+        }]
+      })
+
+    subject.multi_receive([event1])
+  end
+
+  after :each do
+    subject.close
+  end
+
+  describe "when failure logging is enabled for everything" do
+    it "should log a failure on the action" do
+      expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+  describe "when failure logging is disabled for docuemnt exists error" do
+    let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
+
+    it "should log a failure on the action" do
+      expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 4.1.1
+  version: 4.1.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-06-24 00:00:00.000000000 Z
+date: 2016-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -205,6 +205,7 @@ files:
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
+- spec/unit/outputs/error_whitelist_spec.rb
 homepage: http://logstash.net/
 licenses:
 - apache-2.0
@@ -253,3 +254,4 @@ test_files:
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
 - spec/unit/outputs/elasticsearch_ssl_spec.rb
+- spec/unit/outputs/error_whitelist_spec.rb