logstash-output-elasticsearch 0.1.18-java → 0.1.19-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6c15f51be4dc3f93e48d5d7e01e1ee395841778b
-  data.tar.gz: 321c5c48cb9ce8affcd7c9f59c2b8e608e97bdc8
+  metadata.gz: 88af0f35fb188226ed8a3af96835a57aef17cde6
+  data.tar.gz: 4e826e7a0dc993a21ca18a894d59fc2a73670f3d
 SHA512:
-  metadata.gz: a3a3269e0803283108adff84fdbc491ff751f9a76a60970222201855bff5cfbc5c94c3d5418f76ef527a9d835220900272e066630a87246ab3c259a7b6119834
-  data.tar.gz: a06952883180222eef76c94b6b612ec937d5cd6557bcac9d84de00814f9ee1207ba074de53046651ede0dd392481510c013894c0a7b1b0f8e57035d66e64b4d8
+  metadata.gz: 542687e54823d84d895ac468a484ba80f76ff7f31909e424d26f065db1d5b167c20950f7003b6722b39ef9d6eb6aed9a546c5366ef1d249c5daf0b1fc992180b
+  data.tar.gz: f86b66b834d212bbd43ad77b48c273e1e960ff57c216c3b1129edfc1591d3377e660753d084f262cfc391a3999c1b8ae380a7a1cb42460fccb4109c20e384f7c

data/lib/logstash/outputs/elasticsearch.rb

@@ -194,11 +194,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # For more details on actions, check out the http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/docs-bulk.html[Elasticsearch bulk API documentation]
   config :action, :validate => :string, :default => "index"
 
-  # Username and password (HTTP only)
+  # Username and password (only valid when protocol is HTTP; this setting works with HTTP or HTTPS auth)
   config :user, :validate => :string
   config :password, :validate => :password
 
-  # SSL Configurations (HTTP only)
+  # SSL Configurations (only valid when protocol is HTTP)
   #
   # Enable SSL
   config :ssl, :validate => :boolean, :default => false
@@ -425,7 +425,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
           @logger.warn "failed action with response of #{resp_code}, dropping action: #{action}"
         end
       end
-      retry_push(actions_to_retry)
+      retry_push(actions_to_retry) unless actions_to_retry.empty?
     end
   end
 

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -78,6 +78,24 @@ module LogStash::Outputs::Elasticsearch
         Elasticsearch::Client.new client_options
       end
 
+      def self.normalize_bulk_response(bulk_response)
+        if bulk_response["errors"]
+          # The structure of the response from the REST Bulk API is follows:
+          # {"took"=>74, "errors"=>true, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+          #                                                    "_type"=>"logs",
+          #                                                    "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+          #                                                    "_version"=>1,
+          #                                                    "status"=>400,
+          #                                                    "error"=>"MapperParsingException[failed to parse]..."}}]}
+          # where each `item` is a hash of {OPTYPE => Hash[]}. calling first, will retrieve 
+          # this hash as a single array with two elements, where the value is the second element (i.first[1])
+          # then the status of that item is retrieved.
+          {"errors" => true, "statuses" => bulk_response["items"].map { |i| i.first[1]['status'] }}
+        else
+          {"errors" => false}
+        end
+      end
+
       def bulk(actions)
         bulk_response = @client.bulk(:body => actions.collect do |action, args, source|
           if source
@@ -86,11 +104,8 @@ module LogStash::Outputs::Elasticsearch
             next { action => args }
           end
         end.flatten)
-        if bulk_response["errors"]
-          return {"errors" => true, "statuses" => bulk_response["items"].map { |i| i["status"] }}
-        else
-          return {"errors" => false}
-        end
+
+        self.class.normalize_bulk_response(bulk_response)
       end # def bulk
 
       def template_exists?(name)
@@ -187,6 +202,16 @@ module LogStash::Outputs::Elasticsearch
         return nodebuilder.settings(@settings).node.client
       end # def build_client
 
+      def self.normalize_bulk_response(bulk_response)
+        # TODO(talevy): parse item response objects to retrieve correct 200 (OK) or 201(created) status codes
+        if bulk_response.has_failures()
+          {"errors" => true,
+           "statuses" => bulk_response.map { |i| (i.is_failed && i.get_failure.get_status.get_status) || 200 }}
+        else
+          {"errors" => false}
+        end
+      end
+
       def bulk(actions)
         # Actions an array of [ action, action_metadata, source ]
         prep = @client.prepareBulk
@@ -195,14 +220,7 @@ module LogStash::Outputs::Elasticsearch
         end
         response = prep.execute.actionGet()
 
-        if response.has_failures()
-          return {"errors" => true,
-                  "statuses" => response.map { |i| (i.is_failed && i.get_failure.get_status.get_status) || 200 }}
-        else
-          return {"errors" => false}
-        end
-        # returns 200 for all successful actions, represents 201 & 200
-        # TODO(talevy): parse item response objects to retrieve correct 200 (OK) or 201(created) status codes
+        self.class.normalize_bulk_response(response)
       end # def bulk
 
       def build_request(action, args, source)

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '0.1.18'
+  s.version         = '0.1.19'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/outputs/elasticsearch/protocol_spec.rb

@@ -0,0 +1,52 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch/protocol"
+require "java"
+
+describe LogStash::Outputs::Elasticsearch::Protocols::NodeClient do
+  context "successful" do
+    it "should map correctly" do
+      index_response = org.elasticsearch.action.index.IndexResponse.new("my_index", "my_type", "my_id", 123, true)
+      delete_response = org.elasticsearch.action.delete.DeleteResponse.new("my_index", "my_type", "my_id", 123, true)
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", index_response)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", delete_response)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_delete], 0)
+      ret = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { ret } == {"errors" => false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      failure = org.elasticsearch.action.bulk.BulkItemResponse::Failure.new("my_index", "my_type", "my_id", "error message", org.elasticsearch.rest.RestStatus::BAD_REQUEST)
+      bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", failure)
+      bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", failure)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_delete], 0)
+      actual = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors" => true, "statuses" => [400, 400]}
+    end
+  end
+end
+
+describe LogStash::Outputs::Elasticsearch::Protocols::HTTPClient do
+  context "successful" do
+    it "should map correctly" do
+      bulk_response = {"took"=>74, "errors"=>false, "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                                                          "_type"=>"logs", "_id"=>"AUxTS2C55Jrgi-hC6rQF",
+                                                                          "_version"=>1, "status"=>201}}]} 
+      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> false}
+    end
+  end
+
+  context "contains failures" do
+    it "should map correctly" do
+      bulk_response = {"took"=>71, "errors"=>true,
+                       "items"=>[{"create"=>{"_index"=>"logstash-2014.11.17",
+                                             "_type"=>"logs", "_id"=>"AUxTQ_OI5Jrgi-hC6rQB", "status"=>400,
+                                             "error"=>"MapperParsingException[failed to parse]..."}}]}
+      actual = LogStash::Outputs::Elasticsearch::Protocols::HTTPClient.normalize_bulk_response(bulk_response)
+      insist { actual } == {"errors"=> true, "statuses"=> [400]}
+    end
+  end
+end
+

data/spec/outputs/elasticsearch_spec.rb

@@ -478,6 +478,7 @@ describe "outputs/elasticsearch" do
     let(:action1) { ["index", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
     let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
     let(:action2) { ["index", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
+    let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
     let(:max_retries) { 3 }
 
     def mock_actions_with_response(*resp)
@@ -581,6 +582,32 @@ describe "outputs/elasticsearch" do
           subject.buffer_flush(:final => true)
           sleep(3)
         end
+
+        it "non-retryable errors like mapping errors (400) should be dropped and not be retried (unfortunetly)" do
+          subject.register
+          subject.receive(invalid_event)
+          expect(subject).not_to receive(:retry_push)
+          subject.buffer_flush(:final => true)
+
+          @es.indices.refresh
+          Stud::try(10.times) do
+            r = @es.search
+            insist { r["hits"]["total"] } == 0
+          end
+        end
+
+        it "successful requests should not be appended to retry queue" do
+          subject.register
+          subject.receive(event1)
+          expect(subject).not_to receive(:retry_push)
+          subject.buffer_flush(:final => true)
+
+          @es.indices.refresh
+          Stud::try(10.times) do
+            r = @es.search
+            insist { r["hits"]["total"] } == 1
+          end
+        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 0.1.18
+  version: 0.1.19
 platform: java
 authors:
 - Elasticsearch
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-03-02 00:00:00.000000000 Z
+date: 2015-03-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -212,6 +212,7 @@ files:
 - lib/logstash/outputs/elasticsearch/elasticsearch-template.json
 - lib/logstash/outputs/elasticsearch/protocol.rb
 - logstash-output-elasticsearch.gemspec
+- spec/outputs/elasticsearch/protocol_spec.rb
 - spec/outputs/elasticsearch_spec.rb
 homepage: http://logstash.net/
 licenses:
@@ -241,4 +242,5 @@ signing_key:
 specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
+- spec/outputs/elasticsearch/protocol_spec.rb
 - spec/outputs/elasticsearch_spec.rb