logstash-output-elasticsearch 1.0.6-java → 1.0.7-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ac1e8c0d30366df1953643db0a74114fde945b6c
-  data.tar.gz: c204a762798c0168acd124e17610d699e73b2244
+  metadata.gz: 6859ba9736d257c039dd16c08ee0d9e6e233c914
+  data.tar.gz: f645572dac0a860dce40e09d5f81e732b37abab3
 SHA512:
-  metadata.gz: ff724d09052909f07c31175b8606ee0173fd63120ab447fbad768dcece005a59f97fd36297096b4349141f8b2ac8b97dcf86e261d45220fe01b7c180351f8ee7
-  data.tar.gz: 111550510df082e50e2b9a3fe591137a610fd75dca925f2ae032f5c7f62c03c1b8a213ae1191dbcafe3f9e1a6475abb69819475e494854d846ea3391c9bd1300
+  metadata.gz: 3fb6d1a7ecb3c40b2f376cb50f00db79e9573cd2cdad844d71e4aae3523aa76a2d2e018fa72916625d9eab414a4bfdd16733b1a8ec01add0020fe530268622ae
+  data.tar.gz: f05821d1741b5b478090cbcb328aa4e33141d2b796b0dcf02bccaea3ab7c18910564f54bdc631b23effb7063e25d4f2c1b119f8febc5b5cb6c9f9d3d924d079c

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 1.0.7
+ - Add update API support
+
 ## 1.0.6
  - Fix warning about Concurrent lib deprecation
 

data/lib/logstash/outputs/elasticsearch/protocol.rb

@@ -102,6 +102,19 @@ module LogStash::Outputs::Elasticsearch
 
       def bulk(actions)
         bulk_response = @client.bulk(:body => actions.collect do |action, args, source|
+          if action == 'update'
+            if args[:_id]
+              source = { 'doc' => source }
+              if @options[:doc_as_upsert]
+                source['doc_as_upsert'] = true
+              else
+                source['upsert'] = args[:_upsert] if args[:_upsert]
+              end
+            else
+              raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
+            end
+          end
+          args.delete(:_upsert)
           if source
             next [ { action => args }, source ]
           else
@@ -254,9 +267,21 @@ module LogStash::Outputs::Elasticsearch
             else
               raise(LogStash::ConfigurationError, "Specifying action => 'create_unless_exists' without a document '_id' is not supported.")
             end
+          when "update"
+            unless args[:_id].nil?
+              request = org.elasticsearch.action.update.UpdateRequest.new(args[:_index], args[:_type], args[:_id])
+              request.routing(args[:_routing]) if args[:_routing]
+              request.doc(source)
+              if @options[:doc_as_upsert]
+                request.docAsUpsert(true)
+              else
+                request.upsert(args[:_upsert]) if args[:_upsert]
+              end
+            else
+              raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
+            end
           else
             raise(LogStash::ConfigurationError, "action => '#{action_name}' is not currently supported.")
-          #when "update"
         end # case action
 
         request.type(args[:_type]) if args[:_type]

data/lib/logstash/outputs/elasticsearch.rb

@@ -65,7 +65,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   attr_reader :client
 
   include Stud::Buffer
-  RETRYABLE_CODES = [429, 503]
+  RETRYABLE_CODES = [409, 429, 503]
   SUCCESS_CODES = [200, 201]
 
   config_name "elasticsearch"
@@ -257,6 +257,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # - index: indexes a document (an event from Logstash).
   # - delete: deletes a document by id
   # - create: indexes a document, fails if a document by that id already exists in the index.
+  # - update: updates a document by id
   # following action is not supported by HTTP protocol
   # - create_unless_exists: creates a document, fails if no id is provided
   #
@@ -317,6 +318,14 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
   config :proxy
 
+  # Enable doc_as_upsert for update mode
+  # create a new document with source if document_id doesn't exists
+  config :doc_as_upsert, :validate => :boolean, :default => false
+
+  # Set upsert content for update mode
+  # create a new document with this parameter as json string if document_id doesn't exists
+  config :upsert, :validate => :string, :default => ""
+
   public
   def register
     @submit_mutex = Mutex.new
@@ -389,6 +398,13 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
     common_options.merge! setup_basic_auth()
 
+    # Update API setup
+    update_options = {
+      :upsert => @upsert,
+      :doc_as_upsert => @doc_as_upsert
+    }
+    common_options.merge! update_options if @action == 'update'
+
     client_class = case @protocol
       when "transport"
         LogStash::Outputs::Elasticsearch::Protocols::TransportClient
@@ -503,11 +519,16 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
              event["type"] || "logs"
            end
 
-    index = event.sprintf(@index)
+    params = {
+      :_id => @document_id ? event.sprintf(@document_id) : nil,
+      :_index => event.sprintf(@index),
+      :_type => type,
+      :_routing => @routing ? event.sprintf(@routing) : nil
+    }
+    
+    params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
 
-    document_id = @document_id ? event.sprintf(@document_id) : nil
-    routing = @routing ? event.sprintf(@routing) : nil
-    buffer_receive([event.sprintf(@action), { :_id => document_id, :_index => index, :_type => type, :_routing => routing }, event])
+    buffer_receive([event.sprintf(@action), params, event])
   end # def receive
 
   public

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '1.0.6'
+  s.version         = '1.0.7'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/integration/outputs/update_spec.rb

@@ -0,0 +1,87 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "all protocols update actions", :integration => true do
+  require "logstash/outputs/elasticsearch"
+  require "elasticsearch"
+
+  def get_es_output( protocol, id = nil, upsert = nil, doc_as_upsert=nil)
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "protocol" => protocol,
+      "host" => get_host(),
+      "port" => get_port(protocol),
+      "action" => "update"
+    }
+    settings['upsert'] = upsert unless upsert.nil?
+    settings['document_id'] = id unless id.nil?
+    settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
+    LogStash::Outputs::ElasticSearch.new(settings)
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test' }
+    )
+    @es.indices.refresh
+  end
+
+  ["node", "transport", "http"].each do |protocol|
+    context "update only with #{protocol} protocol" do
+      it "should failed without a document_id" do
+        event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
+        action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
+        subject = get_es_output(protocol)
+        subject.register
+        expect { subject.flush([action]) }.to raise_error
+      end
+
+      it "should not create new document" do
+        subject = get_es_output(protocol, "456")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+      end
+
+      it "should update existing document" do
+        subject = get_es_output(protocol, "123")
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "updated message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+        insist { r["_source"]["message"] } == 'updated message here'
+      end
+    end
+
+    context "upsert with #{protocol} protocol" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output(protocol, "456", '{"message": "upsert message"}')
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as upsert" do
+        subject = get_es_output(protocol, "456", nil, true)
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.buffer_flush(:final => true)
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'sample message here'
+      end
+    end
+  end
+end

data/spec/unit/outputs/elasticsearch/protocol_spec.rb

@@ -6,10 +6,12 @@ describe LogStash::Outputs::Elasticsearch::Protocols::NodeClient do
   context "successful" do
     it "should map correctly" do
       index_response = org.elasticsearch.action.index.IndexResponse.new("my_index", "my_type", "my_id", 123, true)
+      update_response = org.elasticsearch.action.update.UpdateResponse.new("my_index", "my_type", "my_id", 123, false)
       delete_response = org.elasticsearch.action.delete.DeleteResponse.new("my_index", "my_type", "my_id", 123, true)
       bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", index_response)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", update_response)
       bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", delete_response)
-      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_delete], 0)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
       ret = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
       insist { ret } == {"errors" => false}
     end
@@ -19,10 +21,11 @@ describe LogStash::Outputs::Elasticsearch::Protocols::NodeClient do
     it "should map correctly" do
       failure = org.elasticsearch.action.bulk.BulkItemResponse::Failure.new("my_index", "my_type", "my_id", "error message", org.elasticsearch.rest.RestStatus::BAD_REQUEST)
       bulk_item_response_index = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "index", failure)
+      bulk_item_response_update = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "update", failure)
       bulk_item_response_delete = org.elasticsearch.action.bulk.BulkItemResponse.new(32, "delete", failure)
-      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_delete], 0)
+      bulk_response = org.elasticsearch.action.bulk.BulkResponse.new([bulk_item_response_index, bulk_item_response_update, bulk_item_response_delete], 0)
       actual = LogStash::Outputs::Elasticsearch::Protocols::NodeClient.normalize_bulk_response(bulk_response)
-      insist { actual } == {"errors" => true, "statuses" => [400, 400]}
+      insist { actual } == {"errors" => true, "statuses" => [400, 400, 400]}
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-08-03 00:00:00.000000000 Z
+date: 2015-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -195,6 +195,7 @@ files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/transport_create_spec.rb
+- spec/integration/outputs/update_spec.rb
 - spec/unit/outputs/elasticsearch/protocol_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb
@@ -251,6 +252,7 @@ test_files:
 - spec/integration/outputs/secure_spec.rb
 - spec/integration/outputs/templates_spec.rb
 - spec/integration/outputs/transport_create_spec.rb
+- spec/integration/outputs/update_spec.rb
 - spec/unit/outputs/elasticsearch/protocol_spec.rb
 - spec/unit/outputs/elasticsearch_proxy_spec.rb
 - spec/unit/outputs/elasticsearch_spec.rb