logstash-output-elasticsearch 2.3.2-java → 2.4.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6adf3c643188045a088ba2644b722587058c4ede
-  data.tar.gz: bcc360c9005ac58487c1b77bb34656c34f94dc07
+  metadata.gz: addba3dc004cd06ac2c27a913a8339d9f6c3568b
+  data.tar.gz: ad5076aabc199484fa42d8f95107e22df94b661f
 SHA512:
-  metadata.gz: 6219cecf9a5087d54264f7554361f1bed8ead48f4d86f509f39ec47a421b629d4ecbadd2df171ac39bedbaae6c8d5f7f0b46183e920a96bf8ffdbc5850762b3a
-  data.tar.gz: 5159a07fcc22e82f4bc6c711f51d95e46cdca302758bb6f51e19dc1897ba4c2df2243bba3b6adcd7372e41c371cbaf7c3f5cade6d99a15d21e342b9b5cca1377
+  metadata.gz: ecddcf3de945eff27ad89305bc12b9b419b41c31758640edde275da7e1d86881b028b26496b0595717085c149d13517d527416ea9ef05a0249e3bc5051feb46e
+  data.tar.gz: cf07a726ed0761fe6b8adefe598e0519c1a85f3b201043e0bb3a9a0cd123be9148717523dfb3e071e69abb660241664809f4433e73751a5a0ffd9e22cf370211

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.4.0
+ - Scripted update support courtesy of @Da-Wei
+
 ## 2.3.2
  - Fix bug where max_retry_interval was not respected for HTTP error codes
 

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -132,7 +132,10 @@ module LogStash; module Outputs; class ElasticSearch;
       }
 
       params[:parent] = event.sprintf(@parent) if @parent
-      params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
+      if @action == 'update'
+        params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
+        params[:_script] = event.sprintf(@script) if @script != ""
+      end
       params
     end
 

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -105,6 +105,24 @@ module LogStash; module Outputs; class ElasticSearch
       # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
       mod.config :max_retries, :validate => :number, :default => 3
 
+      # Set script name for scripted update mode
+      mod.config :script, :validate => :string, :default => ""
+
+      # Define the type of script referenced by "script" variable
+      #  inline : "script" contains inline script
+      #  indexed : "script" contains the name of script directly indexed in elasticsearch
+      #  file    : "script" contains the name of script stored in elasticseach's config directory
+      mod.config :script_type, :validate => ["inline", 'indexed', "file"], :default => ["inline"]
+
+      # Set the language of the used script
+      mod.config :script_lang, :validate => :string, :default => ""
+
+      # Set variable name passed to script (scripted update)
+      mod.config :script_var_name, :validate => :string, :default => "event"
+
+      # if enabled, script is in charge of creating non-existent document (scripted update)
+      mod.config :scripted_upsert, :validate => :boolean, :default => false
+
       # Set max interval between bulk retries.
       mod.config :retry_max_interval, :validate => :number, :default => 2
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -41,20 +41,7 @@ module LogStash; module Outputs; class ElasticSearch;
     def non_threadsafe_bulk(actions)
       return if actions.empty?
       bulk_body = actions.collect do |action, args, source|
-        if action == 'update'
-          if args[:_id]
-            source = { 'doc' => source }
-            if @options[:doc_as_upsert]
-              source['doc_as_upsert'] = true
-            else
-              source['upsert'] = args[:_upsert] if args[:_upsert]
-            end
-          else
-            raise(LogStash::ConfigurationError, "Specifying action => 'update' without a document '_id' is not supported.")
-          end
-        end
-
-        args.delete(:_upsert)
+        args, source = update_action_builder(args, source) if action == 'update'
 
         if source && action != 'delete'
           next [ { action => args }, source ]
@@ -159,5 +146,38 @@ module LogStash; module Outputs; class ElasticSearch;
     def template_put(name, template)
       @client.indices.put_template(:name => name, :body => template)
     end
+
+    # Build a bulk item for an elasticsearch update action
+    def update_action_builder(args, source)
+      if args[:_script]
+        # Use the event as a hash from your script with variable name defined
+        # by script_var_name (default: "event")
+        # Ex: event["@timestamp"]
+        source = { 'script' => {'params' => { @options[:script_var_name] => source }} }
+        if @options[:scripted_upsert]
+          source['scripted_upsert'] = true
+          source['upsert'] = {}
+        else
+          source['upsert'] = args.delete(:_upsert) if args[:_upsert]
+        end
+        case @options[:script_type]
+        when 'indexed'
+          source['script']['id'] = args.delete(:_script)
+        when 'file'
+          source['script']['file'] = args.delete(:_script)
+        when 'inline'
+          source['script']['inline'] = args.delete(:_script)
+        end
+        source['script']['lang'] = @options[:script_lang] if @options[:script_lang] != ''
+      else
+        source = { 'doc' => source }
+        if @options[:doc_as_upsert]
+          source['doc_as_upsert'] = true
+        else
+          source['upsert'] = args.delete(:_upsert) if args[:_upsert]
+        end
+      end
+      [args, source]
+    end
   end
-end end end
+end end end

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -17,10 +17,23 @@ module LogStash; module Outputs; class ElasticSearch;
       client_settings.merge! setup_proxy(logger, params)
       common_options.merge! setup_basic_auth(logger, params)
 
+      # Update API setup
+      raise( Logstash::ConfigurationError,
+        "doc_as_upsert and scripted_upsert are mutually exclusive."
+      ) if params["doc_as_upsert"] and params["scripted_upsert"]
+
+      raise(
+        LogStash::ConfigurationError,
+        "Specifying action => 'update' needs a document_id."
+      ) if params['action'] == 'update' and params.fetch('document_id', '') == ''
+
       # Update API setup
       update_options = {
-        :upsert => params["upsert"],
-        :doc_as_upsert => params["doc_as_upsert"]
+        :doc_as_upsert => params["doc_as_upsert"],
+        :script_var_name => params["script_var_name"],
+        :script_type => params["script_type"],
+        :script_lang => params["script_lang"],
+        :scripted_upsert => params["scripted_upsert"]
       }
       common_options.merge! update_options if params["action"] == 'update'
 

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.3.2'
+  s.version         = '2.4.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/es_spec_helper.rb

@@ -8,7 +8,7 @@ require "logstash/outputs/elasticsearch"
 
 CONTAINER_NAME = "logstash-output-elasticsearch-#{rand(999).to_s}"
 CONTAINER_IMAGE = "elasticsearch"
-CONTAINER_TAG = "1.6"
+CONTAINER_TAG = "2.0"
 
 DOCKER_INTEGRATION = ENV["DOCKER_INTEGRATION"]
 
@@ -45,7 +45,14 @@ RSpec.configure do |config|
         ls = Longshoreman::new
         ls.container.get(CONTAINER_NAME)
       rescue Docker::Error::NotFoundError
-        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME)
+        scriptDir = File.expand_path File.dirname(__FILE__) + '/fixtures/scripts'
+        Longshoreman.new("#{CONTAINER_IMAGE}:#{CONTAINER_TAG}", CONTAINER_NAME, {
+          'Cmd' => [ "-Des.script.inline=on", "-Des.script.indexed=on" ],
+          'HostConfig' => {
+            'Binds' => ["#{scriptDir}:/usr/share/elasticsearch/config/scripts"],
+            'PublishAllPorts' => true
+          }
+        })
         # TODO(talevy): verify ES is running instead of static timeout
         sleep 10
       end

data/spec/fixtures/scripts/scripted_update.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter += event["count"]
+

data/spec/fixtures/scripts/scripted_update_nested.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter += event["data"]["count"]
+

data/spec/fixtures/scripts/scripted_upsert.groovy

@@ -0,0 +1,2 @@
+ctx._source.counter = event["counter"]
+

data/spec/integration/outputs/update_spec.rb

@@ -1,10 +1,10 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "all protocols update actions", :integration => true do
+describe "Update actions", :integration => true do
   require "logstash/outputs/elasticsearch"
   require "elasticsearch"
 
-  def get_es_output(id = nil, upsert = nil, doc_as_upsert=nil)
+  def get_es_output( options={} )
     settings = {
       "manage_template" => true,
       "index" => "logstash-update",
@@ -12,10 +12,7 @@ describe "all protocols update actions", :integration => true do
       "hosts" => get_host_port(),
       "action" => "update"
     }
-    settings['upsert'] = upsert unless upsert.nil?
-    settings['document_id'] = id unless id.nil?
-    settings['doc_as_upsert'] = doc_as_upsert unless doc_as_upsert.nil?
-    LogStash::Outputs::ElasticSearch.new(settings)
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
   end
 
   before :each do
@@ -29,39 +26,87 @@ describe "all protocols update actions", :integration => true do
       :index => 'logstash-update',
       :type => 'logs',
       :id => "123",
-      :body => { :message => 'Test' }
+      :body => { :message => 'Test', :counter => 1 }
     )
     @es.indices.refresh
   end
 
   it "should fail without a document_id" do
-    event = LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0})
-    action = ["update", {:_id=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event]
     subject = get_es_output
-    subject.register
-    expect { subject.flush([action]) }.to raise_error
+    expect { subject.register }.to raise_error(LogStash::ConfigurationError)
   end
 
-  it "should not create new document" do
-    subject = get_es_output("456")
-    subject.register
-    subject.receive(LogStash::Event.new("message" => "sample message here"))
-    subject.flush
-    expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+  context "when update only" do
+    it "should not create new document" do
+      subject = get_es_output({ 'document_id' => "456" } )
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "sample message here"))
+      subject.flush
+      expect {@es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)}.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+    end
+
+    it "should update existing document" do
+      subject = get_es_output({ 'document_id' => "123" })
+      subject.register
+      subject.receive(LogStash::Event.new("message" => "updated message here"))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["message"] } == 'updated message here'
+    end
   end
+    
+  context "when using script" do
+    it "should increment a counter with event/doc 'count' variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
+      subject.register
+      subject.receive(LogStash::Event.new("count" => 2))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
 
-  it "should update existing document" do
-    subject = get_es_output("123")
-    subject.register
-    subject.receive(LogStash::Event.new("message" => "updated message here"))
-    subject.flush
-    r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-    insist { r["_source"]["message"] } == 'updated message here'
+    it "should increment a counter with event/doc '[data][count]' nested variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
+      subject.register
+      subject.receive(LogStash::Event.new("data" => { "count" => 3 }))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'ctx._source.counter += event["count"]',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("count" => 3 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with indexed script" do
+      @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'indexed_update',
+        'script_lang' => 'groovy',
+        'script_type' => 'indexed'
+      })
+      subject.register
+      subject.receive(LogStash::Event.new("count" => 4 ))
+      subject.flush
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 5
+    end
   end
 
-  context "upsert with protocol" do
-    it "should create new documents with upsert content" do
-      subject = get_es_output("456", '{"message": "upsert message"}')
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
       subject.flush
@@ -70,12 +115,32 @@ describe "all protocols update actions", :integration => true do
     end
 
     it "should create new documents with event/doc as upsert" do
-      subject = get_es_output("456", nil, true)
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
       subject.register
       subject.receive(LogStash::Event.new("message" => "sample message here"))
       subject.flush
       r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
       insist { r["_source"]["message"] } == 'sample message here'
     end
+
+    context "when using script" do
+      it "should create new documents with upsert content" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
+        subject.register
+        subject.receive(LogStash::Event.new("message" => "sample message here"))
+        subject.flush
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["message"] } == 'upsert message'
+      end
+
+      it "should create new documents with event/doc as script params" do
+        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
+        subject.register
+        subject.receive(LogStash::Event.new("counter" => 1))
+        subject.flush
+        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+        insist { r["_source"]["counter"] } == 1
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.4.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-05 00:00:00.000000000 Z
+date: 2016-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -210,6 +210,9 @@ files:
 - lib/logstash/outputs/elasticsearch/template_manager.rb
 - logstash-output-elasticsearch.gemspec
 - spec/es_spec_helper.rb
+- spec/fixtures/scripts/scripted_update.groovy
+- spec/fixtures/scripts/scripted_update_nested.groovy
+- spec/fixtures/scripts/scripted_upsert.groovy
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/parent_spec.rb
@@ -252,6 +255,9 @@ specification_version: 4
 summary: Logstash Output to Elasticsearch
 test_files:
 - spec/es_spec_helper.rb
+- spec/fixtures/scripts/scripted_update.groovy
+- spec/fixtures/scripts/scripted_update_nested.groovy
+- spec/fixtures/scripts/scripted_upsert.groovy
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/parent_spec.rb