logstash-output-elasticsearch 7.1.0-java → 7.2.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fd81205081e6b05250ed7f01c930149b6947b7d8
-  data.tar.gz: 66d17e9dae1b92b2053ce814bc3e0e17285afcf0
+  metadata.gz: 8bd235749fa8bfe679c896c6b5e82630b044e354
+  data.tar.gz: 376a1262982ce57c209ab671dd4532d78da68437
 SHA512:
-  metadata.gz: f6263b223df958da29405a47756848072f1fa83cfb50e7e334246e95362e43765e1cce246ab84fc29513b0f78b65f950f41f42205e266d7602b207dab3a10478
-  data.tar.gz: 34956e72d80592e44341f24210d894c9f9ffc2ef007367bd5bfee009b1cb3b17b2b826d3ce168c6088c01896409d3a6d733114b7a359faa0dfd6cc6059ddb5f6
+  metadata.gz: 9f7528c39ce9314e4bcf62a2656e8a850e7000a376fc7aa0f97775841eb6d26c19df13f6aab027aaa9dadfa015a56753a5e7cf2327df58e95c65fa6e569e4e91
+  data.tar.gz: 7d56cc94de48b15c32ee97a7b55deb416d69d2f84c0824d7f3cf8c7da7845d542eabb99cd9d15ae44d8a048102d1471cc794b9daaa0cd5645dd1bb78770eab88

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 7.2.0
+ - Support 6.0.0-alpha1 version of Elasticsearch by adding a separate 6x template
+ - Note: This version is backwards compatible w.r.t. config, but for ES 6.0.0, `_all` has been
+    removed. This BWC issue only affects ES version 6.x; older versions
+    can be used with this plugin as is.
+
 ## 7.1.0
  - Add support to compress requests using the new `http_compression` option.
 

data/lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json

@@ -0,0 +1,45 @@
+{
+  "template" : "logstash-*",
+  "version" : 60001,
+  "settings" : {
+    "index.refresh_interval" : "5s"
+  },
+  "mappings" : {
+    "_default_" : {
+      "dynamic_templates" : [ {
+        "message_field" : {
+          "path_match" : "message",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text",
+            "norms" : false
+          }
+        }
+      }, {
+        "string_fields" : {
+          "match" : "*",
+          "match_mapping_type" : "string",
+          "mapping" : {
+            "type" : "text", "norms" : false,
+            "fields" : {
+              "keyword" : { "type": "keyword" }
+            }
+          }
+        }
+      } ],
+      "properties" : {
+        "@timestamp": { "type": "date"},
+        "@version": { "type": "keyword"},
+        "geoip"  : {
+          "dynamic": true,
+          "properties" : {
+            "ip": { "type": "ip" },
+            "location" : { "type" : "geo_point" },
+            "latitude" : { "type" : "half_float" },
+            "longitude" : { "type" : "half_float" }
+          }
+        }
+      }
+    }
+  }
+}

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -163,8 +163,8 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         return nil
       else
         case major_version(nodes)
-        when 5
-          sniff_5x(nodes)
+        when 5, 6
+          sniff_5x_and_above(nodes)
         when 2
           sniff_2x_1x(nodes)
         when 1
@@ -180,7 +180,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       k,v = nodes.first; v['version'].split('.').first.to_i
     end
     
-    def sniff_5x(nodes)
+    def sniff_5x_and_above(nodes)
       nodes.map do |id,info|
         if info["http"]
           uri = LogStash::Util::SafeURI.new(info["http"]["publish_address"])

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -143,6 +143,11 @@ module LogStash; module Outputs; class ElasticSearch;
       LogStash::Json.load(response.body)
     end
 
+    def get(path)
+      url, response = @pool.get(path, nil)
+      LogStash::Json.load(response.body)
+    end
+
     def close
       @pool.close
     end

data/lib/logstash/outputs/elasticsearch.rb

@@ -147,7 +147,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # It can be either .jks or .p12
   config :keystore, :validate => :path
 
-  # Set the truststore password
+  # Set the keystore password
   config :keystore_password, :validate => :password
 
   # This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list.

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.1.0'
+  s.version         = '7.2.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/fixtures/scripts/painless/scripted_update.painless

@@ -0,0 +1,2 @@
+ctx._source.counter += params.event.count
+

data/spec/fixtures/scripts/painless/scripted_update_nested.painless

@@ -0,0 +1 @@
+ctx._source.counter += params.event.data.count

data/spec/fixtures/scripts/painless/scripted_upsert.painless

@@ -0,0 +1 @@
+ctx._source.counter = params.event.counter

data/spec/integration/outputs/groovy_update_spec.rb

@@ -0,0 +1,148 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Update actions using groovy scripts", :integration => true, :update_tests => 'groovy', :version_greater_than_equal_to_2x => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update",
+      "script_lang" => "groovy"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  context "scripted updates" do
+    it "should increment a counter with event/doc 'count' variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("count" => 2)])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc '[data][count]' nested variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'ctx._source.counter += event["counter"]',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "456",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
+        'script_lang' => 'groovy',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc 'count' variable with indexed script" do
+      @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'indexed_update',
+        'script_lang' => 'groovy',
+        'script_type' => 'indexed'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("count" => 4 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 5
+    end
+  end
+    
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'sample message here'
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "updates with scripted upsert" do
+    it "should create new documents with upsert content" do
+      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as script params" do
+      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 1)])
+      @es.indices.refresh
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["counter"] } == 1
+    end
+  end
+end

data/spec/integration/outputs/{pipeline_spec.rb → ingest_pipeline_spec.rb}

@@ -43,6 +43,7 @@ describe "Ingest pipeline execution behavior", :integration => true, :version_gr
 
     # register pipeline
     req = ftw_client.put(ingest_url, :body => apache_logs_pipeline)
+    req.headers["Content-Type"] = "application/json"
     ftw_client.execute(req)
 
     #TODO: Use esclient

data/spec/integration/outputs/painless_update_spec.rb

@@ -0,0 +1,148 @@
+require_relative "../../../spec/es_spec_helper"
+
+describe "Update actions using painless scripts", :integration => true, :update_tests => 'painless', :version_greater_than_equal_to_5x => true do
+  require "logstash/outputs/elasticsearch"
+
+  def get_es_output( options={} )
+    settings = {
+      "manage_template" => true,
+      "index" => "logstash-update",
+      "template_overwrite" => true,
+      "hosts" => get_host_port(),
+      "action" => "update",
+      "script_lang" => "painless"
+    }
+    LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
+  end
+
+  before :each do
+    @es = get_client
+    # Delete all templates first.
+    # Clean ES of data before we start.
+    @es.indices.delete_template(:name => "*")
+    # This can fail if there are no indexes, ignore failure.
+    @es.indices.delete(:index => "*") rescue nil
+    @es.index(
+      :index => 'logstash-update',
+      :type => 'logs',
+      :id => "123",
+      :body => { :message => 'Test', :counter => 1 }
+    )
+    @es.indices.refresh
+  end
+
+  context "scripted updates" do
+    it "should increment a counter with event/doc 'count' variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("count" => 2)])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc '[data][count]' nested variable" do
+      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'ctx._source.counter += params.event.counter',
+        'script_lang' => 'painless',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "123",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+        'script_lang' => 'painless',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 4
+    end
+
+    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
+      subject = get_es_output({
+        'document_id' => "456",
+        'doc_as_upsert' => true,
+        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += params.event.counter; } else { ctx._source.counter = params.event.counter; }',
+        'script_lang' => 'painless',
+        'script_type' => 'inline'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["counter"] } == 3
+    end
+
+    it "should increment a counter with event/doc 'count' variable with indexed script" do
+      @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
+      subject = get_es_output({
+        'document_id' => "123",
+        'script' => 'indexed_update',
+        'script_lang' => 'painless',
+        'script_type' => 'indexed'
+      })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("count" => 4 )])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+      insist { r["_source"]["counter"] } == 5
+    end
+  end  
+    
+  context "when update with upsert" do
+    it "should create new documents with provided upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as upsert" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'sample message here'
+    end
+
+    it "should fail on documents with event/doc as upsert at external version" do
+      subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
+      expect { subject.register }.to raise_error(LogStash::ConfigurationError)
+    end
+  end
+
+  context "updates with scripted upsert" do
+    it "should create new documents with upsert content" do
+      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["message"] } == 'upsert message'
+    end
+
+    it "should create new documents with event/doc as script params" do
+      subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
+      subject.register
+      subject.multi_receive([LogStash::Event.new("counter" => 1)])
+      @es.indices.refresh
+      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
+      insist { r["_source"]["counter"] } == 1
+    end
+  end
+end

data/spec/integration/outputs/parent_spec.rb

@@ -1,4 +1,5 @@
 require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
 
 shared_examples "a parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
@@ -6,6 +7,9 @@ shared_examples "a parent indexer" do
     let(:event_count) { 10000 + rand(500) }
     let(:parent) { "not_implemented" }
     let(:config) { "not_implemented" }
+    let(:default_headers) {
+      {"Content-Type" => "application/json"}
+    }
     subject { LogStash::Outputs::ElasticSearch.new(config) }
 
     before do
@@ -13,9 +17,9 @@ shared_examples "a parent indexer" do
       index_url = "http://#{get_host_port()}/#{index}"
       ftw = FTW::Agent.new
       mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-      ftw.put!("#{index_url}", :body => mapping.to_json)
+      ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
       pdoc = { "foo" => "bar" }
-      ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
+      ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
 
       subject.register
       subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
@@ -32,7 +36,7 @@ shared_examples "a parent indexer" do
       Stud::try(10.times) do
         query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
         data = ""
-        response = ftw.post!("#{index_url}/_count", :body => query.to_json)
+        response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
         response.read_body { |chunk| data << chunk }
         result = LogStash::Json.load(data)
         cur_count = result["count"]

data/spec/integration/outputs/sniffer_spec.rb

@@ -38,7 +38,7 @@ describe "pool sniffer", :integration => true do
   end
   
   # We do a more thorough check on these versions because we can more reliably guess the ip
-  describe("Complex sniff parsing ES 5x/2x", :version_greater_than_equal_to_2x => true) do
+  describe("Complex sniff parsing ES 6x/5x/2x", :version_greater_than_equal_to_2x => true) do
     include_examples("sniff parsing", true)
   end
 end

data/spec/integration/outputs/update_spec.rb

@@ -1,6 +1,6 @@
 require_relative "../../../spec/es_spec_helper"
 
-describe "Update actions", :integration => true, :version_greater_than_equal_to_2x => true do
+describe "Update actions without scripts", :integration => true, :version_greater_than_equal_to_2x => true do
   require "logstash/outputs/elasticsearch"
 
   def get_es_output( options={} )
@@ -9,8 +9,7 @@ describe "Update actions", :integration => true, :version_greater_than_equal_to_
       "index" => "logstash-update",
       "template_overwrite" => true,
       "hosts" => get_host_port(),
-      "action" => "update",
-      "script_lang" => "groovy"
+      "action" => "update"
     }
     LogStash::Outputs::ElasticSearch.new(settings.merge!(options))
   end
@@ -90,79 +89,6 @@ describe "Update actions", :integration => true, :version_greater_than_equal_to_
 
   end
 
-  context "when using script" do
-    it "should increment a counter with event/doc 'count' variable" do
-      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update', 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("count" => 2)])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 3
-    end
-
-    it "should increment a counter with event/doc '[data][count]' nested variable" do
-      subject = get_es_output({ 'document_id' => "123", 'script' => 'scripted_update_nested', 'script_type' => 'file' })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("data" => { "count" => 3 })])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
-    end
-
-    it "should increment a counter with event/doc 'count' variable with inline script" do
-      subject = get_es_output({
-        'document_id' => "123",
-        'script' => 'ctx._source.counter += event["counter"]',
-        'script_lang' => 'groovy',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
-    end
-
-    it "should increment a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
-        'document_id' => "123",
-        'doc_as_upsert' => true,
-        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["counter"]; } else { ctx._source.counter = event["counter"]; }',
-        'script_lang' => 'groovy',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 4
-    end
-
-    it "should, with new doc, set a counter with event/doc 'count' variable with event/doc as upsert and inline script" do
-      subject = get_es_output({
-        'document_id' => "456",
-        'doc_as_upsert' => true,
-        'script' => 'if( ctx._source.containsKey("counter") ){ ctx._source.counter += event["count"]; } else { ctx._source.counter = event["count"]; }',
-        'script_lang' => 'groovy',
-        'script_type' => 'inline'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("counter" => 3 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-      insist { r["_source"]["counter"] } == 3
-    end
-
-    it "should increment a counter with event/doc 'count' variable with indexed script" do
-      @es.put_script lang: 'groovy', id: 'indexed_update', body: { script: 'ctx._source.counter += event["count"]' }
-      subject = get_es_output({
-        'document_id' => "123",
-        'script' => 'indexed_update',
-        'script_lang' => 'groovy',
-        'script_type' => 'indexed'
-      })
-      subject.register
-      subject.multi_receive([LogStash::Event.new("count" => 4 )])
-      r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-      insist { r["_source"]["counter"] } == 5
-    end
-  end
-
   context "when update with upsert" do
     it "should create new documents with provided upsert" do
       subject = get_es_output({ 'document_id' => "456", 'upsert' => '{"message": "upsert message"}' })
@@ -184,24 +110,5 @@ describe "Update actions", :integration => true, :version_greater_than_equal_to_
       subject = get_es_output({ 'document_id' => "456", 'doc_as_upsert' => true, 'version' => 999, "version_type" => "external" })
       expect { subject.register }.to raise_error(LogStash::ConfigurationError)
     end
-
-    context "when using script" do
-      it "should create new documents with upsert content" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_update', 'upsert' => '{"message": "upsert message"}', 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("message" => "sample message here")])
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["message"] } == 'upsert message'
-      end
-
-      it "should create new documents with event/doc as script params" do
-        subject = get_es_output({ 'document_id' => "456", 'script' => 'scripted_upsert', 'scripted_upsert' => true, 'script_type' => 'file' })
-        subject.register
-        subject.multi_receive([LogStash::Event.new("counter" => 1)])
-        @es.indices.refresh
-        r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "456", :refresh => true)
-        insist { r["_source"]["counter"] } == 1
-      end
-    end
   end
 end

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -121,6 +121,20 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "get" do
+    subject { described_class.new(base_options) }
+    let(:body) { "foobar" }
+    let(:path) { "/hello-id" }
+    let(:get_response) {
+      double("response", :body => LogStash::Json::dump( { "body" => body }))
+    }
+
+    it "returns the hash response" do
+      expect(subject.pool).to receive(:get).with(path, nil).and_return([nil, get_response])
+      expect(subject.get(path)["body"]).to eq(body)
+    end
+  end
+
   describe "join_bulk_responses" do
     subject { described_class.new(base_options) }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.1.0
+  version: 7.2.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-24 00:00:00.000000000 Z
+date: 2017-02-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -213,6 +213,7 @@ files:
 - lib/logstash/outputs/elasticsearch/common_configs.rb
 - lib/logstash/outputs/elasticsearch/elasticsearch-template-es2x.json
 - lib/logstash/outputs/elasticsearch/elasticsearch-template-es5x.json
+- lib/logstash/outputs/elasticsearch/elasticsearch-template-es6x.json
 - lib/logstash/outputs/elasticsearch/http_client.rb
 - lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb
 - lib/logstash/outputs/elasticsearch/http_client/pool.rb
@@ -222,16 +223,21 @@ files:
 - spec/es_spec_helper.rb
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/scripted_update.groovy
-- spec/fixtures/scripts/scripted_update_nested.groovy
-- spec/fixtures/scripts/scripted_upsert.groovy
+- spec/fixtures/scripts/groovy/scripted_update.groovy
+- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
+- spec/fixtures/scripts/groovy/scripted_upsert.groovy
+- spec/fixtures/scripts/painless/scripted_update.painless
+- spec/fixtures/scripts/painless/scripted_update_nested.painless
+- spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/delete_spec.rb
+- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
+- spec/integration/outputs/ingest_pipeline_spec.rb
+- spec/integration/outputs/painless_update_spec.rb
 - spec/integration/outputs/parent_spec.rb
-- spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb
@@ -277,16 +283,21 @@ test_files:
 - spec/es_spec_helper.rb
 - spec/fixtures/htpasswd
 - spec/fixtures/nginx_reverse_proxy.conf
-- spec/fixtures/scripts/scripted_update.groovy
-- spec/fixtures/scripts/scripted_update_nested.groovy
-- spec/fixtures/scripts/scripted_upsert.groovy
+- spec/fixtures/scripts/groovy/scripted_update.groovy
+- spec/fixtures/scripts/groovy/scripted_update_nested.groovy
+- spec/fixtures/scripts/groovy/scripted_upsert.groovy
+- spec/fixtures/scripts/painless/scripted_update.painless
+- spec/fixtures/scripts/painless/scripted_update_nested.painless
+- spec/fixtures/scripts/painless/scripted_upsert.painless
 - spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/delete_spec.rb
+- spec/integration/outputs/groovy_update_spec.rb
 - spec/integration/outputs/index_spec.rb
 - spec/integration/outputs/index_version_spec.rb
+- spec/integration/outputs/ingest_pipeline_spec.rb
+- spec/integration/outputs/painless_update_spec.rb
 - spec/integration/outputs/parent_spec.rb
-- spec/integration/outputs/pipeline_spec.rb
 - spec/integration/outputs/retry_spec.rb
 - spec/integration/outputs/routing_spec.rb
 - spec/integration/outputs/sniffer_spec.rb