logstash-output-elasticsearch 7.4.2-java → 7.4.3-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 770e27fd1c21bc02c120904d170fea45c7891fbd
-  data.tar.gz: 06db893104a68aaaa310efcfe5516bb3ac09b22a
+SHA256:
+  metadata.gz: 138d01e9314f05b8eee7f9bd1115f3c7aeb33117544e840caa83b3b3e2d1ac54
+  data.tar.gz: 7516db608590f0261e1d66b957648fae9878433bec8e4b67b3325207ceb8604a
 SHA512:
-  metadata.gz: de3a1cb1c10024d856ee02498cf3389c55c94c62e8ea42d76a5a830df8446448d772874a0e4568be5f16522b9a637c8a4b0bddd9b662c917f27805ed020243f5
-  data.tar.gz: 07402d951b56e246098da251d58cc64a51ea54e1d532f584569a0ed94543593b788906b48bd01279b38cfd9b7f0644acba106e2a41f1e835d957f9e2a1f804f0
+  metadata.gz: 5f3e2c6287de60def0469b17d4fac31336cd986c35f04f9fd20e7d0f710d8dfdd75163383994da0e627d3f7e78aa61e1fb5d4613e92f0c9f2b2a38e6a147b84a
+  data.tar.gz: 401784c66a69b13c905586f3b5643babc78c37d5996d448c4f7b16d0b3534999236f0efe7717cd56fedd61bc226c12ab0add7b13ec50913e5ea36906a6369ced

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 7.4.3
+  - Avoid infinite loop by checking for zero body size #737
+  - Add support for join based parent indexing (backport of #632) #686
+
 ## 7.4.2
   - Use `#response_body` instead of `#body` when debugging response from the server #679
   - Docs: Add DLQ policy section

data/docs/index.asciidoc

@@ -67,8 +67,8 @@ HTTP requests to the bulk API are expected to return a 200 response code. All ot
 
 The following document errors are handled as follows:
 
-* 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<dlq-policy>> for more info.
-* 409 errors (conflict) are logged as a warning and dropped. 
+  * 400 and 404 errors are sent to the dead letter queue (DLQ), if enabled. If a DLQ is not enabled, a log message will be emitted, and the event will be dropped. See <<dlq-policy>> for more info.
+  * 409 errors (conflict) are logged as a warning and dropped. 
 
 Note that 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 It is more performant for Elasticsearch to retry these exceptions than this plugin.
@@ -495,7 +495,8 @@ Set script name for scripted update mode
   * Value type is <<string,string>>
   * Default value is `"painless"`
 
-Set the language of the used script. If not set, this defaults to painless in ES 5.0
+Set the language of the used script. If not set, this defaults to painless in ES 5.0.
+When using indexed (stored) scripts on Elasticsearch 6 and higher, you must set this parameter to `""` (empty string).
 
 [id="plugins-{type}s-{plugin}-script_type"]
 ===== `script_type` 
@@ -695,4 +696,4 @@ See also https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-in
 
 
 [id="plugins-{type}s-{plugin}-common-options"]
-include::{include_path}/{type}.asciidoc[]
+include::{include_path}/{type}.asciidoc[]

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -176,8 +176,15 @@ module LogStash; module Outputs; class ElasticSearch;
         params[:pipeline] = event.sprintf(@pipeline)
       end
 
-     if @parent
-        params[:parent] = event.sprintf(@parent)
+      if @parent
+        if @join_field
+          join_value = event.get(@join_field)
+          parent_value = event.sprintf(@parent)
+          event.set(@join_field, { "name" => join_value, "parent" => parent_value })
+          params[:_routing] = event.sprintf(@parent)
+        else
+          params[:parent] = event.sprintf(@parent)
+        end
       end
 
       if @action == 'update'

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -78,6 +78,9 @@ module LogStash; module Outputs; class ElasticSearch
       # This can be dynamic using the `%{foo}` syntax.
       mod.config :parent, :validate => :string, :default => nil
 
+      # For child documents, name of the join field
+      mod.config :join_field, :validate => :string, :default => nil
+
       # Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
       # Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
       #     `"127.0.0.1"`

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -50,11 +50,11 @@ module LogStash; module Outputs; class ElasticSearch;
     #   through a special http path, such as using mod_rewrite.
     def initialize(options={})
       @logger = options[:logger]
-      
+
       # Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
       # @options = DEFAULT_OPTIONS.merge(options)
       @options = options
-      
+
       @url_template = build_url_template
 
       @pool = build_pool(@options)
@@ -62,7 +62,7 @@ module LogStash; module Outputs; class ElasticSearch;
       # connection pool at the same time
       @bulk_path = @options[:bulk_path]
     end
-    
+
     def build_url_template
       {
         :scheme => self.scheme,
@@ -106,7 +106,7 @@ module LogStash; module Outputs; class ElasticSearch;
       if http_compression
         body_stream.set_encoding "BINARY"
         stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-      else 
+      else
         stream_writer = body_stream
       end
       bulk_responses = []
@@ -115,7 +115,9 @@ module LogStash; module Outputs; class ElasticSearch;
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        bulk_responses << bulk_send(body_stream) if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
+        if body_stream.size > 0 && (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
+          bulk_responses << bulk_send(body_stream)
+        end
         stream_writer.write(as_json)
       end
       stream_writer.close if http_compression
@@ -133,7 +135,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def bulk_send(body_stream)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
-      # Discard the URL 
+      # Discard the URL
       response = @pool.post(@bulk_path, params, body_stream.string)
       if !body_stream.closed?
         body_stream.truncate(0)
@@ -206,7 +208,7 @@ module LogStash; module Outputs; class ElasticSearch;
                         else
                           nil
                         end
-      
+
       calculated_scheme = calculate_property(uris, :scheme, explicit_scheme, sniffing)
 
       if calculated_scheme && calculated_scheme !~ /https?/
@@ -226,7 +228,7 @@ module LogStash; module Outputs; class ElasticSearch;
       # Enter things like foo:123, bar and wind up with foo:123, bar:9200
       calculate_property(uris, :port, nil, sniffing) || 9200
     end
-    
+
     def uris
       @options[:hosts]
     end
@@ -240,12 +242,12 @@ module LogStash; module Outputs; class ElasticSearch;
     end
 
     def http_compression
-      client_settings.fetch(:http_compression, {})
+      client_settings.fetch(:http_compression, false)
     end
 
     def build_adapter(options)
       timeout = options[:timeout] || 0
-      
+
       adapter_options = {
         :socket_timeout => timeout,
         :request_timeout => timeout,
@@ -266,11 +268,11 @@ module LogStash; module Outputs; class ElasticSearch;
       end
 
       adapter_options[:ssl] = ssl_options if self.scheme == 'https'
-      
+
       adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
       adapter = adapter_class.new(@logger, adapter_options)
     end
-    
+
     def build_pool(options)
       adapter = build_adapter(options)
 
@@ -319,7 +321,7 @@ module LogStash; module Outputs; class ElasticSearch;
                     h.query
                   end
       prefixed_raw_query = raw_query && !raw_query.empty? ? "?#{raw_query}" : nil
-      
+
       raw_url = "#{raw_scheme}://#{postfixed_userinfo}#{raw_host}:#{raw_port}#{prefixed_raw_path}#{prefixed_raw_query}"
 
       ::LogStash::Util::SafeURI.new(raw_url)

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.4.2'
+  s.version         = '7.4.3'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/integration/outputs/painless_update_spec.rb

@@ -93,20 +93,29 @@ if ESHelper.es_version_satisfies?(">= 5")
         insist { r["_source"]["counter"] } == 3
       end
 
-      if ESHelper.es_version_satisfies?('<6')
-        context 'with an indexed script' do
-          it "should increment a counter with event/doc 'count' variable with indexed script" do
-            @es.put_script lang: 'painless', id: 'indexed_update', body: { script: 'ctx._source.counter += params.event.count' }
-            subject = get_es_output({
-              'document_id' => "123",
-              'script' => 'indexed_update',
-              'script_type' => 'indexed'
-            })
-            subject.register
-            subject.multi_receive([LogStash::Event.new("count" => 4 )])
-            r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
-            insist { r["_source"]["counter"] } == 5
+      context 'with an indexed script' do
+        it "should increment a counter with event/doc 'count' variable with indexed script" do
+          if ESHelper.es_version_satisfies?('<6')
+            @es.perform_request(:put, "_scripts/painless/indexed_update", {}, {"script" => "ctx._source.counter += params.event.count" })
+          else
+            @es.perform_request(:put, "_scripts/indexed_update", {}, {"script" => {"source" => "ctx._source.counter += params.event.count", "lang" => "painless"}})
+          end
+
+          plugin_parameters = {
+            'document_id' => "123",
+            'script' => 'indexed_update',
+            'script_type' => 'indexed'
+          }
+
+          if ESHelper.es_version_satisfies?('>= 6.0.0')
+            plugin_parameters.merge!('script_lang' => '')
           end
+
+          subject = get_es_output(plugin_parameters)
+          subject.register
+          subject.multi_receive([LogStash::Event.new("count" => 4 )])
+          r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
+          insist { r["_source"]["counter"] } == 5
         end
       end
      end

data/spec/integration/outputs/parent_spec.rb

@@ -1,7 +1,7 @@
 require_relative "../../../spec/es_spec_helper"
 require "logstash/outputs/elasticsearch"
 
-if ESHelper.es_version_satisfies?("<= 5.x")
+if ESHelper.es_version_satisfies?("< 6")
   context "when using elasticsearch 5.x and before", :integration => true do
     shared_examples "a type based parent indexer" do
       let(:index) { 10.times.collect { rand(10).to_s }.join("") }
@@ -17,11 +17,10 @@ if ESHelper.es_version_satisfies?("<= 5.x")
       before do
         # Add mapping and a parent document
         index_url = "http://#{get_host_port()}/#{index}"
-        ftw = FTW::Agent.new
         mapping = { "mappings" => { "#{type}" => { "_parent" => { "type" => "#{type}_parent" } } } }
-        ftw.put!("#{index_url}", {:body => mapping.to_json, :headers => default_headers})
+        Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
         pdoc = { "foo" => "bar" }
-        ftw.put!("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers})
+        Manticore.put("#{index_url}/#{type}_parent/test", {:body => pdoc.to_json, :headers => default_headers}).call
 
         subject.register
         subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
@@ -31,15 +30,13 @@ if ESHelper.es_version_satisfies?("<= 5.x")
       it "ships events" do
         index_url = "http://#{get_host_port()}/#{index}"
 
-        ftw = FTW::Agent.new
-        ftw.post!("#{index_url}/_refresh")
+        Manticore.post("#{index_url}/_refresh").call
 
         # Wait until all events are available.
         Stud::try(10.times) do
           query = { "query" => { "has_parent" => { "type" => "#{type}_parent", "query" => { "match" => { "foo" => "bar" } } } } }
-          data = ""
-          response = ftw.post!("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
-          response.read_body { |chunk| data << chunk }
+          response = Manticore.post("#{index_url}/_count", {:body => query.to_json, :headers => default_headers})
+          data = response.body
           result = LogStash::Json.load(data)
           cur_count = result["count"]
           insist { cur_count } == event_count
@@ -76,7 +73,6 @@ end
 
 if ESHelper.es_version_satisfies?(">= 5.6")
   context "when using elasticsearch 5.6 and above", :integration => true do
-
     shared_examples "a join field based parent indexer" do
       let(:index) { 10.times.collect { rand(10).to_s }.join("") }
       let(:type) { 10.times.collect { rand(10).to_s }.join("") }
@@ -108,10 +104,7 @@ if ESHelper.es_version_satisfies?(">= 5.6")
           }
         }
         if ESHelper.es_version_satisfies?('<6')
-          mapping.merge!({
-                 "settings" => {
-                   "mapping.single_type" => true
-                 }})
+          mapping.merge!({ "settings" => { "mapping.single_type" => true }})
         end
         Manticore.put("#{index_url}", {:body => mapping.to_json, :headers => default_headers}).call
         pdoc = { "message" => "ohayo", join_field => parent_relation }

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -29,7 +29,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     let(:http_hostname_port) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}") }
     let(:https_hostname_port) { ::LogStash::Util::SafeURI.new("https://#{hostname_port}") }
     let(:http_hostname_port_path) { ::LogStash::Util::SafeURI.new("http://#{hostname_port}/path") }
-    
+
     shared_examples("proper host handling") do
       it "should properly transform a host:port string to a URL" do
         expect(subject.host_to_url(hostname_port_uri).to_s).to eq(http_hostname_port.to_s + "/")
@@ -58,7 +58,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         context "when SSL is false" do
           let(:ssl) { false }
           let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
-          
+
           it "should refuse to handle an https url" do
             expect {
               subject.host_to_url(https_hostname_port)
@@ -72,32 +72,32 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
             subject
             expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
           end
-        end       
+        end
       end
 
       describe "path" do
         let(:url) { http_hostname_port_path }
         let(:base_options) { super.merge(:hosts => [url]) }
-        
+
         it "should allow paths in a url" do
           expect(subject.host_to_url(url)).to eq(url)
         end
 
         context "with the path option set" do
           let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
-          
+
           it "should not allow paths in two places" do
             expect {
               subject.host_to_url(url)
             }.to raise_error(LogStash::ConfigurationError)
           end
         end
-        
+
         context "with a path missing a leading /" do
           let(:url) { http_hostname_port }
           let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
-          
-          
+
+
           it "should automatically insert a / in front of path overlays" do
             expected = url.clone
             expected.path = url.path + "/otherpath"
@@ -180,6 +180,52 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
     end
   end
 
+  describe "#bulk" do
+    subject { described_class.new(base_options) }
+    let(:message) { "hey" }
+    let(:actions) { [
+      ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
+    ]}
+
+    context "if a message is over TARGET_BULK_BYTES" do
+      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+      let(:message) { "a" * (target_bulk_bytes + 1) }
+
+      it "sends the message as its own bulk payload" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once do |data|
+          expect(data.size).to be > target_bulk_bytes
+        end
+        s = subject.send(:bulk, actions)
+      end
+    end
+
+    context "with two messages" do
+      let(:message1) { "hey" }
+      let(:message2) { "you" }
+      let(:actions) { [
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+      ]}
+
+      it "executes one bulk_send operation" do
+        allow(subject).to receive(:join_bulk_responses)
+        expect(subject).to receive(:bulk_send).once
+        s = subject.send(:bulk, actions)
+      end
+
+      context "if one exceeds TARGET_BULK_BYTES" do
+        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+        let(:message1) { "a" * (target_bulk_bytes + 1) }
+        it "executes two bulk_send operations" do
+          allow(subject).to receive(:join_bulk_responses)
+          expect(subject).to receive(:bulk_send).twice
+          s = subject.send(:bulk, actions)
+        end
+      end
+    end
+  end
+
   describe "sniffing" do
     let(:client) { LogStash::Outputs::ElasticSearch::HttpClient.new(base_options.merge(client_opts)) }
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.4.2
+  version: 7.4.3
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-09-22 00:00:00.000000000 Z
+date: 2018-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -196,7 +196,9 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
 email: info@elastic.co
 executables: []
 extensions: []
@@ -280,7 +282,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Logstash Output to Elasticsearch