logstash-output-elasticsearch 10.8.0-java → 10.8.6-java

data/lib/logstash/outputs/elasticsearch.rb

@@ -417,7 +417,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    maximum_seen_major_version < 8
+    # always set type for ES <= 6
+    # for ES 7 only set it if the user defined it
+    (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
   end
 
   def install_template

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -109,27 +109,50 @@ module LogStash; module Outputs; class ElasticSearch;
       body_stream = StringIO.new
       if http_compression
         body_stream.set_encoding "BINARY"
-        stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
-      else 
+        stream_writer = gzip_writer(body_stream)
+      else
         stream_writer = body_stream
       end
       bulk_responses = []
-      bulk_actions.each do |action|
+      batch_actions = []
+      bulk_actions.each_with_index do |action, index|
         as_json = action.is_a?(Array) ?
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-        if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
-          bulk_responses << bulk_send(body_stream) unless body_stream.size == 0
+        if (stream_writer.pos + as_json.bytesize) > TARGET_BULK_BYTES && stream_writer.pos > 0
+          stream_writer.flush # ensure writer has sync'd buffers before reporting sizes
+          logger.debug("Sending partial bulk request for batch with one or more actions remaining.",
+                       :action_count => batch_actions.size,
+                       :payload_size => stream_writer.pos,
+                       :content_length => body_stream.size,
+                       :batch_offset => (index + 1 - batch_actions.size))
+          bulk_responses << bulk_send(body_stream, batch_actions)
+          body_stream.truncate(0) && body_stream.seek(0)
+          stream_writer = gzip_writer(body_stream) if http_compression
+          batch_actions.clear
         end
         stream_writer.write(as_json)
+        batch_actions << action
       end
       stream_writer.close if http_compression
-      bulk_responses << bulk_send(body_stream) if body_stream.size > 0
+      logger.debug("Sending final bulk request for batch.",
+                   :action_count => batch_actions.size,
+                   :payload_size => stream_writer.pos,
+                   :content_length => body_stream.size,
+                   :batch_offset => (actions.size - batch_actions.size))
+      bulk_responses << bulk_send(body_stream, batch_actions) if body_stream.size > 0
       body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end
 
+    def gzip_writer(io)
+      fail(ArgumentError, "Cannot create gzip writer on IO with unread bytes") unless io.eof?
+      fail(ArgumentError, "Cannot create gzip writer on non-empty IO") unless io.pos == 0
+
+      Zlib::GzipWriter.new(io, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+    end
+
     def join_bulk_responses(bulk_responses)
       {
         "errors" => bulk_responses.any? {|r| r["errors"] == true},
@@ -137,25 +160,37 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(body_stream)
+    def bulk_send(body_stream, batch_actions)
       params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
-      # Discard the URL
       response = @pool.post(@bulk_path, params, body_stream.string)
-      if !body_stream.closed?
-        body_stream.truncate(0)
-        body_stream.seek(0)
-      end
 
       @bulk_response_metrics.increment(response.code.to_s)
 
-      if response.code != 200
+      case response.code
+      when 200 # OK
+        LogStash::Json.load(response.body)
+      when 413 # Payload Too Large
+        logger.warn("Bulk request rejected: `413 Payload Too Large`", :action_count => batch_actions.size, :content_length => body_stream.size)
+        emulate_batch_error_response(batch_actions, response.code, 'payload_too_large')
+      else
         url = ::LogStash::Util::SafeURI.new(response.final_url)
         raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(
           response.code, url, body_stream.to_s, response.body
         )
       end
+    end
 
-      LogStash::Json.load(response.body)
+    def emulate_batch_error_response(actions, http_code, reason)
+      {
+          "errors" => true,
+          "items" => actions.map do |action|
+            action = action.first if action.is_a?(Array)
+            request_action, request_parameters = action.first
+            {
+                request_action => {"status" => http_code, "error" => { "type" => reason }}
+            }
+          end
+      }
     end
 
     def get(path)
@@ -400,6 +435,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Build a bulk item for an elasticsearch update action
     def update_action_builder(args, source)
+      args = args.clone()
       if args[:_script]
         # Use the event as a hash from your script with variable name defined
         # by script_var_name (default: "event")

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -70,7 +70,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       @url_info = {}
       @stopping = false
 
-      @license_checker = options.fetch(:license_checker) { LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE }
+      @license_checker = options[:license_checker] || LogStash::PluginMixins::ElasticSearch::NoopLicenseChecker::INSTANCE
     end
 
     def start

data/lib/logstash/plugin_mixins/elasticsearch/common.rb

@@ -204,8 +204,16 @@ module LogStash; module PluginMixins; module ElasticSearch
         return
       end
 
+      responses = bulk_response["items"]
+      if responses.size != actions.size # can not map action -> response reliably
+        # an ES bug (on 7.10.2, 7.11.1) where a _bulk request to index X documents would return Y (> X) items
+        msg = "Sent #{actions.size} documents but Elasticsearch returned #{responses.size} responses"
+        @logger.warn(msg, actions: actions, responses: responses)
+        fail("#{msg} (likely a bug with _bulk endpoint)")
+      end
+
       actions_to_retry = []
-      bulk_response["items"].each_with_index do |response,idx|
+      responses.each_with_index do |response,idx|
         action_type, action_props = response.first
 
         status = action_props["status"]
@@ -291,7 +299,7 @@ module LogStash; module PluginMixins; module ElasticSearch
         retry unless @stopping.true?
       rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
         @bulk_request_metrics.increment(:failures)
-        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s}
+        log_hash = {:code => e.response_code, :url => e.url.sanitized.to_s, :content_length => e.request_body.bytesize}
         log_hash[:body] = e.response_body if @logger.debug? # Generally this is too verbose
         message = "Encountered a retryable error. Will Retry with exponential backoff "
 

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '10.8.0'
+  s.version         = '10.8.6'
 
   s.licenses        = ['apache-2.0']
   s.summary         = "Stores logs in Elasticsearch"

data/spec/integration/outputs/ilm_spec.rb

@@ -5,7 +5,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy with a maximum number of documents' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should rollover when the policy max docs is reached' do
       put_policy(@es, ilm_policy_name, policy)
@@ -54,7 +54,7 @@ shared_examples_for 'an ILM enabled Logstash' do
   context 'with a policy where the maximum number of documents is not reached' do
     let (:policy) { large_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-custom-policy"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should ingest into a single index when max docs is not reached' do
       put_policy(@es,ilm_policy_name, policy)
@@ -119,7 +119,7 @@ shared_examples_for 'an ILM disabled Logstash' do
   context 'with an existing policy that will roll over' do
     let (:policy) { small_max_doc_policy }
     let (:ilm_policy_name) { "logstash-policy-3_docs"}
-    let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+    let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
 
     it 'should not roll over indices' do
       subject.register
@@ -155,7 +155,7 @@ shared_examples_for 'an ILM disabled Logstash' do
 
   context 'with a custom template name' do
     let (:template_name) { "logstash_custom_template_name" }
-    let (:settings) { super.merge('template_name' => template_name)}
+    let (:settings) { super().merge('template_name' => template_name)}
 
     it 'should not write the ILM settings into the template' do
       subject.register
@@ -195,7 +195,7 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   subject { LogStash::Outputs::ElasticSearch.new(settings) }
 
   context 'when ilm is enabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => true }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => true }) }
 
     it 'should raise a configuration error' do
       expect do
@@ -210,13 +210,13 @@ shared_examples_for 'an Elasticsearch instance that does not support index lifec
   end
 
   context 'when ilm is disabled in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => false }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => false }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
 
   context 'when ilm is set to auto in Logstash' do
-    let (:settings) { super.merge!({ 'ilm_enabled' => 'auto' }) }
+    let (:settings) { super().merge!({ 'ilm_enabled' => 'auto' }) }
 
     it_behaves_like 'an ILM disabled Logstash'
   end
@@ -286,7 +286,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using the default policy' do
         context 'with a custom pattern' do
-          let (:settings) { super.merge("ilm_pattern" => "000001")}
+          let (:settings) { super().merge("ilm_pattern" => "000001")}
           it 'should create a rollover alias' do
             expect(@es.indices.exists_alias(name: "logstash")).to be_falsey
             subject.register
@@ -346,7 +346,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when not using the default policy' do
         let (:ilm_policy_name) {"logstash-policy-small"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { small_max_doc_policy }
 
         before do
@@ -363,7 +363,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
       context 'when using a time based policy' do
         let (:ilm_policy_name) {"logstash-policy-time"}
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name)}
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name)}
         let (:policy) { max_age_policy("1d") }
 
         before do
@@ -409,7 +409,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
           let (:template) { "spec/fixtures/template-with-policy-es6x.json" }
         end
 
-        let (:settings) { super.merge("template" => template,
+        let (:settings) { super().merge("template" => template,
                                       "index" => "overwrite-4")}
 
         it 'should not overwrite the index patterns' do
@@ -426,7 +426,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
         let (:ilm_rollover_alias) { "logstash_the_cat_in_the_hat" }
         let (:index) { ilm_rollover_alias }
         let(:expected_index) { index }
-        let (:settings) { super.merge("ilm_policy" => ilm_policy_name,
+        let (:settings) { super().merge("ilm_policy" => ilm_policy_name,
                                       "template" => template,
                                       "ilm_rollover_alias" => ilm_rollover_alias)}
 
@@ -480,7 +480,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
 
         context 'with a different template_name' do
           let (:template_name) { "logstash_custom_template_name" }
-          let (:settings) { super.merge('template_name' => template_name)}
+          let (:settings) { super().merge('template_name' => template_name)}
 
           it_behaves_like 'an ILM enabled Logstash'
 
@@ -514,7 +514,7 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'when ilm_enabled is the default' do
-      let (:settings) { super.tap{|x|x.delete('ilm_enabled')}}
+      let (:settings) { super().tap{|x|x.delete('ilm_enabled')}}
 
       if ESHelper.es_version_satisfies?(">=7.0")
         context 'when Elasticsearch is version 7 or above' do
@@ -530,13 +530,13 @@ if ESHelper.es_version_satisfies?(">= 6.6")
     end
 
     context 'with ilm disabled' do
-      let (:settings) { super.merge('ilm_enabled' => false )}
+      let (:settings) { super().merge('ilm_enabled' => false )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end
 
     context 'with ilm disabled using a string' do
-      let (:settings) { super.merge('ilm_enabled' => 'false' )}
+      let (:settings) { super().merge('ilm_enabled' => 'false' )}
 
       it_behaves_like 'an ILM disabled Logstash'
     end

data/spec/integration/outputs/retry_spec.rb

@@ -4,9 +4,21 @@ require_relative "../../../spec/es_spec_helper"
 describe "failures in bulk class expected behavior", :integration => true do
   let(:template) { '{"template" : "not important, will be updated by :index"}' }
   let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action1) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1]) }
+  let(:action1) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event1])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event1])
+    end
+  end
   let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
-  let(:action2) { ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2]) }
+  let(:action2) do
+    if ESHelper.es_version_satisfies?(">= 6", "< 7")
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17", :_type=> doc_type }, event2])
+    else
+      ESHelper.action_for_version(["index", {:_id=>nil, routing_field_name =>nil, :_index=>"logstash-2014.11.17" }, event2])
+    end
+  end
   let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
 
   def mock_actions_with_response(*resp)

data/spec/unit/http_client_builder_spec.rb

@@ -40,10 +40,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
       context "when setting bulk_path" do
         let(:bulk_path) { "/meh" }
-        let(:options) { super.merge("bulk_path" => bulk_path) }
+        let(:options) { super().merge("bulk_path" => bulk_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq(bulk_path)
@@ -66,7 +66,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets bulk_path to path+_bulk" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:bulk_path]).to eq("#{path}/_bulk")
@@ -88,10 +88,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "healthcheck_path" do
       context "when setting healthcheck_path" do
         let(:healthcheck_path) { "/meh" }
-        let(:options) { super.merge("healthcheck_path" => healthcheck_path) }
+        let(:options) { super().merge("healthcheck_path" => healthcheck_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(healthcheck_path)
@@ -114,7 +114,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets healthcheck_path to path" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:healthcheck_path]).to eq(path)
@@ -136,10 +136,10 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
     describe "sniffing_path" do
       context "when setting sniffing_path" do
         let(:sniffing_path) { "/meh" }
-        let(:options) { super.merge("sniffing_path" => sniffing_path) }
+        let(:options) { super().merge("sniffing_path" => sniffing_path) }
 
         context "when using path" do
-          let(:options) { super.merge("path" => "/path") }
+          let(:options) { super().merge("path" => "/path") }
           it "ignores the path setting" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq(sniffing_path)
@@ -162,7 +162,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClientBuilder do
 
         context "when using path" do
           let(:path) { "/meh" }
-          let(:options) { super.merge("path" => path) }
+          let(:options) { super().merge("path" => path) }
           it "sets sniffing_path to path+_nodes/http" do
             expect(described_class).to receive(:create_http_client) do |options|
               expect(options[:sniffing_path]).to eq("#{path}/_nodes/http")

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -68,7 +68,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
 
       context "and setting healthcheck_path" do
         let(:healthcheck_path) { "/my/health" }
-        let(:options) { super.merge(:healthcheck_path => healthcheck_path) }
+        let(:options) { super().merge(:healthcheck_path => healthcheck_path) }
         it "performs the healthcheck to the healthcheck_path" do
           expect(adapter).to receive(:perform_request) do |url, method, req_path, _, _|
             expect(method).to eq(:head)
@@ -130,7 +130,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     context "when enabled" do
-      let(:options) { super.merge(:sniffing => true)}
+      let(:options) { super().merge(:sniffing => true)}
 
       it "should start the sniffer" do
         expect(subject.sniffer_alive?).to eql(true)
@@ -247,7 +247,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
     end
 
     let(:options) do
-      super.merge(:license_checker => license_checker)
+      super().merge(:license_checker => license_checker)
     end
 
     context 'when LicenseChecker#acceptable_license? returns false' do

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -48,7 +48,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       describe "ssl" do
         context "when SSL is true" do
           let(:ssl) { true }
-          let(:base_options) { super.merge(:hosts => [http_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [http_hostname_port]) }
 
           it "should refuse to handle an http url" do
             expect {
@@ -59,7 +59,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
         context "when SSL is false" do
           let(:ssl) { false }
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           
           it "should refuse to handle an https url" do
             expect {
@@ -69,7 +69,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         end
 
         describe "ssl is nil" do
-          let(:base_options) { super.merge(:hosts => [https_hostname_port]) }
+          let(:base_options) { super().merge(:hosts => [https_hostname_port]) }
           it "should handle an ssl url correctly when SSL is nil" do
             subject
             expect(subject.host_to_url(https_hostname_port).to_s).to eq(https_hostname_port.to_s + "/")
@@ -79,14 +79,14 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
 
       describe "path" do
         let(:url) { http_hostname_port_path }
-        let(:base_options) { super.merge(:hosts => [url]) }
+        let(:base_options) { super().merge(:hosts => [url]) }
         
         it "should allow paths in a url" do
           expect(subject.host_to_url(url)).to eq(url)
         end
 
         context "with the path option set" do
-          let(:base_options) { super.merge(:client_settings => {:path => "/otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "/otherpath"}) }
 
           it "should not allow paths in two places" do
             expect {
@@ -97,7 +97,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
         
         context "with a path missing a leading /" do
           let(:url) { http_hostname_port }
-          let(:base_options) { super.merge(:client_settings => {:path => "otherpath"}) }
+          let(:base_options) { super().merge(:client_settings => {:path => "otherpath"}) }
           
           
           it "should automatically insert a / in front of path overlays" do
@@ -204,7 +204,7 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
   end
 
   describe "#bulk" do
-    subject { described_class.new(base_options) }
+    subject(:http_client) { described_class.new(base_options) }
 
     require "json"
     let(:message) { "hey" }
@@ -212,42 +212,61 @@ describe LogStash::Outputs::ElasticSearch::HttpClient do
       ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message}],
     ]}
 
-    context "if a message is over TARGET_BULK_BYTES" do
-      let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-      let(:message) { "a" * (target_bulk_bytes + 1) }
+    [true,false].each do |http_compression_enabled|
+      context "with `http_compression => #{http_compression_enabled}`" do
 
-      it "should be handled properly" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once do |data|
-          expect(data.size).to be > target_bulk_bytes
+        let(:base_options) { super().merge(:client_settings => {:http_compression => http_compression_enabled}) }
+
+        before(:each) do
+          if http_compression_enabled
+            expect(http_client).to receive(:gzip_writer).at_least(:once).and_call_original
+          else
+            expect(http_client).to_not receive(:gzip_writer)
+          end
         end
-        s = subject.send(:bulk, actions)
-      end
-    end
 
-    context "with two messages" do
-      let(:message1) { "hey" }
-      let(:message2) { "you" }
-      let(:actions) { [
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
-        ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
-      ]}
-      it "executes one bulk_send operation" do
-        allow(subject).to receive(:join_bulk_responses)
-        expect(subject).to receive(:bulk_send).once
-        s = subject.send(:bulk, actions)
-      end
+        context "if a message is over TARGET_BULK_BYTES" do
+          let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+          let(:message) { "a" * (target_bulk_bytes + 1) }
+
+          it "should be handled properly" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once do |data|
+              if !http_compression_enabled
+                expect(data.size).to be > target_bulk_bytes
+              else
+                expect(Zlib::gunzip(data.string).size).to be > target_bulk_bytes
+              end
+            end
+            s = subject.send(:bulk, actions)
+          end
+        end
+
+        context "with two messages" do
+          let(:message1) { "hey" }
+          let(:message2) { "you" }
+          let(:actions) { [
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message1}],
+            ["index", {:_id=>nil, :_index=>"logstash"}, {"message"=> message2}],
+          ]}
+          it "executes one bulk_send operation" do
+            allow(subject).to receive(:join_bulk_responses)
+            expect(subject).to receive(:bulk_send).once
+            s = subject.send(:bulk, actions)
+          end
 
-      context "if one exceeds TARGET_BULK_BYTES" do
-        let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
-        let(:message1) { "a" * (target_bulk_bytes + 1) }
-        it "executes two bulk_send operations" do
-          allow(subject).to receive(:join_bulk_responses)
-          expect(subject).to receive(:bulk_send).twice
-          s = subject.send(:bulk, actions)
+          context "if one exceeds TARGET_BULK_BYTES" do
+            let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+            let(:message1) { "a" * (target_bulk_bytes + 1) }
+            it "executes two bulk_send operations" do
+              allow(subject).to receive(:join_bulk_responses)
+              expect(subject).to receive(:bulk_send).twice
+              s = subject.send(:bulk, actions)
+            end
+          end
         end
-      end
-    end
+       end
+     end
   end
 
   describe "sniffing" do