logstash-output-elasticsearch 7.0.0-java → 7.1.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 76021c7104485b80a897db193e8bcb698bcb3e88
-  data.tar.gz: f4f3bcec4e2f448a1f424b53c03c6c96ca46f5dc
+  metadata.gz: fd81205081e6b05250ed7f01c930149b6947b7d8
+  data.tar.gz: 66d17e9dae1b92b2053ce814bc3e0e17285afcf0
 SHA512:
-  metadata.gz: d1e75e1711a6e2f376d1e69e2bceeb029a617e96195841dd1673a4bf4d151223336816a967fde9a9be6d71edacd2f727f60f0f50162ce48dba17b2fbdf01f770
-  data.tar.gz: 53ace3cb794f69182d6eae22c3041879a587b682a5fb9939c76d4a64f5e5e042928943edcd08966496250903dc1f78f997bddb0f8a80bb676e7d97c7c7f76387
+  metadata.gz: f6263b223df958da29405a47756848072f1fa83cfb50e7e334246e95362e43765e1cce246ab84fc29513b0f78b65f950f41f42205e266d7602b207dab3a10478
+  data.tar.gz: 34956e72d80592e44341f24210d894c9f9ffc2ef007367bd5bfee009b1cb3b17b2b826d3ce168c6088c01896409d3a6d733114b7a359faa0dfd6cc6059ddb5f6

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 7.1.0
+ - Add support to compress requests using the new `http_compression` option.
+
 ## 7.0.0
 - introduce customization of bulk, healthcheck and sniffing paths with the behaviour:
   - if not set: the default value will be used

data/lib/logstash/outputs/elasticsearch.rb

@@ -62,6 +62,17 @@ require "forwardable"
 #
 # Keep in mind that a connection with keepalive enabled will
 # not reevaluate its DNS value while the keepalive is in effect.
+#
+# ==== HTTP Compression
+#
+# This plugin supports request and response compression. Response compression is enabled by default and 
+# for Elasticsearch versions 5.0 and later, the user doesn't have to set any configs in Elasticsearch for 
+# it to send back compressed response. For versions before 5.0, `http.compression` must be set to `true` in 
+# Elasticsearch[https://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http] to take advantage of response compression when using this plugin
+#
+# For requests compression, regardless of the Elasticsearch version, users have to enable `http_compression` 
+# setting in their Logstash config file.
+#
 class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   declare_threadsafe!
 
@@ -202,6 +213,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # See https://hc.apache.org/httpcomponents-client-ga/httpclient/apidocs/org/apache/http/impl/conn/PoolingHttpClientConnectionManager.html#setValidateAfterInactivity(int)[these docs for more info]
   config :validate_after_inactivity, :validate => :number, :default => 10000
 
+  # Enable gzip compression on requests. Note that response compression is on by default for Elasticsearch v5.0 and beyond
+  config :http_compression, :validate => :boolean, :default => false
+
   def build_client
     @client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
   end

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -82,8 +82,7 @@ module LogStash; module Outputs; class ElasticSearch;
         begin
           submit_actions = submit(submit_actions)
           if submit_actions && submit_actions.size > 0
-            @logger.error("Retrying individual actions")
-            submit_actions.each {|action| @logger.error("Action", action) }
+            @logger.info("Retrying individual bulk actions that failed or were rejected by the previous bulk request.", :count => submit_actions.size)
           end
         rescue => e
           @logger.error("Encountered an unexpected error submitting a bulk request! Will retry.",

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -4,6 +4,8 @@ require "base64"
 require 'logstash/outputs/elasticsearch/http_client/pool'
 require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 require 'cgi'
+require 'zlib'
+require 'stringio'
 
 module LogStash; module Outputs; class ElasticSearch;
   # This is a constant instead of a config option because
@@ -101,24 +103,25 @@ module LogStash; module Outputs; class ElasticSearch;
         end
       end
 
-      bulk_body = ""
+      body_stream = StringIO.new
+      if http_compression
+        body_stream.set_encoding "BINARY"
+        stream_writer = Zlib::GzipWriter.new(body_stream, Zlib::DEFAULT_COMPRESSION, Zlib::DEFAULT_STRATEGY)
+      else 
+        stream_writer = body_stream
+      end
       bulk_responses = []
       bulk_actions.each do |action|
         as_json = action.is_a?(Array) ?
                     action.map {|line| LogStash::Json.dump(line)}.join("\n") :
                     LogStash::Json.dump(action)
         as_json << "\n"
-
-        if (bulk_body.bytesize + as_json.bytesize) > TARGET_BULK_BYTES
-          bulk_responses << bulk_send(bulk_body)
-          bulk_body = as_json
-        else
-          bulk_body << as_json
-        end
+        bulk_responses << bulk_send(body_stream) if (body_stream.size + as_json.bytesize) > TARGET_BULK_BYTES
+        stream_writer.write(as_json)
       end
-
-      bulk_responses << bulk_send(bulk_body) if bulk_body.size > 0
-
+      stream_writer.close if http_compression
+      bulk_responses << bulk_send(body_stream) if body_stream.size > 0
+      body_stream.close if !http_compression
       join_bulk_responses(bulk_responses)
     end
 
@@ -129,8 +132,14 @@ module LogStash; module Outputs; class ElasticSearch;
       }
     end
 
-    def bulk_send(bulk_body)
-      _, response = @pool.post(@bulk_path, nil, bulk_body)
+    def bulk_send(body_stream)
+      params = http_compression ? {:headers => {"Content-Encoding" => "gzip"}} : {}
+      # Discard the URL 
+      _, response = @pool.post(@bulk_path, params, body_stream.string)
+      if !body_stream.closed?
+        body_stream.truncate(0)
+        body_stream.seek(0)
+      end
       LogStash::Json.load(response.body)
     end
 
@@ -214,6 +223,10 @@ module LogStash; module Outputs; class ElasticSearch;
       client_settings.fetch(:ssl, {})
     end
 
+    def http_compression
+      client_settings.fetch(:http_compression, {})
+    end
+
     def build_adapter(options)
       timeout = options[:timeout] || 0
       

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -47,7 +47,11 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
     # @see    Transport::Base#perform_request
     #
     def perform_request(url, method, path, params={}, body=nil)
-      params = (params || {}).merge(@client_params)
+      # Perform 2-level deep merge on the params, so if the passed params and client params will both have hashes stored on a key they
+      # will be merged as well, instead of choosing just one of the values
+      params = (params || {}).merge(@client_params) { |key, oldval, newval|
+        (oldval.is_a?(Hash) && newval.is_a?(Hash)) ? oldval.merge(newval) : newval
+      }
       params[:body] = body if body
 
       if url.user

data/lib/logstash/outputs/elasticsearch/http_client_builder.rb

@@ -6,7 +6,8 @@ module LogStash; module Outputs; class ElasticSearch;
       client_settings = {
         :pool_max => params["pool_max"],
         :pool_max_per_route => params["pool_max_per_route"],
-        :check_connection_timeout => params["validate_after_inactivity"]
+        :check_connection_timeout => params["validate_after_inactivity"],
+        :http_compression => params["http_compression"]
       }
       
       client_settings[:proxy] = params["proxy"] if params["proxy"]

data/logstash-output-elasticsearch.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '7.0.0'
+  s.version         = '7.1.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency 'addressable', "~> 2.3.0" # used by FTW. V 2.5.0 is ruby 2.0 only.
   s.add_development_dependency 'logstash-codec-plain'
   s.add_development_dependency 'json' # used by spec/unit/outputs/elasticsearch/http_client/pool_spec.rb
+  s.add_development_dependency 'gzip' # used by spec/integration/outputs/index_spec.rb
 
   if RUBY_PLATFORM == 'java'
     s.platform = RUBY_PLATFORM

data/spec/integration/outputs/compressed_indexing_spec.rb

@@ -0,0 +1,66 @@
+require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
+require "stringio"
+require "gzip"
+
+RSpec::Matchers.define :a_valid_gzip_encoded_string do
+  match { |data|
+    expect { Zlib::GzipReader.new(StringIO.new(data)).read }.not_to raise_error
+  }
+end
+
+describe "indexing with http_compression turned on", :integration => true, :version_greater_than_equal_to_5x => true do
+  let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:event_count) { 10000 + rand(500) }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+    {
+      "hosts" => get_host_port,
+      "index" => index,
+      "http_compression" => true
+    }
+  }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+  let(:es_url) { "http://#{get_host_port}" }
+  let(:index_url) {"#{es_url}/#{index}"}
+  let(:http_client_options) { {} }
+  let(:http_client) do
+    Manticore::Client.new(http_client_options)
+  end
+
+  before do
+    subject.register
+  end
+  
+  shared_examples "an indexer" do
+    it "ships events" do
+      subject.multi_receive(events)
+
+      http_client.post("#{es_url}/_refresh").call
+
+      response = http_client.get("#{index_url}/_count?q=*")
+      result = LogStash::Json.load(response.body)
+      cur_count = result["count"]
+      expect(cur_count).to eq(event_count)
+
+      response = http_client.get("#{index_url}/_search?q=*&size=1000")
+      result = LogStash::Json.load(response.body)
+      result["hits"]["hits"].each do |doc|
+        expect(doc["_type"]).to eq(type)
+        expect(doc["_index"]).to eq(index)
+      end
+    end
+  end
+
+  it "sets the correct content-encoding header and body is compressed" do
+    expect(subject.client.pool.adapter.client).to receive(:send).
+      with(anything, anything, {:headers=>{"Content-Encoding"=>"gzip", "Content-Type"=>"application/json"}, :body => a_valid_gzip_encoded_string}).
+      and_call_original
+    subject.multi_receive(events)
+  end
+
+  it_behaves_like("an indexer")
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 7.0.0
+  version: 7.1.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-02-21 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -120,6 +120,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: gzip
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -211,6 +225,7 @@ files:
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
+- spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/delete_spec.rb
 - spec/integration/outputs/index_spec.rb
@@ -265,6 +280,7 @@ test_files:
 - spec/fixtures/scripts/scripted_update.groovy
 - spec/fixtures/scripts/scripted_update_nested.groovy
 - spec/fixtures/scripts/scripted_upsert.groovy
+- spec/integration/outputs/compressed_indexing_spec.rb
 - spec/integration/outputs/create_spec.rb
 - spec/integration/outputs/delete_spec.rb
 - spec/integration/outputs/index_spec.rb