logstash-output-elasticsearch 5.2.1-java → 5.3.0-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3b251154b11adcfa757987794bdad62c0488d349
-  data.tar.gz: f23e6896387edeb0eeacab8e669acae81c432aa7
+  metadata.gz: 7185b4fb6010e9fc50370f2585835985d867e935
+  data.tar.gz: 799792dd65f34457cacdb14d969d7f69f7a56791
 SHA512:
-  metadata.gz: bb7d268d9129800467d11b13d577fe367b063f6df75317f57e850cb2f0011ed4c7ac49275138c4334d39d8e1776e0002bc2e4cd031e6ec194f34183d85fe9b1a
-  data.tar.gz: 7dcd51f47cafc2cff0dc1633e373f40de486b5be82a9351e693d834e91aeaa63eb91d733950054075ae8cea35bfb340cdf0961a814bfff12ae119187a180aebe
+  metadata.gz: c356227a6c159996b0bffee8998eae408ed9c7c7dffb3c1e5c36558562bb4065b498b7f0c588dfcd70915a85d24c4f98e75847d7e5663129beb0f842a706e849
+  data.tar.gz: 1460ab9245da87f9b60e08779c1a72fe14fcbd2e68d5e1d9de0ab0720de085cab65dac19ec92930cbf7c48d68bede750bb1c888cfc97e3165caa3f804ee18b0e

data/CHANGELOG.md

@@ -1,6 +1,5 @@
-## 5.2.1
-- Hide user/password in connection pool
-- depends on Adressable ~> 2.3.0 to satisfy development dependency of the core ([logstash/#6204](https://github.com/elastic/logstash/issues/6204))
+## 5.3.0
+- Bulk operations will now target 20MB chunks at a time to reduce heap usage
 
 ## 5.2.0
 - Change default lang for scripts to be painless, inline with ES 5.0. Earlier there was no default.

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -19,8 +19,12 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Receive an array of events and immediately attempt to index them (no buffering)
     def multi_receive(events)
-      events.each_slice(@flush_size) do |slice|
-        retrying_submit(slice.map {|e| event_action_tuple(e) })
+      if @flush_size
+        events.each_slice(@flush_size) do |slice|
+          retrying_submit(slice.map {|e| event_action_tuple(e) })
+        end
+      else
+        retrying_submit(events.map {|e| event_action_tuple(e)})
       end
     end
 
@@ -59,14 +63,14 @@ module LogStash; module Outputs; class ElasticSearch;
       VALID_HTTP_ACTIONS
     end
 
-    def retrying_submit(actions)      
+    def retrying_submit(actions)
       # Initially we submit the full list of actions
       submit_actions = actions
 
       sleep_interval = @retry_initial_interval
 
       while submit_actions && submit_actions.length > 0
-        
+
         # We retry with whatever is didn't succeed
         begin
           submit_actions = submit(submit_actions)
@@ -103,7 +107,7 @@ module LogStash; module Outputs; class ElasticSearch;
 
     def submit(actions)
       bulk_response = safe_bulk(actions)
-      
+
       # If the response is nil that means we were in a retry loop
       # and aborted since we're shutting down
       # If it did return and there are no errors we're good as well
@@ -216,7 +220,7 @@ module LogStash; module Outputs; class ElasticSearch;
           sleep_interval = sleep_for_interval(sleep_interval)
           retry unless @stopping.true?
         else
-          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code)
+          @logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code, :body => e.response.body)
         end
       rescue => e
         # Stuff that should never happen
@@ -231,7 +235,7 @@ module LogStash; module Outputs; class ElasticSearch;
         @logger.debug("Failed actions for last bad bulk request!", :actions => actions)
 
         # We retry until there are no errors! Errors should all go to the retry queue
-        sleep_interval = sleep_for_interval(sleep_interval)        
+        sleep_interval = sleep_for_interval(sleep_interval)
         retry unless @stopping.true?
       end
     end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -81,7 +81,7 @@ module LogStash; module Outputs; class ElasticSearch
       # If you specify a number larger than the batch size of your pipeline it will have no effect,
       # save for the case where a filter increases the size of an inflight batch by outputting
       # events.
-      mod.config :flush_size, :validate => :number, :default => 500
+      mod.config :flush_size, :validate => :number, :deprecate => "This setting is no longer necessary as we now try to restrict bulk requests to sane sizes. See the 'Batch Sizes' section of the docs. If you think you still need to restrict payloads based on the number, not size, of events, please open a ticket."
 
       # The amount of time since last flush before a flush is forced.
       #

data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb

@@ -45,7 +45,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
       # but for our current purposes this is correct
       if resp.code < 200 || resp.code > 299 && resp.code != 404
         safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
-        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, safe_url + path, body)
+        raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, safe_url + path, resp.body)
       end
 
       resp

data/lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -261,7 +261,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         new_urls.each do |url|
           # URI objects don't have real hash equality! So, since this isn't perf sensitive we do a linear scan
           unless @url_info.keys.include?(url)
-            state_changes[:added] << url
+            state_changes[:added] << url.to_s
             add_url(url)
           end
         end
@@ -269,24 +269,14 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
         # Delete connections not in the new list
         @url_info.each do |url,_|
           unless new_urls.include?(url)
-            state_changes[:removed] << url
+            state_changes[:removed] << url.to_s
             remove_url(url)
           end
         end
       end
 
       if state_changes[:removed].size > 0 || state_changes[:added].size > 0
-        if logger.info?
-          logger.info("Elasticsearch pool URLs updated", :changes => safe_state_changes(state_changes))
-        end
-      end
-    end
-    
-    def safe_state_changes(state_changes)
-      state_changes.reduce({}) do |acc, kv|
-        k,v = kv
-        acc[k] = v.map(&LogStash::Outputs::ElasticSearch::SafeURL.method(:without_credentials)).map(&:to_s)
-        acc
+        logger.info("Elasticsearch pool URLs updated", :changes => state_changes)
       end
     end
 

data/lib/logstash/outputs/elasticsearch/http_client.rb

@@ -5,6 +5,21 @@ require 'logstash/outputs/elasticsearch/http_client/pool'
 require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
 
 module LogStash; module Outputs; class ElasticSearch;
+  # This is a constant instead of a config option because
+  # there really isn't a good reason to configure it.
+  #
+  # The criteria used are:
+  # 1. We need a number that's less than 100MiB because ES
+  #    won't accept bulks larger than that.
+  # 2. It must be large enough to amortize the connection constant
+  #    across multiple requests.
+  # 3. It must be small enough that even if multiple threads hit this size
+  #    we won't use a lot of heap.
+  #
+  # We wound up agreeing that a number greater than 10 MiB and less than 100MiB
+  # made sense. We picked one on the lowish side to not use too much heap.
+  TARGET_BULK_BYTES = 20 * 1024 * 1024 # 20MiB
+
   class HttpClient
     attr_reader :client, :options, :logger, :pool, :action_count, :recv_count
     # This is here in case we use DEFAULT_OPTIONS in the future
@@ -38,9 +53,11 @@ module LogStash; module Outputs; class ElasticSearch;
     def bulk(actions)
       @action_count ||= 0
       @action_count += actions.size
-      
+
       return if actions.empty?
-      bulk_body = actions.collect do |action, args, source|
+
+
+      bulk_actions = actions.collect do |action, args, source|
         args, source = update_action_builder(args, source) if action == 'update'
 
         if source && action != 'delete'
@@ -48,13 +65,37 @@ module LogStash; module Outputs; class ElasticSearch;
         else
           next { action => args }
         end
-      end.
-      flatten.
-      reduce("") do |acc,line|
-        acc << LogStash::Json.dump(line)
-        acc << "\n"
       end
 
+      bulk_body = ""
+      bulk_responses = []
+      bulk_actions.each do |action|
+        as_json = action.is_a?(Array) ?
+                    action.map {|line| LogStash::Json.dump(line)}.join("\n") :
+                    LogStash::Json.dump(action)
+        as_json << "\n"
+
+        if (bulk_body.size + as_json.size) > TARGET_BULK_BYTES
+          bulk_responses << bulk_send(bulk_body)
+          bulk_body = as_json
+        else
+          bulk_body << as_json
+        end
+      end
+
+      bulk_responses << bulk_send(bulk_body) if bulk_body.size > 0
+
+      join_bulk_responses(bulk_responses)
+    end
+
+    def join_bulk_responses(bulk_responses)
+      {
+        "errors" => bulk_responses.any? {|r| r["errors"] == true},
+        "items" => bulk_responses.reduce([]) {|m,r| m.concat(r["items"])}
+      }
+    end
+
+    def bulk_send(bulk_body)
       # Discard the URL
       url, response = @pool.post("_bulk", nil, bulk_body)
       LogStash::Json.load(response.body)

data/lib/logstash/outputs/elasticsearch.rb

@@ -17,20 +17,20 @@ require "uri" # for escaping user input
 # yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
 # to upgrade Logstash in lock-step. For those still wishing to use the node or transport protocols please see
 # the <<plugins-outputs-elasticsearch_java,elasticsearch_java output plugin>>.
-#
+# 
 # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
 #
 # ==== Template management for Elasticsearch 5.x
-# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0. 
-# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default 
+# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
+# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
 # behavior.
 #
 # ** Users installing ES 5.x and LS 5.x **
 # This change will not affect you and you will continue to use the ES defaults.
 #
 # ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
-# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use 
-# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after 
+# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
+# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
 # the new template is installed.
 #
 # ==== Retry Policy
@@ -48,6 +48,10 @@ require "uri" # for escaping user input
 # NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
 # It is more performant for Elasticsearch to retry these exceptions than this plugin.
 #
+# ==== Batch Sizes ====
+# This plugin attempts to send batches of events as a single request. However, if
+# a request exceeds 20MB we will break it up until multiple batch requests. If a single document exceeds 20MB it will be sent as a single request.
+#
 # ==== DNS Caching
 #
 # This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
@@ -146,7 +150,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # a timeout occurs, the request will be retried.
   config :timeout, :validate => :number, :default => 60
 
-  # Set the Elasticsearch errors in the whitelist that you don't want to log. 
+  # Set the Elasticsearch errors in the whitelist that you don't want to log.
   # A useful example is when you want to skip all 409 errors
   # which are `document_already_exists_exception`.
   config :failure_type_logging_whitelist, :validate => :array, :default => []
@@ -173,7 +177,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # Resurrection is the process by which backend endpoints marked 'down' are checked
   # to see if they have come back to life
   config :resurrect_delay, :validate => :number, :default => 5
-  
+
   # How long to wait before checking if the connection is stale before executing a request on a connection using keepalive.
   # You may want to set this lower, if you get connection errors regularly
   # Quoting the Apache commons docs (this client is based Apache Commmons):

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '5.2.1'
+  s.version         = '5.3.0'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'cabin', ['~> 0.6']
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_development_dependency 'ftw', '~> 0.0.42'
-  s.add_development_dependency 'addressable', "~> 2.3.0" # used by FTW. V 2.5.0 is ruby 2.0 only. 
+  s.add_development_dependency 'addressable', "~> 2.4.0" # used by FTW. V 2.5.0 is ruby 2.0 only. 
   s.add_development_dependency 'logstash-codec-plain'
 
   if RUBY_PLATFORM == 'java'

data/spec/integration/outputs/index_spec.rb

@@ -1,21 +1,21 @@
 require_relative "../../../spec/es_spec_helper"
+require "logstash/outputs/elasticsearch"
 
 shared_examples "an indexer" do
+    let(:event) { LogStash::Event.new("message" => "Hello World!", "type" => type) }
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
     let(:type) { 10.times.collect { rand(10).to_s }.join("") }
     let(:event_count) { 10000 + rand(500) }
-    let(:flush_size) { rand(200) + 1 }
     let(:config) { "not implemented" }
+    let(:events) { event_count.times.map { event }.to_a }
     subject { LogStash::Outputs::ElasticSearch.new(config) }
 
     before do
       subject.register
-      event_count.times do
-        subject.multi_receive([LogStash::Event.new("message" => "Hello World!", "type" => type)])
-      end
     end
 
     it "ships events" do
+      subject.multi_receive(events)
       index_url = "http://#{get_host_port}/#{index}"
 
       ftw = FTW::Agent.new
@@ -42,13 +42,57 @@ shared_examples "an indexer" do
     end
 end
 
+describe "TARGET_BULK_BYTES", :integration => true do
+  let(:target_bulk_bytes) { LogStash::Outputs::ElasticSearch::TARGET_BULK_BYTES }
+  let(:event_count) { 1000 }
+  let(:events) { event_count.times.map { event }.to_a }
+  let(:config) {
+      {
+        "hosts" => get_host_port,
+        "index" => index
+      }
+  }
+  let(:index) { 10.times.collect { rand(10).to_s }.join("") }
+  let(:type) { 10.times.collect { rand(10).to_s }.join("") }
+  subject { LogStash::Outputs::ElasticSearch.new(config) }
+
+
+  before do
+    subject.register
+    allow(subject.client).to receive(:bulk_send).with(any_args).and_call_original
+    subject.multi_receive(events)
+  end
+
+  describe "batches that are too large for one" do
+    let(:event) { LogStash::Event.new("message" => "a " * (((target_bulk_bytes/2) / event_count)+1)) }
+
+    it "should send in two batches" do
+      expect(subject.client).to have_received(:bulk_send).twice do |payload|
+        expect(payload.size).to be <= target_bulk_bytes
+      end
+    end
+
+    describe "batches that fit in one" do
+      # Normally you'd want to generate a request that's just 1 byte below the limit, but it's
+      # impossible to know how many bytes an event will serialize as with bulk proto overhead
+      let(:event) { LogStash::Event.new("message" => "a") }
+
+      it "should send in one batch" do
+        expect(subject.client).to have_received(:bulk_send).once do |payload|
+          expect(payload.size).to be <= target_bulk_bytes
+        end
+      end
+    end
+  end
+end
+
+
 describe "an indexer with custom index_type", :integration => true do
   it_behaves_like "an indexer" do
     let(:config) {
       {
         "hosts" => get_host_port,
-        "index" => index,
-        "flush_size" => flush_size
+        "index" => index
       }
     }
   end
@@ -60,8 +104,7 @@ describe "an indexer with no type value set (default to logs)", :integration =>
     let(:config) {
       {
         "hosts" => get_host_port,
-        "index" => index,
-        "flush_size" => flush_size
+        "index" => index
       }
     }
   end

data/spec/integration/outputs/parent_spec.rb

@@ -4,7 +4,6 @@ shared_examples "a parent indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
     let(:type) { 10.times.collect { rand(10).to_s }.join("") }
     let(:event_count) { 10000 + rand(500) }
-    let(:flush_size) { rand(200) + 1 }
     let(:parent) { "not_implemented" }
     let(:config) { "not_implemented" }
     subject { LogStash::Outputs::ElasticSearch.new(config) }
@@ -17,7 +16,7 @@ shared_examples "a parent indexer" do
       ftw.put!("#{index_url}", :body => mapping.to_json)
       pdoc = { "foo" => "bar" }
       ftw.put!("#{index_url}/#{type}_parent/test", :body => pdoc.to_json)
-      
+
       subject.register
       subject.multi_receive(event_count.times.map { LogStash::Event.new("link_to" => "test", "message" => "Hello World!", "type" => type) })
     end
@@ -49,7 +48,6 @@ describe "(http protocol) index events with static parent", :integration => true
       {
         "hosts" => get_host_port,
         "index" => index,
-        "flush_size" => flush_size,
         "parent" => parent
       }
     }
@@ -62,10 +60,8 @@ describe "(http_protocol) index events with fieldref in parent value", :integrat
       {
         "hosts" => get_host_port,
         "index" => index,
-        "flush_size" => flush_size,
         "parent" => "%{link_to}"
       }
     }
   end
 end
-

data/spec/integration/outputs/routing_spec.rb

@@ -4,7 +4,6 @@ shared_examples "a routing indexer" do
     let(:index) { 10.times.collect { rand(10).to_s }.join("") }
     let(:type) { 10.times.collect { rand(10).to_s }.join("") }
     let(:event_count) { 10000 + rand(500) }
-    let(:flush_size) { rand(200) + 1 }
     let(:routing) { "not_implemented" }
     let(:config) { "not_implemented" }
     subject { LogStash::Outputs::ElasticSearch.new(config) }
@@ -43,7 +42,6 @@ describe "(http protocol) index events with static routing", :integration => tru
       {
         "hosts" => get_host_port,
         "index" => index,
-        "flush_size" => flush_size,
         "routing" => routing
       }
     }
@@ -57,10 +55,8 @@ describe "(http_protocol) index events with fieldref in routing value", :integra
       {
         "hosts" => get_host_port,
         "index" => index,
-        "flush_size" => flush_size,
         "routing" => "%{message}"
       }
     }
   end
 end
-

data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb

@@ -93,21 +93,6 @@ describe LogStash::Outputs::ElasticSearch::HttpClient::Pool do
       expect(subject).to have_received(:in_use_connections).twice
     end
   end
-  
-  describe "safe_state_changes" do
-    let(:state_changes) do 
-      {
-        :added => [URI.parse("http://sekretu:sekretp@foo1")],
-        :removed => [URI.parse("http://sekretu:sekretp@foo2")]
-      }
-    end
-    let(:processed) { subject.safe_state_changes(state_changes)}
-    
-    it "should hide passwords" do
-      expect(processed[:added].any? {|p| p =~ /sekretp/ }).to be false
-      expect(processed[:removed].any? {|p| p =~ /sekretp/ }).to be false
-    end
-  end
 
   describe "connection management" do
     context "with only one URL in the list" do

data/spec/unit/outputs/elasticsearch/http_client_spec.rb

@@ -3,7 +3,7 @@ require "logstash/outputs/elasticsearch/http_client"
 require "java"
 
 describe LogStash::Outputs::ElasticSearch::HttpClient do
-  let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get }}
+  let(:base_options) { {:hosts => ["127.0.0.1"], :logger => Cabin::Channel.get} }
 
   describe "Host/URL Parsing" do
     subject { described_class.new(base_options) }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 5.2.1
+  version: 5.3.0
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-11-11 00:00:00.000000000 Z
+date: 2016-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -83,7 +83,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
   name: addressable
   prerelease: false
   type: :development
@@ -91,7 +91,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.3.0
+        version: 2.4.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements: