logstash-output-elasticsearch 2.4.1-java → 2.4.2-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9f8d5b26eb15f7a372ac43583a962009103f7a4a
-  data.tar.gz: 0b98bd6bf747ec4c576ec7426d67d93a7dc583a9
+  metadata.gz: 22b9b67e7eada407308a8da6bb3ab1c9dabe4e11
+  data.tar.gz: e434712af48381f69dbd6eb75a3556614ea40e93
 SHA512:
-  metadata.gz: dba81882f5f3fa79f7dd4b9b307f8a95dd53d5ec1876112226fe3e67a2991bad3a03260d704e13e905b194e9bdb05f7284453a1415b03631e9ac9a472fea78e7
-  data.tar.gz: bc25879d936c938d8c21fe85cc98b45fd7f9671a2ea0cd3c2e7120ec910b5a8ea6719ec8160b45af87800b24810a2cf59572a9fa13980332a3eccd5ef02be3f5
+  metadata.gz: b4046883d73f45496ad777cf3e3708a619e4172a07d7c8c682e459a859eac562ce130346f0164de8b8477ce14080d3bfc1bf4b5584111ac861ee0eff961d4b89
+  data.tar.gz: 1ece9c0def2d670ebd7ffd49308c6838aa1b0c4f1c12a3aef226cda00ebf0f3e348673b68720cc396ad4ddd45f9fa7200d6017afc3867cd6bc5342c17a9c488a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 2.4.2
+ - Make flush_size actually cap the batch size in LS 2.2+
+
 ## 2.4.1
  - Used debug level instead of info when emitting flush log message
  - Updated docs about template

data/lib/logstash/outputs/elasticsearch/common.rb

@@ -25,7 +25,9 @@ module LogStash; module Outputs; class ElasticSearch;
 
     # Receive an array of events and immediately attempt to index them (no buffering)
     def multi_receive(events)
-      retrying_submit(events.map {|e| event_action_tuple(e) })
+      events.each_slice(@flush_size) do |slice|
+        retrying_submit(slice.map {|e| event_action_tuple(e) })
+      end
     end
 
     # Convert the event into a 3-tuple of action, params, and event
@@ -190,4 +192,4 @@ module LogStash; module Outputs; class ElasticSearch;
       raise e
     end
   end
-end; end; end
+end; end; end

data/lib/logstash/outputs/elasticsearch/common_configs.rb

@@ -5,7 +5,9 @@ module LogStash; module Outputs; class ElasticSearch
       # The default value will partition your indices by day so you can more easily
       # delete old data or only search specific date ranges.
       # Indexes may not contain uppercase characters.
-      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
+      # For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}.
+      # LS uses Joda to format the index pattern from event timestamp. 
+      # Joda formats are defined http://www.joda.org/joda-time/apidocs/org/joda/time/format/DateTimeFormat.html[here]. 
       mod.config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
 
       # The index type to write events to. Generally you should try to write only
@@ -44,15 +46,15 @@ module LogStash; module Outputs; class ElasticSearch
       # If not set, the included template will be used.
       mod.config :template, :validate => :path
 
-      # The template_overwrite option will always overwrite the indicated template 
-      # in Elasticsearch with either the one indicated by template or the included one. 
-      # This option is set to false by default. If you always want to stay up to date 
-      # with the template provided by Logstash, this option could be very useful to you. 
-      # Likewise, if you have your own template file managed by puppet, for example, and 
+      # The template_overwrite option will always overwrite the indicated template
+      # in Elasticsearch with either the one indicated by template or the included one.
+      # This option is set to false by default. If you always want to stay up to date
+      # with the template provided by Logstash, this option could be very useful to you.
+      # Likewise, if you have your own template file managed by puppet, for example, and
       # you wanted to be able to update it regularly, this option could help there as well.
-      # 
-      # Please note that if you are using your own customized version of the Logstash 
-      # template (logstash), setting this to true will make Logstash to overwrite 
+      #
+      # Please note that if you are using your own customized version of the Logstash
+      # template (logstash), setting this to true will make Logstash to overwrite
       # the "logstash" template (i.e. removing all customized settings)
       mod.config :template_overwrite, :validate => :boolean, :default => false
 
@@ -83,6 +85,15 @@ module LogStash; module Outputs; class ElasticSearch
       mod.config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
 
       # This plugin uses the bulk index API for improved indexing performance.
+      # In Logstashes >= 2.2 this setting defines the maximum sized bulk request Logstash will make
+      # You you may want to increase this to be in line with your pipeline's batch size.
+      # If you specify a number larger than the batch size of your pipeline it will have no effect,
+      # save for the case where a filter increases the size of an inflight batch by outputting
+      # events.
+      #
+      # In Logstashes <= 2.1 this plugin uses its own internal buffer of events.
+      # This config option sets that size. In these older logstashes this size may
+      # have a significant impact on heap usage, whereas in 2.2+ it will never increase it.
       # To make efficient bulk API calls, we will buffer a certain number of
       # events before flushing that out to Elasticsearch. This setting
       # controls how many events will be buffered before sending a batch
@@ -139,4 +150,4 @@ module LogStash; module Outputs; class ElasticSearch
       mod.config :retry_max_items, :validate => :number, :default => 500, :deprecated => true
     end
   end
-end end end
+end end end

data/logstash-output-elasticsearch.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-output-elasticsearch'
-  s.version         = '2.4.1'
+  s.version         = '2.4.2'
   s.licenses        = ['apache-2.0']
   s.summary         = "Logstash Output to Elasticsearch"
   s.description     = "Output events to elasticsearch"

data/spec/unit/outputs/elasticsearch_spec.rb

@@ -92,6 +92,28 @@ describe "outputs/elasticsearch" do
         expect(manticore_host).to include("9202")
       end
     end
+
+    describe "#multi_receive" do
+      let(:events) { [double("one"), double("two"), double("three")] }
+      let(:events_tuples) { [double("one t"), double("two t"), double("three t")] }
+      let(:options) { super.merge("flush_size" => 2) }
+
+      before do
+        allow(eso).to receive(:retrying_submit).with(anything)
+        events.each_with_index do |e,i|
+          et = events_tuples[i]
+          allow(eso).to receive(:event_action_tuple).with(e).and_return(et)
+        end
+        eso.multi_receive(events)
+      end
+
+      it "should receive an array of events and invoke retrying_submit with them, split by flush_size" do
+        expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(0,2))
+        expect(eso).to have_received(:retrying_submit).with(events_tuples.slice(2,3))
+      end
+
+    end
+
   end
 
   # TODO(sissel): Improve this. I'm not a fan of using message expectations (expect().to receive...)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.4.1
+  version: 2.4.2
 platform: java
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-01-29 00:00:00.000000000 Z
+date: 2016-02-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby