fluent-plugin-elasticsearch 3.5.0 → 3.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e569ad392bb2cd8d948582b76aaae250d4acb47d48ffe1f533a113062b75e100
-  data.tar.gz: a44cc9cdacc8f742c19ee81a3a7792ce7444a7170d6c4f3c71dbd631b5679f37
+  metadata.gz: 157eb6e8f0c5412c1eec69c86be920e6737c2ed172cfd62618d284ae99bb68fc
+  data.tar.gz: 30d6c01d4626bbc016fe97e95caf529e1be40afa48196e806aebbc02457b7119
 SHA512:
-  metadata.gz: 2d3b8ba9dde0905e89c0f51170eedd84d7d6ff687a26341b2361a8982c7a47517ba01f86a217bc041cb8ba59340281928b8c56a834f8ff3cd4a5069ace176dae
-  data.tar.gz: 85be138322cf8db2843d16cf550490c0a275c37534d93a98e2c8b15016e10bcd5ba823b60f2918fe3116d5980c074b1bfdc8be76d26cb0b78eaa8dc6b4a97ca6
+  metadata.gz: 5b2830f040fda0e697a7b6e54eff371d78858637476eae95e8bbce7edc7334cb38b29977961b0ccf4f52aebb084638e4b4a8f3a8f5023766d28ec1781ab5b400
+  data.tar.gz: 8518ef11cdeca51a115e0be1201df72df0639c33e00f2b200148ede002e32d2a734cbba4e7b3accc16d25eed91120615277ba1ac40e39a901198f0d60425443a

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 3.5.1
+- Configurable split request size threshold (#586)
+
 ### 3.5.0
 - Adopt Elasticsearch ruby client v7 loggable class (#583)
 

data/README.md

@@ -85,6 +85,7 @@ Current maintainers: @cosmo0920
   + [suppress_doc_wrap](#suppress_doc_wrap)
   + [ignore_exceptions](#ignore_exceptions)
   + [exception_backup](#exception_backup)
+  + [bulk_message_request_threshold](#bulk_message_request_threshold)
 * [Troubleshooting](#troubleshooting)
   + [Cannot send events to elasticsearch](#cannot-send-events-to-elasticsearch)
   + [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
@@ -1064,6 +1065,14 @@ Indicates whether to backup chunk when ignore exception occurs.
 
 Default value is `true`.
 
+## bulk_message_request_threshold
+
+Configure `bulk_message` request splitting threshold size.
+
+Default value is `20MB`. (20 * 1024 * 1024)
+
+If you specify this size as negative number, `bulk_message` request splitting feature will be disabled.
+
 ## Troubleshooting
 
 ### Cannot send events to Elasticsearch

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.5.0'
+  s.version       = '3.5.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -138,6 +138,7 @@ EOC
     config_param :suppress_doc_wrap, :bool, :default => false
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
+    config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
 
     config_section :buffer do
       config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -296,6 +297,16 @@ EOC
           Object.const_get(exception)
         end
       end.compact
+
+      if @bulk_message_request_threshold < 0
+        class << self
+          alias_method :split_request?, :split_request_size_uncheck?
+        end
+      else
+        class << self
+          alias_method :split_request?, :split_request_size_check?
+        end
+      end
     end
 
     def placeholder?(name, param)
@@ -592,7 +603,7 @@ EOC
 
           if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
             bulk_message_count[info] += 1;
-            if bulk_message[info].size > TARGET_BULK_BYTES
+            if split_request?(bulk_message, info)
               bulk_message.each do |info, msgs|
                 send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info) unless msgs.empty?
                 msgs.clear
@@ -619,6 +630,18 @@ EOC
       end
     end
 
+    def split_request?(bulk_message, info)
+      # For safety.
+    end
+
+    def split_request_size_check?(bulk_message, info)
+      bulk_message[info].size > @bulk_message_request_threshold
+    end
+
+    def split_request_size_uncheck?(bulk_message, info)
+      false
+    end
+
     def process_message(tag, meta, header, time, record, extracted_values)
       logstash_prefix, index_name, type_name = extracted_values
 

data/test/plugin/test_out_elasticsearch.rb

@@ -1083,6 +1083,27 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_requested(request, times: 2)
   end
 
+  def test_writes_with_huge_records_but_uncheck
+    driver.configure(Fluent::Config::Element.new(
+                       'ROOT', '', {
+                         '@type' => 'elasticsearch',
+                         'bulk_message_request_threshold' => -1,
+                       }, [
+                         Fluent::Config::Element.new('buffer', 'tag', {
+                                                       'chunk_keys' => ['tag', 'time'],
+                                                       'chunk_limit_size' => '64MB',
+                                                     }, [])
+                       ]
+                     ))
+    request = stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
+      driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
+    end
+    assert_false(driver.instance.split_request?({}, nil))
+    assert_requested(request, times: 1)
+  end
+
   class IndexNamePlaceholdersTest < self
     def test_writes_to_speficied_index_with_tag_placeholder
       driver.configure("index_name myindex.${tag}\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.5.0
+  version: 3.5.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-08 00:00:00.000000000 Z
+date: 2019-05-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd