fluent-plugin-elasticsearch 3.1.1 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 75885bc5381810a40869a729ac3a7b6987f94b5083aad30e2111778982d239d7
-  data.tar.gz: a07a3c9a043e21d638b1b2cab1faa8746cead0b4757690553a34ffd7058e1442
+  metadata.gz: 75f13a6667d8bb5183b4a30c48db1d5d3ec24140aeebe9bf6c1ee022c11ba93a
+  data.tar.gz: cd25702d61748ea15b0296ff8158626031c1e11cdb44ca98ad2c389014ce867c
 SHA512:
-  metadata.gz: ea912aa3c8d707a3490cb30ba4ae01f8a071e699525957a765bd084f34077d2f9f7564232b2372645ca3fc9eb46e408ecd6bb6f42e06d19e276be9bd10d3b129
-  data.tar.gz: 3f6b3eceaa5da555fa58c1718a7c43fbe2cb61957e771058605f5956d618874fccc6f6330eb48314808529f50175bed5facb76f10e0f4a501c0677db9871ca86
+  metadata.gz: 9ae46bd72823ac77aed8168f6e96df224b80ee72a10a53e902ba924533933e5fccf5a3b8c7a7e6d4c5c54d87dd0d37e10f7f1f62034c1365ac0c6648a0bb698b
+  data.tar.gz: 1f99abfa263a0d4cb437371d030b4802bb3865d2032c286275ac829509136cd005587604bc09073598acf0e2d7ad1bd35850859c8f0fecd37e8e07b755afcd14

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 3.2.0
+- Split huge record requests (#539)
+
 ### 3.1.1
 - Add document for custom_headers (#538)
 - out_elasticsearch: Add custom_headers parameter (#529)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.1.1'
+  s.version       = '3.2.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -51,6 +51,7 @@ module Fluent::Plugin
     DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
     DEFAULT_TYPE_NAME = "fluentd".freeze
     DEFAULT_RELOAD_AFTER = -1
+    TARGET_BULK_BYTES = 20 * 1024 * 1024
 
     config_param :host, :string,  :default => 'localhost'
     config_param :port, :integer, :default => 9200
@@ -528,6 +529,15 @@ EOC
 
           if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
             bulk_message_count[info] += 1;
+            if bulk_message[info].size > TARGET_BULK_BYTES
+              bulk_message.each do |info, msgs|
+                send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
+                msgs.clear
+                # Clear bulk_message_count for this info.
+                bulk_message_count[info] = 0;
+                next
+              end
+            end
           else
             if @emit_error_for_missing_id
               raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"

data/test/plugin/test_out_elasticsearch.rb

@@ -231,6 +231,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_true instance.verify_es_version_at_startup
     assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
     assert_false instance.log_es_400_reason
+    assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
   end
 
   test 'configure Content-Type' do
@@ -1014,6 +1015,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal('myindex', index_cmds.first['index']['_index'])
   end
 
+  def test_writes_with_huge_records
+    driver.configure(Fluent::Config::Element.new(
+                       'ROOT', '', {
+                         '@type' => 'elasticsearch',
+                       }, [
+                         Fluent::Config::Element.new('buffer', 'tag', {
+                                                       'chunk_keys' => ['tag', 'time'],
+                                                       'chunk_limit_size' => '64MB',
+                                                     }, [])
+                       ]
+                     ))
+    request = stub_elastic
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
+      driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
+    end
+    assert_requested(request, times: 2)
+  end
+
   class IndexNamePlaceholdersTest < self
     def test_writes_to_speficied_index_with_tag_placeholder
       driver.configure("index_name myindex.${tag}\n")

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.1.1
+  version: 3.2.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-04 00:00:00.000000000 Z
+date: 2019-02-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd