fluent-plugin-elasticsearch 3.1.1 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 75885bc5381810a40869a729ac3a7b6987f94b5083aad30e2111778982d239d7
4
- data.tar.gz: a07a3c9a043e21d638b1b2cab1faa8746cead0b4757690553a34ffd7058e1442
3
+ metadata.gz: 75f13a6667d8bb5183b4a30c48db1d5d3ec24140aeebe9bf6c1ee022c11ba93a
4
+ data.tar.gz: cd25702d61748ea15b0296ff8158626031c1e11cdb44ca98ad2c389014ce867c
5
5
  SHA512:
6
- metadata.gz: ea912aa3c8d707a3490cb30ba4ae01f8a071e699525957a765bd084f34077d2f9f7564232b2372645ca3fc9eb46e408ecd6bb6f42e06d19e276be9bd10d3b129
7
- data.tar.gz: 3f6b3eceaa5da555fa58c1718a7c43fbe2cb61957e771058605f5956d618874fccc6f6330eb48314808529f50175bed5facb76f10e0f4a501c0677db9871ca86
6
+ metadata.gz: 9ae46bd72823ac77aed8168f6e96df224b80ee72a10a53e902ba924533933e5fccf5a3b8c7a7e6d4c5c54d87dd0d37e10f7f1f62034c1365ac0c6648a0bb698b
7
+ data.tar.gz: 1f99abfa263a0d4cb437371d030b4802bb3865d2032c286275ac829509136cd005587604bc09073598acf0e2d7ad1bd35850859c8f0fecd37e8e07b755afcd14
data/History.md CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 3.2.0
6
+ - Split huge record requests (#539)
7
+
5
8
  ### 3.1.1
6
9
  - Add document for custom_headers (#538)
7
10
  - out_elasticsearch: Add custom_headers parameter (#529)
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '3.1.1'
6
+ s.version = '3.2.0'
7
7
  s.authors = ['diogo', 'pitr']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -51,6 +51,7 @@ module Fluent::Plugin
51
51
  DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
52
52
  DEFAULT_TYPE_NAME = "fluentd".freeze
53
53
  DEFAULT_RELOAD_AFTER = -1
54
+ TARGET_BULK_BYTES = 20 * 1024 * 1024
54
55
 
55
56
  config_param :host, :string, :default => 'localhost'
56
57
  config_param :port, :integer, :default => 9200
@@ -528,6 +529,15 @@ EOC
528
529
 
529
530
  if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
530
531
  bulk_message_count[info] += 1;
532
+ if bulk_message[info].size > TARGET_BULK_BYTES
533
+ bulk_message.each do |info, msgs|
534
+ send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
535
+ msgs.clear
536
+ # Clear bulk_message_count for this info.
537
+ bulk_message_count[info] = 0;
538
+ next
539
+ end
540
+ end
531
541
  else
532
542
  if @emit_error_for_missing_id
533
543
  raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
@@ -231,6 +231,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
231
231
  assert_true instance.verify_es_version_at_startup
232
232
  assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
233
233
  assert_false instance.log_es_400_reason
234
+ assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
234
235
  end
235
236
 
236
237
  test 'configure Content-Type' do
@@ -1014,6 +1015,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
1014
1015
  assert_equal('myindex', index_cmds.first['index']['_index'])
1015
1016
  end
1016
1017
 
1018
+ def test_writes_with_huge_records
1019
+ driver.configure(Fluent::Config::Element.new(
1020
+ 'ROOT', '', {
1021
+ '@type' => 'elasticsearch',
1022
+ }, [
1023
+ Fluent::Config::Element.new('buffer', 'tag', {
1024
+ 'chunk_keys' => ['tag', 'time'],
1025
+ 'chunk_limit_size' => '64MB',
1026
+ }, [])
1027
+ ]
1028
+ ))
1029
+ request = stub_elastic
1030
+ driver.run(default_tag: 'test') do
1031
+ driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
1032
+ driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
1033
+ end
1034
+ assert_requested(request, times: 2)
1035
+ end
1036
+
1017
1037
  class IndexNamePlaceholdersTest < self
1018
1038
  def test_writes_to_speficied_index_with_tag_placeholder
1019
1039
  driver.configure("index_name myindex.${tag}\n")
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 3.1.1
4
+ version: 3.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2019-02-04 00:00:00.000000000 Z
12
+ date: 2019-02-05 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd