fluent-plugin-elasticsearch 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +3 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +10 -0
- data/test/plugin/test_out_elasticsearch.rb +20 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 75f13a6667d8bb5183b4a30c48db1d5d3ec24140aeebe9bf6c1ee022c11ba93a
|
4
|
+
data.tar.gz: cd25702d61748ea15b0296ff8158626031c1e11cdb44ca98ad2c389014ce867c
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9ae46bd72823ac77aed8168f6e96df224b80ee72a10a53e902ba924533933e5fccf5a3b8c7a7e6d4c5c54d87dd0d37e10f7f1f62034c1365ac0c6648a0bb698b
|
7
|
+
data.tar.gz: 1f99abfa263a0d4cb437371d030b4802bb3865d2032c286275ac829509136cd005587604bc09073598acf0e2d7ad1bd35850859c8f0fecd37e8e07b755afcd14
|
data/History.md
CHANGED
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '3.
|
6
|
+
s.version = '3.2.0'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -51,6 +51,7 @@ module Fluent::Plugin
|
|
51
51
|
DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
|
52
52
|
DEFAULT_TYPE_NAME = "fluentd".freeze
|
53
53
|
DEFAULT_RELOAD_AFTER = -1
|
54
|
+
TARGET_BULK_BYTES = 20 * 1024 * 1024
|
54
55
|
|
55
56
|
config_param :host, :string, :default => 'localhost'
|
56
57
|
config_param :port, :integer, :default => 9200
|
@@ -528,6 +529,15 @@ EOC
|
|
528
529
|
|
529
530
|
if append_record_to_messages(@write_operation, meta, header, record, bulk_message[info])
|
530
531
|
bulk_message_count[info] += 1;
|
532
|
+
if bulk_message[info].size > TARGET_BULK_BYTES
|
533
|
+
bulk_message.each do |info, msgs|
|
534
|
+
send_bulk(msgs, tag, chunk, bulk_message_count[info], extracted_values, info.index) unless msgs.empty?
|
535
|
+
msgs.clear
|
536
|
+
# Clear bulk_message_count for this info.
|
537
|
+
bulk_message_count[info] = 0;
|
538
|
+
next
|
539
|
+
end
|
540
|
+
end
|
531
541
|
else
|
532
542
|
if @emit_error_for_missing_id
|
533
543
|
raise MissingIdFieldError, "Missing '_id' field. Write operation is #{@write_operation}"
|
@@ -231,6 +231,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
231
231
|
assert_true instance.verify_es_version_at_startup
|
232
232
|
assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
|
233
233
|
assert_false instance.log_es_400_reason
|
234
|
+
assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
|
234
235
|
end
|
235
236
|
|
236
237
|
test 'configure Content-Type' do
|
@@ -1014,6 +1015,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1014
1015
|
assert_equal('myindex', index_cmds.first['index']['_index'])
|
1015
1016
|
end
|
1016
1017
|
|
1018
|
+
def test_writes_with_huge_records
|
1019
|
+
driver.configure(Fluent::Config::Element.new(
|
1020
|
+
'ROOT', '', {
|
1021
|
+
'@type' => 'elasticsearch',
|
1022
|
+
}, [
|
1023
|
+
Fluent::Config::Element.new('buffer', 'tag', {
|
1024
|
+
'chunk_keys' => ['tag', 'time'],
|
1025
|
+
'chunk_limit_size' => '64MB',
|
1026
|
+
}, [])
|
1027
|
+
]
|
1028
|
+
))
|
1029
|
+
request = stub_elastic
|
1030
|
+
driver.run(default_tag: 'test') do
|
1031
|
+
driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
|
1032
|
+
driver.feed(sample_record('huge_record' => ("a" * 20 * 1024 * 1024)))
|
1033
|
+
end
|
1034
|
+
assert_requested(request, times: 2)
|
1035
|
+
end
|
1036
|
+
|
1017
1037
|
class IndexNamePlaceholdersTest < self
|
1018
1038
|
def test_writes_to_speficied_index_with_tag_placeholder
|
1019
1039
|
driver.configure("index_name myindex.${tag}\n")
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.
|
4
|
+
version: 3.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-02-
|
12
|
+
date: 2019-02-05 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|