fluent-plugin-elasticsearch 3.6.0 → 3.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +5 -0
- data/README.md +8 -0
- data/fluent-plugin-elasticsearch.gemspec +5 -1
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +2 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +58 -3
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +15 -3
- data/test/plugin/test_elasticsearch_error_handler.rb +123 -1
- data/test/plugin/test_out_elasticsearch.rb +138 -17
- data/test/plugin/test_out_elasticsearch_dynamic.rb +78 -0
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1073f1770d5c380f496b57f111e1792812b870392c5996111465ce9cd2cfa0c6
|
4
|
+
data.tar.gz: 1f8e70ec6516ba252d7f2b722ec7c85d3830e3eb0b616b67c909663ac5db9461
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 291e4a936373fc096718c4fa9db816874a7e53d2f950b9a8e5bfa3f7a6830d98a4d79ab4b36d16b5615cfd831e364b5e545ff91a49866f26805a5aae73670338
|
7
|
+
data.tar.gz: ea66679ef5406d950923e9f255532116f5378619b712f063c64b01997313aecf5ec53c3e04a7c1313125cda4370e058c200d539bd741c2afd55a08015a42c26b
|
data/History.md
CHANGED
@@ -1,6 +1,11 @@
|
|
1
1
|
## Changelog [[tags]](https://github.com/uken/fluent-plugin-elasticsearch/tags)
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
|
+
### 3.6.1
|
5
|
+
- retry upsert on recoverable error. (#667)
|
6
|
+
- Allow `_index` in chunk_keys (#665)
|
7
|
+
- Support compression feature (#664)
|
8
|
+
|
4
9
|
### 3.6.0
|
5
10
|
- Set order in newly created templates (#660)
|
6
11
|
- Merge Support index lifecycle management into master (#659)
|
data/README.md
CHANGED
@@ -66,6 +66,7 @@ Current maintainers: @cosmo0920
|
|
66
66
|
+ [include_index_in_url](#include_index_in_url)
|
67
67
|
+ [http_backend](#http_backend)
|
68
68
|
+ [prefer_oj_serializer](#prefer_oj_serializer)
|
69
|
+
+ [compression_level](#compression_level)
|
69
70
|
+ [Client/host certificate options](#clienthost-certificate-options)
|
70
71
|
+ [Proxy Support](#proxy-support)
|
71
72
|
+ [Buffer options](#buffer-options)
|
@@ -792,6 +793,13 @@ Default value is `excon` which is default http_backend of elasticsearch plugin.
|
|
792
793
|
http_backend typhoeus
|
793
794
|
```
|
794
795
|
|
796
|
+
### compression_level
|
797
|
+
You can add gzip compression of output data. In this case `default_compression`, `best_compression` or `best speed` option should be chosen.
|
798
|
+
By default there is no compression, default value for this option is `no_compression`
|
799
|
+
```
|
800
|
+
compression_level best_compression
|
801
|
+
```
|
802
|
+
|
795
803
|
### prefer_oj_serializer
|
796
804
|
|
797
805
|
With default behavior, Elasticsearch client uses `Yajl` as JSON encoder/decoder.
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '3.6.
|
6
|
+
s.version = '3.6.1'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -16,6 +16,10 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
17
17
|
s.require_paths = ['lib']
|
18
18
|
|
19
|
+
if s.respond_to?(:metadata)
|
20
|
+
s.metadata["changelog_uri"] = "https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md"
|
21
|
+
end
|
22
|
+
|
19
23
|
s.required_ruby_version = Gem::Requirement.new(">= 2.0".freeze)
|
20
24
|
|
21
25
|
s.add_runtime_dependency 'fluentd', '>= 0.14.22'
|
@@ -60,6 +60,8 @@ class Fluent::Plugin::ElasticsearchErrorHandler
|
|
60
60
|
write_operation = @plugin.write_operation
|
61
61
|
elsif INDEX_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(CREATE_OP)
|
62
62
|
write_operation = CREATE_OP
|
63
|
+
elsif UPSERT_OP == @plugin.write_operation && item.is_a?(Hash) && item.has_key?(UPDATE_OP)
|
64
|
+
write_operation = UPDATE_OP
|
63
65
|
elsif item.nil?
|
64
66
|
stats[:errors_nil_resp] += 1
|
65
67
|
next
|
@@ -18,6 +18,7 @@ require 'fluent/event'
|
|
18
18
|
require 'fluent/error'
|
19
19
|
require 'fluent/time'
|
20
20
|
require 'fluent/log-ext'
|
21
|
+
require 'zlib'
|
21
22
|
require_relative 'elasticsearch_constants'
|
22
23
|
require_relative 'elasticsearch_error'
|
23
24
|
require_relative 'elasticsearch_error_handler'
|
@@ -148,6 +149,7 @@ EOC
|
|
148
149
|
config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
|
149
150
|
config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
|
150
151
|
config_param :bulk_message_request_threshold, :size, :default => TARGET_BULK_BYTES
|
152
|
+
config_param :compression_level, :enum, {list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression}
|
151
153
|
config_param :enable_ilm, :bool, :default => false
|
152
154
|
config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
|
153
155
|
config_param :ilm_policy, :hash, :default => {}
|
@@ -170,7 +172,7 @@ EOC
|
|
170
172
|
compat_parameters_convert(conf, :buffer)
|
171
173
|
|
172
174
|
super
|
173
|
-
raise Fluent::ConfigError, "'tag' in chunk_keys is required." if not @
|
175
|
+
raise Fluent::ConfigError, "'tag' or '_index' in chunk_keys is required." if not @buffer_config.chunk_keys.include? "tag" and not @buffer_config.chunk_keys.include? "_index"
|
174
176
|
|
175
177
|
@time_parser = create_time_parser
|
176
178
|
@backend_options = backend_options
|
@@ -327,6 +329,18 @@ EOC
|
|
327
329
|
alias_method :split_request?, :split_request_size_check?
|
328
330
|
end
|
329
331
|
end
|
332
|
+
|
333
|
+
version_arr = Elasticsearch::Transport::VERSION.split('.')
|
334
|
+
|
335
|
+
if (version_arr[0].to_i < 7) || (version_arr[0].to_i == 7 && version_arr[1].to_i < 2)
|
336
|
+
if compression
|
337
|
+
raise Fluent::ConfigError, <<-EOC
|
338
|
+
Cannot use compression with elasticsearch-transport plugin version < 7.2.0
|
339
|
+
Your elasticsearch-transport plugin version version is #{Elasticsearch::Transport::VERSION}.
|
340
|
+
Please consider to upgrade ES client.
|
341
|
+
EOC
|
342
|
+
end
|
343
|
+
end
|
330
344
|
end
|
331
345
|
|
332
346
|
def placeholder?(name, param)
|
@@ -338,6 +352,23 @@ EOC
|
|
338
352
|
end
|
339
353
|
end
|
340
354
|
|
355
|
+
def compression
|
356
|
+
!(@compression_level == :no_compression)
|
357
|
+
end
|
358
|
+
|
359
|
+
def compression_strategy
|
360
|
+
case @compression_level
|
361
|
+
when :default_compression
|
362
|
+
Zlib::DEFAULT_COMPRESSION
|
363
|
+
when :best_compression
|
364
|
+
Zlib::BEST_COMPRESSION
|
365
|
+
when :best_speed
|
366
|
+
Zlib::BEST_SPEED
|
367
|
+
else
|
368
|
+
Zlib::NO_COMPRESSION
|
369
|
+
end
|
370
|
+
end
|
371
|
+
|
341
372
|
def backend_options
|
342
373
|
case @http_backend
|
343
374
|
when :excon
|
@@ -438,7 +469,14 @@ EOC
|
|
438
469
|
if local_reload_connections && @reload_after > DEFAULT_RELOAD_AFTER
|
439
470
|
local_reload_connections = @reload_after
|
440
471
|
end
|
441
|
-
|
472
|
+
|
473
|
+
gzip_headers = if compression
|
474
|
+
{'Content-Encoding' => 'gzip'}
|
475
|
+
else
|
476
|
+
{}
|
477
|
+
end
|
478
|
+
headers = { 'Content-Type' => @content_type.to_s }.merge(@custom_headers).merge(gzip_headers)
|
479
|
+
|
442
480
|
transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
|
443
481
|
options: {
|
444
482
|
reload_connections: local_reload_connections,
|
@@ -456,6 +494,7 @@ EOC
|
|
456
494
|
},
|
457
495
|
sniffer_class: @sniffer_class,
|
458
496
|
serializer_class: @serializer_class,
|
497
|
+
compression: compression,
|
459
498
|
}), &adapter_conf)
|
460
499
|
Elasticsearch::Client.new transport: transport
|
461
500
|
end
|
@@ -768,6 +807,15 @@ EOC
|
|
768
807
|
[parent_object, path[-1]]
|
769
808
|
end
|
770
809
|
|
810
|
+
# gzip compress data
|
811
|
+
def gzip(string)
|
812
|
+
wio = StringIO.new("w")
|
813
|
+
w_gz = Zlib::GzipWriter.new(wio, strategy = compression_strategy)
|
814
|
+
w_gz.write(string)
|
815
|
+
w_gz.close
|
816
|
+
wio.string
|
817
|
+
end
|
818
|
+
|
771
819
|
# send_bulk given a specific bulk request, the original tag,
|
772
820
|
# chunk, and bulk_message_count
|
773
821
|
def send_bulk(data, tag, chunk, bulk_message_count, extracted_values, info)
|
@@ -791,7 +839,14 @@ EOC
|
|
791
839
|
begin
|
792
840
|
|
793
841
|
log.on_trace { log.trace "bulk request: #{data}" }
|
794
|
-
|
842
|
+
|
843
|
+
prepared_data = if compression
|
844
|
+
gzip(data)
|
845
|
+
else
|
846
|
+
data
|
847
|
+
end
|
848
|
+
|
849
|
+
response = client(info.host).bulk body: prepared_data, index: info.index
|
795
850
|
log.on_trace { log.trace "bulk response: #{response}" }
|
796
851
|
|
797
852
|
if response['errors']
|
@@ -44,6 +44,12 @@ module Fluent::Plugin
|
|
44
44
|
@_es ||= begin
|
45
45
|
@current_config = connection_options[:hosts].clone
|
46
46
|
adapter_conf = lambda {|f| f.adapter @http_backend, @backend_options }
|
47
|
+
gzip_headers = if compression
|
48
|
+
{'Content-Encoding' => 'gzip'}
|
49
|
+
else
|
50
|
+
{}
|
51
|
+
end
|
52
|
+
headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
|
47
53
|
transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(connection_options.merge(
|
48
54
|
options: {
|
49
55
|
reload_connections: @reload_connections,
|
@@ -51,14 +57,15 @@ module Fluent::Plugin
|
|
51
57
|
resurrect_after: @resurrect_after,
|
52
58
|
logger: @transport_logger,
|
53
59
|
transport_options: {
|
54
|
-
headers:
|
60
|
+
headers: headers,
|
55
61
|
request: { timeout: @request_timeout },
|
56
62
|
ssl: { verify: @ssl_verify, ca_file: @ca_file, version: @ssl_version }
|
57
63
|
},
|
58
64
|
http: {
|
59
65
|
user: @user,
|
60
66
|
password: @password
|
61
|
-
}
|
67
|
+
},
|
68
|
+
compression: compression,
|
62
69
|
}), &adapter_conf)
|
63
70
|
Elasticsearch::Client.new transport: transport
|
64
71
|
end
|
@@ -210,7 +217,12 @@ module Fluent::Plugin
|
|
210
217
|
|
211
218
|
def send_bulk(data, host, index)
|
212
219
|
begin
|
213
|
-
|
220
|
+
prepared_data = if compression
|
221
|
+
gzip(data)
|
222
|
+
else
|
223
|
+
data
|
224
|
+
end
|
225
|
+
response = client(host).bulk body: prepared_data, index: index
|
214
226
|
if response['errors']
|
215
227
|
log.error "Could not push log to Elasticsearch: #{response}"
|
216
228
|
end
|
@@ -7,9 +7,10 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
7
7
|
|
8
8
|
class TestPlugin
|
9
9
|
attr_reader :log
|
10
|
-
attr_reader :
|
10
|
+
attr_reader :error_events
|
11
11
|
attr_accessor :unrecoverable_error_types
|
12
12
|
attr_accessor :log_es_400_reason
|
13
|
+
attr_accessor :write_operation
|
13
14
|
def initialize(log, log_es_400_reason = false)
|
14
15
|
@log = log
|
15
16
|
@write_operation = 'index'
|
@@ -522,4 +523,125 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
522
523
|
|
523
524
|
end
|
524
525
|
|
526
|
+
def test_retry_error_upsert
|
527
|
+
@plugin.write_operation = 'upsert'
|
528
|
+
records = []
|
529
|
+
error_records = Hash.new(false)
|
530
|
+
error_records.merge!({0=>true, 4=>true, 9=>true})
|
531
|
+
10.times do |i|
|
532
|
+
records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
|
533
|
+
end
|
534
|
+
chunk = MockChunk.new(records)
|
535
|
+
|
536
|
+
response = parse_response(%({
|
537
|
+
"took" : 1,
|
538
|
+
"errors" : true,
|
539
|
+
"items" : [
|
540
|
+
{
|
541
|
+
"update" : {
|
542
|
+
"_index" : "foo",
|
543
|
+
"_type" : "bar",
|
544
|
+
"_id" : "1",
|
545
|
+
"status" : 201
|
546
|
+
}
|
547
|
+
},
|
548
|
+
{
|
549
|
+
"update" : {
|
550
|
+
"_index" : "foo",
|
551
|
+
"_type" : "bar",
|
552
|
+
"_id" : "2",
|
553
|
+
"status" : 500,
|
554
|
+
"error" : {
|
555
|
+
"type" : "some unrecognized type",
|
556
|
+
"reason":"unrecognized error"
|
557
|
+
}
|
558
|
+
}
|
559
|
+
},
|
560
|
+
{
|
561
|
+
"update" : {
|
562
|
+
"_index" : "foo",
|
563
|
+
"_type" : "bar",
|
564
|
+
"_id" : "3",
|
565
|
+
"status" : 409,
|
566
|
+
"error" : {
|
567
|
+
"type":"version_conflict_engine_exception",
|
568
|
+
"reason":"document already exists"
|
569
|
+
}
|
570
|
+
}
|
571
|
+
},
|
572
|
+
{
|
573
|
+
"update" : {
|
574
|
+
"_index" : "foo",
|
575
|
+
"_type" : "bar",
|
576
|
+
"_id" : "5",
|
577
|
+
"status" : 500,
|
578
|
+
"error" : {
|
579
|
+
"reason":"unrecognized error - no type field"
|
580
|
+
}
|
581
|
+
}
|
582
|
+
},
|
583
|
+
{
|
584
|
+
"update" : {
|
585
|
+
"_index" : "foo",
|
586
|
+
"_type" : "bar",
|
587
|
+
"_id" : "6",
|
588
|
+
"status" : 400,
|
589
|
+
"error" : {
|
590
|
+
"type" : "mapper_parsing_exception",
|
591
|
+
"reason":"failed to parse"
|
592
|
+
}
|
593
|
+
}
|
594
|
+
},
|
595
|
+
{
|
596
|
+
"update" : {
|
597
|
+
"_index" : "foo",
|
598
|
+
"_type" : "bar",
|
599
|
+
"_id" : "7",
|
600
|
+
"status" : 400,
|
601
|
+
"error" : {
|
602
|
+
"type" : "some unrecognized type",
|
603
|
+
"reason":"unrecognized error"
|
604
|
+
}
|
605
|
+
}
|
606
|
+
},
|
607
|
+
{
|
608
|
+
"update" : {
|
609
|
+
"_index" : "foo",
|
610
|
+
"_type" : "bar",
|
611
|
+
"_id" : "8",
|
612
|
+
"status" : 500,
|
613
|
+
"error" : {
|
614
|
+
"type" : "some unrecognized type",
|
615
|
+
"reason":"unrecognized error"
|
616
|
+
}
|
617
|
+
}
|
618
|
+
}
|
619
|
+
]
|
620
|
+
}))
|
621
|
+
|
622
|
+
begin
|
623
|
+
failed = false
|
624
|
+
dummy_extracted_values = []
|
625
|
+
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
|
626
|
+
rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
|
627
|
+
failed = true
|
628
|
+
records = [].tap do |records|
|
629
|
+
next unless e.respond_to?(:retry_stream)
|
630
|
+
e.retry_stream.each {|time, record| records << record}
|
631
|
+
end
|
632
|
+
puts records
|
633
|
+
assert_equal 3, records.length
|
634
|
+
assert_equal 2, records[0]['_id']
|
635
|
+
# upsert is retried in case of conflict error.
|
636
|
+
assert_equal 3, records[1]['_id']
|
637
|
+
assert_equal 8, records[2]['_id']
|
638
|
+
error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
|
639
|
+
assert_equal 3, error_ids.length
|
640
|
+
assert_equal [5, 6, 7], error_ids
|
641
|
+
@plugin.error_events.collect {|h| h[:error]}.each do |e|
|
642
|
+
assert_true e.respond_to?(:backtrace)
|
643
|
+
end
|
644
|
+
end
|
645
|
+
assert_true failed
|
646
|
+
end
|
525
647
|
end
|
@@ -232,6 +232,44 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
232
232
|
assert_equal Fluent::Plugin::ElasticsearchOutput::DEFAULT_ELASTICSEARCH_VERSION, instance.default_elasticsearch_version
|
233
233
|
assert_false instance.log_es_400_reason
|
234
234
|
assert_equal 20 * 1024 * 1024, Fluent::Plugin::ElasticsearchOutput::TARGET_BULK_BYTES
|
235
|
+
assert_false instance.compression
|
236
|
+
assert_equal :no_compression, instance.compression_level
|
237
|
+
end
|
238
|
+
|
239
|
+
test 'configure compression' do
|
240
|
+
config = %{
|
241
|
+
compression_level best_compression
|
242
|
+
}
|
243
|
+
instance = driver(config).instance
|
244
|
+
|
245
|
+
assert_equal true, instance.compression
|
246
|
+
end
|
247
|
+
|
248
|
+
test 'check compression strategy' do
|
249
|
+
config = %{
|
250
|
+
compression_level best_speed
|
251
|
+
}
|
252
|
+
instance = driver(config).instance
|
253
|
+
|
254
|
+
assert_equal Zlib::BEST_SPEED, instance.compression_strategy
|
255
|
+
end
|
256
|
+
|
257
|
+
test 'check content-encoding header with compression' do
|
258
|
+
config = %{
|
259
|
+
compression_level best_compression
|
260
|
+
}
|
261
|
+
instance = driver(config).instance
|
262
|
+
|
263
|
+
assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
264
|
+
end
|
265
|
+
|
266
|
+
test 'check compression option is passed to transport' do
|
267
|
+
config = %{
|
268
|
+
compression_level best_compression
|
269
|
+
}
|
270
|
+
instance = driver(config).instance
|
271
|
+
|
272
|
+
assert_equal true, instance.client.transport.options[:compression]
|
235
273
|
end
|
236
274
|
|
237
275
|
test 'configure Content-Type' do
|
@@ -341,23 +379,65 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
341
379
|
end
|
342
380
|
end
|
343
381
|
|
344
|
-
|
345
|
-
|
346
|
-
|
347
|
-
|
348
|
-
'
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
382
|
+
sub_test_case 'chunk_keys requirement' do
|
383
|
+
test 'tag in chunk_keys' do
|
384
|
+
assert_nothing_raised do
|
385
|
+
driver(Fluent::Config::Element.new(
|
386
|
+
'ROOT', '', {
|
387
|
+
'@type' => 'elasticsearch',
|
388
|
+
'host' => 'log.google.com',
|
389
|
+
'port' => 777,
|
390
|
+
'scheme' => 'https',
|
391
|
+
'path' => '/es/',
|
392
|
+
'user' => 'john',
|
393
|
+
'password' => 'doe',
|
394
|
+
}, [
|
395
|
+
Fluent::Config::Element.new('buffer', 'tag', {
|
396
|
+
'chunk_keys' => 'tag'
|
397
|
+
}, [])
|
398
|
+
]
|
399
|
+
))
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
test '_index in chunk_keys' do
|
404
|
+
assert_nothing_raised do
|
405
|
+
driver(Fluent::Config::Element.new(
|
406
|
+
'ROOT', '', {
|
407
|
+
'@type' => 'elasticsearch',
|
408
|
+
'host' => 'log.google.com',
|
409
|
+
'port' => 777,
|
410
|
+
'scheme' => 'https',
|
411
|
+
'path' => '/es/',
|
412
|
+
'user' => 'john',
|
413
|
+
'password' => 'doe',
|
414
|
+
}, [
|
415
|
+
Fluent::Config::Element.new('buffer', '_index', {
|
416
|
+
'chunk_keys' => '_index'
|
417
|
+
}, [])
|
418
|
+
]
|
419
|
+
))
|
420
|
+
end
|
421
|
+
end
|
422
|
+
|
423
|
+
test 'lack of tag and _index in chunk_keys' do
|
424
|
+
assert_raise_message(/'tag' or '_index' in chunk_keys is required./) do
|
425
|
+
driver(Fluent::Config::Element.new(
|
426
|
+
'ROOT', '', {
|
427
|
+
'@type' => 'elasticsearch',
|
428
|
+
'host' => 'log.google.com',
|
429
|
+
'port' => 777,
|
430
|
+
'scheme' => 'https',
|
431
|
+
'path' => '/es/',
|
432
|
+
'user' => 'john',
|
433
|
+
'password' => 'doe',
|
434
|
+
}, [
|
435
|
+
Fluent::Config::Element.new('buffer', 'mykey', {
|
436
|
+
'chunk_keys' => 'mykey'
|
437
|
+
}, [])
|
438
|
+
]
|
439
|
+
))
|
440
|
+
end
|
361
441
|
end
|
362
442
|
end
|
363
443
|
|
@@ -1762,6 +1842,47 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
1762
1842
|
assert_equal(index_name, index_cmds.first['index']['_index'])
|
1763
1843
|
end
|
1764
1844
|
|
1845
|
+
# gzip compress data
|
1846
|
+
def gzip(string, strategy)
|
1847
|
+
wio = StringIO.new("w")
|
1848
|
+
w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
|
1849
|
+
w_gz.write(string)
|
1850
|
+
w_gz.close
|
1851
|
+
wio.string
|
1852
|
+
end
|
1853
|
+
|
1854
|
+
|
1855
|
+
def test_writes_to_default_index_with_compression
|
1856
|
+
config = %[
|
1857
|
+
compression_level default_compression
|
1858
|
+
]
|
1859
|
+
|
1860
|
+
bodystr = %({
|
1861
|
+
"took" : 500,
|
1862
|
+
"errors" : false,
|
1863
|
+
"items" : [
|
1864
|
+
{
|
1865
|
+
"create": {
|
1866
|
+
"_index" : "fluentd",
|
1867
|
+
"_type" : "fluentd"
|
1868
|
+
}
|
1869
|
+
}
|
1870
|
+
]
|
1871
|
+
})
|
1872
|
+
|
1873
|
+
compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
|
1874
|
+
|
1875
|
+
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
1876
|
+
to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
|
1877
|
+
|
1878
|
+
driver(config)
|
1879
|
+
driver.run(default_tag: 'test') do
|
1880
|
+
driver.feed(sample_record)
|
1881
|
+
end
|
1882
|
+
|
1883
|
+
assert_requested(elastic_request)
|
1884
|
+
end
|
1885
|
+
|
1765
1886
|
data('Elasticsearch 6' => [6, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME],
|
1766
1887
|
'Elasticsearch 7' => [7, Fluent::Plugin::ElasticsearchOutput::DEFAULT_TYPE_NAME_ES_7x],
|
1767
1888
|
'Elasticsearch 8' => [8, nil],
|
@@ -104,6 +104,44 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
104
104
|
assert_false instance.with_transporter_log
|
105
105
|
assert_equal :"application/json", instance.content_type
|
106
106
|
assert_equal :excon, instance.http_backend
|
107
|
+
assert_false instance.compression
|
108
|
+
assert_equal :no_compression, instance.compression_level
|
109
|
+
end
|
110
|
+
|
111
|
+
test 'configure compression' do
|
112
|
+
config = %{
|
113
|
+
compression_level best_compression
|
114
|
+
}
|
115
|
+
instance = driver(config).instance
|
116
|
+
|
117
|
+
assert_equal true, instance.compression
|
118
|
+
end
|
119
|
+
|
120
|
+
test 'check compression strategy' do
|
121
|
+
config = %{
|
122
|
+
compression_level best_speed
|
123
|
+
}
|
124
|
+
instance = driver(config).instance
|
125
|
+
|
126
|
+
assert_equal Zlib::BEST_SPEED, instance.compression_strategy
|
127
|
+
end
|
128
|
+
|
129
|
+
test 'check content-encoding header with compression' do
|
130
|
+
config = %{
|
131
|
+
compression_level best_compression
|
132
|
+
}
|
133
|
+
instance = driver(config).instance
|
134
|
+
|
135
|
+
assert_equal "gzip", instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
136
|
+
end
|
137
|
+
|
138
|
+
test 'check compression option is passed to transport' do
|
139
|
+
config = %{
|
140
|
+
compression_level best_compression
|
141
|
+
}
|
142
|
+
instance = driver(config).instance
|
143
|
+
|
144
|
+
assert_equal true, instance.client.transport.options[:compression]
|
107
145
|
end
|
108
146
|
|
109
147
|
test 'configure Content-Type' do
|
@@ -305,6 +343,46 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
305
343
|
assert_equal('fluentd', index_cmds.first['index']['_index'])
|
306
344
|
end
|
307
345
|
|
346
|
+
# gzip compress data
|
347
|
+
def gzip(string, strategy)
|
348
|
+
wio = StringIO.new("w")
|
349
|
+
w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
|
350
|
+
w_gz.write(string)
|
351
|
+
w_gz.close
|
352
|
+
wio.string
|
353
|
+
end
|
354
|
+
|
355
|
+
def test_writes_to_default_index_with_compression
|
356
|
+
config = %[
|
357
|
+
compression_level default_compression
|
358
|
+
]
|
359
|
+
|
360
|
+
bodystr = %({
|
361
|
+
"took" : 500,
|
362
|
+
"errors" : false,
|
363
|
+
"items" : [
|
364
|
+
{
|
365
|
+
"create": {
|
366
|
+
"_index" : "fluentd",
|
367
|
+
"_type" : "fluentd"
|
368
|
+
}
|
369
|
+
}
|
370
|
+
]
|
371
|
+
})
|
372
|
+
|
373
|
+
compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
|
374
|
+
|
375
|
+
elastic_request = stub_request(:post, "http://localhost:9200/_bulk").
|
376
|
+
to_return(:status => 200, :headers => {'Content-Type' => 'Application/json'}, :body => compressed_body)
|
377
|
+
|
378
|
+
driver(config)
|
379
|
+
driver.run(default_tag: 'test') do
|
380
|
+
driver.feed(sample_record)
|
381
|
+
end
|
382
|
+
|
383
|
+
assert_requested(elastic_request)
|
384
|
+
end
|
385
|
+
|
308
386
|
def test_writes_to_default_type
|
309
387
|
stub_elastic
|
310
388
|
driver.run(default_tag: 'test') do
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.6.
|
4
|
+
version: 3.6.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2019-11-
|
12
|
+
date: 2019-11-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -170,7 +170,8 @@ files:
|
|
170
170
|
homepage: https://github.com/uken/fluent-plugin-elasticsearch
|
171
171
|
licenses:
|
172
172
|
- Apache-2.0
|
173
|
-
metadata:
|
173
|
+
metadata:
|
174
|
+
changelog_uri: https://github.com/uken/fluent-plugin-elasticsearch/blob/master/History.md
|
174
175
|
post_install_message:
|
175
176
|
rdoc_options: []
|
176
177
|
require_paths:
|