fluent-plugin-elasticsearch 4.0.5 → 4.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -1
- data/Gemfile +2 -1
- data/History.md +6 -0
- data/README.md +18 -25
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/gemfiles/Gemfile.elasticsearch.v6 +1 -1
- data/gemfiles/{Gemfile.ilm → Gemfile.without.ilm} +1 -2
- data/lib/fluent/plugin/elasticsearch_index_template.rb +1 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +16 -8
- data/test/plugin/test_elasticsearch_error_handler.rb +1 -1
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +1 -1
- data/test/plugin/test_filter_elasticsearch_genid.rb +1 -1
- data/test/plugin/test_in_elasticsearch.rb +1 -1
- data/test/plugin/test_out_elasticsearch.rb +59 -3
- data/test/plugin/test_out_elasticsearch_dynamic.rb +2 -2
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c2f2143534244444b8377964ee5b1b116aca96f5c0d9cd57c7bec5ddb6a61983
|
4
|
+
data.tar.gz: 89068eedecb66309e9440b2e7e1166366ef7f2414980a7bff59d308000b78f49
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6b4796d25b2c88393c1a346d69f86e7998343a8224ded9c61dce2dbdf019672e50642306fdffaa5ec94b98ebaaa9a01d6a1f9a0cc1cc4863dc29f97fc79cd6cb
|
7
|
+
data.tar.gz: e25a9736833589adcff5663ec6e8c054bcd5b799ecdb170bbcef750e4d778fbd1f6aa1a26a0ee716b25e4b3f98d22d1523a4f0ee620a46cd8f90f12df3acfc4d
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -4,7 +4,8 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec
|
5
5
|
|
6
6
|
gem 'simplecov', require: false
|
7
|
-
gem 'coveralls', require: false
|
7
|
+
gem 'coveralls', ">= 0.8.0", require: false
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
+
gem "elasticsearch-xpack"
|
10
11
|
gem "oj"
|
data/History.md
CHANGED
@@ -2,6 +2,12 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 4.0.6
|
6
|
+
- Add fallback mechanism for handling to detect es version (#730)
|
7
|
+
- Remove needless section (#728)
|
8
|
+
- Handle exception if index already exists (#727)
|
9
|
+
- Tweak test cases (#726)
|
10
|
+
|
5
11
|
### 4.0.5
|
6
12
|
- add logstash_dateformat as placeholder (#718)
|
7
13
|
- Tweak travis.yml for suppressing validator warnings and add CI for Linux Arm64 architecture and macOS 10.14 (#724)
|
data/README.md
CHANGED
@@ -45,6 +45,7 @@ Current maintainers: @cosmo0920
|
|
45
45
|
+ [templates](#templates)
|
46
46
|
+ [max_retry_putting_template](#max_retry_putting_template)
|
47
47
|
+ [fail_on_putting_template_retry_exceed](#fail_on_putting_template_retry_exceed)
|
48
|
+
+ [fail_on_detecting_es_version_retry_exceed](#fail_on_detecting_es_version_retry_exceed)
|
48
49
|
+ [max_retry_get_es_version](#max_retry_get_es_version)
|
49
50
|
+ [request_timeout](#request_timeout)
|
50
51
|
+ [reload_connections](#reload_connections)
|
@@ -99,7 +100,6 @@ Current maintainers: @cosmo0920
|
|
99
100
|
+ [Cannot see detailed failure log](#cannot-see-detailed-failure-log)
|
100
101
|
+ [Cannot connect TLS enabled reverse Proxy](#cannot-connect-tls-enabled-reverse-proxy)
|
101
102
|
+ [Declined logs are resubmitted forever, why?](#declined-logs-are-resubmitted-forever-why)
|
102
|
-
+ [Suggested to increase flush_thread_count, why?](#suggested-to-increase-flush_thread_count-why)
|
103
103
|
+ [Suggested to install typhoeus gem, why?](#suggested-to-install-typhoeus-gem-why)
|
104
104
|
+ [Stopped to send events on k8s, why?](#stopped-to-send-events-on-k8s-why)
|
105
105
|
+ [Random 400 - Rejected by Elasticsearch is occured, why?](#random-400---rejected-by-elasticsearch-is-occured-why)
|
@@ -502,6 +502,23 @@ If you have multiple output plugin, you could use this property to do not fail o
|
|
502
502
|
fail_on_putting_template_retry_exceed false # defaults to true
|
503
503
|
```
|
504
504
|
|
505
|
+
### fail_on_detecting_es_version_retry_exceed
|
506
|
+
|
507
|
+
Indicates whether to fail when `max_retry_get_es_version` is exceeded.
|
508
|
+
If you want to use fallback mechanism for obtaining ELasticsearch version, you could use this property to do not fail on fluentd statup.
|
509
|
+
|
510
|
+
```
|
511
|
+
fail_on_detecting_es_version_retry_exceed false
|
512
|
+
```
|
513
|
+
|
514
|
+
And the following parameters should be working with:
|
515
|
+
|
516
|
+
```
|
517
|
+
verify_es_version_at_startup true
|
518
|
+
max_retry_get_es_version 2 # greater than 0.
|
519
|
+
default_elasticsearch_version 7 # This version is used when occurring fallback.
|
520
|
+
```
|
521
|
+
|
505
522
|
### max_retry_get_es_version
|
506
523
|
|
507
524
|
You can specify times of retry obtaining Elasticsearch version.
|
@@ -1445,30 +1462,6 @@ The following configuration uses label:
|
|
1445
1462
|
</label>
|
1446
1463
|
```
|
1447
1464
|
|
1448
|
-
### Suggested to increase flush_thread_count, why?
|
1449
|
-
|
1450
|
-
fluent-plugin-elasticsearch default behavior has a possibility to cause events traffic jam.
|
1451
|
-
When users use `flush_thread_count` = 1, ES plugin retries to send events if connection errors are disappeared.
|
1452
|
-
|
1453
|
-
To prevent the following warning and sending events blocking, you must specify `flush_thread_count` >= 2:
|
1454
|
-
|
1455
|
-
```log
|
1456
|
-
2018-12-24 14:32:06 +0900 [warn]: #0 To prevent events traffic jam, you should specify 2 or more 'flush_thread_count'.
|
1457
|
-
```
|
1458
|
-
|
1459
|
-
```aconf
|
1460
|
-
<match out.elasticsearch.**>
|
1461
|
-
@type elasticsearch
|
1462
|
-
host localhost
|
1463
|
-
port 9200
|
1464
|
-
# ...
|
1465
|
-
<buffer tag>
|
1466
|
-
@type memory # or file
|
1467
|
-
flush_thread_count 4
|
1468
|
-
</buffer>
|
1469
|
-
</match>
|
1470
|
-
```
|
1471
|
-
|
1472
1465
|
### Suggested to install typhoeus gem, why?
|
1473
1466
|
|
1474
1467
|
fluent-plugin-elasticsearch doesn't depend on typhoeus gem by default.
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0.
|
6
|
+
s.version = '4.0.6'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -4,7 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec :path => "../"
|
5
5
|
|
6
6
|
gem 'simplecov', require: false
|
7
|
-
gem 'coveralls', require: false
|
7
|
+
gem 'coveralls', ">= 0.8.0", require: false
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
10
|
gem "elasticsearch", "~> 6.8.1"
|
@@ -4,8 +4,7 @@ source 'https://rubygems.org'
|
|
4
4
|
gemspec :path => "../"
|
5
5
|
|
6
6
|
gem 'simplecov', require: false
|
7
|
-
gem 'coveralls', require: false
|
7
|
+
gem 'coveralls', ">= 0.8.0", require: false
|
8
8
|
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
9
|
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
-
gem "elasticsearch-xpack"
|
11
10
|
gem "oj"
|
@@ -58,7 +58,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
58
58
|
def indexcreation(index_name, host = nil)
|
59
59
|
client(host).indices.create(:index => index_name)
|
60
60
|
rescue Elasticsearch::Transport::Transport::Error => e
|
61
|
-
if e.message =~ /"already exists"/
|
61
|
+
if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
|
62
62
|
log.debug("Index #{index_name} already exists")
|
63
63
|
else
|
64
64
|
log.error("Error while index creation - #{index_name}: #{e.inspect}")
|
@@ -125,6 +125,7 @@ EOC
|
|
125
125
|
config_param :templates, :hash, :default => nil
|
126
126
|
config_param :max_retry_putting_template, :integer, :default => 10
|
127
127
|
config_param :fail_on_putting_template_retry_exceed, :bool, :default => true
|
128
|
+
config_param :fail_on_detecting_es_version_retry_exceed, :bool, :default => true
|
128
129
|
config_param :max_retry_get_es_version, :integer, :default => 15
|
129
130
|
config_param :include_tag_key, :bool, :default => false
|
130
131
|
config_param :tag_key, :string, :default => 'tag'
|
@@ -290,14 +291,11 @@ EOC
|
|
290
291
|
raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
|
291
292
|
end
|
292
293
|
|
293
|
-
@last_seen_major_version =
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
else
|
299
|
-
@default_elasticsearch_version
|
300
|
-
end
|
294
|
+
@last_seen_major_version = if major_version = handle_last_seen_es_major_version
|
295
|
+
major_version
|
296
|
+
else
|
297
|
+
@default_elasticsearch_version
|
298
|
+
end
|
301
299
|
if @last_seen_major_version == 6 && @type_name != DEFAULT_TYPE_NAME_ES_7x
|
302
300
|
log.info "Detected ES 6.x: ES 7.x will only accept `_doc` in type_name."
|
303
301
|
end
|
@@ -421,6 +419,16 @@ EOC
|
|
421
419
|
raise Fluent::ConfigError, "You must install #{@http_backend} gem. Exception: #{ex}"
|
422
420
|
end
|
423
421
|
|
422
|
+
def handle_last_seen_es_major_version
|
423
|
+
if @verify_es_version_at_startup && !dry_run?
|
424
|
+
retry_operate(@max_retry_get_es_version, @fail_on_detecting_es_version_retry_exceed) do
|
425
|
+
detect_es_major_version
|
426
|
+
end
|
427
|
+
else
|
428
|
+
nil
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
424
432
|
def detect_es_major_version
|
425
433
|
@_es_info ||= client.info
|
426
434
|
@_es_info["version"]["number"].to_i
|
@@ -4,6 +4,7 @@ require 'fluent/test/helpers'
|
|
4
4
|
require 'json'
|
5
5
|
require 'fluent/test/driver/input'
|
6
6
|
require 'flexmock/test_unit'
|
7
|
+
require 'fluent/plugin/in_elasticsearch'
|
7
8
|
|
8
9
|
class ElasticsearchInputTest < Test::Unit::TestCase
|
9
10
|
include FlexMock::TestCase
|
@@ -16,7 +17,6 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
16
17
|
|
17
18
|
def setup
|
18
19
|
Fluent::Test.setup
|
19
|
-
require 'fluent/plugin/in_elasticsearch'
|
20
20
|
@driver = nil
|
21
21
|
log = Fluent::Engine.log
|
22
22
|
log.out.logs.slice!(0, log.out.logs.length)
|
@@ -1,11 +1,12 @@
|
|
1
|
-
|
1
|
+
require_relative '../helper'
|
2
2
|
require 'date'
|
3
3
|
require 'fluent/test/helpers'
|
4
4
|
require 'json'
|
5
5
|
require 'fluent/test/driver/output'
|
6
6
|
require 'flexmock/test_unit'
|
7
|
+
require 'fluent/plugin/out_elasticsearch'
|
7
8
|
|
8
|
-
class
|
9
|
+
class ElasticsearchOutputTest < Test::Unit::TestCase
|
9
10
|
include FlexMock::TestCase
|
10
11
|
include Fluent::Test::Helpers
|
11
12
|
|
@@ -13,7 +14,6 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
13
14
|
|
14
15
|
def setup
|
15
16
|
Fluent::Test.setup
|
16
|
-
require 'fluent/plugin/out_elasticsearch'
|
17
17
|
@driver = nil
|
18
18
|
log = Fluent::Engine.log
|
19
19
|
log.out.logs.slice!(0, log.out.logs.length)
|
@@ -535,6 +535,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
535
535
|
"template_name_placeholder" => ["template_name", "logstash-${mykey}"],
|
536
536
|
"customize_template" => ["customize_template", '{"<<TAG>>":"${mykey}"}'],
|
537
537
|
"logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${mykey}"],
|
538
|
+
"logstash_dateformat_placeholder" => ["logstash_dateformat", "${mykey}"],
|
538
539
|
"deflector_alias_placeholder" => ["deflector_alias", "fluentd-${mykey}"],
|
539
540
|
"application_name_placeholder" => ["application_name", "fluentd-${mykey}"],
|
540
541
|
)
|
@@ -560,6 +561,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
560
561
|
"template_name_placeholder" => ["template_name", "logstash-${tag}-%Y%m%d"],
|
561
562
|
"customize_template" => ["customize_template", '{"<<TAG>>":"${es_index}"}'],
|
562
563
|
"logstash_prefix_placeholder" => ["logstash_prefix", "fluentd-${es_index}-%Y%m%d"],
|
564
|
+
"logstash_dataformat_placeholder" => ["logstash_dateformat", "${es_index}"],
|
563
565
|
"deflector_alias_placeholder" => ["deflector_alias", "fluentd-%Y%m%d"],
|
564
566
|
"application_name_placeholder" => ["application_name", "fluentd-${tag}-${es_index}-%Y%m%d"],
|
565
567
|
)
|
@@ -710,6 +712,60 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
710
712
|
end
|
711
713
|
end
|
712
714
|
|
715
|
+
class GetElasticsearchVersionWithFallbackTest < self
|
716
|
+
def create_driver(conf='', client_version="\"5.0\"")
|
717
|
+
# For request stub to detect compatibility.
|
718
|
+
@client_version ||= client_version
|
719
|
+
# Ensure original implementation existence.
|
720
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
721
|
+
def detect_es_major_version
|
722
|
+
@_es_info ||= client.info
|
723
|
+
@_es_info["version"]["number"].to_i
|
724
|
+
end
|
725
|
+
CODE
|
726
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
727
|
+
def client_library_version
|
728
|
+
#{@client_version}
|
729
|
+
end
|
730
|
+
CODE
|
731
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
|
732
|
+
end
|
733
|
+
|
734
|
+
data("Elasticsearch 5" => ["5.0", 5],
|
735
|
+
"Elasticsearch 6" => ["6.0", 6],
|
736
|
+
"Elasticsearch 7" => ["7.0", 7],
|
737
|
+
"Elasticsearch 8" => ["8.0", 8])
|
738
|
+
def test_retry_get_es_version_without_fail_on_detecting_es_version_retry_exceeded(data)
|
739
|
+
client_version, es_major_version = data
|
740
|
+
config = %{
|
741
|
+
host logs.google.com
|
742
|
+
port 778
|
743
|
+
scheme https
|
744
|
+
path /es/
|
745
|
+
user john
|
746
|
+
password doe
|
747
|
+
verify_es_version_at_startup true
|
748
|
+
max_retry_get_es_version 2
|
749
|
+
fail_on_detecting_es_version_retry_exceed false
|
750
|
+
default_elasticsearch_version #{es_major_version}
|
751
|
+
@log_level info
|
752
|
+
}
|
753
|
+
|
754
|
+
connection_resets = 0
|
755
|
+
stub_request(:get, "https://logs.google.com:778/es//").
|
756
|
+
with(basic_auth: ['john', 'doe']) do |req|
|
757
|
+
connection_resets += 1
|
758
|
+
raise Faraday::ConnectionFailed, "Test message"
|
759
|
+
end
|
760
|
+
|
761
|
+
d = create_driver(config, client_version)
|
762
|
+
|
763
|
+
assert_equal es_major_version, d.instance.default_elasticsearch_version
|
764
|
+
assert_equal 3, connection_resets
|
765
|
+
assert_equal es_major_version, d.instance.instance_variable_get(:@last_seen_major_version)
|
766
|
+
end
|
767
|
+
end
|
768
|
+
|
713
769
|
def test_template_already_present
|
714
770
|
config = %{
|
715
771
|
host logs.google.com
|
@@ -1,8 +1,9 @@
|
|
1
|
-
|
1
|
+
require_relative '../helper'
|
2
2
|
require 'date'
|
3
3
|
require 'fluent/test/helpers'
|
4
4
|
require 'fluent/test/driver/output'
|
5
5
|
require 'flexmock/test_unit'
|
6
|
+
require 'fluent/plugin/out_elasticsearch_dynamic'
|
6
7
|
|
7
8
|
class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
8
9
|
include FlexMock::TestCase
|
@@ -12,7 +13,6 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
12
13
|
|
13
14
|
def setup
|
14
15
|
Fluent::Test.setup
|
15
|
-
require 'fluent/plugin/out_elasticsearch_dynamic'
|
16
16
|
@driver = nil
|
17
17
|
end
|
18
18
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0.
|
4
|
+
version: 4.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-03-
|
13
|
+
date: 2020-03-12 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -149,7 +149,7 @@ files:
|
|
149
149
|
- appveyor.yml
|
150
150
|
- fluent-plugin-elasticsearch.gemspec
|
151
151
|
- gemfiles/Gemfile.elasticsearch.v6
|
152
|
-
- gemfiles/Gemfile.ilm
|
152
|
+
- gemfiles/Gemfile.without.ilm
|
153
153
|
- lib/fluent/log-ext.rb
|
154
154
|
- lib/fluent/plugin/default-ilm-policy.json
|
155
155
|
- lib/fluent/plugin/elasticsearch_constants.rb
|