fluent-plugin-elasticsearch 4.1.1 → 5.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/issue-auto-closer.yml +2 -2
- data/.github/workflows/linux.yml +5 -2
- data/.github/workflows/macos.yml +5 -2
- data/.github/workflows/windows.yml +5 -2
- data/Gemfile +1 -2
- data/History.md +146 -0
- data/README.ElasticsearchGenID.md +4 -4
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +260 -550
- data/fluent-plugin-elasticsearch.gemspec +4 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/helper.rb +0 -4
- data/test/plugin/mock_chunk.dat +0 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
- data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
- data/test/plugin/test_elasticsearch_tls.rb +8 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +51 -21
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +2118 -704
- data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
- metadata +62 -10
- data/.coveralls.yml +0 -2
- data/.travis.yml +0 -44
- data/appveyor.yml +0 -20
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.
|
6
|
+
s.version = '5.4.3'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -23,11 +23,14 @@ Gem::Specification.new do |s|
|
|
23
23
|
s.required_ruby_version = Gem::Requirement.new(">= 2.3".freeze)
|
24
24
|
|
25
25
|
s.add_runtime_dependency 'fluentd', '>= 0.14.22'
|
26
|
+
s.add_runtime_dependency 'faraday', '>= 2.0.0'
|
27
|
+
s.add_runtime_dependency 'faraday-excon', '>= 2.0.0'
|
26
28
|
s.add_runtime_dependency 'excon', '>= 0'
|
27
29
|
s.add_runtime_dependency 'elasticsearch'
|
28
30
|
|
29
31
|
|
30
32
|
s.add_development_dependency 'rake', '>= 0'
|
33
|
+
s.add_development_dependency 'webrick', '~> 1.7.0'
|
31
34
|
s.add_development_dependency 'webmock', '~> 3'
|
32
35
|
s.add_development_dependency 'test-unit', '~> 3.3.0'
|
33
36
|
s.add_development_dependency 'minitest', '~> 5.8'
|
@@ -0,0 +1,31 @@
|
|
1
|
+
begin
|
2
|
+
require 'elastic/transport'
|
3
|
+
::TRANSPORT_CLASS = Elastic::Transport
|
4
|
+
rescue LoadError
|
5
|
+
end
|
6
|
+
begin
|
7
|
+
require 'elasticsearch/transport'
|
8
|
+
::TRANSPORT_CLASS = Elasticsearch::Transport
|
9
|
+
rescue LoadError
|
10
|
+
end
|
11
|
+
if Gem::Version.new(Elasticsearch::VERSION) < Gem::Version.new("8.0.0")
|
12
|
+
begin
|
13
|
+
require 'elasticsearch/xpack'
|
14
|
+
rescue LoadError
|
15
|
+
require 'elasticsearch/api' # For elasticsearch-ruby 8 or later
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
begin
|
20
|
+
require 'elastic/transport/transport/connections/selector'
|
21
|
+
::SELECTOR_CLASS = Elastic::Transport::Transport::Connections::Selector
|
22
|
+
rescue LoadError
|
23
|
+
end
|
24
|
+
begin
|
25
|
+
require 'elasticsearch/transport/transport/connections/selector'
|
26
|
+
::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
|
27
|
+
rescue LoadError
|
28
|
+
end
|
29
|
+
unless defined?(::Elasticsearch::UnsupportedProductError)
|
30
|
+
class ::Elasticsearch::UnsupportedProductError < StandardError; end
|
31
|
+
end
|
@@ -23,6 +23,10 @@ class Fluent::Plugin::ElasticsearchErrorHandler
|
|
23
23
|
unrecoverable_error_types.include?(type)
|
24
24
|
end
|
25
25
|
|
26
|
+
def unrecoverable_record_error?(type)
|
27
|
+
['json_parse_exception'].include?(type)
|
28
|
+
end
|
29
|
+
|
26
30
|
def log_es_400_reason(&block)
|
27
31
|
if @plugin.log_es_400_reason
|
28
32
|
block.call
|
@@ -31,7 +35,7 @@ class Fluent::Plugin::ElasticsearchErrorHandler
|
|
31
35
|
end
|
32
36
|
end
|
33
37
|
|
34
|
-
def handle_error(response, tag, chunk, bulk_message_count, extracted_values)
|
38
|
+
def handle_error(response, tag, chunk, bulk_message_count, extracted_values, unpacked_msg_arr)
|
35
39
|
items = response['items']
|
36
40
|
if items.nil? || !items.is_a?(Array)
|
37
41
|
raise ElasticsearchVersionMismatch, "The response format was unrecognized: #{response}"
|
@@ -43,15 +47,21 @@ class Fluent::Plugin::ElasticsearchErrorHandler
|
|
43
47
|
stats = Hash.new(0)
|
44
48
|
meta = {}
|
45
49
|
header = {}
|
46
|
-
|
50
|
+
affinity_target_indices = @plugin.get_affinity_target_indices(chunk)
|
51
|
+
|
52
|
+
unpacked_msg_arr.each do |msg|
|
53
|
+
time = msg[:time]
|
54
|
+
rawrecord = msg[:record]
|
55
|
+
|
47
56
|
bulk_message = ''
|
48
57
|
next unless rawrecord.is_a? Hash
|
49
58
|
begin
|
50
59
|
# we need a deep copy for process_message to alter
|
51
60
|
processrecord = Marshal.load(Marshal.dump(rawrecord))
|
52
|
-
meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, extracted_values)
|
61
|
+
meta, header, record = @plugin.process_message(tag, meta, header, time, processrecord, affinity_target_indices, extracted_values)
|
53
62
|
next unless @plugin.append_record_to_messages(@plugin.write_operation, meta, header, record, bulk_message)
|
54
63
|
rescue => e
|
64
|
+
@plugin.log.debug("Exception in error handler during deep copy: #{e}")
|
55
65
|
stats[:bad_chunk_record] += 1
|
56
66
|
next
|
57
67
|
end
|
@@ -105,10 +115,15 @@ class Fluent::Plugin::ElasticsearchErrorHandler
|
|
105
115
|
elsif item[write_operation].has_key?('error') && item[write_operation]['error'].has_key?('type')
|
106
116
|
type = item[write_operation]['error']['type']
|
107
117
|
stats[type] += 1
|
108
|
-
retry_stream.add(time, rawrecord)
|
109
118
|
if unrecoverable_error?(type)
|
110
119
|
raise ElasticsearchRequestAbortError, "Rejected Elasticsearch due to #{type}"
|
111
120
|
end
|
121
|
+
if unrecoverable_record_error?(type)
|
122
|
+
@plugin.router.emit_error_event(tag, time, rawrecord, ElasticsearchError.new("#{status} - #{type}: #{reason}"))
|
123
|
+
next
|
124
|
+
else
|
125
|
+
retry_stream.add(time, rawrecord) unless unrecoverable_record_error?(type)
|
126
|
+
end
|
112
127
|
else
|
113
128
|
# When we don't have a type field, something changed in the API
|
114
129
|
# expected return values (ES 2.x)
|
@@ -1,7 +1,7 @@
|
|
1
|
-
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
2
|
|
3
3
|
class Fluent::Plugin::ElasticseatchFallbackSelector
|
4
|
-
include
|
4
|
+
include SELECTOR_CLASS::Base
|
5
5
|
|
6
6
|
def select(options={})
|
7
7
|
connections.first
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require_relative 'elasticsearch_compat'
|
2
|
+
|
1
3
|
module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
2
4
|
ILM_DEFAULT_POLICY_PATH = "default-ilm-policy.json"
|
3
5
|
|
@@ -21,7 +23,7 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
21
23
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not available in your Elasticsearch" unless ilm['available']
|
22
24
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not enabled in your Elasticsearch" unless ilm['enabled']
|
23
25
|
|
24
|
-
rescue
|
26
|
+
rescue ::TRANSPORT_CLASS::Transport::Error => e
|
25
27
|
raise Fluent::ConfigError, "Index Lifecycle management is enabled in Fluentd, but not installed on your Elasticsearch", error: e
|
26
28
|
end
|
27
29
|
end
|
@@ -43,12 +45,20 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
43
45
|
end
|
44
46
|
|
45
47
|
def get_ilm_policy
|
46
|
-
|
48
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
|
49
|
+
client.ilm.get_lifecycle
|
50
|
+
else
|
51
|
+
client.ilm.get_policy
|
52
|
+
end
|
47
53
|
end
|
48
54
|
|
49
55
|
def ilm_policy_exists?(policy_id)
|
50
56
|
begin
|
51
|
-
|
57
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
|
58
|
+
client.ilm.get_lifecycle(policy: policy_id)
|
59
|
+
else
|
60
|
+
client.ilm.get_policy(policy_id: policy_id)
|
61
|
+
end
|
52
62
|
true
|
53
63
|
rescue
|
54
64
|
false
|
@@ -57,7 +67,11 @@ module Fluent::Plugin::ElasticsearchIndexLifecycleManagement
|
|
57
67
|
|
58
68
|
def ilm_policy_put(policy_id, policy)
|
59
69
|
log.info("Installing ILM policy: #{policy}")
|
60
|
-
|
70
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
|
71
|
+
client.ilm.put_lifecycle(policy: policy_id, body: policy)
|
72
|
+
else
|
73
|
+
client.ilm.put_policy(policy_id: policy_id, body: policy)
|
74
|
+
end
|
61
75
|
end
|
62
76
|
|
63
77
|
def default_policy_payload
|
@@ -3,7 +3,7 @@ require_relative './elasticsearch_error'
|
|
3
3
|
|
4
4
|
module Fluent::ElasticsearchIndexTemplate
|
5
5
|
def get_template(template_file)
|
6
|
-
if !File.
|
6
|
+
if !File.exist?(template_file)
|
7
7
|
raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
|
8
8
|
end
|
9
9
|
file_contents = IO.read(template_file).gsub(/\n/,'')
|
@@ -11,7 +11,7 @@ module Fluent::ElasticsearchIndexTemplate
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def get_custom_template(template_file, customize_template)
|
14
|
-
if !File.
|
14
|
+
if !File.exist?(template_file)
|
15
15
|
raise "If you specify a template_name you must specify a valid template file (checked '#{template_file}')!"
|
16
16
|
end
|
17
17
|
file_contents = IO.read(template_file).gsub(/\n/,'')
|
@@ -22,18 +22,39 @@ module Fluent::ElasticsearchIndexTemplate
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def template_exists?(name, host = nil)
|
25
|
-
|
25
|
+
if @use_legacy_template
|
26
|
+
client(host).indices.get_template(:name => name)
|
27
|
+
else
|
28
|
+
client(host).indices.get_index_template(:name => name)
|
29
|
+
end
|
26
30
|
return true
|
27
|
-
rescue
|
31
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
28
32
|
return false
|
29
33
|
end
|
30
34
|
|
31
|
-
def
|
35
|
+
def host_unreachable_exceptions
|
36
|
+
if Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
37
|
+
# elasticsearch-ruby 8.0.0's elastic-transport uses
|
38
|
+
# direct callable #host_unreachable_exceptions again.
|
39
|
+
client.transport.host_unreachable_exceptions
|
40
|
+
elsif Gem::Version.new(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
41
|
+
# elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
|
42
|
+
# Elasticsearch class on Transport.
|
43
|
+
# This is why #host_unreachable_exceptions is not callable directly
|
44
|
+
# via transport (not transport's transport instance accessor) any more.
|
45
|
+
client.transport.transport.host_unreachable_exceptions
|
46
|
+
else
|
47
|
+
client.transport.host_unreachable_exceptions
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
|
32
52
|
return unless block_given?
|
33
53
|
retries = 0
|
54
|
+
transport_errors = TRANSPORT_CLASS::Transport::Errors.constants.map{ |c| TRANSPORT_CLASS::Transport::Errors.const_get c } if catch_trasport_exceptions
|
34
55
|
begin
|
35
56
|
yield
|
36
|
-
rescue *
|
57
|
+
rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
|
37
58
|
@_es = nil
|
38
59
|
@_es_info = nil
|
39
60
|
if retries < max_retries
|
@@ -52,27 +73,32 @@ module Fluent::ElasticsearchIndexTemplate
|
|
52
73
|
end
|
53
74
|
|
54
75
|
def template_put(name, template, host = nil)
|
55
|
-
|
76
|
+
if @use_legacy_template
|
77
|
+
client(host).indices.put_template(:name => name, :body => template)
|
78
|
+
else
|
79
|
+
client(host).indices.put_index_template(:name => name, :body => template)
|
80
|
+
end
|
56
81
|
end
|
57
82
|
|
58
83
|
def indexcreation(index_name, host = nil)
|
59
84
|
client(host).indices.create(:index => index_name)
|
60
|
-
rescue
|
85
|
+
rescue TRANSPORT_CLASS::Transport::Error => e
|
61
86
|
if e.message =~ /"already exists"/ || e.message =~ /resource_already_exists_exception/
|
62
87
|
log.debug("Index #{index_name} already exists")
|
63
88
|
else
|
64
|
-
log.error("Error while index creation - #{index_name}:
|
89
|
+
log.error("Error while index creation - #{index_name}", error: e)
|
65
90
|
end
|
66
91
|
end
|
67
92
|
|
68
|
-
def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil)
|
93
|
+
def template_install(name, template_file, overwrite, enable_ilm = false, deflector_alias_name = nil, ilm_policy_id = nil, host = nil, target_index = nil, index_separator = '-')
|
69
94
|
inject_template_name = get_template_name(enable_ilm, name, deflector_alias_name)
|
70
95
|
if overwrite
|
71
96
|
template_put(inject_template_name,
|
72
97
|
enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
|
73
98
|
target_index,
|
74
99
|
ilm_policy_id,
|
75
|
-
get_template(template_file)
|
100
|
+
get_template(template_file),
|
101
|
+
index_separator) :
|
76
102
|
get_template(template_file), host)
|
77
103
|
|
78
104
|
log.debug("Template '#{inject_template_name}' overwritten with #{template_file}.")
|
@@ -83,7 +109,8 @@ module Fluent::ElasticsearchIndexTemplate
|
|
83
109
|
enable_ilm ? inject_ilm_settings_to_template(deflector_alias_name,
|
84
110
|
target_index,
|
85
111
|
ilm_policy_id,
|
86
|
-
get_template(template_file)
|
112
|
+
get_template(template_file),
|
113
|
+
index_separator) :
|
87
114
|
get_template(template_file), host)
|
88
115
|
log.info("Template configured, but no template installed. Installed '#{inject_template_name}' from #{template_file}.")
|
89
116
|
else
|
@@ -91,14 +118,15 @@ module Fluent::ElasticsearchIndexTemplate
|
|
91
118
|
end
|
92
119
|
end
|
93
120
|
|
94
|
-
def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index)
|
121
|
+
def template_custom_install(template_name, template_file, overwrite, customize_template, enable_ilm, deflector_alias_name, ilm_policy_id, host, target_index, index_separator)
|
95
122
|
template_custom_name = get_template_name(enable_ilm, template_name, deflector_alias_name)
|
96
123
|
custom_template = if enable_ilm
|
97
124
|
inject_ilm_settings_to_template(deflector_alias_name,
|
98
125
|
target_index,
|
99
126
|
ilm_policy_id,
|
100
127
|
get_custom_template(template_file,
|
101
|
-
customize_template)
|
128
|
+
customize_template),
|
129
|
+
index_separator)
|
102
130
|
else
|
103
131
|
get_custom_template(template_file, customize_template)
|
104
132
|
end
|
@@ -119,15 +147,31 @@ module Fluent::ElasticsearchIndexTemplate
|
|
119
147
|
enable_ilm ? deflector_alias_name : template_name
|
120
148
|
end
|
121
149
|
|
122
|
-
def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template)
|
150
|
+
def inject_ilm_settings_to_template(deflector_alias, target_index, ilm_policy_id, template, index_separator)
|
123
151
|
log.debug("Overwriting index patterns when Index Lifecycle Management is enabled.")
|
124
|
-
template
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
152
|
+
template['index_patterns'] = "#{target_index}#{index_separator}*"
|
153
|
+
if @use_legacy_template
|
154
|
+
template.delete('template') if template.include?('template')
|
155
|
+
# Prepare settings Hash
|
156
|
+
if !template.key?('settings')
|
157
|
+
template['settings'] = {}
|
158
|
+
end
|
159
|
+
if template['settings'] && (template['settings']['index.lifecycle.name'] || template['settings']['index.lifecycle.rollover_alias'])
|
160
|
+
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
161
|
+
end
|
162
|
+
template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
163
|
+
template['order'] = template['order'] ? template['order'] + target_index.count(index_separator) + 1 : 51 + target_index.count(index_separator)
|
164
|
+
else
|
165
|
+
# Prepare template.settings Hash
|
166
|
+
if !template['template'].key?('settings')
|
167
|
+
template['template']['settings'] = {}
|
168
|
+
end
|
169
|
+
if template['template']['settings'] && (template['template']['settings']['index.lifecycle.name'] || template['template']['settings']['index.lifecycle.rollover_alias'])
|
170
|
+
log.debug("Overwriting index lifecycle name and rollover alias when Index Lifecycle Management is enabled.")
|
171
|
+
end
|
172
|
+
template['template']['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
173
|
+
template['priority'] = template['priority'] ? template['priority'] + target_index.count(index_separator) + 1 : 101 + target_index.count(index_separator)
|
129
174
|
end
|
130
|
-
template['settings'].update({ 'index.lifecycle.name' => ilm_policy_id, 'index.lifecycle.rollover_alias' => deflector_alias})
|
131
175
|
template
|
132
176
|
end
|
133
177
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
require 'elasticsearch'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
|
-
class Fluent::Plugin::ElasticsearchSimpleSniffer <
|
4
|
+
class Fluent::Plugin::ElasticsearchSimpleSniffer < TRANSPORT_CLASS::Transport::Sniffer
|
4
5
|
|
5
6
|
def hosts
|
6
7
|
@transport.logger.debug "In Fluent::Plugin::ElasticsearchSimpleSniffer hosts #{@transport.hosts}" if @transport.logger
|
@@ -53,7 +53,7 @@ module Fluent::Plugin
|
|
53
53
|
seed += tag + separator if @include_tag_in_seed
|
54
54
|
seed += time.to_s + separator if @include_time_in_seed
|
55
55
|
if @use_entire_record
|
56
|
-
record.each {|k
|
56
|
+
record.keys.sort.each {|k| seed += "|#{k}|#{record[k]}"}
|
57
57
|
else
|
58
58
|
seed += record_keys.map {|k| record[k]}.join(separator)
|
59
59
|
end
|
@@ -1,8 +1,10 @@
|
|
1
1
|
require 'elasticsearch'
|
2
2
|
|
3
|
+
require 'faraday/excon'
|
3
4
|
require 'fluent/log-ext'
|
4
5
|
require 'fluent/plugin/input'
|
5
6
|
require_relative 'elasticsearch_constants'
|
7
|
+
require_relative 'elasticsearch_compat'
|
6
8
|
|
7
9
|
module Fluent::Plugin
|
8
10
|
class ElasticsearchInput < Input
|
@@ -218,7 +220,7 @@ module Fluent::Plugin
|
|
218
220
|
|
219
221
|
headers = { 'Content-Type' => "application/json" }.merge(@custom_headers)
|
220
222
|
|
221
|
-
transport =
|
223
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(
|
222
224
|
connection_options.merge(
|
223
225
|
options: {
|
224
226
|
reload_connections: local_reload_connections,
|
@@ -284,7 +286,11 @@ module Fluent::Plugin
|
|
284
286
|
end
|
285
287
|
|
286
288
|
router.emit_stream(@tag, es)
|
287
|
-
|
289
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("7.0.0")
|
290
|
+
client.clear_scroll(body: {scroll_id: scroll_id}) if scroll_id
|
291
|
+
else
|
292
|
+
client.clear_scroll(scroll_id: scroll_id) if scroll_id
|
293
|
+
end
|
288
294
|
end
|
289
295
|
|
290
296
|
def process_scroll_request(scroll_id)
|
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'oj'
|
2
|
+
require_relative 'elasticsearch_compat'
|
2
3
|
|
3
4
|
module Fluent::Plugin
|
4
5
|
module Serializer
|
5
6
|
|
6
7
|
class Oj
|
7
|
-
include
|
8
|
+
include TRANSPORT_CLASS::Transport::Serializer::Base
|
8
9
|
|
9
10
|
# De-serialize a Hash from JSON string
|
10
11
|
#
|