fluent-plugin-elasticsearch 4.1.1 → 5.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/issue-auto-closer.yml +2 -2
- data/.github/workflows/linux.yml +5 -2
- data/.github/workflows/macos.yml +5 -2
- data/.github/workflows/windows.yml +5 -2
- data/Gemfile +1 -2
- data/History.md +146 -0
- data/README.ElasticsearchGenID.md +4 -4
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +260 -550
- data/fluent-plugin-elasticsearch.gemspec +4 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/helper.rb +0 -4
- data/test/plugin/mock_chunk.dat +0 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
- data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
- data/test/plugin/test_elasticsearch_tls.rb +8 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +51 -21
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +2118 -704
- data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
- metadata +62 -10
- data/.coveralls.yml +0 -2
- data/.travis.yml +0 -44
- data/appveyor.yml +0 -20
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -0,0 +1,298 @@
|
|
1
|
+
|
2
|
+
require_relative 'out_elasticsearch'
|
3
|
+
|
4
|
+
module Fluent::Plugin
|
5
|
+
class ElasticsearchOutputDataStream < ElasticsearchOutput
|
6
|
+
|
7
|
+
Fluent::Plugin.register_output('elasticsearch_data_stream', self)
|
8
|
+
|
9
|
+
helpers :event_emitter
|
10
|
+
|
11
|
+
config_param :data_stream_name, :string
|
12
|
+
config_param :data_stream_ilm_name, :string, :default => nil
|
13
|
+
config_param :data_stream_template_name, :string, :default => nil
|
14
|
+
config_param :data_stream_ilm_policy, :string, :default => nil
|
15
|
+
config_param :data_stream_ilm_policy_overwrite, :bool, :default => false
|
16
|
+
config_param :data_stream_template_use_index_patterns_wildcard, :bool, :default => true
|
17
|
+
|
18
|
+
# Elasticsearch 7.9 or later always support new style of index template.
|
19
|
+
config_set_default :use_legacy_template, false
|
20
|
+
|
21
|
+
INVALID_START_CHRACTERS = ["-", "_", "+", "."]
|
22
|
+
INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
|
23
|
+
|
24
|
+
def configure(conf)
|
25
|
+
super
|
26
|
+
|
27
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) < Gem::Version.new("8.0.0")
|
28
|
+
begin
|
29
|
+
require 'elasticsearch/api'
|
30
|
+
require 'elasticsearch/xpack'
|
31
|
+
rescue LoadError
|
32
|
+
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
33
|
+
end
|
34
|
+
else
|
35
|
+
begin
|
36
|
+
require 'elasticsearch/api'
|
37
|
+
rescue LoadError
|
38
|
+
raise Fluent::ConfigError, "'elasticsearch/api is required for <@elasticsearch_data_stream>."
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
@data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
|
43
|
+
@data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
|
44
|
+
@data_stream_ilm_policy = File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json")) if @data_stream_ilm_policy.nil?
|
45
|
+
|
46
|
+
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
47
|
+
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
48
|
+
validate_data_stream_parameters
|
49
|
+
else
|
50
|
+
@use_placeholder = true
|
51
|
+
@data_stream_names = []
|
52
|
+
end
|
53
|
+
|
54
|
+
unless @use_placeholder
|
55
|
+
begin
|
56
|
+
@data_stream_names = [@data_stream_name]
|
57
|
+
retry_operate(@max_retry_putting_template,
|
58
|
+
@fail_on_putting_template_retry_exceed,
|
59
|
+
@catch_transport_exception_on_retry) do
|
60
|
+
create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
61
|
+
create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name)
|
62
|
+
create_data_stream(@data_stream_name)
|
63
|
+
end
|
64
|
+
rescue => e
|
65
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def validate_data_stream_parameters
|
71
|
+
{"data_stream_name" => @data_stream_name,
|
72
|
+
"data_stream_template_name"=> @data_stream_template_name,
|
73
|
+
"data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
|
74
|
+
unless valid_data_stream_parameters?(value)
|
75
|
+
unless start_with_valid_characters?(value)
|
76
|
+
if not_dots?(value)
|
77
|
+
raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
|
78
|
+
else
|
79
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
unless valid_characters?(value)
|
83
|
+
raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
|
84
|
+
end
|
85
|
+
unless lowercase_only?(value)
|
86
|
+
raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
|
87
|
+
end
|
88
|
+
if value.bytes.size > 255
|
89
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
def create_ilm_policy(datastream_name, template_name, ilm_name, host = nil)
|
96
|
+
unless @data_stream_ilm_policy_overwrite
|
97
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name, host)
|
98
|
+
end
|
99
|
+
|
100
|
+
params = {
|
101
|
+
body: @data_stream_ilm_policy
|
102
|
+
}
|
103
|
+
retry_operate(@max_retry_putting_template,
|
104
|
+
@fail_on_putting_template_retry_exceed,
|
105
|
+
@catch_transport_exception_on_retry) do
|
106
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
|
107
|
+
client(host).ilm.put_lifecycle(params.merge(policy: ilm_name))
|
108
|
+
else
|
109
|
+
client(host).xpack.ilm.put_policy(params.merge(policy_id: ilm_name))
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def create_index_template(datastream_name, template_name, ilm_name, host = nil)
|
115
|
+
return if data_stream_exist?(datastream_name, host) or template_exists?(template_name, host)
|
116
|
+
wildcard = @data_stream_template_use_index_patterns_wildcard ? '*' : ''
|
117
|
+
body = {
|
118
|
+
"index_patterns" => ["#{datastream_name}#{wildcard}"],
|
119
|
+
"data_stream" => {},
|
120
|
+
"template" => {
|
121
|
+
"settings" => {
|
122
|
+
"index.lifecycle.name" => "#{ilm_name}"
|
123
|
+
}
|
124
|
+
}
|
125
|
+
}
|
126
|
+
params = {
|
127
|
+
name: template_name,
|
128
|
+
body: body
|
129
|
+
}
|
130
|
+
retry_operate(@max_retry_putting_template,
|
131
|
+
@fail_on_putting_template_retry_exceed,
|
132
|
+
@catch_transport_exception_on_retry) do
|
133
|
+
client(host).indices.put_index_template(params)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def data_stream_exist?(datastream_name, host = nil)
|
138
|
+
params = {
|
139
|
+
name: datastream_name
|
140
|
+
}
|
141
|
+
begin
|
142
|
+
response = client(host).indices.get_data_stream(params)
|
143
|
+
return (not response.is_a?(TRANSPORT_CLASS::Transport::Errors::NotFound))
|
144
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound => e
|
145
|
+
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
146
|
+
return false
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
def create_data_stream(datastream_name, host = nil)
|
151
|
+
return if data_stream_exist?(datastream_name, host)
|
152
|
+
params = {
|
153
|
+
name: datastream_name
|
154
|
+
}
|
155
|
+
retry_operate(@max_retry_putting_template,
|
156
|
+
@fail_on_putting_template_retry_exceed,
|
157
|
+
@catch_transport_exception_on_retry) do
|
158
|
+
client(host).indices.create_data_stream(params)
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def ilm_policy_exists?(policy_id, host = nil)
|
163
|
+
begin
|
164
|
+
if Gem::Version.new(Elasticsearch::VERSION) >= Gem::Version.new("8.0.0")
|
165
|
+
client(host).ilm.get_lifecycle(policy: policy_id)
|
166
|
+
else
|
167
|
+
client(host).ilm.get_policy(policy_id: policy_id)
|
168
|
+
end
|
169
|
+
true
|
170
|
+
rescue
|
171
|
+
false
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def template_exists?(name, host = nil)
|
176
|
+
if @use_legacy_template
|
177
|
+
client(host).indices.get_template(:name => name)
|
178
|
+
else
|
179
|
+
client(host).indices.get_index_template(:name => name)
|
180
|
+
end
|
181
|
+
return true
|
182
|
+
rescue TRANSPORT_CLASS::Transport::Errors::NotFound
|
183
|
+
return false
|
184
|
+
end
|
185
|
+
|
186
|
+
def valid_data_stream_parameters?(data_stream_parameter)
|
187
|
+
lowercase_only?(data_stream_parameter) and
|
188
|
+
valid_characters?(data_stream_parameter) and
|
189
|
+
start_with_valid_characters?(data_stream_parameter) and
|
190
|
+
not_dots?(data_stream_parameter) and
|
191
|
+
data_stream_parameter.bytes.size <= 255
|
192
|
+
end
|
193
|
+
|
194
|
+
def lowercase_only?(data_stream_parameter)
|
195
|
+
data_stream_parameter.downcase == data_stream_parameter
|
196
|
+
end
|
197
|
+
|
198
|
+
def valid_characters?(data_stream_parameter)
|
199
|
+
not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
|
200
|
+
end
|
201
|
+
|
202
|
+
def start_with_valid_characters?(data_stream_parameter)
|
203
|
+
not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
|
204
|
+
end
|
205
|
+
|
206
|
+
def not_dots?(data_stream_parameter)
|
207
|
+
not (data_stream_parameter == "." or data_stream_parameter == "..")
|
208
|
+
end
|
209
|
+
|
210
|
+
def client_library_version
|
211
|
+
Elasticsearch::VERSION
|
212
|
+
end
|
213
|
+
|
214
|
+
def multi_workers_ready?
|
215
|
+
true
|
216
|
+
end
|
217
|
+
|
218
|
+
def write(chunk)
|
219
|
+
data_stream_name = @data_stream_name
|
220
|
+
data_stream_template_name = @data_stream_template_name
|
221
|
+
data_stream_ilm_name = @data_stream_ilm_name
|
222
|
+
host = nil
|
223
|
+
if @use_placeholder
|
224
|
+
host = if @hosts
|
225
|
+
extract_placeholders(@hosts, chunk)
|
226
|
+
else
|
227
|
+
extract_placeholders(@host, chunk)
|
228
|
+
end
|
229
|
+
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
230
|
+
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
|
231
|
+
data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
|
232
|
+
unless @data_stream_names.include?(data_stream_name)
|
233
|
+
begin
|
234
|
+
create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
235
|
+
create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
236
|
+
create_data_stream(data_stream_name)
|
237
|
+
@data_stream_names << data_stream_name
|
238
|
+
rescue => e
|
239
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
bulk_message = ""
|
245
|
+
headers = {
|
246
|
+
CREATE_OP => {}
|
247
|
+
}
|
248
|
+
tag = chunk.metadata.tag
|
249
|
+
chunk.msgpack_each do |time, record|
|
250
|
+
next unless record.is_a? Hash
|
251
|
+
|
252
|
+
if @include_tag_key
|
253
|
+
record[@tag_key] = tag
|
254
|
+
end
|
255
|
+
|
256
|
+
begin
|
257
|
+
unless record.has_key?("@timestamp")
|
258
|
+
record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
|
259
|
+
end
|
260
|
+
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
261
|
+
rescue => e
|
262
|
+
router.emit_error_event(tag, time, record, e)
|
263
|
+
end
|
264
|
+
end
|
265
|
+
|
266
|
+
prepared_data = if compression
|
267
|
+
gzip(bulk_message)
|
268
|
+
else
|
269
|
+
bulk_message
|
270
|
+
end
|
271
|
+
|
272
|
+
params = {
|
273
|
+
index: data_stream_name,
|
274
|
+
body: prepared_data
|
275
|
+
}
|
276
|
+
begin
|
277
|
+
response = client(host, compression).bulk(params)
|
278
|
+
if response['errors']
|
279
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
280
|
+
@num_errors_metrics.inc
|
281
|
+
end
|
282
|
+
rescue => e
|
283
|
+
raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
|
284
|
+
end
|
285
|
+
end
|
286
|
+
|
287
|
+
def append_record_to_messages(op, meta, header, record, msgs)
|
288
|
+
header[CREATE_OP] = meta
|
289
|
+
msgs << @dump_proc.call(header) << BODY_DELIMITER
|
290
|
+
msgs << @dump_proc.call(record) << BODY_DELIMITER
|
291
|
+
msgs
|
292
|
+
end
|
293
|
+
|
294
|
+
def retry_stream_retryable?
|
295
|
+
@buffer.storable?
|
296
|
+
end
|
297
|
+
end
|
298
|
+
end
|
@@ -28,6 +28,8 @@ module Fluent::Plugin
|
|
28
28
|
@dynamic_config[key] = value.to_s
|
29
29
|
}
|
30
30
|
# end eval all configs
|
31
|
+
|
32
|
+
log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
|
31
33
|
end
|
32
34
|
|
33
35
|
def create_meta_config_map
|
@@ -53,7 +55,7 @@ module Fluent::Plugin
|
|
53
55
|
end
|
54
56
|
headers = { 'Content-Type' => @content_type.to_s, }.merge(gzip_headers)
|
55
57
|
ssl_options = { verify: @ssl_verify, ca_file: @ca_file}.merge(@ssl_version_options)
|
56
|
-
transport =
|
58
|
+
transport = TRANSPORT_CLASS::Transport::HTTP::Faraday.new(connection_options.merge(
|
57
59
|
options: {
|
58
60
|
reload_connections: @reload_connections,
|
59
61
|
reload_on_failure: @reload_on_failure,
|
data/test/helper.rb
CHANGED
@@ -5,9 +5,6 @@ SimpleCov.start do
|
|
5
5
|
end
|
6
6
|
end
|
7
7
|
|
8
|
-
require 'coveralls'
|
9
|
-
Coveralls.wear!
|
10
|
-
|
11
8
|
# needs to be after simplecov but before test/unit, because fluentd sets default
|
12
9
|
# encoding to ASCII-8BIT, but coverall might load git data which could contain a
|
13
10
|
# UTF-8 character
|
@@ -18,7 +15,6 @@ end
|
|
18
15
|
|
19
16
|
require 'test/unit'
|
20
17
|
require 'fluent/test'
|
21
|
-
require 'minitest/pride'
|
22
18
|
|
23
19
|
require 'webmock/test_unit'
|
24
20
|
WebMock.disable_net_connect!
|
Binary file
|
@@ -2,6 +2,7 @@ require_relative '../helper'
|
|
2
2
|
require 'fluent/plugin/out_elasticsearch'
|
3
3
|
require 'fluent/plugin/elasticsearch_error_handler'
|
4
4
|
require 'json'
|
5
|
+
require 'msgpack'
|
5
6
|
|
6
7
|
class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
7
8
|
|
@@ -27,13 +28,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
27
28
|
@error_events << {:tag => tag, :time=>time, :record=>record, :error=>e}
|
28
29
|
end
|
29
30
|
|
30
|
-
def process_message(tag, meta, header, time, record, extracted_values)
|
31
|
+
def process_message(tag, meta, header, time, record, affinity_target_indices, extracted_values)
|
31
32
|
return [meta, header, record]
|
32
33
|
end
|
33
34
|
|
35
|
+
def get_affinity_target_indices(chunk)
|
36
|
+
indices = Hash.new
|
37
|
+
indices
|
38
|
+
end
|
39
|
+
|
34
40
|
def append_record_to_messages(op, meta, header, record, msgs)
|
35
41
|
if record.has_key?('raise') && record['raise']
|
36
|
-
raise
|
42
|
+
raise 'process_message'
|
37
43
|
end
|
38
44
|
return true
|
39
45
|
end
|
@@ -49,6 +55,27 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
49
55
|
end
|
50
56
|
end
|
51
57
|
|
58
|
+
class MockMsgpackChunk
|
59
|
+
def initialize(chunk)
|
60
|
+
@chunk = chunk
|
61
|
+
@factory = MessagePack::Factory.new
|
62
|
+
@factory.register_type(Fluent::EventTime::TYPE, Fluent::EventTime)
|
63
|
+
end
|
64
|
+
|
65
|
+
def msgpack_each
|
66
|
+
@factory.unpacker(@chunk).each { |time, record| yield(time, record) }
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
class MockUnpackedMsg
|
71
|
+
def initialize(records)
|
72
|
+
@records = records
|
73
|
+
end
|
74
|
+
def each
|
75
|
+
@records.each { |item| yield({:time => item[:time], :record => item[:record]}) }
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
52
79
|
def setup
|
53
80
|
Fluent::Test.setup
|
54
81
|
@log_device = Fluent::Test::DummyLogDevice.new
|
@@ -93,8 +120,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
93
120
|
]
|
94
121
|
}))
|
95
122
|
chunk = MockChunk.new(records)
|
123
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
96
124
|
dummy_extracted_values = []
|
97
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
125
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
98
126
|
assert_equal(1, @plugin.error_events.size)
|
99
127
|
expected_log = "failed to parse"
|
100
128
|
exception_message = @plugin.error_events.first[:error].message
|
@@ -135,8 +163,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
135
163
|
]
|
136
164
|
}))
|
137
165
|
chunk = MockChunk.new(records)
|
166
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
138
167
|
dummy_extracted_values = []
|
139
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
168
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
140
169
|
assert_equal(1, @plugin.error_events.size)
|
141
170
|
expected_log = "failed to parse"
|
142
171
|
exception_message = @plugin.error_events.first[:error].message
|
@@ -154,8 +183,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
154
183
|
"items" : [{}]
|
155
184
|
}))
|
156
185
|
chunk = MockChunk.new(records)
|
186
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
157
187
|
dummy_extracted_values = []
|
158
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
188
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
159
189
|
assert_equal(0, @plugin.error_events.size)
|
160
190
|
assert_nil(@plugin.error_events[0])
|
161
191
|
end
|
@@ -176,8 +206,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
176
206
|
]
|
177
207
|
}))
|
178
208
|
chunk = MockChunk.new(records)
|
209
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
179
210
|
dummy_extracted_values = []
|
180
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
211
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
181
212
|
assert_equal(1, @plugin.error_events.size)
|
182
213
|
assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
|
183
214
|
end
|
@@ -199,8 +230,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
199
230
|
]
|
200
231
|
}))
|
201
232
|
chunk = MockChunk.new(records)
|
233
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
202
234
|
dummy_extracted_values = []
|
203
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
235
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
204
236
|
assert_equal(1, @plugin.error_events.size)
|
205
237
|
assert_true(@plugin.error_events[0][:error].respond_to?(:backtrace))
|
206
238
|
end
|
@@ -225,10 +257,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
225
257
|
]
|
226
258
|
}))
|
227
259
|
|
228
|
-
|
229
|
-
|
260
|
+
chunk = MockChunk.new(records)
|
261
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
262
|
+
dummy_extracted_values = []
|
230
263
|
assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
|
231
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
264
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
232
265
|
end
|
233
266
|
end
|
234
267
|
|
@@ -252,10 +285,11 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
252
285
|
]
|
253
286
|
}))
|
254
287
|
|
255
|
-
|
256
|
-
|
288
|
+
chunk = MockChunk.new(records)
|
289
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
290
|
+
dummy_extracted_values = []
|
257
291
|
assert_raise(Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError) do
|
258
|
-
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values)
|
292
|
+
@handler.handle_error(response, 'atag', chunk, records.length, dummy_extracted_values, unpacked_msg_arr)
|
259
293
|
end
|
260
294
|
end
|
261
295
|
|
@@ -285,8 +319,9 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
285
319
|
begin
|
286
320
|
failed = false
|
287
321
|
chunk = MockChunk.new(records)
|
322
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
288
323
|
dummy_extracted_values = []
|
289
|
-
handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
|
324
|
+
handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
|
290
325
|
rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
|
291
326
|
failed = true
|
292
327
|
records = [].tap do |records|
|
@@ -302,11 +337,12 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
302
337
|
def test_retry_error
|
303
338
|
records = []
|
304
339
|
error_records = Hash.new(false)
|
305
|
-
error_records.merge!({0=>true, 4=>true
|
340
|
+
error_records.merge!({0=>true, 4=>true})
|
306
341
|
10.times do |i|
|
307
342
|
records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
|
308
343
|
end
|
309
344
|
chunk = MockChunk.new(records)
|
345
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
310
346
|
|
311
347
|
response = parse_response(%({
|
312
348
|
"took" : 1,
|
@@ -386,6 +422,18 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
386
422
|
"reason":"unrecognized error"
|
387
423
|
}
|
388
424
|
}
|
425
|
+
},
|
426
|
+
{
|
427
|
+
"create" : {
|
428
|
+
"_index" : "foo",
|
429
|
+
"_type" : "bar",
|
430
|
+
"_id" : "9",
|
431
|
+
"status" : 500,
|
432
|
+
"error" : {
|
433
|
+
"type" : "json_parse_exception",
|
434
|
+
"reason":"Invalid UTF-8 start byte 0x92\\n at [Source: org.elasticsearch.transport.netty4.ByteBufStreamInput@204fe9c9; line: 1, column: 81]"
|
435
|
+
}
|
436
|
+
}
|
389
437
|
}
|
390
438
|
]
|
391
439
|
}))
|
@@ -393,19 +441,19 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
393
441
|
begin
|
394
442
|
failed = false
|
395
443
|
dummy_extracted_values = []
|
396
|
-
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
|
444
|
+
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
|
397
445
|
rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
|
398
446
|
failed = true
|
399
447
|
records = [].tap do |records|
|
400
448
|
next unless e.respond_to?(:retry_stream)
|
401
449
|
e.retry_stream.each {|time, record| records << record}
|
402
450
|
end
|
403
|
-
assert_equal 2, records.length
|
404
|
-
assert_equal 2, records[0]['_id']
|
405
|
-
assert_equal 8, records[1]['_id']
|
451
|
+
assert_equal 2, records.length, "Exp. retry_stream to contain records"
|
452
|
+
assert_equal 2, records[0]['_id'], "Exp record with given ID to in retry_stream"
|
453
|
+
assert_equal 8, records[1]['_id'], "Exp record with given ID to in retry_stream"
|
406
454
|
error_ids = @plugin.error_events.collect {|h| h[:record]['_id']}
|
407
|
-
assert_equal
|
408
|
-
assert_equal [5, 6, 7], error_ids
|
455
|
+
assert_equal 4, error_ids.length, "Exp. a certain number of records to be dropped from retry_stream"
|
456
|
+
assert_equal [5, 6, 7, 9], error_ids, "Exp. specific records to be dropped from retry_stream"
|
409
457
|
@plugin.error_events.collect {|h| h[:error]}.each do |e|
|
410
458
|
assert_true e.respond_to?(:backtrace)
|
411
459
|
end
|
@@ -422,6 +470,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
422
470
|
records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
|
423
471
|
end
|
424
472
|
chunk = MockChunk.new(records)
|
473
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
425
474
|
|
426
475
|
response = parse_response(%({
|
427
476
|
"took" : 1,
|
@@ -509,7 +558,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
509
558
|
begin
|
510
559
|
failed = false
|
511
560
|
dummy_extracted_values = []
|
512
|
-
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
|
561
|
+
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
|
513
562
|
rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
|
514
563
|
failed = true
|
515
564
|
records = [].tap do |records|
|
@@ -532,6 +581,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
532
581
|
records << {time: 12345, record: {"message"=>"record #{i}","_id"=>i,"raise"=>error_records[i]}}
|
533
582
|
end
|
534
583
|
chunk = MockChunk.new(records)
|
584
|
+
unpacked_msg_arr = MockUnpackedMsg.new(records)
|
535
585
|
|
536
586
|
response = parse_response(%({
|
537
587
|
"took" : 1,
|
@@ -622,7 +672,7 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
622
672
|
begin
|
623
673
|
failed = false
|
624
674
|
dummy_extracted_values = []
|
625
|
-
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values)
|
675
|
+
@handler.handle_error(response, 'atag', chunk, response['items'].length, dummy_extracted_values, unpacked_msg_arr)
|
626
676
|
rescue Fluent::Plugin::ElasticsearchErrorHandler::ElasticsearchRequestAbortError, Fluent::Plugin::ElasticsearchOutput::RetryStreamError=>e
|
627
677
|
failed = true
|
628
678
|
records = [].tap do |records|
|
@@ -643,4 +693,61 @@ class TestElasticsearchErrorHandler < Test::Unit::TestCase
|
|
643
693
|
end
|
644
694
|
assert_true failed
|
645
695
|
end
|
696
|
+
|
697
|
+
def test_nested_msgpack_each
|
698
|
+
cwd = File.dirname(__FILE__)
|
699
|
+
chunk_path = File.join(cwd, 'mock_chunk.dat')
|
700
|
+
chunk_file = File.open(chunk_path, 'rb', 0644)
|
701
|
+
chunk_file.seek(0, IO::SEEK_SET)
|
702
|
+
|
703
|
+
chunk = MockMsgpackChunk.new(chunk_file)
|
704
|
+
|
705
|
+
unpacked_msg_arr = []
|
706
|
+
msg_count = 0
|
707
|
+
count_to_trigger_error_handle = 0
|
708
|
+
chunk.msgpack_each do |time, record|
|
709
|
+
next unless record.is_a? Hash
|
710
|
+
|
711
|
+
unpacked_msg_arr << {:time => time, :record => record}
|
712
|
+
msg_count += 1
|
713
|
+
|
714
|
+
record.each_key do |k|
|
715
|
+
if k != 'aaa' && k != 'bbb' && k != 'ccc' && k != 'log_path'
|
716
|
+
assert_equal(:impossible, k)
|
717
|
+
end
|
718
|
+
end
|
719
|
+
|
720
|
+
if msg_count % 55 == 0
|
721
|
+
if count_to_trigger_error_handle == 1
|
722
|
+
begin
|
723
|
+
response = {}
|
724
|
+
response['errors'] = true
|
725
|
+
response['items'] = []
|
726
|
+
item = {}
|
727
|
+
item['index'] = {}
|
728
|
+
item['index']['status'] = 429
|
729
|
+
item['index']['error'] = {}
|
730
|
+
item['index']['error']['type'] = "es_rejected_execution_exception"
|
731
|
+
abc = 0
|
732
|
+
while abc < unpacked_msg_arr.length
|
733
|
+
abc += 1
|
734
|
+
response['items'] << item
|
735
|
+
end
|
736
|
+
|
737
|
+
dummy_extracted_values = []
|
738
|
+
@handler.handle_error(response, 'atag', chunk, unpacked_msg_arr.length, dummy_extracted_values, unpacked_msg_arr)
|
739
|
+
assert_equal(0, @plugin.error_events.size)
|
740
|
+
assert_nil(@plugin.error_events[0])
|
741
|
+
rescue => e
|
742
|
+
# capture ElasticsearchRequestAbortError, beacuse es_rejected_execution_exception is unrecoverable.
|
743
|
+
end
|
744
|
+
end
|
745
|
+
|
746
|
+
count_to_trigger_error_handle += 1
|
747
|
+
unpacked_msg_arr.clear
|
748
|
+
end # end if
|
749
|
+
end # end chunk.msgpack_each
|
750
|
+
|
751
|
+
chunk_file.close
|
752
|
+
end
|
646
753
|
end
|
@@ -15,20 +15,28 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
15
15
|
def stub_elastic(url="http://localhost:9200/_bulk")
|
16
16
|
stub_request(:post, url).with do |req|
|
17
17
|
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
18
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
19
|
+
end
|
20
|
+
|
21
|
+
def elasticsearch_version
|
22
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
23
|
+
TRANSPORT_CLASS::VERSION
|
24
|
+
else
|
25
|
+
'6.4.2'.freeze
|
18
26
|
end
|
19
27
|
end
|
20
28
|
|
21
|
-
def stub_elastic_info(url="http://localhost:9200/", version=
|
22
|
-
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
23
|
-
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
29
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version)
|
30
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
31
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
24
32
|
end
|
25
33
|
|
26
|
-
def stub_elastic_info_not_found(url="http://localhost:9200/", version=
|
27
|
-
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
34
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version=elasticsearch_version)
|
35
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
28
36
|
end
|
29
37
|
|
30
|
-
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=
|
31
|
-
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
38
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version=elasticsearch_version)
|
39
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"], headers: {'x-elastic-product' => 'Elasticsearch' })
|
32
40
|
end
|
33
41
|
|
34
42
|
def sample_record(content={})
|
@@ -58,8 +66,9 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
58
66
|
with_transporter_log true
|
59
67
|
reload_connections true
|
60
68
|
reload_after 10
|
69
|
+
catch_transport_exception_on_retry false # For fallback testing
|
61
70
|
]
|
62
|
-
assert_raise(
|
71
|
+
assert_raise(TRANSPORT_CLASS::Transport::Errors::NotFound) do
|
63
72
|
driver(config)
|
64
73
|
end
|
65
74
|
driver.run(default_tag: 'test') do
|