fluent-plugin-elasticsearch 4.1.1 → 5.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/ISSUE_TEMPLATE/bug_report.md +37 -0
- data/.github/ISSUE_TEMPLATE/feature_request.md +24 -0
- data/.github/dependabot.yml +6 -0
- data/.github/workflows/issue-auto-closer.yml +2 -2
- data/.github/workflows/linux.yml +5 -2
- data/.github/workflows/macos.yml +5 -2
- data/.github/workflows/windows.yml +5 -2
- data/Gemfile +1 -2
- data/History.md +146 -0
- data/README.ElasticsearchGenID.md +4 -4
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +260 -550
- data/fluent-plugin-elasticsearch.gemspec +4 -1
- data/lib/fluent/plugin/elasticsearch_compat.rb +31 -0
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +19 -4
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +2 -2
- data/lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb +18 -4
- data/lib/fluent/plugin/elasticsearch_index_template.rb +65 -21
- data/lib/fluent/plugin/elasticsearch_simple_sniffer.rb +2 -1
- data/lib/fluent/plugin/filter_elasticsearch_genid.rb +1 -1
- data/lib/fluent/plugin/in_elasticsearch.rb +8 -2
- data/lib/fluent/plugin/oj_serializer.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +192 -36
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +298 -0
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +3 -1
- data/test/helper.rb +0 -4
- data/test/plugin/mock_chunk.dat +0 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +130 -23
- data/test/plugin/test_elasticsearch_fallback_selector.rb +17 -8
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +57 -18
- data/test/plugin/test_elasticsearch_tls.rb +8 -2
- data/test/plugin/test_filter_elasticsearch_genid.rb +16 -16
- data/test/plugin/test_in_elasticsearch.rb +51 -21
- data/test/plugin/test_index_alias_template.json +11 -0
- data/test/plugin/test_index_template.json +25 -0
- data/test/plugin/test_out_elasticsearch.rb +2118 -704
- data/test/plugin/test_out_elasticsearch_data_stream.rb +1199 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +170 -31
- metadata +62 -10
- data/.coveralls.yml +0 -2
- data/.travis.yml +0 -44
- data/appveyor.yml +0 -20
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -0,0 +1,1199 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'date'
|
3
|
+
require 'fluent/test/helpers'
|
4
|
+
require 'fluent/test/driver/output'
|
5
|
+
require 'flexmock/test_unit'
|
6
|
+
require 'fluent/plugin/out_elasticsearch_data_stream'
|
7
|
+
|
8
|
+
class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
9
|
+
include FlexMock::TestCase
|
10
|
+
include Fluent::Test::Helpers
|
11
|
+
|
12
|
+
attr_accessor :bulk_records
|
13
|
+
|
14
|
+
REQUIRED_ELASTIC_MESSAGE = "Elasticsearch 7.9.0 or later is needed."
|
15
|
+
ELASTIC_DATA_STREAM_TYPE = "elasticsearch_data_stream"
|
16
|
+
|
17
|
+
def setup
|
18
|
+
Fluent::Test.setup
|
19
|
+
@driver = nil
|
20
|
+
log = Fluent::Engine.log
|
21
|
+
log.out.logs.slice!(0, log.out.logs.length)
|
22
|
+
@bulk_records = []
|
23
|
+
end
|
24
|
+
|
25
|
+
def elasticsearch_version
|
26
|
+
if Gem::Version.new(TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
27
|
+
TRANSPORT_CLASS::VERSION
|
28
|
+
else
|
29
|
+
'5.0.0'.freeze
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def ilm_endpoint
|
34
|
+
'_ilm'.freeze
|
35
|
+
end
|
36
|
+
|
37
|
+
def index_template_endpoint
|
38
|
+
'_index_template'.freeze
|
39
|
+
end
|
40
|
+
|
41
|
+
def driver(conf='', es_version=elasticsearch_version.to_i, client_version=elasticsearch_version)
|
42
|
+
# For request stub to detect compatibility.
|
43
|
+
@es_version ||= es_version
|
44
|
+
@client_version ||= client_version
|
45
|
+
Fluent::Plugin::ElasticsearchOutputDataStream.module_eval(<<-CODE)
|
46
|
+
def detect_es_major_version
|
47
|
+
#{@es_version}
|
48
|
+
end
|
49
|
+
CODE
|
50
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutputDataStream) {
|
51
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
52
|
+
if !defined?(Fluent::Plugin::Output)
|
53
|
+
def format(tag, time, record)
|
54
|
+
[time, record].to_msgpack
|
55
|
+
end
|
56
|
+
end
|
57
|
+
}.configure(conf)
|
58
|
+
end
|
59
|
+
|
60
|
+
def elasticsearch_transport_layer_decoupling?
|
61
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("7.14.0")
|
62
|
+
end
|
63
|
+
|
64
|
+
def elastic_transport_layer?
|
65
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.new("8.0.0")
|
66
|
+
end
|
67
|
+
|
68
|
+
def sample_data_stream
|
69
|
+
{
|
70
|
+
'data_streams': [
|
71
|
+
{
|
72
|
+
'name' => 'foo',
|
73
|
+
'timestamp_field' => {
|
74
|
+
'name' => '@timestamp'
|
75
|
+
}
|
76
|
+
}
|
77
|
+
]
|
78
|
+
}
|
79
|
+
end
|
80
|
+
|
81
|
+
SAMPLE_RECORD_TIMESTAMP = Time.now.iso8601
|
82
|
+
def sample_record
|
83
|
+
{'@timestamp' => SAMPLE_RECORD_TIMESTAMP, 'message' => 'Sample record'}
|
84
|
+
end
|
85
|
+
|
86
|
+
def sample_record_no_timestamp
|
87
|
+
{'message' => 'Sample record no timestamp'}
|
88
|
+
end
|
89
|
+
|
90
|
+
RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
|
91
|
+
DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
|
92
|
+
NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
|
93
|
+
|
94
|
+
def stub_ilm_policy(name="foo_ilm_policy", url="http://localhost:9200")
|
95
|
+
stub_request(:put, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
96
|
+
end
|
97
|
+
|
98
|
+
def stub_index_template(name="foo_tpl", url="http://localhost:9200")
|
99
|
+
stub_request(:put, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
100
|
+
end
|
101
|
+
|
102
|
+
def stub_data_stream(name="foo", url="http://localhost:9200")
|
103
|
+
stub_request(:put, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
104
|
+
end
|
105
|
+
|
106
|
+
def stub_existent_data_stream?(name="foo", url="http://localhost:9200")
|
107
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
108
|
+
end
|
109
|
+
|
110
|
+
def stub_existent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
111
|
+
|
112
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
113
|
+
end
|
114
|
+
|
115
|
+
def stub_existent_template?(name="foo_tpl", url="http://localhost:9200")
|
116
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
117
|
+
end
|
118
|
+
|
119
|
+
def stub_nonexistent_data_stream?(name="foo", url="http://localhost:9200")
|
120
|
+
stub_request(:get, "#{url}/_data_stream/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
121
|
+
end
|
122
|
+
|
123
|
+
def stub_nonexistent_ilm?(name="foo_ilm_policy", url="http://localhost:9200")
|
124
|
+
stub_request(:get, "#{url}/#{ilm_endpoint}/policy/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
125
|
+
end
|
126
|
+
|
127
|
+
def stub_nonexistent_template?(name="foo_tpl", url="http://localhost:9200")
|
128
|
+
stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound], :headers => {'x-elastic-product' => 'Elasticsearch'})
|
129
|
+
end
|
130
|
+
|
131
|
+
def push_bulk_request(req_body)
|
132
|
+
# bulk data must be pair of OP and records
|
133
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
134
|
+
# {"@timestamp": ...}
|
135
|
+
ops = req_body.split("\n")
|
136
|
+
@bulk_records += ops.values_at(
|
137
|
+
* ops.each_index.select {|i| i.odd? }
|
138
|
+
).map{ |i| JSON.parse(i) }
|
139
|
+
end
|
140
|
+
|
141
|
+
def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
|
142
|
+
stub_request(:get, "#{url}/_index_template/#{name}").
|
143
|
+
to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}', :headers => {'x-elastic-product' => 'Elasticsearch'} })
|
144
|
+
end
|
145
|
+
|
146
|
+
def stub_bulk_feed(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", url="http://localhost:9200")
|
147
|
+
stub_request(:post, "#{url}/#{datastream_name}/_bulk").with do |req|
|
148
|
+
# bulk data must be pair of OP and records
|
149
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
150
|
+
# {"@timestamp": ...}
|
151
|
+
push_bulk_request(req.body)
|
152
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
153
|
+
stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
|
154
|
+
# bulk data must be pair of OP and records
|
155
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
156
|
+
# {"@timestamp": ...}
|
157
|
+
push_bulk_request(req.body)
|
158
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
159
|
+
stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
|
160
|
+
# bulk data must be pair of OP and records
|
161
|
+
# {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
|
162
|
+
# {"@timestamp": ...}
|
163
|
+
push_bulk_request(req.body)
|
164
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
165
|
+
end
|
166
|
+
|
167
|
+
def stub_elastic_info(url="http://localhost:9200/", version=elasticsearch_version, headers={})
|
168
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
169
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' }.merge(headers) })
|
170
|
+
end
|
171
|
+
|
172
|
+
def stub_default(datastream_name="foo", ilm_name="foo_ilm_policy", template_name="foo_tpl", host="http://localhost:9200")
|
173
|
+
stub_elastic_info(host)
|
174
|
+
stub_nonexistent_ilm?(ilm_name)
|
175
|
+
stub_ilm_policy(ilm_name)
|
176
|
+
stub_nonexistent_template?(template_name)
|
177
|
+
stub_index_template(template_name)
|
178
|
+
stub_nonexistent_data_stream?(datastream_name)
|
179
|
+
stub_data_stream(datastream_name)
|
180
|
+
end
|
181
|
+
|
182
|
+
def stub_elastic_with_store_index_command_counts(url="http://localhost:9200/_bulk")
|
183
|
+
if @index_command_counts == nil
|
184
|
+
@index_command_counts = {}
|
185
|
+
@index_command_counts.default = 0
|
186
|
+
end
|
187
|
+
|
188
|
+
stub_request(:post, url).with do |req|
|
189
|
+
index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
190
|
+
@index_command_counts[url] += index_cmds.size
|
191
|
+
end.to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
192
|
+
end
|
193
|
+
|
194
|
+
def data_stream_supported?
|
195
|
+
Gem::Version.create(::TRANSPORT_CLASS::VERSION) >= Gem::Version.create("7.9.0")
|
196
|
+
end
|
197
|
+
|
198
|
+
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
199
|
+
class DataStreamNameTest < self
|
200
|
+
|
201
|
+
def test_missing_data_stream_name
|
202
|
+
conf = config_element(
|
203
|
+
'ROOT', '', {
|
204
|
+
'@type' => 'elasticsearch_datastream'
|
205
|
+
})
|
206
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' parameter is required") do
|
207
|
+
driver(conf).run
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
sub_test_case "invalid uppercase" do
|
212
|
+
def test_stream_name
|
213
|
+
conf = config_element(
|
214
|
+
'ROOT', '', {
|
215
|
+
'@type' => 'elasticsearch_datastream',
|
216
|
+
'data_stream_name' => 'TEST',
|
217
|
+
'data_stream_ilm_name' => 'default-policy',
|
218
|
+
'data_stream_template_name' => 'template'
|
219
|
+
})
|
220
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
|
221
|
+
driver(conf)
|
222
|
+
end
|
223
|
+
end
|
224
|
+
def test_stream_ilm_name
|
225
|
+
conf = config_element(
|
226
|
+
'ROOT', '', {
|
227
|
+
'@type' => 'elasticsearch_datastream',
|
228
|
+
'data_stream_name' => 'data_stream',
|
229
|
+
'data_stream_ilm_name' => 'TEST-ILM',
|
230
|
+
'data_stream_template_name' => 'template'
|
231
|
+
})
|
232
|
+
assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must be lowercase only: <TEST-ILM>") do
|
233
|
+
driver(conf)
|
234
|
+
end
|
235
|
+
end
|
236
|
+
def test_stream_template_name
|
237
|
+
conf = config_element(
|
238
|
+
'ROOT', '', {
|
239
|
+
'@type' => 'elasticsearch_datastream',
|
240
|
+
'data_stream_name' => 'default',
|
241
|
+
'data_stream_ilm_name' => 'default-policy',
|
242
|
+
'data_stream_template_name' => 'TEST-TPL'
|
243
|
+
})
|
244
|
+
assert_raise Fluent::ConfigError.new("'data_stream_template_name' must be lowercase only: <TEST-TPL>") do
|
245
|
+
driver(conf)
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
sub_test_case "invalid parameters" do
|
251
|
+
data("backslash" => "\\",
|
252
|
+
"slash" => "/",
|
253
|
+
"asterisk" => "*",
|
254
|
+
"question" => "?",
|
255
|
+
"doublequote" => "\"",
|
256
|
+
"lt" => "<",
|
257
|
+
"gt" => ">",
|
258
|
+
"bar" => "|",
|
259
|
+
"space" => " ",
|
260
|
+
"comma" => ",",
|
261
|
+
"sharp" => "#",
|
262
|
+
"colon" => ":")
|
263
|
+
def test_stream_name(data)
|
264
|
+
c, _ = data
|
265
|
+
conf = config_element(
|
266
|
+
'ROOT', '', {
|
267
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
268
|
+
'data_stream_name' => "TEST#{c}",
|
269
|
+
'data_stream_ilm_name' => "default_policy",
|
270
|
+
'data_stream_template_name' => "data_stream"
|
271
|
+
})
|
272
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
|
273
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
|
274
|
+
driver(conf)
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
data("backslash" => "\\",
|
279
|
+
"slash" => "/",
|
280
|
+
"asterisk" => "*",
|
281
|
+
"question" => "?",
|
282
|
+
"doublequote" => "\"",
|
283
|
+
"lt" => "<",
|
284
|
+
"gt" => ">",
|
285
|
+
"bar" => "|",
|
286
|
+
"space" => " ",
|
287
|
+
"comma" => ",",
|
288
|
+
"sharp" => "#",
|
289
|
+
"colon" => ":")
|
290
|
+
def test_stream_ilm_name(data)
|
291
|
+
c, _ = data
|
292
|
+
conf = config_element(
|
293
|
+
'ROOT', '', {
|
294
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
295
|
+
'data_stream_name' => "default",
|
296
|
+
'data_stream_ilm_name' => "TEST#{c}",
|
297
|
+
'data_stream_template_name' => "data_stream"
|
298
|
+
})
|
299
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
|
300
|
+
assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not contain invalid characters #{label}: <TEST#{c}>") do
|
301
|
+
driver(conf)
|
302
|
+
end
|
303
|
+
end
|
304
|
+
|
305
|
+
data("backslash" => "\\",
|
306
|
+
"slash" => "/",
|
307
|
+
"asterisk" => "*",
|
308
|
+
"question" => "?",
|
309
|
+
"doublequote" => "\"",
|
310
|
+
"lt" => "<",
|
311
|
+
"gt" => ">",
|
312
|
+
"bar" => "|",
|
313
|
+
"space" => " ",
|
314
|
+
"comma" => ",",
|
315
|
+
"sharp" => "#",
|
316
|
+
"colon" => ":")
|
317
|
+
def test_stream_template_name(data)
|
318
|
+
c, _ = data
|
319
|
+
conf = config_element(
|
320
|
+
'ROOT', '', {
|
321
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
322
|
+
'data_stream_name' => "default",
|
323
|
+
'data_stream_ilm_name' => "default_policy",
|
324
|
+
'data_stream_template_name' => "TEST#{c}"
|
325
|
+
})
|
326
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
|
327
|
+
assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not contain invalid characters #{label}: <TEST#{c}>") do
|
328
|
+
driver(conf)
|
329
|
+
end
|
330
|
+
end
|
331
|
+
end
|
332
|
+
|
333
|
+
sub_test_case "invalid start characters" do
|
334
|
+
data("hyphen" => "-",
|
335
|
+
"underscore" => "_",
|
336
|
+
"plus" => "+",
|
337
|
+
"period" => ".")
|
338
|
+
def test_stream_name(data)
|
339
|
+
c, _ = data
|
340
|
+
conf = config_element(
|
341
|
+
'ROOT', '', {
|
342
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
343
|
+
'data_stream_name' => "#{c}TEST",
|
344
|
+
'data_stream_ilm_name' => "default-policy",
|
345
|
+
'data_stream_template_name' => "template"
|
346
|
+
})
|
347
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
|
348
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
|
349
|
+
driver(conf)
|
350
|
+
end
|
351
|
+
end
|
352
|
+
data("hyphen" => "-",
|
353
|
+
"underscore" => "_",
|
354
|
+
"plus" => "+",
|
355
|
+
"period" => ".")
|
356
|
+
def test_stream_ilm_name(data)
|
357
|
+
c, _ = data
|
358
|
+
conf = config_element(
|
359
|
+
'ROOT', '', {
|
360
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
361
|
+
'data_stream_name' => "default",
|
362
|
+
'data_stream_ilm_name' => "#{c}TEST",
|
363
|
+
'data_stream_template_name' => "template"
|
364
|
+
})
|
365
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
|
366
|
+
assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not start with #{label}: <#{c}TEST>") do
|
367
|
+
driver(conf)
|
368
|
+
end
|
369
|
+
end
|
370
|
+
data("hyphen" => "-",
|
371
|
+
"underscore" => "_",
|
372
|
+
"plus" => "+",
|
373
|
+
"period" => ".")
|
374
|
+
def test_stream_template_name(data)
|
375
|
+
c, _ = data
|
376
|
+
conf = config_element(
|
377
|
+
'ROOT', '', {
|
378
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
379
|
+
'data_stream_name' => "default",
|
380
|
+
'data_stream_ilm_name' => "default-policy",
|
381
|
+
'data_stream_template_name' => "#{c}TEST"
|
382
|
+
})
|
383
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
|
384
|
+
assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not start with #{label}: <#{c}TEST>") do
|
385
|
+
driver(conf)
|
386
|
+
end
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
390
|
+
sub_test_case "invalid dots" do
|
391
|
+
data("current" => ".",
|
392
|
+
"parents" => "..")
|
393
|
+
def test_stream_name
|
394
|
+
c, _ = data
|
395
|
+
conf = config_element(
|
396
|
+
'ROOT', '', {
|
397
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
398
|
+
'data_stream_name' => "#{c}",
|
399
|
+
'data_stream_ilm_name' => "default-policy",
|
400
|
+
'data_stream_template_name' => "template"
|
401
|
+
})
|
402
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
|
403
|
+
driver(conf)
|
404
|
+
end
|
405
|
+
end
|
406
|
+
|
407
|
+
data("current" => ".",
|
408
|
+
"parents" => "..")
|
409
|
+
def test_stream_ilm_name
|
410
|
+
c, _ = data
|
411
|
+
conf = config_element(
|
412
|
+
'ROOT', '', {
|
413
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
414
|
+
'data_stream_name' => "default",
|
415
|
+
'data_stream_ilm_name' => "#{c}",
|
416
|
+
'data_stream_template_name' => "template"
|
417
|
+
})
|
418
|
+
assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be . or ..: <#{c}>") do
|
419
|
+
driver(conf)
|
420
|
+
end
|
421
|
+
end
|
422
|
+
|
423
|
+
data("current" => ".",
|
424
|
+
"parents" => "..")
|
425
|
+
def test_stream_template_name
|
426
|
+
c, _ = data
|
427
|
+
conf = config_element(
|
428
|
+
'ROOT', '', {
|
429
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
430
|
+
'data_stream_name' => "default",
|
431
|
+
'data_stream_ilm_name' => "default-policy",
|
432
|
+
'data_stream_template_name' => "#{c}"
|
433
|
+
})
|
434
|
+
assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be . or ..: <#{c}>") do
|
435
|
+
driver(conf)
|
436
|
+
end
|
437
|
+
end
|
438
|
+
end
|
439
|
+
|
440
|
+
sub_test_case "invalid length" do
|
441
|
+
def test_stream_name
|
442
|
+
c = "a" * 256
|
443
|
+
conf = config_element(
|
444
|
+
'ROOT', '', {
|
445
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
446
|
+
'data_stream_name' => "#{c}",
|
447
|
+
'data_stream_ilm_name' => "default-policy",
|
448
|
+
'data_stream_template_name' => "template"
|
449
|
+
})
|
450
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
|
451
|
+
driver(conf)
|
452
|
+
end
|
453
|
+
end
|
454
|
+
def test_stream_ilm_name
|
455
|
+
c = "a" * 256
|
456
|
+
conf = config_element(
|
457
|
+
'ROOT', '', {
|
458
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
459
|
+
'data_stream_name' => "default",
|
460
|
+
'data_stream_ilm_name' => "#{c}",
|
461
|
+
'data_stream_template_name' => "template"
|
462
|
+
})
|
463
|
+
assert_raise Fluent::ConfigError.new("'data_stream_ilm_name' must not be longer than 255 bytes: <#{c}>") do
|
464
|
+
driver(conf)
|
465
|
+
end
|
466
|
+
end
|
467
|
+
def test_stream_template_name
|
468
|
+
c = "a" * 256
|
469
|
+
conf = config_element(
|
470
|
+
'ROOT', '', {
|
471
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
472
|
+
'data_stream_name' => "default",
|
473
|
+
'data_stream_ilm_name' => "default-policy",
|
474
|
+
'data_stream_template_name' => "#{c}"
|
475
|
+
})
|
476
|
+
assert_raise Fluent::ConfigError.new("'data_stream_template_name' must not be longer than 255 bytes: <#{c}>") do
|
477
|
+
driver(conf)
|
478
|
+
end
|
479
|
+
end
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
def test_datastream_configure
|
484
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
485
|
+
|
486
|
+
stub_default
|
487
|
+
conf = config_element(
|
488
|
+
'ROOT', '', {
|
489
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
490
|
+
'data_stream_name' => 'foo',
|
491
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
492
|
+
'data_stream_template_name' => "foo_tpl",
|
493
|
+
'data_stream_template_use_index_patterns_wildcard' => false
|
494
|
+
})
|
495
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
496
|
+
end
|
497
|
+
|
498
|
+
def test_datastream_configure_retry
|
499
|
+
stub_elastic_info
|
500
|
+
stub_nonexistent_ilm?
|
501
|
+
stub_ilm_policy
|
502
|
+
stub_nonexistent_template_retry?
|
503
|
+
stub_index_template
|
504
|
+
stub_nonexistent_data_stream?
|
505
|
+
stub_data_stream
|
506
|
+
conf = config_element(
|
507
|
+
'ROOT', '', {
|
508
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
509
|
+
'data_stream_name' => 'foo',
|
510
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
511
|
+
'data_stream_template_name' => "foo_tpl"
|
512
|
+
})
|
513
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
514
|
+
end
|
515
|
+
|
516
|
+
def test_hosts_list_configure
|
517
|
+
config = %{
|
518
|
+
hosts https://john:password@host1:443/elastic/,http://host2
|
519
|
+
path /default_path
|
520
|
+
user default_user
|
521
|
+
password default_password
|
522
|
+
data_stream_name default
|
523
|
+
}
|
524
|
+
stub_elastic_info("https://host1:443/elastic//", elasticsearch_version,
|
525
|
+
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
526
|
+
stub_elastic_info("http://host2/default_path/_data_stream/default", elasticsearch_version,
|
527
|
+
{'Authorization'=>"Basic #{Base64.encode64('john:password').split.first}"})
|
528
|
+
stub_existent_data_stream?("default", "https://host1/elastic/")
|
529
|
+
instance = driver(config).instance
|
530
|
+
|
531
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
532
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
533
|
+
|
534
|
+
assert_equal 'host1', host1[:host]
|
535
|
+
assert_equal 443, host1[:port]
|
536
|
+
assert_equal 'https', host1[:scheme]
|
537
|
+
assert_equal 'john', host1[:user]
|
538
|
+
assert_equal 'password', host1[:password]
|
539
|
+
assert_equal '/elastic/', host1[:path]
|
540
|
+
|
541
|
+
assert_equal 'host2', host2[:host]
|
542
|
+
assert_equal 'http', host2[:scheme]
|
543
|
+
assert_equal 'default_user', host2[:user]
|
544
|
+
assert_equal 'default_password', host2[:password]
|
545
|
+
assert_equal '/default_path', host2[:path]
|
546
|
+
end
|
547
|
+
|
548
|
+
def test_configure_compression
|
549
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
550
|
+
|
551
|
+
config = %{
|
552
|
+
data_stream_name foo
|
553
|
+
data_stream_template_name foo_tpl
|
554
|
+
data_stream_ilm_name foo_ilm_policy
|
555
|
+
compression_level best_compression
|
556
|
+
}
|
557
|
+
|
558
|
+
stub_default
|
559
|
+
|
560
|
+
assert_equal true, driver(config).instance.compression
|
561
|
+
end
|
562
|
+
|
563
|
+
def test_configure_wildcard_usage
|
564
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
565
|
+
|
566
|
+
config = %{
|
567
|
+
data_stream_name foo
|
568
|
+
data_stream_template_name foo_tpl
|
569
|
+
data_stream_ilm_name foo_ilm_policy
|
570
|
+
data_stream_template_use_index_patterns_wildcard false
|
571
|
+
}
|
572
|
+
|
573
|
+
stub_default
|
574
|
+
|
575
|
+
assert_equal false, driver(config).instance.data_stream_template_use_index_patterns_wildcard
|
576
|
+
end
|
577
|
+
|
578
|
+
def test_check_compression_strategy
|
579
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
580
|
+
|
581
|
+
config = %{
|
582
|
+
data_stream_name foo
|
583
|
+
data_stream_template_name foo_tpl
|
584
|
+
data_stream_ilm_name foo_ilm_policy
|
585
|
+
compression_level best_speed
|
586
|
+
}
|
587
|
+
|
588
|
+
stub_default
|
589
|
+
stub_bulk_feed
|
590
|
+
|
591
|
+
assert_equal Zlib::BEST_SPEED, driver(config).instance.compression_strategy
|
592
|
+
end
|
593
|
+
|
594
|
+
def test_check_content_encoding_header_with_compression
|
595
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
596
|
+
|
597
|
+
config = %{
|
598
|
+
data_stream_name foo
|
599
|
+
data_stream_template_name foo_tpl
|
600
|
+
data_stream_ilm_name foo_ilm_policy
|
601
|
+
compression_level best_compression
|
602
|
+
}
|
603
|
+
|
604
|
+
stub_default
|
605
|
+
stub_request(:post, "http://localhost:9200/foo/_bulk").
|
606
|
+
to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
|
607
|
+
|
608
|
+
instance = driver(config).instance
|
609
|
+
|
610
|
+
if elastic_transport_layer?
|
611
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
612
|
+
elsif elasticsearch_transport_layer_decoupling?
|
613
|
+
assert_equal nil, instance.client.transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
614
|
+
else
|
615
|
+
assert_equal nil, instance.client.transport.options[:transport_options][:headers]["Content-Encoding"]
|
616
|
+
end
|
617
|
+
|
618
|
+
driver.run(default_tag: 'test') do
|
619
|
+
driver.feed(sample_record)
|
620
|
+
end
|
621
|
+
compressable = instance.compressable_connection
|
622
|
+
|
623
|
+
if elastic_transport_layer?
|
624
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
625
|
+
elsif elasticsearch_transport_layer_decoupling?
|
626
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.transport.options[:transport_options][:headers]["Content-Encoding"]
|
627
|
+
else
|
628
|
+
assert_equal "gzip", instance.client(nil, compressable).transport.options[:transport_options][:headers]["Content-Encoding"]
|
629
|
+
end
|
630
|
+
end
|
631
|
+
|
632
|
+
def test_check_compression_option_is_passed_to_transport
|
633
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
634
|
+
|
635
|
+
config = %{
|
636
|
+
data_stream_name foo
|
637
|
+
data_stream_template_name foo_tpl
|
638
|
+
data_stream_ilm_name foo_ilm_policy
|
639
|
+
compression_level best_compression
|
640
|
+
}
|
641
|
+
|
642
|
+
stub_default
|
643
|
+
stub_request(:post, "http://localhost:9200/foo/_bulk").
|
644
|
+
to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
|
645
|
+
|
646
|
+
instance = driver(config).instance
|
647
|
+
|
648
|
+
if elastic_transport_layer?
|
649
|
+
assert_equal false, instance.client.transport.options[:compression]
|
650
|
+
elsif elasticsearch_transport_layer_decoupling?
|
651
|
+
assert_equal false, instance.client.transport.transport.options[:compression]
|
652
|
+
else
|
653
|
+
assert_equal false, instance.client.transport.options[:compression]
|
654
|
+
end
|
655
|
+
|
656
|
+
|
657
|
+
driver.run(default_tag: 'test') do
|
658
|
+
driver.feed(sample_record)
|
659
|
+
end
|
660
|
+
compressable = instance.compressable_connection
|
661
|
+
|
662
|
+
if elastic_transport_layer?
|
663
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
664
|
+
elsif elasticsearch_transport_layer_decoupling?
|
665
|
+
assert_equal true, instance.client(nil, compressable).transport.transport.options[:compression]
|
666
|
+
else
|
667
|
+
assert_equal true, instance.client(nil, compressable).transport.options[:compression]
|
668
|
+
end
|
669
|
+
end
|
670
|
+
|
671
|
+
def test_existent_data_stream
|
672
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
673
|
+
|
674
|
+
stub_ilm_policy
|
675
|
+
stub_index_template
|
676
|
+
stub_existent_data_stream?
|
677
|
+
stub_data_stream
|
678
|
+
stub_elastic_info
|
679
|
+
conf = config_element(
|
680
|
+
'ROOT', '', {
|
681
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
682
|
+
'data_stream_name' => 'foo',
|
683
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
684
|
+
'data_stream_template_name' => "foo_tpl"
|
685
|
+
})
|
686
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
687
|
+
end
|
688
|
+
|
689
|
+
def test_template_unset
|
690
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
691
|
+
|
692
|
+
stub_ilm_policy
|
693
|
+
stub_index_template
|
694
|
+
stub_existent_data_stream?
|
695
|
+
stub_data_stream
|
696
|
+
stub_elastic_info
|
697
|
+
conf = config_element(
|
698
|
+
'ROOT', '', {
|
699
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
700
|
+
'data_stream_name' => 'foo',
|
701
|
+
'data_stream_ilm_name' => "foo_ilm_policy",
|
702
|
+
})
|
703
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
704
|
+
assert_equal "foo_ilm_policy", driver(conf).instance.data_stream_ilm_name
|
705
|
+
assert_equal "foo_template", driver(conf).instance.data_stream_template_name
|
706
|
+
end
|
707
|
+
|
708
|
+
def test_ilm_unset
|
709
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
710
|
+
|
711
|
+
stub_ilm_policy
|
712
|
+
stub_index_template
|
713
|
+
stub_existent_data_stream?
|
714
|
+
stub_data_stream
|
715
|
+
stub_elastic_info
|
716
|
+
conf = config_element(
|
717
|
+
'ROOT', '', {
|
718
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
719
|
+
'data_stream_name' => 'foo',
|
720
|
+
'data_stream_template_name' => "foo_tpl"
|
721
|
+
})
|
722
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
723
|
+
assert_equal "foo_tpl", driver(conf).instance.data_stream_template_name
|
724
|
+
end
|
725
|
+
|
726
|
+
def test_template_and_ilm_unset
|
727
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
728
|
+
|
729
|
+
stub_ilm_policy
|
730
|
+
stub_index_template
|
731
|
+
stub_existent_data_stream?
|
732
|
+
stub_data_stream
|
733
|
+
stub_elastic_info
|
734
|
+
conf = config_element(
|
735
|
+
'ROOT', '', {
|
736
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
737
|
+
'data_stream_name' => 'foo',
|
738
|
+
})
|
739
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
740
|
+
assert_equal "foo_template", driver(conf).instance.data_stream_template_name
|
741
|
+
assert_equal "foo_policy", driver(conf).instance.data_stream_ilm_name
|
742
|
+
end
|
743
|
+
|
744
|
+
def test_template_index_patterns_with_wildcard
|
745
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
746
|
+
|
747
|
+
stub_existent_ilm?
|
748
|
+
|
749
|
+
stub_nonexistent_data_stream?
|
750
|
+
stub_data_stream
|
751
|
+
|
752
|
+
stub_nonexistent_template?("foo_template")
|
753
|
+
stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
|
754
|
+
# bulk data must be pair of OP and records
|
755
|
+
# {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
|
756
|
+
# {"@timestamp": ...}
|
757
|
+
push_bulk_request(req.body)
|
758
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
759
|
+
|
760
|
+
conf = config_element(
|
761
|
+
'ROOT', '', {
|
762
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
763
|
+
'data_stream_name' => 'foo',
|
764
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
765
|
+
'data_stream_template_use_index_patterns_wildcard' => false
|
766
|
+
})
|
767
|
+
|
768
|
+
assert_nothing_raised {
|
769
|
+
driver(conf)
|
770
|
+
}
|
771
|
+
|
772
|
+
assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
|
773
|
+
body: '{"index_patterns":["foo"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
|
774
|
+
times: 1)
|
775
|
+
end
|
776
|
+
|
777
|
+
def test_template_index_patterns_without_wildcard
|
778
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
779
|
+
|
780
|
+
stub_existent_ilm?
|
781
|
+
|
782
|
+
stub_nonexistent_data_stream?
|
783
|
+
stub_data_stream
|
784
|
+
|
785
|
+
stub_nonexistent_template?("foo_template")
|
786
|
+
stub_request(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template").with do |req|
|
787
|
+
# bulk data must be pair of OP and records
|
788
|
+
# {"create": {}}\nhttp://localhost:9200/_index_template//foo_template
|
789
|
+
# {"@timestamp": ...}
|
790
|
+
push_bulk_request(req.body)
|
791
|
+
end.to_return({:status => 200, :body => "{}", :headers => { 'Content-Type' => 'json', 'x-elastic-product' => 'Elasticsearch' } })
|
792
|
+
|
793
|
+
conf = config_element(
|
794
|
+
'ROOT', '', {
|
795
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
796
|
+
'data_stream_name' => 'foo',
|
797
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
798
|
+
'data_stream_template_use_index_patterns_wildcard' => true
|
799
|
+
})
|
800
|
+
|
801
|
+
assert_nothing_raised {
|
802
|
+
driver(conf)
|
803
|
+
}
|
804
|
+
|
805
|
+
assert_requested(:put, "http://localhost:9200/#{index_template_endpoint}/foo_template",
|
806
|
+
body: '{"index_patterns":["foo*"],"data_stream":{},"template":{"settings":{"index.lifecycle.name":"foo_ilm_policy"}}}',
|
807
|
+
times: 1)
|
808
|
+
end
|
809
|
+
|
810
|
+
def test_placeholder
|
811
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
812
|
+
|
813
|
+
dsname = "foo_test"
|
814
|
+
ilmname = "foo_ilm_test"
|
815
|
+
tplname = "foo_tpl_test"
|
816
|
+
stub_default(dsname, ilmname, tplname)
|
817
|
+
stub_bulk_feed(dsname, ilmname, tplname)
|
818
|
+
conf = config_element(
|
819
|
+
'ROOT', '', {
|
820
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
821
|
+
'data_stream_name' => 'foo_${tag}',
|
822
|
+
'data_stream_ilm_name' => "foo_ilm_${tag}",
|
823
|
+
'data_stream_template_name' => "foo_tpl_${tag}"
|
824
|
+
})
|
825
|
+
driver(conf).run(default_tag: 'test') do
|
826
|
+
driver.feed(sample_record)
|
827
|
+
end
|
828
|
+
assert_equal 1, @bulk_records.length
|
829
|
+
end
|
830
|
+
|
831
|
+
def test_placeholder_params_unset
|
832
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
833
|
+
|
834
|
+
dsname = "foo_test"
|
835
|
+
ilmname = "foo_test_policy"
|
836
|
+
tplname = "foo_test_template"
|
837
|
+
stub_default(dsname, ilmname, tplname)
|
838
|
+
stub_bulk_feed(dsname, ilmname, tplname)
|
839
|
+
conf = config_element(
|
840
|
+
'ROOT', '', {
|
841
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
842
|
+
'data_stream_name' => 'foo_${tag}',
|
843
|
+
})
|
844
|
+
driver(conf).run(default_tag: 'test') do
|
845
|
+
driver.feed(sample_record)
|
846
|
+
end
|
847
|
+
assert_equal 1, @bulk_records.length
|
848
|
+
end
|
849
|
+
|
850
|
+
|
851
|
+
def test_time_placeholder
|
852
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
853
|
+
|
854
|
+
time = Time.now
|
855
|
+
dsname = "foo_#{time.strftime("%Y%m%d")}"
|
856
|
+
ilmname = "foo_ilm_#{time.strftime("%Y%m%d")}"
|
857
|
+
tplname = "foo_tpl_#{time.strftime("%Y%m%d")}"
|
858
|
+
stub_default(dsname, ilmname, tplname)
|
859
|
+
stub_bulk_feed(dsname, ilmname, tplname)
|
860
|
+
conf = config_element(
|
861
|
+
'ROOT', '', {
|
862
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
863
|
+
'data_stream_name' => 'foo_%Y%m%d',
|
864
|
+
'data_stream_ilm_name' => 'foo_ilm_%Y%m%d',
|
865
|
+
'data_stream_template_name' => 'foo_tpl_%Y%m%d'
|
866
|
+
}, [config_element('buffer', 'time', {
|
867
|
+
'timekey' => '1d'
|
868
|
+
}, [])]
|
869
|
+
)
|
870
|
+
driver(conf).run(default_tag: 'test') do
|
871
|
+
driver.feed(sample_record)
|
872
|
+
end
|
873
|
+
assert_equal 1, @bulk_records.length
|
874
|
+
end
|
875
|
+
|
876
|
+
def test_custom_record_placeholder
|
877
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
878
|
+
|
879
|
+
keys = ["bar", "baz"]
|
880
|
+
keys.each do |key|
|
881
|
+
dsname = "foo_#{key}"
|
882
|
+
ilmname = "foo_ilm_#{key}"
|
883
|
+
tplname = "foo_tpl_#{key}"
|
884
|
+
stub_default(dsname, ilmname, tplname)
|
885
|
+
stub_bulk_feed(dsname, ilmname, tplname)
|
886
|
+
end
|
887
|
+
conf = config_element(
|
888
|
+
'ROOT', '', {
|
889
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
890
|
+
'data_stream_name' => 'foo_${key1}',
|
891
|
+
'data_stream_ilm_name' => 'foo_ilm_${key1}',
|
892
|
+
'data_stream_template_name' => 'foo_tpl_${key1}'
|
893
|
+
}, [config_element('buffer', 'tag,key1', {
|
894
|
+
'timekey' => '1d'
|
895
|
+
}, [])]
|
896
|
+
)
|
897
|
+
driver(conf).run(default_tag: 'test') do
|
898
|
+
keys.each do |key|
|
899
|
+
record = sample_record.merge({"key1" => key})
|
900
|
+
driver.feed(record)
|
901
|
+
end
|
902
|
+
end
|
903
|
+
assert_equal keys.count, @bulk_records.length
|
904
|
+
end
|
905
|
+
|
906
|
+
def test_bulk_insert_feed
|
907
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
908
|
+
|
909
|
+
stub_default
|
910
|
+
stub_bulk_feed
|
911
|
+
conf = config_element(
|
912
|
+
'ROOT', '', {
|
913
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
914
|
+
'data_stream_name' => 'foo',
|
915
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
916
|
+
'data_stream_template_name' => 'foo_tpl'
|
917
|
+
})
|
918
|
+
driver(conf).run(default_tag: 'test') do
|
919
|
+
driver.feed(sample_record)
|
920
|
+
end
|
921
|
+
assert_equal 1, @bulk_records.length
|
922
|
+
end
|
923
|
+
|
924
|
+
def test_placeholder_writes_to_multi_hosts
|
925
|
+
stub_default("foo_bar", "foo_tpl_bar")
|
926
|
+
hosts = [['192.168.33.50', 9201], ['192.168.33.51', 9201], ['192.168.33.52', 9201]]
|
927
|
+
hosts_string = hosts.map {|x| "#{x[0]}:#{x[1]}"}.compact.join(',')
|
928
|
+
hosts.each do |host_info|
|
929
|
+
host, port = host_info
|
930
|
+
stub_elastic_with_store_index_command_counts("http://#{host}:#{port}/foo_bar/_bulk")
|
931
|
+
stub_elastic_info("http://#{host}:#{port}/")
|
932
|
+
stub_request(:get, "http://#{host}:#{port}/_data_stream/foo_bar").
|
933
|
+
to_return(status: 200, body: "", headers: {'x-elastic-product' => 'Elasticsearch'})
|
934
|
+
end
|
935
|
+
|
936
|
+
conf = config_element(
|
937
|
+
'ROOT', '', {
|
938
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
939
|
+
'data_stream_name' => 'foo_${key1}',
|
940
|
+
'data_stream_template_name' => 'foo_tpl_${key1}',
|
941
|
+
'hosts' => "#{hosts_string}"
|
942
|
+
}, [config_element('buffer', 'tag,key1', {
|
943
|
+
'timekey' => '1d'
|
944
|
+
}, [])])
|
945
|
+
driver(conf).run(default_tag: 'test') do
|
946
|
+
hashes = {
|
947
|
+
'age' => rand(100),
|
948
|
+
'key1' => 'bar'
|
949
|
+
}
|
950
|
+
1000.times do
|
951
|
+
driver.feed(sample_record.merge(hashes))
|
952
|
+
end
|
953
|
+
end
|
954
|
+
|
955
|
+
# @note: we cannot make multi chunks with options (flush_interval, buffer_chunk_limit)
|
956
|
+
# it's Fluentd test driver's constraint
|
957
|
+
# so @index_command_counts.size is always 1
|
958
|
+
assert(@index_command_counts.size > 0, "not working with hosts options")
|
959
|
+
|
960
|
+
total = 0
|
961
|
+
@index_command_counts.each do |_, count|
|
962
|
+
total += count
|
963
|
+
end
|
964
|
+
assert_equal(2000, total)
|
965
|
+
end
|
966
|
+
|
967
|
+
# gzip compress data
|
968
|
+
def gzip(string, strategy)
|
969
|
+
wio = StringIO.new("w")
|
970
|
+
w_gz = Zlib::GzipWriter.new(wio, strategy = strategy)
|
971
|
+
w_gz.write(string)
|
972
|
+
w_gz.close
|
973
|
+
wio.string
|
974
|
+
end
|
975
|
+
|
976
|
+
def test_writes_to_data_stream_with_compression
|
977
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
978
|
+
|
979
|
+
config = %{
|
980
|
+
data_stream_name foo
|
981
|
+
data_stream_template_name foo_tpl
|
982
|
+
data_stream_ilm_name foo_ilm_policy
|
983
|
+
compression_level default_compression
|
984
|
+
}
|
985
|
+
|
986
|
+
bodystr = %({
|
987
|
+
"took" : 500,
|
988
|
+
"errors" : false,
|
989
|
+
"items" : [
|
990
|
+
{
|
991
|
+
"create": {
|
992
|
+
"_index" : "fluentd",
|
993
|
+
"_type" : "fluentd"
|
994
|
+
}
|
995
|
+
}
|
996
|
+
]
|
997
|
+
})
|
998
|
+
|
999
|
+
compressed_body = gzip(bodystr, Zlib::DEFAULT_COMPRESSION)
|
1000
|
+
|
1001
|
+
stub_default
|
1002
|
+
elastic_request = stub_request(:post, "http://localhost:9200/foo/_bulk").
|
1003
|
+
to_return(:status => 200, :headers => {'Content-Type' => 'application/json', 'x-elastic-product' => 'Elasticsearch'}, :body => compressed_body)
|
1004
|
+
|
1005
|
+
driver(config)
|
1006
|
+
driver.run(default_tag: 'test') do
|
1007
|
+
driver.feed(sample_record)
|
1008
|
+
end
|
1009
|
+
|
1010
|
+
assert_requested(elastic_request)
|
1011
|
+
end
|
1012
|
+
|
1013
|
+
def test_doesnt_update_ilm_policy_if_overwrite_unset
|
1014
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1015
|
+
|
1016
|
+
config = %{
|
1017
|
+
data_stream_name foo
|
1018
|
+
data_stream_ilm_name foo_ilm_policy
|
1019
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
1020
|
+
}
|
1021
|
+
|
1022
|
+
stub_elastic_info
|
1023
|
+
stub_index_template
|
1024
|
+
stub_existent_data_stream?
|
1025
|
+
stub_existent_ilm?
|
1026
|
+
stub_data_stream
|
1027
|
+
|
1028
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
1029
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1030
|
+
|
1031
|
+
assert_nothing_raised {
|
1032
|
+
driver(config)
|
1033
|
+
}
|
1034
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 0)
|
1035
|
+
end
|
1036
|
+
|
1037
|
+
def test_updates_ilm_policy_if_overwrite_set
|
1038
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1039
|
+
|
1040
|
+
config = %{
|
1041
|
+
data_stream_name foo
|
1042
|
+
data_stream_ilm_name foo_ilm_policy
|
1043
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
1044
|
+
data_stream_ilm_policy_overwrite true
|
1045
|
+
}
|
1046
|
+
|
1047
|
+
stub_elastic_info
|
1048
|
+
stub_index_template
|
1049
|
+
stub_existent_data_stream?
|
1050
|
+
stub_existent_ilm?
|
1051
|
+
stub_data_stream
|
1052
|
+
|
1053
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
1054
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1055
|
+
|
1056
|
+
assert_nothing_raised {
|
1057
|
+
driver(config)
|
1058
|
+
}
|
1059
|
+
|
1060
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
1061
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
1062
|
+
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
1063
|
+
times: 1)
|
1064
|
+
end
|
1065
|
+
|
1066
|
+
def test_creates_custom_ilm_policy_if_none_exists
|
1067
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1068
|
+
|
1069
|
+
config = %{
|
1070
|
+
data_stream_name foo
|
1071
|
+
data_stream_ilm_name foo_ilm_policy
|
1072
|
+
data_stream_ilm_policy {"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}
|
1073
|
+
}
|
1074
|
+
|
1075
|
+
stub_elastic_info
|
1076
|
+
stub_index_template("foo_template")
|
1077
|
+
stub_data_stream
|
1078
|
+
stub_nonexistent_data_stream?
|
1079
|
+
stub_nonexistent_ilm?
|
1080
|
+
stub_nonexistent_template?("foo_template")
|
1081
|
+
|
1082
|
+
stub_request(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy").
|
1083
|
+
to_return(:status => 200, :body => "", :headers => {'x-elastic-product' => 'Elasticsearch'})
|
1084
|
+
|
1085
|
+
assert_nothing_raised {
|
1086
|
+
driver(config)
|
1087
|
+
}
|
1088
|
+
|
1089
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy", times: 1)
|
1090
|
+
assert_requested(:put, "http://localhost:9200/#{ilm_endpoint}/policy/foo_ilm_policy",
|
1091
|
+
body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
|
1092
|
+
times: 1)
|
1093
|
+
end
|
1094
|
+
|
1095
|
+
def test_doesnt_add_tag_key_when_not_configured
|
1096
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1097
|
+
|
1098
|
+
config = %{
|
1099
|
+
data_stream_name foo
|
1100
|
+
data_stream_template_name foo_tpl
|
1101
|
+
data_stream_ilm_name foo_ilm_policy
|
1102
|
+
}
|
1103
|
+
|
1104
|
+
stub_default
|
1105
|
+
stub_bulk_feed
|
1106
|
+
driver(config)
|
1107
|
+
driver.run(default_tag: 'mytag') do
|
1108
|
+
driver.feed(sample_record)
|
1109
|
+
end
|
1110
|
+
|
1111
|
+
assert_equal(1, @bulk_records.length)
|
1112
|
+
assert_false(@bulk_records[0].has_key?('tag'))
|
1113
|
+
end
|
1114
|
+
|
1115
|
+
|
1116
|
+
def test_adds_tag_key_when_configured
|
1117
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1118
|
+
|
1119
|
+
config = %{
|
1120
|
+
data_stream_name foo
|
1121
|
+
data_stream_template_name foo_tpl
|
1122
|
+
data_stream_ilm_name foo_ilm_policy
|
1123
|
+
include_tag_key true
|
1124
|
+
}
|
1125
|
+
|
1126
|
+
stub_default
|
1127
|
+
stub_bulk_feed
|
1128
|
+
driver(config)
|
1129
|
+
driver.run(default_tag: 'mytag') do
|
1130
|
+
driver.feed(sample_record)
|
1131
|
+
end
|
1132
|
+
|
1133
|
+
assert_equal(1, @bulk_records.length)
|
1134
|
+
assert(@bulk_records[0].has_key?('tag'))
|
1135
|
+
assert_equal('mytag', @bulk_records[0]['tag'])
|
1136
|
+
end
|
1137
|
+
|
1138
|
+
def test_adds_custom_tag_key_when_configured
|
1139
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1140
|
+
|
1141
|
+
config = %{
|
1142
|
+
data_stream_name foo
|
1143
|
+
data_stream_template_name foo_tpl
|
1144
|
+
data_stream_ilm_name foo_ilm_policy
|
1145
|
+
include_tag_key true
|
1146
|
+
tag_key custom_tag_key
|
1147
|
+
}
|
1148
|
+
|
1149
|
+
stub_default
|
1150
|
+
stub_bulk_feed
|
1151
|
+
driver(config)
|
1152
|
+
driver.run(default_tag: 'mytag') do
|
1153
|
+
driver.feed(sample_record)
|
1154
|
+
end
|
1155
|
+
|
1156
|
+
assert_equal(1, @bulk_records.length)
|
1157
|
+
assert(@bulk_records[0].has_key?('custom_tag_key'))
|
1158
|
+
assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
|
1159
|
+
end
|
1160
|
+
|
1161
|
+
def test_use_record_timestamp_if_present
|
1162
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1163
|
+
|
1164
|
+
stub_default
|
1165
|
+
stub_bulk_feed
|
1166
|
+
conf = config_element(
|
1167
|
+
'ROOT', '', {
|
1168
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
1169
|
+
'data_stream_name' => 'foo',
|
1170
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
1171
|
+
'data_stream_template_name' => 'foo_tpl'
|
1172
|
+
})
|
1173
|
+
driver(conf).run(default_tag: 'test') do
|
1174
|
+
driver.feed(sample_record)
|
1175
|
+
end
|
1176
|
+
assert_equal 1, @bulk_records.length
|
1177
|
+
assert(@bulk_records[0].has_key?('@timestamp'))
|
1178
|
+
assert_equal SAMPLE_RECORD_TIMESTAMP, @bulk_records[0]['@timestamp']
|
1179
|
+
end
|
1180
|
+
|
1181
|
+
def test_add_timestamp_if_not_present_in_record
|
1182
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
1183
|
+
|
1184
|
+
stub_default
|
1185
|
+
stub_bulk_feed
|
1186
|
+
conf = config_element(
|
1187
|
+
'ROOT', '', {
|
1188
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
1189
|
+
'data_stream_name' => 'foo',
|
1190
|
+
'data_stream_ilm_name' => 'foo_ilm_policy',
|
1191
|
+
'data_stream_template_name' => 'foo_tpl'
|
1192
|
+
})
|
1193
|
+
driver(conf).run(default_tag: 'test') do
|
1194
|
+
driver.feed(sample_record_no_timestamp)
|
1195
|
+
end
|
1196
|
+
assert_equal 1, @bulk_records.length
|
1197
|
+
assert(@bulk_records[0].has_key?('@timestamp'))
|
1198
|
+
end
|
1199
|
+
end
|