fluent-plugin-elasticsearch 4.3.3 → 5.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/.travis.yml +0 -4
- data/History.md +24 -0
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +115 -586
- data/fluent-plugin-elasticsearch.gemspec +2 -1
- data/lib/fluent/plugin/elasticsearch_error_handler.rb +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +56 -3
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +218 -0
- data/test/plugin/test_elasticsearch_error_handler.rb +6 -1
- data/test/plugin/test_out_elasticsearch.rb +296 -1
- data/test/plugin/test_out_elasticsearch_data_stream.rb +337 -0
- metadata +21 -4
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -0,0 +1,337 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'date'
|
3
|
+
require 'fluent/test/helpers'
|
4
|
+
require 'fluent/test/driver/output'
|
5
|
+
require 'flexmock/test_unit'
|
6
|
+
require 'fluent/plugin/out_elasticsearch_data_stream'
|
7
|
+
|
8
|
+
class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
|
9
|
+
include FlexMock::TestCase
|
10
|
+
include Fluent::Test::Helpers
|
11
|
+
|
12
|
+
attr_accessor :bulk_records
|
13
|
+
|
14
|
+
REQUIRED_ELASTIC_MESSAGE = "Elasticsearch 7.9.0 or later is needed."
|
15
|
+
ELASTIC_DATA_STREAM_TYPE = "elasticsearch_data_stream"
|
16
|
+
|
17
|
+
def setup
|
18
|
+
Fluent::Test.setup
|
19
|
+
@driver = nil
|
20
|
+
log = Fluent::Engine.log
|
21
|
+
log.out.logs.slice!(0, log.out.logs.length)
|
22
|
+
@bulk_records = 0
|
23
|
+
end
|
24
|
+
|
25
|
+
def driver(conf='', es_version=5, client_version="\"5.0\"")
|
26
|
+
# For request stub to detect compatibility.
|
27
|
+
@es_version ||= es_version
|
28
|
+
@client_version ||= client_version
|
29
|
+
Fluent::Plugin::ElasticsearchOutputDataStream.module_eval(<<-CODE)
|
30
|
+
def detect_es_major_version
|
31
|
+
#{@es_version}
|
32
|
+
end
|
33
|
+
CODE
|
34
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutputDataStream) {
|
35
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
36
|
+
if !defined?(Fluent::Plugin::Output)
|
37
|
+
def format(tag, time, record)
|
38
|
+
[time, record].to_msgpack
|
39
|
+
end
|
40
|
+
end
|
41
|
+
}.configure(conf)
|
42
|
+
end
|
43
|
+
|
44
|
+
def sample_data_stream
|
45
|
+
{
|
46
|
+
'data_streams': [
|
47
|
+
{
|
48
|
+
'name' => 'my-data-stream',
|
49
|
+
'timestamp_field' => {
|
50
|
+
'name' => '@timestamp'
|
51
|
+
}
|
52
|
+
}
|
53
|
+
]
|
54
|
+
}
|
55
|
+
end
|
56
|
+
|
57
|
+
def sample_record
|
58
|
+
{'@timestamp' => Time.now.iso8601, 'message' => 'Sample record'}
|
59
|
+
end
|
60
|
+
|
61
|
+
RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
|
62
|
+
DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
|
63
|
+
NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
|
64
|
+
|
65
|
+
def stub_ilm_policy(name="foo")
|
66
|
+
stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}_policy").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
67
|
+
end
|
68
|
+
|
69
|
+
def stub_index_template(name="foo")
|
70
|
+
stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
71
|
+
end
|
72
|
+
|
73
|
+
def stub_data_stream(name="foo")
|
74
|
+
stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
75
|
+
end
|
76
|
+
|
77
|
+
def stub_existent_data_stream?(name="foo")
|
78
|
+
stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
|
79
|
+
end
|
80
|
+
|
81
|
+
def stub_nonexistent_data_stream?(name="foo")
|
82
|
+
stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
|
83
|
+
end
|
84
|
+
|
85
|
+
def stub_bulk_feed(name="foo")
|
86
|
+
stub_request(:post, "http://localhost:9200/#{name}/_bulk").with do |req|
|
87
|
+
# bulk data must be pair of OP and records
|
88
|
+
# {"create": {}}\n
|
89
|
+
# {"@timestamp": ...}
|
90
|
+
@bulk_records += req.body.split("\n").size / 2
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
def stub_default(name="foo")
|
95
|
+
stub_ilm_policy(name)
|
96
|
+
stub_index_template(name)
|
97
|
+
stub_nonexistent_data_stream?(name)
|
98
|
+
stub_data_stream(name)
|
99
|
+
end
|
100
|
+
|
101
|
+
def data_stream_supported?
|
102
|
+
Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.9.0")
|
103
|
+
end
|
104
|
+
|
105
|
+
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
106
|
+
class DataStreamNameTest < self
|
107
|
+
|
108
|
+
def test_missing_data_stream_name
|
109
|
+
conf = config_element(
|
110
|
+
'ROOT', '', {
|
111
|
+
'@type' => 'elasticsearch_datastream'
|
112
|
+
})
|
113
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' parameter is required") do
|
114
|
+
driver(conf).run
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def test_invalid_uppercase
|
119
|
+
conf = config_element(
|
120
|
+
'ROOT', '', {
|
121
|
+
'@type' => 'elasticsearch_datastream',
|
122
|
+
'data_stream_name' => 'TEST'
|
123
|
+
})
|
124
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
|
125
|
+
driver(conf)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
data("backslash" => "\\",
|
130
|
+
"slash" => "/",
|
131
|
+
"asterisk" => "*",
|
132
|
+
"question" => "?",
|
133
|
+
"doublequote" => "\"",
|
134
|
+
"lt" => "<",
|
135
|
+
"gt" => ">",
|
136
|
+
"bar" => "|",
|
137
|
+
"space" => " ",
|
138
|
+
"comma" => ",",
|
139
|
+
"sharp" => "#",
|
140
|
+
"colon" => ":")
|
141
|
+
def test_invalid_characters(data)
|
142
|
+
c, _ = data
|
143
|
+
conf = config_element(
|
144
|
+
'ROOT', '', {
|
145
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
146
|
+
'data_stream_name' => "TEST#{c}"
|
147
|
+
})
|
148
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
|
149
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
|
150
|
+
driver(conf)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
data("hyphen" => "-",
|
155
|
+
"underscore" => "_",
|
156
|
+
"plus" => "+",
|
157
|
+
"period" => ".")
|
158
|
+
def test_invalid_start_characters(data)
|
159
|
+
c, _ = data
|
160
|
+
conf = config_element(
|
161
|
+
'ROOT', '', {
|
162
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
163
|
+
'data_stream_name' => "#{c}TEST"
|
164
|
+
})
|
165
|
+
label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
|
166
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
|
167
|
+
driver(conf)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
data("current" => ".",
|
172
|
+
"parents" => "..")
|
173
|
+
def test_invalid_dots
|
174
|
+
c, _ = data
|
175
|
+
conf = config_element(
|
176
|
+
'ROOT', '', {
|
177
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
178
|
+
'data_stream_name' => "#{c}"
|
179
|
+
})
|
180
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
|
181
|
+
driver(conf)
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def test_invalid_length
|
186
|
+
c = "a" * 256
|
187
|
+
conf = config_element(
|
188
|
+
'ROOT', '', {
|
189
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
190
|
+
'data_stream_name' => "#{c}"
|
191
|
+
})
|
192
|
+
assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
|
193
|
+
driver(conf)
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
def test_datastream_configure
|
199
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
200
|
+
|
201
|
+
stub_default
|
202
|
+
conf = config_element(
|
203
|
+
'ROOT', '', {
|
204
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
205
|
+
'data_stream_name' => 'foo'
|
206
|
+
})
|
207
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
208
|
+
end
|
209
|
+
|
210
|
+
def test_existent_data_stream
|
211
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
212
|
+
|
213
|
+
stub_ilm_policy
|
214
|
+
stub_index_template
|
215
|
+
stub_existent_data_stream?
|
216
|
+
stub_data_stream
|
217
|
+
conf = config_element(
|
218
|
+
'ROOT', '', {
|
219
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
220
|
+
'data_stream_name' => 'foo'
|
221
|
+
})
|
222
|
+
assert_equal "foo", driver(conf).instance.data_stream_name
|
223
|
+
end
|
224
|
+
|
225
|
+
def test_placeholder
|
226
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
227
|
+
|
228
|
+
name = "foo_test"
|
229
|
+
stub_default(name)
|
230
|
+
stub_bulk_feed(name)
|
231
|
+
conf = config_element(
|
232
|
+
'ROOT', '', {
|
233
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
234
|
+
'data_stream_name' => 'foo_${tag}'
|
235
|
+
})
|
236
|
+
driver(conf).run(default_tag: 'test') do
|
237
|
+
driver.feed(sample_record)
|
238
|
+
end
|
239
|
+
assert_equal 1, @bulk_records
|
240
|
+
end
|
241
|
+
|
242
|
+
def test_time_placeholder
|
243
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
244
|
+
|
245
|
+
time = Time.now
|
246
|
+
name = "foo_#{time.strftime("%Y%m%d")}"
|
247
|
+
stub_default(name)
|
248
|
+
stub_bulk_feed(name)
|
249
|
+
conf = config_element(
|
250
|
+
'ROOT', '', {
|
251
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
252
|
+
'data_stream_name' => 'foo_%Y%m%d'
|
253
|
+
}, [config_element('buffer', 'time', {
|
254
|
+
'timekey' => '1d'
|
255
|
+
}, [])]
|
256
|
+
)
|
257
|
+
driver(conf).run(default_tag: 'test') do
|
258
|
+
driver.feed(sample_record)
|
259
|
+
end
|
260
|
+
assert_equal 1, @bulk_records
|
261
|
+
end
|
262
|
+
|
263
|
+
def test_custom_record_placeholder
|
264
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
265
|
+
|
266
|
+
keys = ["bar", "baz"]
|
267
|
+
keys.each do |key|
|
268
|
+
name = "foo_#{key}"
|
269
|
+
stub_default(name)
|
270
|
+
stub_bulk_feed(name)
|
271
|
+
end
|
272
|
+
conf = config_element(
|
273
|
+
'ROOT', '', {
|
274
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
275
|
+
'data_stream_name' => 'foo_${key1}'
|
276
|
+
}, [config_element('buffer', 'tag,key1', {
|
277
|
+
'timekey' => '1d'
|
278
|
+
}, [])]
|
279
|
+
)
|
280
|
+
driver(conf).run(default_tag: 'test') do
|
281
|
+
keys.each do |key|
|
282
|
+
record = sample_record.merge({"key1" => key})
|
283
|
+
driver.feed(record)
|
284
|
+
end
|
285
|
+
end
|
286
|
+
assert_equal keys.count, @bulk_records
|
287
|
+
end
|
288
|
+
|
289
|
+
def test_bulk_insert_feed
|
290
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
291
|
+
|
292
|
+
stub_default
|
293
|
+
stub_bulk_feed
|
294
|
+
conf = config_element(
|
295
|
+
'ROOT', '', {
|
296
|
+
'@type' => ELASTIC_DATA_STREAM_TYPE,
|
297
|
+
'data_stream_name' => 'foo'
|
298
|
+
})
|
299
|
+
driver(conf).run(default_tag: 'test') do
|
300
|
+
driver.feed(sample_record)
|
301
|
+
end
|
302
|
+
assert_equal 1, @bulk_records
|
303
|
+
end
|
304
|
+
|
305
|
+
def test_template_retry_install_fails
|
306
|
+
omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
|
307
|
+
|
308
|
+
cwd = File.dirname(__FILE__)
|
309
|
+
template_file = File.join(cwd, 'test_index_template.json')
|
310
|
+
|
311
|
+
config = %{
|
312
|
+
host logs.google.com
|
313
|
+
port 778
|
314
|
+
scheme https
|
315
|
+
data_stream_name foo
|
316
|
+
user john
|
317
|
+
password doe
|
318
|
+
template_name logstash
|
319
|
+
template_file #{template_file}
|
320
|
+
max_retry_putting_template 3
|
321
|
+
}
|
322
|
+
|
323
|
+
connection_resets = 0
|
324
|
+
# check if template exists
|
325
|
+
stub_request(:get, "https://logs.google.com:778/_index_template/logstash")
|
326
|
+
.with(basic_auth: ['john', 'doe']) do |req|
|
327
|
+
connection_resets += 1
|
328
|
+
raise Faraday::ConnectionFailed, "Test message"
|
329
|
+
end
|
330
|
+
|
331
|
+
assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
|
332
|
+
driver(config)
|
333
|
+
end
|
334
|
+
|
335
|
+
assert_equal(4, connection_resets)
|
336
|
+
end
|
337
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 5.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date:
|
13
|
+
date: 2021-06-07 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -68,6 +68,20 @@ dependencies:
|
|
68
68
|
- - ">="
|
69
69
|
- !ruby/object:Gem::Version
|
70
70
|
version: '0'
|
71
|
+
- !ruby/object:Gem::Dependency
|
72
|
+
name: webrick
|
73
|
+
requirement: !ruby/object:Gem::Requirement
|
74
|
+
requirements:
|
75
|
+
- - "~>"
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 1.7.0
|
78
|
+
type: :development
|
79
|
+
prerelease: false
|
80
|
+
version_requirements: !ruby/object:Gem::Requirement
|
81
|
+
requirements:
|
82
|
+
- - "~>"
|
83
|
+
- !ruby/object:Gem::Version
|
84
|
+
version: 1.7.0
|
71
85
|
- !ruby/object:Gem::Dependency
|
72
86
|
name: webmock
|
73
87
|
requirement: !ruby/object:Gem::Requirement
|
@@ -151,12 +165,12 @@ files:
|
|
151
165
|
- PULL_REQUEST_TEMPLATE.md
|
152
166
|
- README.ElasticsearchGenID.md
|
153
167
|
- README.ElasticsearchInput.md
|
168
|
+
- README.Troubleshooting.md
|
154
169
|
- README.md
|
155
170
|
- Rakefile
|
156
171
|
- appveyor.yml
|
157
172
|
- fluent-plugin-elasticsearch.gemspec
|
158
173
|
- gemfiles/Gemfile.elasticsearch.v6
|
159
|
-
- gemfiles/Gemfile.without.ilm
|
160
174
|
- lib/fluent/log-ext.rb
|
161
175
|
- lib/fluent/plugin/default-ilm-policy.json
|
162
176
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
@@ -171,6 +185,7 @@ files:
|
|
171
185
|
- lib/fluent/plugin/in_elasticsearch.rb
|
172
186
|
- lib/fluent/plugin/oj_serializer.rb
|
173
187
|
- lib/fluent/plugin/out_elasticsearch.rb
|
188
|
+
- lib/fluent/plugin/out_elasticsearch_data_stream.rb
|
174
189
|
- lib/fluent/plugin/out_elasticsearch_dynamic.rb
|
175
190
|
- test/helper.rb
|
176
191
|
- test/plugin/test_alias_template.json
|
@@ -184,6 +199,7 @@ files:
|
|
184
199
|
- test/plugin/test_index_template.json
|
185
200
|
- test/plugin/test_oj_serializer.rb
|
186
201
|
- test/plugin/test_out_elasticsearch.rb
|
202
|
+
- test/plugin/test_out_elasticsearch_data_stream.rb
|
187
203
|
- test/plugin/test_out_elasticsearch_dynamic.rb
|
188
204
|
- test/plugin/test_template.json
|
189
205
|
- test/test_log-ext.rb
|
@@ -207,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
207
223
|
- !ruby/object:Gem::Version
|
208
224
|
version: '0'
|
209
225
|
requirements: []
|
210
|
-
rubygems_version: 3.
|
226
|
+
rubygems_version: 3.2.3
|
211
227
|
signing_key:
|
212
228
|
specification_version: 4
|
213
229
|
summary: Elasticsearch output plugin for Fluent event collector
|
@@ -224,6 +240,7 @@ test_files:
|
|
224
240
|
- test/plugin/test_index_template.json
|
225
241
|
- test/plugin/test_oj_serializer.rb
|
226
242
|
- test/plugin/test_out_elasticsearch.rb
|
243
|
+
- test/plugin/test_out_elasticsearch_data_stream.rb
|
227
244
|
- test/plugin/test_out_elasticsearch_dynamic.rb
|
228
245
|
- test/plugin/test_template.json
|
229
246
|
- test/test_log-ext.rb
|
@@ -1,10 +0,0 @@
|
|
1
|
-
source 'https://rubygems.org'
|
2
|
-
|
3
|
-
# Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
|
4
|
-
gemspec :path => "../"
|
5
|
-
|
6
|
-
gem 'simplecov', require: false
|
7
|
-
gem 'coveralls', ">= 0.8.0", require: false
|
8
|
-
gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
|
9
|
-
gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
|
10
|
-
gem "oj"
|