fluent-plugin-elasticsearch 4.3.3 → 5.0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,337 @@
1
+ require_relative '../helper'
2
+ require 'date'
3
+ require 'fluent/test/helpers'
4
+ require 'fluent/test/driver/output'
5
+ require 'flexmock/test_unit'
6
+ require 'fluent/plugin/out_elasticsearch_data_stream'
7
+
8
+ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
9
+ include FlexMock::TestCase
10
+ include Fluent::Test::Helpers
11
+
12
+ attr_accessor :bulk_records
13
+
14
+ REQUIRED_ELASTIC_MESSAGE = "Elasticsearch 7.9.0 or later is needed."
15
+ ELASTIC_DATA_STREAM_TYPE = "elasticsearch_data_stream"
16
+
17
+ def setup
18
+ Fluent::Test.setup
19
+ @driver = nil
20
+ log = Fluent::Engine.log
21
+ log.out.logs.slice!(0, log.out.logs.length)
22
+ @bulk_records = 0
23
+ end
24
+
25
+ def driver(conf='', es_version=5, client_version="\"5.0\"")
26
+ # For request stub to detect compatibility.
27
+ @es_version ||= es_version
28
+ @client_version ||= client_version
29
+ Fluent::Plugin::ElasticsearchOutputDataStream.module_eval(<<-CODE)
30
+ def detect_es_major_version
31
+ #{@es_version}
32
+ end
33
+ CODE
34
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutputDataStream) {
35
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
36
+ if !defined?(Fluent::Plugin::Output)
37
+ def format(tag, time, record)
38
+ [time, record].to_msgpack
39
+ end
40
+ end
41
+ }.configure(conf)
42
+ end
43
+
44
+ def sample_data_stream
45
+ {
46
+ 'data_streams': [
47
+ {
48
+ 'name' => 'my-data-stream',
49
+ 'timestamp_field' => {
50
+ 'name' => '@timestamp'
51
+ }
52
+ }
53
+ ]
54
+ }
55
+ end
56
+
57
+ def sample_record
58
+ {'@timestamp' => Time.now.iso8601, 'message' => 'Sample record'}
59
+ end
60
+
61
+ RESPONSE_ACKNOWLEDGED = {"acknowledged": true}
62
+ DUPLICATED_DATA_STREAM_EXCEPTION = {"error": {}, "status": 400}
63
+ NONEXISTENT_DATA_STREAM_EXCEPTION = {"error": {}, "status": 404}
64
+
65
+ def stub_ilm_policy(name="foo")
66
+ stub_request(:put, "http://localhost:9200/_ilm/policy/#{name}_policy").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
67
+ end
68
+
69
+ def stub_index_template(name="foo")
70
+ stub_request(:put, "http://localhost:9200/_index_template/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
71
+ end
72
+
73
+ def stub_data_stream(name="foo")
74
+ stub_request(:put, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
75
+ end
76
+
77
+ def stub_existent_data_stream?(name="foo")
78
+ stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [200, RESPONSE_ACKNOWLEDGED])
79
+ end
80
+
81
+ def stub_nonexistent_data_stream?(name="foo")
82
+ stub_request(:get, "http://localhost:9200/_data_stream/#{name}").to_return(:status => [404, Elasticsearch::Transport::Transport::Errors::NotFound])
83
+ end
84
+
85
+ def stub_bulk_feed(name="foo")
86
+ stub_request(:post, "http://localhost:9200/#{name}/_bulk").with do |req|
87
+ # bulk data must be pair of OP and records
88
+ # {"create": {}}\n
89
+ # {"@timestamp": ...}
90
+ @bulk_records += req.body.split("\n").size / 2
91
+ end
92
+ end
93
+
94
+ def stub_default(name="foo")
95
+ stub_ilm_policy(name)
96
+ stub_index_template(name)
97
+ stub_nonexistent_data_stream?(name)
98
+ stub_data_stream(name)
99
+ end
100
+
101
+ def data_stream_supported?
102
+ Gem::Version.create(::Elasticsearch::Transport::VERSION) >= Gem::Version.create("7.9.0")
103
+ end
104
+
105
+ # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
106
+ class DataStreamNameTest < self
107
+
108
+ def test_missing_data_stream_name
109
+ conf = config_element(
110
+ 'ROOT', '', {
111
+ '@type' => 'elasticsearch_datastream'
112
+ })
113
+ assert_raise Fluent::ConfigError.new("'data_stream_name' parameter is required") do
114
+ driver(conf).run
115
+ end
116
+ end
117
+
118
+ def test_invalid_uppercase
119
+ conf = config_element(
120
+ 'ROOT', '', {
121
+ '@type' => 'elasticsearch_datastream',
122
+ 'data_stream_name' => 'TEST'
123
+ })
124
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must be lowercase only: <TEST>") do
125
+ driver(conf)
126
+ end
127
+ end
128
+
129
+ data("backslash" => "\\",
130
+ "slash" => "/",
131
+ "asterisk" => "*",
132
+ "question" => "?",
133
+ "doublequote" => "\"",
134
+ "lt" => "<",
135
+ "gt" => ">",
136
+ "bar" => "|",
137
+ "space" => " ",
138
+ "comma" => ",",
139
+ "sharp" => "#",
140
+ "colon" => ":")
141
+ def test_invalid_characters(data)
142
+ c, _ = data
143
+ conf = config_element(
144
+ 'ROOT', '', {
145
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
146
+ 'data_stream_name' => "TEST#{c}"
147
+ })
148
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_CHARACTERS.join(',')
149
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not contain invalid characters #{label}: <TEST#{c}>") do
150
+ driver(conf)
151
+ end
152
+ end
153
+
154
+ data("hyphen" => "-",
155
+ "underscore" => "_",
156
+ "plus" => "+",
157
+ "period" => ".")
158
+ def test_invalid_start_characters(data)
159
+ c, _ = data
160
+ conf = config_element(
161
+ 'ROOT', '', {
162
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
163
+ 'data_stream_name' => "#{c}TEST"
164
+ })
165
+ label = Fluent::Plugin::ElasticsearchOutputDataStream::INVALID_START_CHRACTERS.join(',')
166
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not start with #{label}: <#{c}TEST>") do
167
+ driver(conf)
168
+ end
169
+ end
170
+
171
+ data("current" => ".",
172
+ "parents" => "..")
173
+ def test_invalid_dots
174
+ c, _ = data
175
+ conf = config_element(
176
+ 'ROOT', '', {
177
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
178
+ 'data_stream_name' => "#{c}"
179
+ })
180
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not be . or ..: <#{c}>") do
181
+ driver(conf)
182
+ end
183
+ end
184
+
185
+ def test_invalid_length
186
+ c = "a" * 256
187
+ conf = config_element(
188
+ 'ROOT', '', {
189
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
190
+ 'data_stream_name' => "#{c}"
191
+ })
192
+ assert_raise Fluent::ConfigError.new("'data_stream_name' must not be longer than 255 bytes: <#{c}>") do
193
+ driver(conf)
194
+ end
195
+ end
196
+ end
197
+
198
+ def test_datastream_configure
199
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
200
+
201
+ stub_default
202
+ conf = config_element(
203
+ 'ROOT', '', {
204
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
205
+ 'data_stream_name' => 'foo'
206
+ })
207
+ assert_equal "foo", driver(conf).instance.data_stream_name
208
+ end
209
+
210
+ def test_existent_data_stream
211
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
212
+
213
+ stub_ilm_policy
214
+ stub_index_template
215
+ stub_existent_data_stream?
216
+ stub_data_stream
217
+ conf = config_element(
218
+ 'ROOT', '', {
219
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
220
+ 'data_stream_name' => 'foo'
221
+ })
222
+ assert_equal "foo", driver(conf).instance.data_stream_name
223
+ end
224
+
225
+ def test_placeholder
226
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
227
+
228
+ name = "foo_test"
229
+ stub_default(name)
230
+ stub_bulk_feed(name)
231
+ conf = config_element(
232
+ 'ROOT', '', {
233
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
234
+ 'data_stream_name' => 'foo_${tag}'
235
+ })
236
+ driver(conf).run(default_tag: 'test') do
237
+ driver.feed(sample_record)
238
+ end
239
+ assert_equal 1, @bulk_records
240
+ end
241
+
242
+ def test_time_placeholder
243
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
244
+
245
+ time = Time.now
246
+ name = "foo_#{time.strftime("%Y%m%d")}"
247
+ stub_default(name)
248
+ stub_bulk_feed(name)
249
+ conf = config_element(
250
+ 'ROOT', '', {
251
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
252
+ 'data_stream_name' => 'foo_%Y%m%d'
253
+ }, [config_element('buffer', 'time', {
254
+ 'timekey' => '1d'
255
+ }, [])]
256
+ )
257
+ driver(conf).run(default_tag: 'test') do
258
+ driver.feed(sample_record)
259
+ end
260
+ assert_equal 1, @bulk_records
261
+ end
262
+
263
+ def test_custom_record_placeholder
264
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
265
+
266
+ keys = ["bar", "baz"]
267
+ keys.each do |key|
268
+ name = "foo_#{key}"
269
+ stub_default(name)
270
+ stub_bulk_feed(name)
271
+ end
272
+ conf = config_element(
273
+ 'ROOT', '', {
274
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
275
+ 'data_stream_name' => 'foo_${key1}'
276
+ }, [config_element('buffer', 'tag,key1', {
277
+ 'timekey' => '1d'
278
+ }, [])]
279
+ )
280
+ driver(conf).run(default_tag: 'test') do
281
+ keys.each do |key|
282
+ record = sample_record.merge({"key1" => key})
283
+ driver.feed(record)
284
+ end
285
+ end
286
+ assert_equal keys.count, @bulk_records
287
+ end
288
+
289
+ def test_bulk_insert_feed
290
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
291
+
292
+ stub_default
293
+ stub_bulk_feed
294
+ conf = config_element(
295
+ 'ROOT', '', {
296
+ '@type' => ELASTIC_DATA_STREAM_TYPE,
297
+ 'data_stream_name' => 'foo'
298
+ })
299
+ driver(conf).run(default_tag: 'test') do
300
+ driver.feed(sample_record)
301
+ end
302
+ assert_equal 1, @bulk_records
303
+ end
304
+
305
+ def test_template_retry_install_fails
306
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
307
+
308
+ cwd = File.dirname(__FILE__)
309
+ template_file = File.join(cwd, 'test_index_template.json')
310
+
311
+ config = %{
312
+ host logs.google.com
313
+ port 778
314
+ scheme https
315
+ data_stream_name foo
316
+ user john
317
+ password doe
318
+ template_name logstash
319
+ template_file #{template_file}
320
+ max_retry_putting_template 3
321
+ }
322
+
323
+ connection_resets = 0
324
+ # check if template exists
325
+ stub_request(:get, "https://logs.google.com:778/_index_template/logstash")
326
+ .with(basic_auth: ['john', 'doe']) do |req|
327
+ connection_resets += 1
328
+ raise Faraday::ConnectionFailed, "Test message"
329
+ end
330
+
331
+ assert_raise(Fluent::Plugin::ElasticsearchError::RetryableOperationExhaustedFailure) do
332
+ driver(config)
333
+ end
334
+
335
+ assert_equal(4, connection_resets)
336
+ end
337
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.3.3
4
+ version: 5.0.4
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-12-18 00:00:00.000000000 Z
13
+ date: 2021-06-07 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -68,6 +68,20 @@ dependencies:
68
68
  - - ">="
69
69
  - !ruby/object:Gem::Version
70
70
  version: '0'
71
+ - !ruby/object:Gem::Dependency
72
+ name: webrick
73
+ requirement: !ruby/object:Gem::Requirement
74
+ requirements:
75
+ - - "~>"
76
+ - !ruby/object:Gem::Version
77
+ version: 1.7.0
78
+ type: :development
79
+ prerelease: false
80
+ version_requirements: !ruby/object:Gem::Requirement
81
+ requirements:
82
+ - - "~>"
83
+ - !ruby/object:Gem::Version
84
+ version: 1.7.0
71
85
  - !ruby/object:Gem::Dependency
72
86
  name: webmock
73
87
  requirement: !ruby/object:Gem::Requirement
@@ -151,12 +165,12 @@ files:
151
165
  - PULL_REQUEST_TEMPLATE.md
152
166
  - README.ElasticsearchGenID.md
153
167
  - README.ElasticsearchInput.md
168
+ - README.Troubleshooting.md
154
169
  - README.md
155
170
  - Rakefile
156
171
  - appveyor.yml
157
172
  - fluent-plugin-elasticsearch.gemspec
158
173
  - gemfiles/Gemfile.elasticsearch.v6
159
- - gemfiles/Gemfile.without.ilm
160
174
  - lib/fluent/log-ext.rb
161
175
  - lib/fluent/plugin/default-ilm-policy.json
162
176
  - lib/fluent/plugin/elasticsearch_constants.rb
@@ -171,6 +185,7 @@ files:
171
185
  - lib/fluent/plugin/in_elasticsearch.rb
172
186
  - lib/fluent/plugin/oj_serializer.rb
173
187
  - lib/fluent/plugin/out_elasticsearch.rb
188
+ - lib/fluent/plugin/out_elasticsearch_data_stream.rb
174
189
  - lib/fluent/plugin/out_elasticsearch_dynamic.rb
175
190
  - test/helper.rb
176
191
  - test/plugin/test_alias_template.json
@@ -184,6 +199,7 @@ files:
184
199
  - test/plugin/test_index_template.json
185
200
  - test/plugin/test_oj_serializer.rb
186
201
  - test/plugin/test_out_elasticsearch.rb
202
+ - test/plugin/test_out_elasticsearch_data_stream.rb
187
203
  - test/plugin/test_out_elasticsearch_dynamic.rb
188
204
  - test/plugin/test_template.json
189
205
  - test/test_log-ext.rb
@@ -207,7 +223,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
207
223
  - !ruby/object:Gem::Version
208
224
  version: '0'
209
225
  requirements: []
210
- rubygems_version: 3.1.2
226
+ rubygems_version: 3.2.3
211
227
  signing_key:
212
228
  specification_version: 4
213
229
  summary: Elasticsearch output plugin for Fluent event collector
@@ -224,6 +240,7 @@ test_files:
224
240
  - test/plugin/test_index_template.json
225
241
  - test/plugin/test_oj_serializer.rb
226
242
  - test/plugin/test_out_elasticsearch.rb
243
+ - test/plugin/test_out_elasticsearch_data_stream.rb
227
244
  - test/plugin/test_out_elasticsearch_dynamic.rb
228
245
  - test/plugin/test_template.json
229
246
  - test/test_log-ext.rb
@@ -1,10 +0,0 @@
1
- source 'https://rubygems.org'
2
-
3
- # Specify your gem's dependencies in fluent-plugin-elasticsearch.gemspec
4
- gemspec :path => "../"
5
-
6
- gem 'simplecov', require: false
7
- gem 'coveralls', ">= 0.8.0", require: false
8
- gem 'strptime', require: false if RUBY_ENGINE == "ruby" && RUBY_VERSION =~ /^2/
9
- gem "irb" if RUBY_ENGINE == "ruby" && RUBY_VERSION >= "2.6"
10
- gem "oj"