fluent-plugin-elasticsearch 4.3.2 → 5.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/.travis.yml +0 -4
- data/History.md +21 -0
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +45 -586
- data/fluent-plugin-elasticsearch.gemspec +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +12 -2
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +218 -0
- data/test/plugin/test_out_elasticsearch.rb +167 -0
- data/test/plugin/test_out_elasticsearch_data_stream.rb +337 -0
- metadata +21 -4
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '
|
6
|
+
s.version = '5.0.3'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
|
|
28
28
|
|
29
29
|
|
30
30
|
s.add_development_dependency 'rake', '>= 0'
|
31
|
+
s.add_development_dependency 'webrick', '~> 1.7.0'
|
31
32
|
s.add_development_dependency 'webmock', '~> 3'
|
32
33
|
s.add_development_dependency 'test-unit', '~> 3.3.0'
|
33
34
|
s.add_development_dependency 'minitest', '~> 5.8'
|
@@ -13,6 +13,7 @@ begin
|
|
13
13
|
require 'strptime'
|
14
14
|
rescue LoadError
|
15
15
|
end
|
16
|
+
require 'resolv'
|
16
17
|
|
17
18
|
require 'fluent/plugin/output'
|
18
19
|
require 'fluent/event'
|
@@ -493,7 +494,12 @@ EOC
|
|
493
494
|
|
494
495
|
def detect_es_major_version
|
495
496
|
@_es_info ||= client.info
|
496
|
-
|
497
|
+
begin
|
498
|
+
unless version = @_es_info.dig("version", "number")
|
499
|
+
version = @default_elasticsearch_version
|
500
|
+
end
|
501
|
+
rescue NoMethodError => e
|
502
|
+
log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
|
497
503
|
version = @default_elasticsearch_version
|
498
504
|
end
|
499
505
|
version.to_i
|
@@ -663,7 +669,11 @@ EOC
|
|
663
669
|
end
|
664
670
|
end.compact
|
665
671
|
else
|
666
|
-
|
672
|
+
if Resolv::IPv6::Regex.match(@host)
|
673
|
+
[{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
|
674
|
+
else
|
675
|
+
[{host: @host, port: @port, scheme: @scheme.to_s}]
|
676
|
+
end
|
667
677
|
end.each do |host|
|
668
678
|
host.merge!(user: @user, password: @password) if !host[:user] && @user
|
669
679
|
host.merge!(path: @path) if !host[:path] && @path
|
@@ -0,0 +1,218 @@
|
|
1
|
+
require_relative 'out_elasticsearch'
|
2
|
+
|
3
|
+
module Fluent::Plugin
|
4
|
+
class ElasticsearchOutputDataStream < ElasticsearchOutput
|
5
|
+
|
6
|
+
Fluent::Plugin.register_output('elasticsearch_data_stream', self)
|
7
|
+
|
8
|
+
helpers :event_emitter
|
9
|
+
|
10
|
+
config_param :data_stream_name, :string
|
11
|
+
# Elasticsearch 7.9 or later always support new style of index template.
|
12
|
+
config_set_default :use_legacy_template, false
|
13
|
+
|
14
|
+
INVALID_START_CHRACTERS = ["-", "_", "+", "."]
|
15
|
+
INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
|
16
|
+
|
17
|
+
def configure(conf)
|
18
|
+
super
|
19
|
+
|
20
|
+
begin
|
21
|
+
require 'elasticsearch/api'
|
22
|
+
require 'elasticsearch/xpack'
|
23
|
+
rescue LoadError
|
24
|
+
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
25
|
+
end
|
26
|
+
|
27
|
+
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
28
|
+
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
29
|
+
validate_data_stream_name
|
30
|
+
else
|
31
|
+
@use_placeholder = true
|
32
|
+
@data_stream_names = []
|
33
|
+
end
|
34
|
+
|
35
|
+
@client = client
|
36
|
+
unless @use_placeholder
|
37
|
+
begin
|
38
|
+
@data_stream_names = [@data_stream_name]
|
39
|
+
create_ilm_policy(@data_stream_name)
|
40
|
+
create_index_template(@data_stream_name)
|
41
|
+
create_data_stream(@data_stream_name)
|
42
|
+
rescue => e
|
43
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def validate_data_stream_name
|
49
|
+
unless valid_data_stream_name?
|
50
|
+
unless start_with_valid_characters?
|
51
|
+
if not_dots?
|
52
|
+
raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
|
53
|
+
else
|
54
|
+
raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
unless valid_characters?
|
58
|
+
raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
|
59
|
+
end
|
60
|
+
unless lowercase_only?
|
61
|
+
raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
|
62
|
+
end
|
63
|
+
if @data_stream_name.bytes.size > 255
|
64
|
+
raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def create_ilm_policy(name)
|
70
|
+
return if data_stream_exist?(name)
|
71
|
+
params = {
|
72
|
+
policy_id: "#{name}_policy",
|
73
|
+
body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
|
74
|
+
}
|
75
|
+
retry_operate(@max_retry_putting_template,
|
76
|
+
@fail_on_putting_template_retry_exceed,
|
77
|
+
@catch_transport_exception_on_retry) do
|
78
|
+
@client.xpack.ilm.put_policy(params)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def create_index_template(name)
|
83
|
+
return if data_stream_exist?(name)
|
84
|
+
body = {
|
85
|
+
"index_patterns" => ["#{name}*"],
|
86
|
+
"data_stream" => {},
|
87
|
+
"template" => {
|
88
|
+
"settings" => {
|
89
|
+
"index.lifecycle.name" => "#{name}_policy"
|
90
|
+
}
|
91
|
+
}
|
92
|
+
}
|
93
|
+
params = {
|
94
|
+
name: name,
|
95
|
+
body: body
|
96
|
+
}
|
97
|
+
retry_operate(@max_retry_putting_template,
|
98
|
+
@fail_on_putting_template_retry_exceed,
|
99
|
+
@catch_transport_exception_on_retry) do
|
100
|
+
@client.indices.put_index_template(params)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def data_stream_exist?(name)
|
105
|
+
params = {
|
106
|
+
"name": name
|
107
|
+
}
|
108
|
+
begin
|
109
|
+
response = @client.indices.get_data_stream(params)
|
110
|
+
return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
|
111
|
+
rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
|
112
|
+
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
113
|
+
return false
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def create_data_stream(name)
|
118
|
+
return if data_stream_exist?(name)
|
119
|
+
params = {
|
120
|
+
"name": name
|
121
|
+
}
|
122
|
+
retry_operate(@max_retry_putting_template,
|
123
|
+
@fail_on_putting_template_retry_exceed,
|
124
|
+
@catch_transport_exception_on_retry) do
|
125
|
+
@client.indices.create_data_stream(params)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def valid_data_stream_name?
|
130
|
+
lowercase_only? and
|
131
|
+
valid_characters? and
|
132
|
+
start_with_valid_characters? and
|
133
|
+
not_dots? and
|
134
|
+
@data_stream_name.bytes.size <= 255
|
135
|
+
end
|
136
|
+
|
137
|
+
def lowercase_only?
|
138
|
+
@data_stream_name.downcase == @data_stream_name
|
139
|
+
end
|
140
|
+
|
141
|
+
def valid_characters?
|
142
|
+
not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
|
143
|
+
end
|
144
|
+
|
145
|
+
def start_with_valid_characters?
|
146
|
+
not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
|
147
|
+
end
|
148
|
+
|
149
|
+
def not_dots?
|
150
|
+
not (@data_stream_name == "." or @data_stream_name == "..")
|
151
|
+
end
|
152
|
+
|
153
|
+
def client_library_version
|
154
|
+
Elasticsearch::VERSION
|
155
|
+
end
|
156
|
+
|
157
|
+
def multi_workers_ready?
|
158
|
+
true
|
159
|
+
end
|
160
|
+
|
161
|
+
def write(chunk)
|
162
|
+
data_stream_name = @data_stream_name
|
163
|
+
if @use_placeholder
|
164
|
+
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
165
|
+
unless @data_stream_names.include?(data_stream_name)
|
166
|
+
begin
|
167
|
+
create_ilm_policy(data_stream_name)
|
168
|
+
create_index_template(data_stream_name)
|
169
|
+
create_data_stream(data_stream_name)
|
170
|
+
@data_stream_names << data_stream_name
|
171
|
+
rescue => e
|
172
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
bulk_message = ""
|
178
|
+
headers = {
|
179
|
+
CREATE_OP => {}
|
180
|
+
}
|
181
|
+
tag = chunk.metadata.tag
|
182
|
+
chunk.msgpack_each do |time, record|
|
183
|
+
next unless record.is_a? Hash
|
184
|
+
|
185
|
+
begin
|
186
|
+
record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
|
187
|
+
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
188
|
+
rescue => e
|
189
|
+
router.emit_error_event(tag, time, record, e)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
params = {
|
194
|
+
index: data_stream_name,
|
195
|
+
body: bulk_message
|
196
|
+
}
|
197
|
+
begin
|
198
|
+
response = @client.bulk(params)
|
199
|
+
if response['errors']
|
200
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
201
|
+
end
|
202
|
+
rescue => e
|
203
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def append_record_to_messages(op, meta, header, record, msgs)
|
208
|
+
header[CREATE_OP] = meta
|
209
|
+
msgs << @dump_proc.call(header) << BODY_DELIMITER
|
210
|
+
msgs << @dump_proc.call(record) << BODY_DELIMITER
|
211
|
+
msgs
|
212
|
+
end
|
213
|
+
|
214
|
+
def retry_stream_retryable?
|
215
|
+
@buffer.storable?
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
@@ -370,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
370
370
|
}
|
371
371
|
end
|
372
372
|
|
373
|
+
sub_test_case 'Check client.info response' do
|
374
|
+
def create_driver(conf='', es_version=5, client_version="\"5.0\"")
|
375
|
+
# For request stub to detect compatibility.
|
376
|
+
@client_version ||= client_version
|
377
|
+
@default_elasticsearch_version ||= es_version
|
378
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
379
|
+
def detect_es_major_version
|
380
|
+
@_es_info ||= client.info
|
381
|
+
begin
|
382
|
+
unless version = @_es_info.dig("version", "number")
|
383
|
+
version = @default_elasticsearch_version
|
384
|
+
end
|
385
|
+
rescue NoMethodError => e
|
386
|
+
log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
|
387
|
+
version = @default_elasticsearch_version
|
388
|
+
end
|
389
|
+
version.to_i
|
390
|
+
end
|
391
|
+
CODE
|
392
|
+
|
393
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
394
|
+
def client_library_version
|
395
|
+
#{@client_version}
|
396
|
+
end
|
397
|
+
CODE
|
398
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
|
399
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
400
|
+
if !defined?(Fluent::Plugin::Output)
|
401
|
+
def format(tag, time, record)
|
402
|
+
[time, record].to_msgpack
|
403
|
+
end
|
404
|
+
end
|
405
|
+
}.configure(conf)
|
406
|
+
end
|
407
|
+
|
408
|
+
def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
|
409
|
+
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
410
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
|
411
|
+
end
|
412
|
+
|
413
|
+
test 'handle invalid client.info' do
|
414
|
+
stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
|
415
|
+
config = %{
|
416
|
+
host logs.fluentd.com
|
417
|
+
port 24225
|
418
|
+
scheme https
|
419
|
+
path /es/
|
420
|
+
user john
|
421
|
+
password doe
|
422
|
+
default_elasticsearch_version 7
|
423
|
+
scheme https
|
424
|
+
@log_level info
|
425
|
+
}
|
426
|
+
d = create_driver(config, 7, "\"7.10.1\"")
|
427
|
+
logs = d.logs
|
428
|
+
assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
373
432
|
sub_test_case 'Check TLS handshake stuck warning log' do
|
374
433
|
test 'warning TLS log' do
|
375
434
|
config = %{
|
@@ -3553,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3553
3612
|
assert_equal '/default_path', host2[:path]
|
3554
3613
|
end
|
3555
3614
|
|
3615
|
+
class IPv6AdressStringHostsTest < self
|
3616
|
+
def test_legacy_hosts_list
|
3617
|
+
config = %{
|
3618
|
+
hosts "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
|
3619
|
+
scheme https
|
3620
|
+
path /es/
|
3621
|
+
port 123
|
3622
|
+
}
|
3623
|
+
instance = driver(config).instance
|
3624
|
+
|
3625
|
+
assert_raise(URI::InvalidURIError) do
|
3626
|
+
instance.get_connection_options[:hosts].length
|
3627
|
+
end
|
3628
|
+
end
|
3629
|
+
|
3630
|
+
def test_hosts_list
|
3631
|
+
config = %{
|
3632
|
+
hosts https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
|
3633
|
+
path /default_path
|
3634
|
+
user default_user
|
3635
|
+
password default_password
|
3636
|
+
}
|
3637
|
+
instance = driver(config).instance
|
3638
|
+
|
3639
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
3640
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
3641
|
+
|
3642
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3643
|
+
assert_equal 443, host1[:port]
|
3644
|
+
assert_equal 'https', host1[:scheme]
|
3645
|
+
assert_equal 'john', host1[:user]
|
3646
|
+
assert_equal 'password', host1[:password]
|
3647
|
+
assert_equal '/elastic/', host1[:path]
|
3648
|
+
|
3649
|
+
assert_equal 'host2', host2[:host]
|
3650
|
+
assert_equal 'http', host2[:scheme]
|
3651
|
+
assert_equal 'default_user', host2[:user]
|
3652
|
+
assert_equal 'default_password', host2[:password]
|
3653
|
+
assert_equal '/default_path', host2[:path]
|
3654
|
+
end
|
3655
|
+
|
3656
|
+
def test_hosts_list_with_escape_placeholders
|
3657
|
+
config = %{
|
3658
|
+
hosts https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
|
3659
|
+
path /default_path
|
3660
|
+
user default_user
|
3661
|
+
password default_password
|
3662
|
+
}
|
3663
|
+
instance = driver(config).instance
|
3664
|
+
|
3665
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
3666
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
3667
|
+
|
3668
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3669
|
+
assert_equal 443, host1[:port]
|
3670
|
+
assert_equal 'https', host1[:scheme]
|
3671
|
+
assert_equal 'j%2Bhn', host1[:user]
|
3672
|
+
assert_equal 'passw%40rd', host1[:password]
|
3673
|
+
assert_equal '/elastic/', host1[:path]
|
3674
|
+
|
3675
|
+
assert_equal 'host2', host2[:host]
|
3676
|
+
assert_equal 'http', host2[:scheme]
|
3677
|
+
assert_equal 'default_user', host2[:user]
|
3678
|
+
assert_equal 'default_password', host2[:password]
|
3679
|
+
assert_equal '/default_path', host2[:path]
|
3680
|
+
end
|
3681
|
+
end
|
3682
|
+
|
3556
3683
|
def test_single_host_params_and_defaults
|
3557
3684
|
config = %{
|
3558
3685
|
host logs.google.com
|
@@ -3606,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3606
3733
|
assert(ports.none? { |p| p == 9200 })
|
3607
3734
|
end
|
3608
3735
|
|
3736
|
+
class IPv6AdressStringHostTest < self
|
3737
|
+
def test_single_host_params_and_defaults
|
3738
|
+
config = %{
|
3739
|
+
host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
|
3740
|
+
user john
|
3741
|
+
password doe
|
3742
|
+
}
|
3743
|
+
instance = driver(config).instance
|
3744
|
+
|
3745
|
+
assert_equal 1, instance.get_connection_options[:hosts].length
|
3746
|
+
host1 = instance.get_connection_options[:hosts][0]
|
3747
|
+
|
3748
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3749
|
+
assert_equal 9200, host1[:port]
|
3750
|
+
assert_equal 'http', host1[:scheme]
|
3751
|
+
assert_equal 'john', host1[:user]
|
3752
|
+
assert_equal 'doe', host1[:password]
|
3753
|
+
assert_equal nil, host1[:path]
|
3754
|
+
end
|
3755
|
+
|
3756
|
+
def test_single_host_params_and_defaults_with_escape_placeholders
|
3757
|
+
config = %{
|
3758
|
+
host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
|
3759
|
+
user %{j+hn}
|
3760
|
+
password %{d@e}
|
3761
|
+
}
|
3762
|
+
instance = driver(config).instance
|
3763
|
+
|
3764
|
+
assert_equal 1, instance.get_connection_options[:hosts].length
|
3765
|
+
host1 = instance.get_connection_options[:hosts][0]
|
3766
|
+
|
3767
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3768
|
+
assert_equal 9200, host1[:port]
|
3769
|
+
assert_equal 'http', host1[:scheme]
|
3770
|
+
assert_equal 'j%2Bhn', host1[:user]
|
3771
|
+
assert_equal 'd%40e', host1[:password]
|
3772
|
+
assert_equal nil, host1[:path]
|
3773
|
+
end
|
3774
|
+
end
|
3775
|
+
|
3609
3776
|
def test_password_is_required_if_specify_user
|
3610
3777
|
config = %{
|
3611
3778
|
user john
|