fluent-plugin-elasticsearch 4.3.2 → 5.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/linux.yml +1 -1
- data/.github/workflows/macos.yml +1 -1
- data/.github/workflows/windows.yml +1 -1
- data/.travis.yml +0 -4
- data/History.md +21 -0
- data/README.ElasticsearchInput.md +1 -1
- data/README.Troubleshooting.md +692 -0
- data/README.md +45 -586
- data/fluent-plugin-elasticsearch.gemspec +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +12 -2
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +218 -0
- data/test/plugin/test_out_elasticsearch.rb +167 -0
- data/test/plugin/test_out_elasticsearch_data_stream.rb +337 -0
- metadata +21 -4
- data/gemfiles/Gemfile.without.ilm +0 -10
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '
|
6
|
+
s.version = '5.0.3'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
|
|
28
28
|
|
29
29
|
|
30
30
|
s.add_development_dependency 'rake', '>= 0'
|
31
|
+
s.add_development_dependency 'webrick', '~> 1.7.0'
|
31
32
|
s.add_development_dependency 'webmock', '~> 3'
|
32
33
|
s.add_development_dependency 'test-unit', '~> 3.3.0'
|
33
34
|
s.add_development_dependency 'minitest', '~> 5.8'
|
@@ -13,6 +13,7 @@ begin
|
|
13
13
|
require 'strptime'
|
14
14
|
rescue LoadError
|
15
15
|
end
|
16
|
+
require 'resolv'
|
16
17
|
|
17
18
|
require 'fluent/plugin/output'
|
18
19
|
require 'fluent/event'
|
@@ -493,7 +494,12 @@ EOC
|
|
493
494
|
|
494
495
|
def detect_es_major_version
|
495
496
|
@_es_info ||= client.info
|
496
|
-
|
497
|
+
begin
|
498
|
+
unless version = @_es_info.dig("version", "number")
|
499
|
+
version = @default_elasticsearch_version
|
500
|
+
end
|
501
|
+
rescue NoMethodError => e
|
502
|
+
log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
|
497
503
|
version = @default_elasticsearch_version
|
498
504
|
end
|
499
505
|
version.to_i
|
@@ -663,7 +669,11 @@ EOC
|
|
663
669
|
end
|
664
670
|
end.compact
|
665
671
|
else
|
666
|
-
|
672
|
+
if Resolv::IPv6::Regex.match(@host)
|
673
|
+
[{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
|
674
|
+
else
|
675
|
+
[{host: @host, port: @port, scheme: @scheme.to_s}]
|
676
|
+
end
|
667
677
|
end.each do |host|
|
668
678
|
host.merge!(user: @user, password: @password) if !host[:user] && @user
|
669
679
|
host.merge!(path: @path) if !host[:path] && @path
|
@@ -0,0 +1,218 @@
|
|
1
|
+
require_relative 'out_elasticsearch'
|
2
|
+
|
3
|
+
module Fluent::Plugin
|
4
|
+
class ElasticsearchOutputDataStream < ElasticsearchOutput
|
5
|
+
|
6
|
+
Fluent::Plugin.register_output('elasticsearch_data_stream', self)
|
7
|
+
|
8
|
+
helpers :event_emitter
|
9
|
+
|
10
|
+
config_param :data_stream_name, :string
|
11
|
+
# Elasticsearch 7.9 or later always support new style of index template.
|
12
|
+
config_set_default :use_legacy_template, false
|
13
|
+
|
14
|
+
INVALID_START_CHRACTERS = ["-", "_", "+", "."]
|
15
|
+
INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
|
16
|
+
|
17
|
+
def configure(conf)
|
18
|
+
super
|
19
|
+
|
20
|
+
begin
|
21
|
+
require 'elasticsearch/api'
|
22
|
+
require 'elasticsearch/xpack'
|
23
|
+
rescue LoadError
|
24
|
+
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
25
|
+
end
|
26
|
+
|
27
|
+
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
28
|
+
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
29
|
+
validate_data_stream_name
|
30
|
+
else
|
31
|
+
@use_placeholder = true
|
32
|
+
@data_stream_names = []
|
33
|
+
end
|
34
|
+
|
35
|
+
@client = client
|
36
|
+
unless @use_placeholder
|
37
|
+
begin
|
38
|
+
@data_stream_names = [@data_stream_name]
|
39
|
+
create_ilm_policy(@data_stream_name)
|
40
|
+
create_index_template(@data_stream_name)
|
41
|
+
create_data_stream(@data_stream_name)
|
42
|
+
rescue => e
|
43
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def validate_data_stream_name
|
49
|
+
unless valid_data_stream_name?
|
50
|
+
unless start_with_valid_characters?
|
51
|
+
if not_dots?
|
52
|
+
raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
|
53
|
+
else
|
54
|
+
raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
|
55
|
+
end
|
56
|
+
end
|
57
|
+
unless valid_characters?
|
58
|
+
raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
|
59
|
+
end
|
60
|
+
unless lowercase_only?
|
61
|
+
raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
|
62
|
+
end
|
63
|
+
if @data_stream_name.bytes.size > 255
|
64
|
+
raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def create_ilm_policy(name)
|
70
|
+
return if data_stream_exist?(name)
|
71
|
+
params = {
|
72
|
+
policy_id: "#{name}_policy",
|
73
|
+
body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
|
74
|
+
}
|
75
|
+
retry_operate(@max_retry_putting_template,
|
76
|
+
@fail_on_putting_template_retry_exceed,
|
77
|
+
@catch_transport_exception_on_retry) do
|
78
|
+
@client.xpack.ilm.put_policy(params)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def create_index_template(name)
|
83
|
+
return if data_stream_exist?(name)
|
84
|
+
body = {
|
85
|
+
"index_patterns" => ["#{name}*"],
|
86
|
+
"data_stream" => {},
|
87
|
+
"template" => {
|
88
|
+
"settings" => {
|
89
|
+
"index.lifecycle.name" => "#{name}_policy"
|
90
|
+
}
|
91
|
+
}
|
92
|
+
}
|
93
|
+
params = {
|
94
|
+
name: name,
|
95
|
+
body: body
|
96
|
+
}
|
97
|
+
retry_operate(@max_retry_putting_template,
|
98
|
+
@fail_on_putting_template_retry_exceed,
|
99
|
+
@catch_transport_exception_on_retry) do
|
100
|
+
@client.indices.put_index_template(params)
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def data_stream_exist?(name)
|
105
|
+
params = {
|
106
|
+
"name": name
|
107
|
+
}
|
108
|
+
begin
|
109
|
+
response = @client.indices.get_data_stream(params)
|
110
|
+
return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
|
111
|
+
rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
|
112
|
+
log.info "Specified data stream does not exist. Will be created: <#{e}>"
|
113
|
+
return false
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
117
|
+
def create_data_stream(name)
|
118
|
+
return if data_stream_exist?(name)
|
119
|
+
params = {
|
120
|
+
"name": name
|
121
|
+
}
|
122
|
+
retry_operate(@max_retry_putting_template,
|
123
|
+
@fail_on_putting_template_retry_exceed,
|
124
|
+
@catch_transport_exception_on_retry) do
|
125
|
+
@client.indices.create_data_stream(params)
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def valid_data_stream_name?
|
130
|
+
lowercase_only? and
|
131
|
+
valid_characters? and
|
132
|
+
start_with_valid_characters? and
|
133
|
+
not_dots? and
|
134
|
+
@data_stream_name.bytes.size <= 255
|
135
|
+
end
|
136
|
+
|
137
|
+
def lowercase_only?
|
138
|
+
@data_stream_name.downcase == @data_stream_name
|
139
|
+
end
|
140
|
+
|
141
|
+
def valid_characters?
|
142
|
+
not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
|
143
|
+
end
|
144
|
+
|
145
|
+
def start_with_valid_characters?
|
146
|
+
not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
|
147
|
+
end
|
148
|
+
|
149
|
+
def not_dots?
|
150
|
+
not (@data_stream_name == "." or @data_stream_name == "..")
|
151
|
+
end
|
152
|
+
|
153
|
+
def client_library_version
|
154
|
+
Elasticsearch::VERSION
|
155
|
+
end
|
156
|
+
|
157
|
+
def multi_workers_ready?
|
158
|
+
true
|
159
|
+
end
|
160
|
+
|
161
|
+
def write(chunk)
|
162
|
+
data_stream_name = @data_stream_name
|
163
|
+
if @use_placeholder
|
164
|
+
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
165
|
+
unless @data_stream_names.include?(data_stream_name)
|
166
|
+
begin
|
167
|
+
create_ilm_policy(data_stream_name)
|
168
|
+
create_index_template(data_stream_name)
|
169
|
+
create_data_stream(data_stream_name)
|
170
|
+
@data_stream_names << data_stream_name
|
171
|
+
rescue => e
|
172
|
+
raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
bulk_message = ""
|
178
|
+
headers = {
|
179
|
+
CREATE_OP => {}
|
180
|
+
}
|
181
|
+
tag = chunk.metadata.tag
|
182
|
+
chunk.msgpack_each do |time, record|
|
183
|
+
next unless record.is_a? Hash
|
184
|
+
|
185
|
+
begin
|
186
|
+
record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
|
187
|
+
bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
|
188
|
+
rescue => e
|
189
|
+
router.emit_error_event(tag, time, record, e)
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
params = {
|
194
|
+
index: data_stream_name,
|
195
|
+
body: bulk_message
|
196
|
+
}
|
197
|
+
begin
|
198
|
+
response = @client.bulk(params)
|
199
|
+
if response['errors']
|
200
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
201
|
+
end
|
202
|
+
rescue => e
|
203
|
+
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
|
204
|
+
end
|
205
|
+
end
|
206
|
+
|
207
|
+
def append_record_to_messages(op, meta, header, record, msgs)
|
208
|
+
header[CREATE_OP] = meta
|
209
|
+
msgs << @dump_proc.call(header) << BODY_DELIMITER
|
210
|
+
msgs << @dump_proc.call(record) << BODY_DELIMITER
|
211
|
+
msgs
|
212
|
+
end
|
213
|
+
|
214
|
+
def retry_stream_retryable?
|
215
|
+
@buffer.storable?
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
@@ -370,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
370
370
|
}
|
371
371
|
end
|
372
372
|
|
373
|
+
sub_test_case 'Check client.info response' do
|
374
|
+
def create_driver(conf='', es_version=5, client_version="\"5.0\"")
|
375
|
+
# For request stub to detect compatibility.
|
376
|
+
@client_version ||= client_version
|
377
|
+
@default_elasticsearch_version ||= es_version
|
378
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
379
|
+
def detect_es_major_version
|
380
|
+
@_es_info ||= client.info
|
381
|
+
begin
|
382
|
+
unless version = @_es_info.dig("version", "number")
|
383
|
+
version = @default_elasticsearch_version
|
384
|
+
end
|
385
|
+
rescue NoMethodError => e
|
386
|
+
log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
|
387
|
+
version = @default_elasticsearch_version
|
388
|
+
end
|
389
|
+
version.to_i
|
390
|
+
end
|
391
|
+
CODE
|
392
|
+
|
393
|
+
Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
|
394
|
+
def client_library_version
|
395
|
+
#{@client_version}
|
396
|
+
end
|
397
|
+
CODE
|
398
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
|
399
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
400
|
+
if !defined?(Fluent::Plugin::Output)
|
401
|
+
def format(tag, time, record)
|
402
|
+
[time, record].to_msgpack
|
403
|
+
end
|
404
|
+
end
|
405
|
+
}.configure(conf)
|
406
|
+
end
|
407
|
+
|
408
|
+
def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
|
409
|
+
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
410
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
|
411
|
+
end
|
412
|
+
|
413
|
+
test 'handle invalid client.info' do
|
414
|
+
stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
|
415
|
+
config = %{
|
416
|
+
host logs.fluentd.com
|
417
|
+
port 24225
|
418
|
+
scheme https
|
419
|
+
path /es/
|
420
|
+
user john
|
421
|
+
password doe
|
422
|
+
default_elasticsearch_version 7
|
423
|
+
scheme https
|
424
|
+
@log_level info
|
425
|
+
}
|
426
|
+
d = create_driver(config, 7, "\"7.10.1\"")
|
427
|
+
logs = d.logs
|
428
|
+
assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
|
429
|
+
end
|
430
|
+
end
|
431
|
+
|
373
432
|
sub_test_case 'Check TLS handshake stuck warning log' do
|
374
433
|
test 'warning TLS log' do
|
375
434
|
config = %{
|
@@ -3553,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3553
3612
|
assert_equal '/default_path', host2[:path]
|
3554
3613
|
end
|
3555
3614
|
|
3615
|
+
class IPv6AdressStringHostsTest < self
|
3616
|
+
def test_legacy_hosts_list
|
3617
|
+
config = %{
|
3618
|
+
hosts "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
|
3619
|
+
scheme https
|
3620
|
+
path /es/
|
3621
|
+
port 123
|
3622
|
+
}
|
3623
|
+
instance = driver(config).instance
|
3624
|
+
|
3625
|
+
assert_raise(URI::InvalidURIError) do
|
3626
|
+
instance.get_connection_options[:hosts].length
|
3627
|
+
end
|
3628
|
+
end
|
3629
|
+
|
3630
|
+
def test_hosts_list
|
3631
|
+
config = %{
|
3632
|
+
hosts https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
|
3633
|
+
path /default_path
|
3634
|
+
user default_user
|
3635
|
+
password default_password
|
3636
|
+
}
|
3637
|
+
instance = driver(config).instance
|
3638
|
+
|
3639
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
3640
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
3641
|
+
|
3642
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3643
|
+
assert_equal 443, host1[:port]
|
3644
|
+
assert_equal 'https', host1[:scheme]
|
3645
|
+
assert_equal 'john', host1[:user]
|
3646
|
+
assert_equal 'password', host1[:password]
|
3647
|
+
assert_equal '/elastic/', host1[:path]
|
3648
|
+
|
3649
|
+
assert_equal 'host2', host2[:host]
|
3650
|
+
assert_equal 'http', host2[:scheme]
|
3651
|
+
assert_equal 'default_user', host2[:user]
|
3652
|
+
assert_equal 'default_password', host2[:password]
|
3653
|
+
assert_equal '/default_path', host2[:path]
|
3654
|
+
end
|
3655
|
+
|
3656
|
+
def test_hosts_list_with_escape_placeholders
|
3657
|
+
config = %{
|
3658
|
+
hosts https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
|
3659
|
+
path /default_path
|
3660
|
+
user default_user
|
3661
|
+
password default_password
|
3662
|
+
}
|
3663
|
+
instance = driver(config).instance
|
3664
|
+
|
3665
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
3666
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
3667
|
+
|
3668
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3669
|
+
assert_equal 443, host1[:port]
|
3670
|
+
assert_equal 'https', host1[:scheme]
|
3671
|
+
assert_equal 'j%2Bhn', host1[:user]
|
3672
|
+
assert_equal 'passw%40rd', host1[:password]
|
3673
|
+
assert_equal '/elastic/', host1[:path]
|
3674
|
+
|
3675
|
+
assert_equal 'host2', host2[:host]
|
3676
|
+
assert_equal 'http', host2[:scheme]
|
3677
|
+
assert_equal 'default_user', host2[:user]
|
3678
|
+
assert_equal 'default_password', host2[:password]
|
3679
|
+
assert_equal '/default_path', host2[:path]
|
3680
|
+
end
|
3681
|
+
end
|
3682
|
+
|
3556
3683
|
def test_single_host_params_and_defaults
|
3557
3684
|
config = %{
|
3558
3685
|
host logs.google.com
|
@@ -3606,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
|
|
3606
3733
|
assert(ports.none? { |p| p == 9200 })
|
3607
3734
|
end
|
3608
3735
|
|
3736
|
+
class IPv6AdressStringHostTest < self
|
3737
|
+
def test_single_host_params_and_defaults
|
3738
|
+
config = %{
|
3739
|
+
host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
|
3740
|
+
user john
|
3741
|
+
password doe
|
3742
|
+
}
|
3743
|
+
instance = driver(config).instance
|
3744
|
+
|
3745
|
+
assert_equal 1, instance.get_connection_options[:hosts].length
|
3746
|
+
host1 = instance.get_connection_options[:hosts][0]
|
3747
|
+
|
3748
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3749
|
+
assert_equal 9200, host1[:port]
|
3750
|
+
assert_equal 'http', host1[:scheme]
|
3751
|
+
assert_equal 'john', host1[:user]
|
3752
|
+
assert_equal 'doe', host1[:password]
|
3753
|
+
assert_equal nil, host1[:path]
|
3754
|
+
end
|
3755
|
+
|
3756
|
+
def test_single_host_params_and_defaults_with_escape_placeholders
|
3757
|
+
config = %{
|
3758
|
+
host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
|
3759
|
+
user %{j+hn}
|
3760
|
+
password %{d@e}
|
3761
|
+
}
|
3762
|
+
instance = driver(config).instance
|
3763
|
+
|
3764
|
+
assert_equal 1, instance.get_connection_options[:hosts].length
|
3765
|
+
host1 = instance.get_connection_options[:hosts][0]
|
3766
|
+
|
3767
|
+
assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
|
3768
|
+
assert_equal 9200, host1[:port]
|
3769
|
+
assert_equal 'http', host1[:scheme]
|
3770
|
+
assert_equal 'j%2Bhn', host1[:user]
|
3771
|
+
assert_equal 'd%40e', host1[:password]
|
3772
|
+
assert_equal nil, host1[:path]
|
3773
|
+
end
|
3774
|
+
end
|
3775
|
+
|
3609
3776
|
def test_password_is_required_if_specify_user
|
3610
3777
|
config = %{
|
3611
3778
|
user john
|