fluent-plugin-elasticsearch 4.3.2 → 5.0.3

Sign up to get free protection for your applications and to get access to all the features.
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.3.2'
6
+ s.version = '5.0.3'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
28
28
 
29
29
 
30
30
  s.add_development_dependency 'rake', '>= 0'
31
+ s.add_development_dependency 'webrick', '~> 1.7.0'
31
32
  s.add_development_dependency 'webmock', '~> 3'
32
33
  s.add_development_dependency 'test-unit', '~> 3.3.0'
33
34
  s.add_development_dependency 'minitest', '~> 5.8'
@@ -13,6 +13,7 @@ begin
13
13
  require 'strptime'
14
14
  rescue LoadError
15
15
  end
16
+ require 'resolv'
16
17
 
17
18
  require 'fluent/plugin/output'
18
19
  require 'fluent/event'
@@ -493,7 +494,12 @@ EOC
493
494
 
494
495
  def detect_es_major_version
495
496
  @_es_info ||= client.info
496
- unless version = @_es_info.dig("version", "number")
497
+ begin
498
+ unless version = @_es_info.dig("version", "number")
499
+ version = @default_elasticsearch_version
500
+ end
501
+ rescue NoMethodError => e
502
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
497
503
  version = @default_elasticsearch_version
498
504
  end
499
505
  version.to_i
@@ -663,7 +669,11 @@ EOC
663
669
  end
664
670
  end.compact
665
671
  else
666
- [{host: @host, port: @port, scheme: @scheme.to_s}]
672
+ if Resolv::IPv6::Regex.match(@host)
673
+ [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
674
+ else
675
+ [{host: @host, port: @port, scheme: @scheme.to_s}]
676
+ end
667
677
  end.each do |host|
668
678
  host.merge!(user: @user, password: @password) if !host[:user] && @user
669
679
  host.merge!(path: @path) if !host[:path] && @path
@@ -0,0 +1,218 @@
1
+ require_relative 'out_elasticsearch'
2
+
3
+ module Fluent::Plugin
4
+ class ElasticsearchOutputDataStream < ElasticsearchOutput
5
+
6
+ Fluent::Plugin.register_output('elasticsearch_data_stream', self)
7
+
8
+ helpers :event_emitter
9
+
10
+ config_param :data_stream_name, :string
11
+ # Elasticsearch 7.9 or later always support new style of index template.
12
+ config_set_default :use_legacy_template, false
13
+
14
+ INVALID_START_CHRACTERS = ["-", "_", "+", "."]
15
+ INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
16
+
17
+ def configure(conf)
18
+ super
19
+
20
+ begin
21
+ require 'elasticsearch/api'
22
+ require 'elasticsearch/xpack'
23
+ rescue LoadError
24
+ raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
25
+ end
26
+
27
+ # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
28
+ unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
29
+ validate_data_stream_name
30
+ else
31
+ @use_placeholder = true
32
+ @data_stream_names = []
33
+ end
34
+
35
+ @client = client
36
+ unless @use_placeholder
37
+ begin
38
+ @data_stream_names = [@data_stream_name]
39
+ create_ilm_policy(@data_stream_name)
40
+ create_index_template(@data_stream_name)
41
+ create_data_stream(@data_stream_name)
42
+ rescue => e
43
+ raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
44
+ end
45
+ end
46
+ end
47
+
48
+ def validate_data_stream_name
49
+ unless valid_data_stream_name?
50
+ unless start_with_valid_characters?
51
+ if not_dots?
52
+ raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
53
+ else
54
+ raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
55
+ end
56
+ end
57
+ unless valid_characters?
58
+ raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
59
+ end
60
+ unless lowercase_only?
61
+ raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
62
+ end
63
+ if @data_stream_name.bytes.size > 255
64
+ raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
65
+ end
66
+ end
67
+ end
68
+
69
+ def create_ilm_policy(name)
70
+ return if data_stream_exist?(name)
71
+ params = {
72
+ policy_id: "#{name}_policy",
73
+ body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
74
+ }
75
+ retry_operate(@max_retry_putting_template,
76
+ @fail_on_putting_template_retry_exceed,
77
+ @catch_transport_exception_on_retry) do
78
+ @client.xpack.ilm.put_policy(params)
79
+ end
80
+ end
81
+
82
+ def create_index_template(name)
83
+ return if data_stream_exist?(name)
84
+ body = {
85
+ "index_patterns" => ["#{name}*"],
86
+ "data_stream" => {},
87
+ "template" => {
88
+ "settings" => {
89
+ "index.lifecycle.name" => "#{name}_policy"
90
+ }
91
+ }
92
+ }
93
+ params = {
94
+ name: name,
95
+ body: body
96
+ }
97
+ retry_operate(@max_retry_putting_template,
98
+ @fail_on_putting_template_retry_exceed,
99
+ @catch_transport_exception_on_retry) do
100
+ @client.indices.put_index_template(params)
101
+ end
102
+ end
103
+
104
+ def data_stream_exist?(name)
105
+ params = {
106
+ "name": name
107
+ }
108
+ begin
109
+ response = @client.indices.get_data_stream(params)
110
+ return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
111
+ rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
112
+ log.info "Specified data stream does not exist. Will be created: <#{e}>"
113
+ return false
114
+ end
115
+ end
116
+
117
+ def create_data_stream(name)
118
+ return if data_stream_exist?(name)
119
+ params = {
120
+ "name": name
121
+ }
122
+ retry_operate(@max_retry_putting_template,
123
+ @fail_on_putting_template_retry_exceed,
124
+ @catch_transport_exception_on_retry) do
125
+ @client.indices.create_data_stream(params)
126
+ end
127
+ end
128
+
129
+ def valid_data_stream_name?
130
+ lowercase_only? and
131
+ valid_characters? and
132
+ start_with_valid_characters? and
133
+ not_dots? and
134
+ @data_stream_name.bytes.size <= 255
135
+ end
136
+
137
+ def lowercase_only?
138
+ @data_stream_name.downcase == @data_stream_name
139
+ end
140
+
141
+ def valid_characters?
142
+ not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
143
+ end
144
+
145
+ def start_with_valid_characters?
146
+ not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
147
+ end
148
+
149
+ def not_dots?
150
+ not (@data_stream_name == "." or @data_stream_name == "..")
151
+ end
152
+
153
+ def client_library_version
154
+ Elasticsearch::VERSION
155
+ end
156
+
157
+ def multi_workers_ready?
158
+ true
159
+ end
160
+
161
+ def write(chunk)
162
+ data_stream_name = @data_stream_name
163
+ if @use_placeholder
164
+ data_stream_name = extract_placeholders(@data_stream_name, chunk)
165
+ unless @data_stream_names.include?(data_stream_name)
166
+ begin
167
+ create_ilm_policy(data_stream_name)
168
+ create_index_template(data_stream_name)
169
+ create_data_stream(data_stream_name)
170
+ @data_stream_names << data_stream_name
171
+ rescue => e
172
+ raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
173
+ end
174
+ end
175
+ end
176
+
177
+ bulk_message = ""
178
+ headers = {
179
+ CREATE_OP => {}
180
+ }
181
+ tag = chunk.metadata.tag
182
+ chunk.msgpack_each do |time, record|
183
+ next unless record.is_a? Hash
184
+
185
+ begin
186
+ record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
187
+ bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
188
+ rescue => e
189
+ router.emit_error_event(tag, time, record, e)
190
+ end
191
+ end
192
+
193
+ params = {
194
+ index: data_stream_name,
195
+ body: bulk_message
196
+ }
197
+ begin
198
+ response = @client.bulk(params)
199
+ if response['errors']
200
+ log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
201
+ end
202
+ rescue => e
203
+ log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
204
+ end
205
+ end
206
+
207
+ def append_record_to_messages(op, meta, header, record, msgs)
208
+ header[CREATE_OP] = meta
209
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
210
+ msgs << @dump_proc.call(record) << BODY_DELIMITER
211
+ msgs
212
+ end
213
+
214
+ def retry_stream_retryable?
215
+ @buffer.storable?
216
+ end
217
+ end
218
+ end
@@ -370,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
370
370
  }
371
371
  end
372
372
 
373
+ sub_test_case 'Check client.info response' do
374
+ def create_driver(conf='', es_version=5, client_version="\"5.0\"")
375
+ # For request stub to detect compatibility.
376
+ @client_version ||= client_version
377
+ @default_elasticsearch_version ||= es_version
378
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
379
+ def detect_es_major_version
380
+ @_es_info ||= client.info
381
+ begin
382
+ unless version = @_es_info.dig("version", "number")
383
+ version = @default_elasticsearch_version
384
+ end
385
+ rescue NoMethodError => e
386
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
387
+ version = @default_elasticsearch_version
388
+ end
389
+ version.to_i
390
+ end
391
+ CODE
392
+
393
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
394
+ def client_library_version
395
+ #{@client_version}
396
+ end
397
+ CODE
398
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
399
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
400
+ if !defined?(Fluent::Plugin::Output)
401
+ def format(tag, time, record)
402
+ [time, record].to_msgpack
403
+ end
404
+ end
405
+ }.configure(conf)
406
+ end
407
+
408
+ def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
409
+ body ="{\"version\":{\"number\":\"#{version}\"}}"
410
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
411
+ end
412
+
413
+ test 'handle invalid client.info' do
414
+ stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
415
+ config = %{
416
+ host logs.fluentd.com
417
+ port 24225
418
+ scheme https
419
+ path /es/
420
+ user john
421
+ password doe
422
+ default_elasticsearch_version 7
423
+ scheme https
424
+ @log_level info
425
+ }
426
+ d = create_driver(config, 7, "\"7.10.1\"")
427
+ logs = d.logs
428
+ assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
429
+ end
430
+ end
431
+
373
432
  sub_test_case 'Check TLS handshake stuck warning log' do
374
433
  test 'warning TLS log' do
375
434
  config = %{
@@ -3553,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3553
3612
  assert_equal '/default_path', host2[:path]
3554
3613
  end
3555
3614
 
3615
+ class IPv6AdressStringHostsTest < self
3616
+ def test_legacy_hosts_list
3617
+ config = %{
3618
+ hosts "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
3619
+ scheme https
3620
+ path /es/
3621
+ port 123
3622
+ }
3623
+ instance = driver(config).instance
3624
+
3625
+ assert_raise(URI::InvalidURIError) do
3626
+ instance.get_connection_options[:hosts].length
3627
+ end
3628
+ end
3629
+
3630
+ def test_hosts_list
3631
+ config = %{
3632
+ hosts https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
3633
+ path /default_path
3634
+ user default_user
3635
+ password default_password
3636
+ }
3637
+ instance = driver(config).instance
3638
+
3639
+ assert_equal 2, instance.get_connection_options[:hosts].length
3640
+ host1, host2 = instance.get_connection_options[:hosts]
3641
+
3642
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3643
+ assert_equal 443, host1[:port]
3644
+ assert_equal 'https', host1[:scheme]
3645
+ assert_equal 'john', host1[:user]
3646
+ assert_equal 'password', host1[:password]
3647
+ assert_equal '/elastic/', host1[:path]
3648
+
3649
+ assert_equal 'host2', host2[:host]
3650
+ assert_equal 'http', host2[:scheme]
3651
+ assert_equal 'default_user', host2[:user]
3652
+ assert_equal 'default_password', host2[:password]
3653
+ assert_equal '/default_path', host2[:path]
3654
+ end
3655
+
3656
+ def test_hosts_list_with_escape_placeholders
3657
+ config = %{
3658
+ hosts https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
3659
+ path /default_path
3660
+ user default_user
3661
+ password default_password
3662
+ }
3663
+ instance = driver(config).instance
3664
+
3665
+ assert_equal 2, instance.get_connection_options[:hosts].length
3666
+ host1, host2 = instance.get_connection_options[:hosts]
3667
+
3668
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3669
+ assert_equal 443, host1[:port]
3670
+ assert_equal 'https', host1[:scheme]
3671
+ assert_equal 'j%2Bhn', host1[:user]
3672
+ assert_equal 'passw%40rd', host1[:password]
3673
+ assert_equal '/elastic/', host1[:path]
3674
+
3675
+ assert_equal 'host2', host2[:host]
3676
+ assert_equal 'http', host2[:scheme]
3677
+ assert_equal 'default_user', host2[:user]
3678
+ assert_equal 'default_password', host2[:password]
3679
+ assert_equal '/default_path', host2[:path]
3680
+ end
3681
+ end
3682
+
3556
3683
  def test_single_host_params_and_defaults
3557
3684
  config = %{
3558
3685
  host logs.google.com
@@ -3606,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3606
3733
  assert(ports.none? { |p| p == 9200 })
3607
3734
  end
3608
3735
 
3736
+ class IPv6AdressStringHostTest < self
3737
+ def test_single_host_params_and_defaults
3738
+ config = %{
3739
+ host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
3740
+ user john
3741
+ password doe
3742
+ }
3743
+ instance = driver(config).instance
3744
+
3745
+ assert_equal 1, instance.get_connection_options[:hosts].length
3746
+ host1 = instance.get_connection_options[:hosts][0]
3747
+
3748
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3749
+ assert_equal 9200, host1[:port]
3750
+ assert_equal 'http', host1[:scheme]
3751
+ assert_equal 'john', host1[:user]
3752
+ assert_equal 'doe', host1[:password]
3753
+ assert_equal nil, host1[:path]
3754
+ end
3755
+
3756
+ def test_single_host_params_and_defaults_with_escape_placeholders
3757
+ config = %{
3758
+ host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
3759
+ user %{j+hn}
3760
+ password %{d@e}
3761
+ }
3762
+ instance = driver(config).instance
3763
+
3764
+ assert_equal 1, instance.get_connection_options[:hosts].length
3765
+ host1 = instance.get_connection_options[:hosts][0]
3766
+
3767
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3768
+ assert_equal 9200, host1[:port]
3769
+ assert_equal 'http', host1[:scheme]
3770
+ assert_equal 'j%2Bhn', host1[:user]
3771
+ assert_equal 'd%40e', host1[:password]
3772
+ assert_equal nil, host1[:path]
3773
+ end
3774
+ end
3775
+
3609
3776
  def test_password_is_required_if_specify_user
3610
3777
  config = %{
3611
3778
  user john