fluent-plugin-elasticsearch 4.3.2 → 5.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.3.2'
6
+ s.version = '5.0.3'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
28
28
 
29
29
 
30
30
  s.add_development_dependency 'rake', '>= 0'
31
+ s.add_development_dependency 'webrick', '~> 1.7.0'
31
32
  s.add_development_dependency 'webmock', '~> 3'
32
33
  s.add_development_dependency 'test-unit', '~> 3.3.0'
33
34
  s.add_development_dependency 'minitest', '~> 5.8'
@@ -13,6 +13,7 @@ begin
13
13
  require 'strptime'
14
14
  rescue LoadError
15
15
  end
16
+ require 'resolv'
16
17
 
17
18
  require 'fluent/plugin/output'
18
19
  require 'fluent/event'
@@ -493,7 +494,12 @@ EOC
493
494
 
494
495
  def detect_es_major_version
495
496
  @_es_info ||= client.info
496
- unless version = @_es_info.dig("version", "number")
497
+ begin
498
+ unless version = @_es_info.dig("version", "number")
499
+ version = @default_elasticsearch_version
500
+ end
501
+ rescue NoMethodError => e
502
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
497
503
  version = @default_elasticsearch_version
498
504
  end
499
505
  version.to_i
@@ -663,7 +669,11 @@ EOC
663
669
  end
664
670
  end.compact
665
671
  else
666
- [{host: @host, port: @port, scheme: @scheme.to_s}]
672
+ if Resolv::IPv6::Regex.match(@host)
673
+ [{host: "[#{@host}]", scheme: @scheme.to_s, port: @port}]
674
+ else
675
+ [{host: @host, port: @port, scheme: @scheme.to_s}]
676
+ end
667
677
  end.each do |host|
668
678
  host.merge!(user: @user, password: @password) if !host[:user] && @user
669
679
  host.merge!(path: @path) if !host[:path] && @path
@@ -0,0 +1,218 @@
1
+ require_relative 'out_elasticsearch'
2
+
3
+ module Fluent::Plugin
4
+ class ElasticsearchOutputDataStream < ElasticsearchOutput
5
+
6
+ Fluent::Plugin.register_output('elasticsearch_data_stream', self)
7
+
8
+ helpers :event_emitter
9
+
10
+ config_param :data_stream_name, :string
11
+ # Elasticsearch 7.9 or later always support new style of index template.
12
+ config_set_default :use_legacy_template, false
13
+
14
+ INVALID_START_CHRACTERS = ["-", "_", "+", "."]
15
+ INVALID_CHARACTERS = ["\\", "/", "*", "?", "\"", "<", ">", "|", " ", ",", "#", ":"]
16
+
17
+ def configure(conf)
18
+ super
19
+
20
+ begin
21
+ require 'elasticsearch/api'
22
+ require 'elasticsearch/xpack'
23
+ rescue LoadError
24
+ raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
25
+ end
26
+
27
+ # ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
28
+ unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
29
+ validate_data_stream_name
30
+ else
31
+ @use_placeholder = true
32
+ @data_stream_names = []
33
+ end
34
+
35
+ @client = client
36
+ unless @use_placeholder
37
+ begin
38
+ @data_stream_names = [@data_stream_name]
39
+ create_ilm_policy(@data_stream_name)
40
+ create_index_template(@data_stream_name)
41
+ create_data_stream(@data_stream_name)
42
+ rescue => e
43
+ raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
44
+ end
45
+ end
46
+ end
47
+
48
+ def validate_data_stream_name
49
+ unless valid_data_stream_name?
50
+ unless start_with_valid_characters?
51
+ if not_dots?
52
+ raise Fluent::ConfigError, "'data_stream_name' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{@data_stream_name}>"
53
+ else
54
+ raise Fluent::ConfigError, "'data_stream_name' must not be . or ..: <#{@data_stream_name}>"
55
+ end
56
+ end
57
+ unless valid_characters?
58
+ raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
59
+ end
60
+ unless lowercase_only?
61
+ raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
62
+ end
63
+ if @data_stream_name.bytes.size > 255
64
+ raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
65
+ end
66
+ end
67
+ end
68
+
69
+ def create_ilm_policy(name)
70
+ return if data_stream_exist?(name)
71
+ params = {
72
+ policy_id: "#{name}_policy",
73
+ body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
74
+ }
75
+ retry_operate(@max_retry_putting_template,
76
+ @fail_on_putting_template_retry_exceed,
77
+ @catch_transport_exception_on_retry) do
78
+ @client.xpack.ilm.put_policy(params)
79
+ end
80
+ end
81
+
82
+ def create_index_template(name)
83
+ return if data_stream_exist?(name)
84
+ body = {
85
+ "index_patterns" => ["#{name}*"],
86
+ "data_stream" => {},
87
+ "template" => {
88
+ "settings" => {
89
+ "index.lifecycle.name" => "#{name}_policy"
90
+ }
91
+ }
92
+ }
93
+ params = {
94
+ name: name,
95
+ body: body
96
+ }
97
+ retry_operate(@max_retry_putting_template,
98
+ @fail_on_putting_template_retry_exceed,
99
+ @catch_transport_exception_on_retry) do
100
+ @client.indices.put_index_template(params)
101
+ end
102
+ end
103
+
104
+ def data_stream_exist?(name)
105
+ params = {
106
+ "name": name
107
+ }
108
+ begin
109
+ response = @client.indices.get_data_stream(params)
110
+ return (not response.is_a?(Elasticsearch::Transport::Transport::Errors::NotFound))
111
+ rescue Elasticsearch::Transport::Transport::Errors::NotFound => e
112
+ log.info "Specified data stream does not exist. Will be created: <#{e}>"
113
+ return false
114
+ end
115
+ end
116
+
117
+ def create_data_stream(name)
118
+ return if data_stream_exist?(name)
119
+ params = {
120
+ "name": name
121
+ }
122
+ retry_operate(@max_retry_putting_template,
123
+ @fail_on_putting_template_retry_exceed,
124
+ @catch_transport_exception_on_retry) do
125
+ @client.indices.create_data_stream(params)
126
+ end
127
+ end
128
+
129
+ def valid_data_stream_name?
130
+ lowercase_only? and
131
+ valid_characters? and
132
+ start_with_valid_characters? and
133
+ not_dots? and
134
+ @data_stream_name.bytes.size <= 255
135
+ end
136
+
137
+ def lowercase_only?
138
+ @data_stream_name.downcase == @data_stream_name
139
+ end
140
+
141
+ def valid_characters?
142
+ not (INVALID_CHARACTERS.each.any? do |v| @data_stream_name.include?(v) end)
143
+ end
144
+
145
+ def start_with_valid_characters?
146
+ not (INVALID_START_CHRACTERS.each.any? do |v| @data_stream_name.start_with?(v) end)
147
+ end
148
+
149
+ def not_dots?
150
+ not (@data_stream_name == "." or @data_stream_name == "..")
151
+ end
152
+
153
+ def client_library_version
154
+ Elasticsearch::VERSION
155
+ end
156
+
157
+ def multi_workers_ready?
158
+ true
159
+ end
160
+
161
+ def write(chunk)
162
+ data_stream_name = @data_stream_name
163
+ if @use_placeholder
164
+ data_stream_name = extract_placeholders(@data_stream_name, chunk)
165
+ unless @data_stream_names.include?(data_stream_name)
166
+ begin
167
+ create_ilm_policy(data_stream_name)
168
+ create_index_template(data_stream_name)
169
+ create_data_stream(data_stream_name)
170
+ @data_stream_names << data_stream_name
171
+ rescue => e
172
+ raise Fluent::ConfigError, "Failed to create data stream: <#{data_stream_name}> #{e.message}"
173
+ end
174
+ end
175
+ end
176
+
177
+ bulk_message = ""
178
+ headers = {
179
+ CREATE_OP => {}
180
+ }
181
+ tag = chunk.metadata.tag
182
+ chunk.msgpack_each do |time, record|
183
+ next unless record.is_a? Hash
184
+
185
+ begin
186
+ record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
187
+ bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
188
+ rescue => e
189
+ router.emit_error_event(tag, time, record, e)
190
+ end
191
+ end
192
+
193
+ params = {
194
+ index: data_stream_name,
195
+ body: bulk_message
196
+ }
197
+ begin
198
+ response = @client.bulk(params)
199
+ if response['errors']
200
+ log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
201
+ end
202
+ rescue => e
203
+ log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{e.message}"
204
+ end
205
+ end
206
+
207
+ def append_record_to_messages(op, meta, header, record, msgs)
208
+ header[CREATE_OP] = meta
209
+ msgs << @dump_proc.call(header) << BODY_DELIMITER
210
+ msgs << @dump_proc.call(record) << BODY_DELIMITER
211
+ msgs
212
+ end
213
+
214
+ def retry_stream_retryable?
215
+ @buffer.storable?
216
+ end
217
+ end
218
+ end
@@ -370,6 +370,65 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
370
370
  }
371
371
  end
372
372
 
373
+ sub_test_case 'Check client.info response' do
374
+ def create_driver(conf='', es_version=5, client_version="\"5.0\"")
375
+ # For request stub to detect compatibility.
376
+ @client_version ||= client_version
377
+ @default_elasticsearch_version ||= es_version
378
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
379
+ def detect_es_major_version
380
+ @_es_info ||= client.info
381
+ begin
382
+ unless version = @_es_info.dig("version", "number")
383
+ version = @default_elasticsearch_version
384
+ end
385
+ rescue NoMethodError => e
386
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
387
+ version = @default_elasticsearch_version
388
+ end
389
+ version.to_i
390
+ end
391
+ CODE
392
+
393
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
394
+ def client_library_version
395
+ #{@client_version}
396
+ end
397
+ CODE
398
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
399
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
400
+ if !defined?(Fluent::Plugin::Output)
401
+ def format(tag, time, record)
402
+ [time, record].to_msgpack
403
+ end
404
+ end
405
+ }.configure(conf)
406
+ end
407
+
408
+ def stub_elastic_info_bad(url="http://localhost:9200/", version="6.4.2")
409
+ body ="{\"version\":{\"number\":\"#{version}\"}}"
410
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'text/plain' } })
411
+ end
412
+
413
+ test 'handle invalid client.info' do
414
+ stub_elastic_info_bad("https://logs.fluentd.com:24225/es//", "7.7.1")
415
+ config = %{
416
+ host logs.fluentd.com
417
+ port 24225
418
+ scheme https
419
+ path /es/
420
+ user john
421
+ password doe
422
+ default_elasticsearch_version 7
423
+ scheme https
424
+ @log_level info
425
+ }
426
+ d = create_driver(config, 7, "\"7.10.1\"")
427
+ logs = d.logs
428
+ assert_logs_include(logs, /can not dig version information. Assuming Elasticsearch 7/)
429
+ end
430
+ end
431
+
373
432
  sub_test_case 'Check TLS handshake stuck warning log' do
374
433
  test 'warning TLS log' do
375
434
  config = %{
@@ -3553,6 +3612,74 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3553
3612
  assert_equal '/default_path', host2[:path]
3554
3613
  end
3555
3614
 
3615
+ class IPv6AdressStringHostsTest < self
3616
+ def test_legacy_hosts_list
3617
+ config = %{
3618
+ hosts "[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:50,host2:100,host3"
3619
+ scheme https
3620
+ path /es/
3621
+ port 123
3622
+ }
3623
+ instance = driver(config).instance
3624
+
3625
+ assert_raise(URI::InvalidURIError) do
3626
+ instance.get_connection_options[:hosts].length
3627
+ end
3628
+ end
3629
+
3630
+ def test_hosts_list
3631
+ config = %{
3632
+ hosts https://john:password@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
3633
+ path /default_path
3634
+ user default_user
3635
+ password default_password
3636
+ }
3637
+ instance = driver(config).instance
3638
+
3639
+ assert_equal 2, instance.get_connection_options[:hosts].length
3640
+ host1, host2 = instance.get_connection_options[:hosts]
3641
+
3642
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3643
+ assert_equal 443, host1[:port]
3644
+ assert_equal 'https', host1[:scheme]
3645
+ assert_equal 'john', host1[:user]
3646
+ assert_equal 'password', host1[:password]
3647
+ assert_equal '/elastic/', host1[:path]
3648
+
3649
+ assert_equal 'host2', host2[:host]
3650
+ assert_equal 'http', host2[:scheme]
3651
+ assert_equal 'default_user', host2[:user]
3652
+ assert_equal 'default_password', host2[:password]
3653
+ assert_equal '/default_path', host2[:path]
3654
+ end
3655
+
3656
+ def test_hosts_list_with_escape_placeholders
3657
+ config = %{
3658
+ hosts https://%{j+hn}:%{passw@rd}@[2404:7a80:d440:3000:192a:a292:bd7f:ca19]:443/elastic/,http://host2
3659
+ path /default_path
3660
+ user default_user
3661
+ password default_password
3662
+ }
3663
+ instance = driver(config).instance
3664
+
3665
+ assert_equal 2, instance.get_connection_options[:hosts].length
3666
+ host1, host2 = instance.get_connection_options[:hosts]
3667
+
3668
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3669
+ assert_equal 443, host1[:port]
3670
+ assert_equal 'https', host1[:scheme]
3671
+ assert_equal 'j%2Bhn', host1[:user]
3672
+ assert_equal 'passw%40rd', host1[:password]
3673
+ assert_equal '/elastic/', host1[:path]
3674
+
3675
+ assert_equal 'host2', host2[:host]
3676
+ assert_equal 'http', host2[:scheme]
3677
+ assert_equal 'default_user', host2[:user]
3678
+ assert_equal 'default_password', host2[:password]
3679
+ assert_equal '/default_path', host2[:path]
3680
+ end
3681
+ end
3682
+
3556
3683
  def test_single_host_params_and_defaults
3557
3684
  config = %{
3558
3685
  host logs.google.com
@@ -3606,6 +3733,46 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
3606
3733
  assert(ports.none? { |p| p == 9200 })
3607
3734
  end
3608
3735
 
3736
+ class IPv6AdressStringHostTest < self
3737
+ def test_single_host_params_and_defaults
3738
+ config = %{
3739
+ host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
3740
+ user john
3741
+ password doe
3742
+ }
3743
+ instance = driver(config).instance
3744
+
3745
+ assert_equal 1, instance.get_connection_options[:hosts].length
3746
+ host1 = instance.get_connection_options[:hosts][0]
3747
+
3748
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3749
+ assert_equal 9200, host1[:port]
3750
+ assert_equal 'http', host1[:scheme]
3751
+ assert_equal 'john', host1[:user]
3752
+ assert_equal 'doe', host1[:password]
3753
+ assert_equal nil, host1[:path]
3754
+ end
3755
+
3756
+ def test_single_host_params_and_defaults_with_escape_placeholders
3757
+ config = %{
3758
+ host 2404:7a80:d440:3000:192a:a292:bd7f:ca19
3759
+ user %{j+hn}
3760
+ password %{d@e}
3761
+ }
3762
+ instance = driver(config).instance
3763
+
3764
+ assert_equal 1, instance.get_connection_options[:hosts].length
3765
+ host1 = instance.get_connection_options[:hosts][0]
3766
+
3767
+ assert_equal '[2404:7a80:d440:3000:192a:a292:bd7f:ca19]', host1[:host]
3768
+ assert_equal 9200, host1[:port]
3769
+ assert_equal 'http', host1[:scheme]
3770
+ assert_equal 'j%2Bhn', host1[:user]
3771
+ assert_equal 'd%40e', host1[:password]
3772
+ assert_equal nil, host1[:path]
3773
+ end
3774
+ end
3775
+
3609
3776
  def test_password_is_required_if_specify_user
3610
3777
  config = %{
3611
3778
  user john