fluent-plugin-elasticsearch 5.2.0 → 5.2.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4c4d872817c5c5f788b40c89adfddec72dc5765666ab90046a38d5e881ba4f2d
4
- data.tar.gz: 8e83854fa3eea144ba97e44ba52885dd5f28ef3801fe293c69cf5158e7839789
3
+ metadata.gz: 425652fd5cb31671a87378fdaaf8fa911aa6bda0743fe44c53d7631d5018aebb
4
+ data.tar.gz: d3fb4697a32f8448fdd9995adeef50a96839eb15b10f984a03618044c7a9ff35
5
5
  SHA512:
6
- metadata.gz: 65eb51a5cd710eb39c54f822f3748bb6ba202ebc87d9aab563adf4befec9696279219f452cc99a1d7528e4c484e03807f195cba8695ec16cc0b3525793a872be
7
- data.tar.gz: 3ce726708a2b1900f27999518a56bbf857c28337ce1e3d82ab0acd8fbdb58d92488ddb7e9ccf5a8482768b2c4da1c261b96835f02237321b1a941c6d3b0eb2dc
6
+ metadata.gz: d8b6b4e919d79de0153707bfb84033ea403c0a88f7d76a7f1d986a45e67a0fec340762bd0a5c8776e2f6dff18b267550623fe516018a634fa3c56b9f63a3937c
7
+ data.tar.gz: 40b798377a202b4e7c0c0e6f7e84910cf7574711a83a0919c274ff5b8b9eb0dd5b90bd3e5c682ba2be9d684384c29bbbfe67d62d9307e8d0df050ee92d460c0b
data/History.md CHANGED
@@ -2,6 +2,11 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 5.2.1
6
+ - respect include\_tag\_key and tag\_key setting when using data streams (#936)
7
+ - Handle unsupported version error (#956)
8
+ - Display deprecated warning on ES dynamic plugin (#955)
9
+
5
10
  ### 5.2.0
6
11
  - Migrate to handle Elasticsearch 8 (#949)
7
12
 
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '5.2.0'
6
+ s.version = '5.2.1'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -25,3 +25,6 @@ begin
25
25
  ::SELECTOR_CLASS = Elasticsearch::Transport::Transport::Connections::Selector
26
26
  rescue LoadError
27
27
  end
28
+ unless defined?(Elasticsearch::UnsupportedProductError)
29
+ class Elasticsearch::UnsupportedProductError < StandardError; end
30
+ end
@@ -492,7 +492,11 @@ EOC
492
492
  end
493
493
 
494
494
  def detect_es_major_version
495
- @_es_info ||= client.info
495
+ begin
496
+ @_es_info ||= client.info
497
+ rescue Elasticsearch::UnsupportedProductError => e
498
+ raise Fluent::ConfigError, "Using Elasticsearch client #{client_library_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
499
+ end
496
500
  begin
497
501
  unless version = @_es_info.dig("version", "number")
498
502
  version = @default_elasticsearch_version
@@ -243,6 +243,10 @@ module Fluent::Plugin
243
243
  chunk.msgpack_each do |time, record|
244
244
  next unless record.is_a? Hash
245
245
 
246
+ if @include_tag_key
247
+ record[@tag_key] = tag
248
+ end
249
+
246
250
  begin
247
251
  record.merge!({"@timestamp" => Time.at(time).iso8601(@time_precision)})
248
252
  bulk_message = append_record_to_messages(CREATE_OP, {}, headers, record, bulk_message)
@@ -28,6 +28,8 @@ module Fluent::Plugin
28
28
  @dynamic_config[key] = value.to_s
29
29
  }
30
30
  # end eval all configs
31
+
32
+ log.warn "Elasticsearch dynamic plugin will be deprecated and removed in the future. Please consider to use normal Elasticsearch plugin"
31
33
  end
32
34
 
33
35
  def create_meta_config_map
@@ -920,6 +920,67 @@ class ElasticsearchOutputTest < Test::Unit::TestCase
920
920
  end
921
921
  end
922
922
 
923
+ class GetElasticsearchIncompatibleVersionTest < self
924
+ def create_driver(conf='', client_version="7.14")
925
+ # For request stub to detect compatibility.
926
+ @client_version ||= client_version
927
+ # Ensure original implementation existence.
928
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
929
+ def detect_es_major_version
930
+ begin
931
+ @_es_info ||= client.info
932
+ rescue Elasticsearch::UnsupportedProductError => e
933
+ raise Fluent::ConfigError, "Using Elasticsearch client #{@client_version} is not compatible for your Elasticsearch server. Please check your using elasticsearch gem version and Elasticsearch server."
934
+ end
935
+ begin
936
+ unless version = @_es_info.dig("version", "number")
937
+ version = @default_elasticsearch_version
938
+ end
939
+ rescue NoMethodError => e
940
+ log.warn "#{@_es_info} can not dig version information. Assuming Elasticsearch #{@default_elasticsearch_version}", error: e
941
+ version = @default_elasticsearch_version
942
+ end
943
+ version.to_i
944
+ end
945
+ CODE
946
+ Fluent::Plugin::ElasticsearchOutput.module_eval(<<-CODE)
947
+ def client_library_version
948
+ #{@client_version}
949
+ end
950
+ CODE
951
+ Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput).configure(conf)
952
+ end
953
+
954
+ def test_incompatible_es_version
955
+ if Gem::Version.create(::TRANSPORT_CLASS::VERSION) < Gem::Version.create("7.14.0")
956
+ omit "This test is not effective before elasticsearch 7.14"
957
+ end
958
+ config = %{
959
+ host logs.google.com
960
+ port 778
961
+ scheme https
962
+ path /es/
963
+ user john
964
+ password doe
965
+ verify_es_version_at_startup true
966
+ max_retry_get_es_version 1
967
+ }
968
+
969
+ connection_resets = 0
970
+ stub_request(:get, "https://logs.google.com:778/es//").
971
+ with(basic_auth: ['john', 'doe']) do |req|
972
+ connection_resets += 1
973
+ raise Elasticsearch::UnsupportedProductError
974
+ end
975
+
976
+ assert_raise(Fluent::ConfigError) do
977
+ create_driver(config)
978
+ end
979
+
980
+ assert_equal(1, connection_resets)
981
+ end
982
+ end
983
+
923
984
  class GetElasticsearchVersionWithFallbackTest < self
924
985
  def create_driver(conf='', client_version="\"5.0\"")
925
986
  # For request stub to detect compatibility.
@@ -19,7 +19,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
19
19
  @driver = nil
20
20
  log = Fluent::Engine.log
21
21
  log.out.logs.slice!(0, log.out.logs.length)
22
- @bulk_records = 0
22
+ @bulk_records = []
23
23
  end
24
24
 
25
25
  def elasticsearch_version
@@ -115,6 +115,17 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
115
115
  stub_request(:get, "#{url}/_index_template/#{name}").to_return(:status => [404, TRANSPORT_CLASS::Transport::Errors::NotFound])
116
116
  end
117
117
 
118
+
119
+ def push_bulk_request(req_body)
120
+ # bulk data must be pair of OP and records
121
+ # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
122
+ # {"@timestamp": ...}
123
+ ops = req_body.split("\n")
124
+ @bulk_records += ops.values_at(
125
+ * ops.each_index.select {|i| i.odd? }
126
+ ).map{ |i| JSON.parse(i) }
127
+ end
128
+
118
129
  def stub_nonexistent_template_retry?(name="foo_tpl", url="http://localhost:9200")
119
130
  stub_request(:get, "#{url}/_index_template/#{name}").
120
131
  to_return({ status: 500, body: 'Internal Server Error' }, { status: 404, body: '{}' })
@@ -125,19 +136,19 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
125
136
  # bulk data must be pair of OP and records
126
137
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
127
138
  # {"@timestamp": ...}
128
- @bulk_records += req.body.split("\n").size / 2
139
+ push_bulk_request(req.body)
129
140
  end
130
141
  stub_request(:post, "#{url}/#{ilm_name}/_bulk").with do |req|
131
142
  # bulk data must be pair of OP and records
132
143
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
133
144
  # {"@timestamp": ...}
134
- @bulk_records += req.body.split("\n").size / 2
145
+ push_bulk_request(req.body)
135
146
  end
136
147
  stub_request(:post, "#{url}/#{template_name}/_bulk").with do |req|
137
148
  # bulk data must be pair of OP and records
138
149
  # {"create": {}}\nhttp://localhost:9200/_ilm/policy/foo_ilm_bar
139
150
  # {"@timestamp": ...}
140
- @bulk_records += req.body.split("\n").size / 2
151
+ push_bulk_request(req.body)
141
152
  end
142
153
  end
143
154
 
@@ -600,7 +611,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
600
611
  driver(conf).run(default_tag: 'test') do
601
612
  driver.feed(sample_record)
602
613
  end
603
- assert_equal 1, @bulk_records
614
+ assert_equal 1, @bulk_records.length
604
615
  end
605
616
 
606
617
  def test_placeholder_params_unset
@@ -619,7 +630,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
619
630
  driver(conf).run(default_tag: 'test') do
620
631
  driver.feed(sample_record)
621
632
  end
622
- assert_equal 1, @bulk_records
633
+ assert_equal 1, @bulk_records.length
623
634
  end
624
635
 
625
636
 
@@ -645,7 +656,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
645
656
  driver(conf).run(default_tag: 'test') do
646
657
  driver.feed(sample_record)
647
658
  end
648
- assert_equal 1, @bulk_records
659
+ assert_equal 1, @bulk_records.length
649
660
  end
650
661
 
651
662
  def test_custom_record_placeholder
@@ -675,7 +686,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
675
686
  driver.feed(record)
676
687
  end
677
688
  end
678
- assert_equal keys.count, @bulk_records
689
+ assert_equal keys.count, @bulk_records.length
679
690
  end
680
691
 
681
692
  def test_bulk_insert_feed
@@ -693,7 +704,7 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
693
704
  driver(conf).run(default_tag: 'test') do
694
705
  driver.feed(sample_record)
695
706
  end
696
- assert_equal 1, @bulk_records
707
+ assert_equal 1, @bulk_records.length
697
708
  end
698
709
 
699
710
  def test_template_retry_install_fails
@@ -813,4 +824,70 @@ class ElasticsearchOutputDataStreamTest < Test::Unit::TestCase
813
824
  body: '{"policy":{"phases":{"hot":{"actions":{"rollover":{"max_age":"15d"}}}}}}',
814
825
  times: 1)
815
826
  end
827
+
828
+ def test_doesnt_add_tag_key_when_not_configured
829
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
830
+
831
+ config = %{
832
+ data_stream_name foo
833
+ data_stream_template_name foo_tpl
834
+ data_stream_ilm_name foo_ilm_policy
835
+ }
836
+
837
+ stub_default
838
+ stub_bulk_feed
839
+ driver(config)
840
+ driver.run(default_tag: 'mytag') do
841
+ driver.feed(sample_record)
842
+ end
843
+
844
+ assert_equal(1, @bulk_records.length)
845
+ assert_false(@bulk_records[0].has_key?('tag'))
846
+ end
847
+
848
+
849
+ def test_adds_tag_key_when_configured
850
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
851
+
852
+ config = %{
853
+ data_stream_name foo
854
+ data_stream_template_name foo_tpl
855
+ data_stream_ilm_name foo_ilm_policy
856
+ include_tag_key true
857
+ }
858
+
859
+ stub_default
860
+ stub_bulk_feed
861
+ driver(config)
862
+ driver.run(default_tag: 'mytag') do
863
+ driver.feed(sample_record)
864
+ end
865
+
866
+ assert_equal(1, @bulk_records.length)
867
+ assert(@bulk_records[0].has_key?('tag'))
868
+ assert_equal('mytag', @bulk_records[0]['tag'])
869
+ end
870
+
871
+ def test_adds_custom_tag_key_when_configured
872
+ omit REQUIRED_ELASTIC_MESSAGE unless data_stream_supported?
873
+
874
+ config = %{
875
+ data_stream_name foo
876
+ data_stream_template_name foo_tpl
877
+ data_stream_ilm_name foo_ilm_policy
878
+ include_tag_key true
879
+ tag_key custom_tag_key
880
+ }
881
+
882
+ stub_default
883
+ stub_bulk_feed
884
+ driver(config)
885
+ driver.run(default_tag: 'mytag') do
886
+ driver.feed(sample_record)
887
+ end
888
+
889
+ assert_equal(1, @bulk_records.length)
890
+ assert(@bulk_records[0].has_key?('custom_tag_key'))
891
+ assert_equal('mytag', @bulk_records[0]['custom_tag_key'])
892
+ end
816
893
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.2.0
4
+ version: 5.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2022-02-21 00:00:00.000000000 Z
13
+ date: 2022-03-16 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd