fluent-plugin-elasticsearch 4.0.11 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c90e0b347a5c90ec23231df927384dadcc4b91b813a4e2c11df23539dfe05628
4
- data.tar.gz: d6e296c8006870a5f85bd0b97294c69d797fc3f40b574ede7645fcd808ae6356
3
+ metadata.gz: 12f88bef6f61ff3acbbd654c63a6402f7374678e78cf438f2026551246e9e0fc
4
+ data.tar.gz: 7c8ffbd9626bf8204820326ed7a3bb9157f0e3e6b406dbba51f2cb66c899b6ca
5
5
  SHA512:
6
- metadata.gz: 8466c4e3a13ef5c5ca1bda536a0baf7044fdaa0bc3ae1e1c7e6d0e6cf3dc7596d2b68e3f61ed2fb788f8ef2d8392939b8f63fd6aedc53e5818725bc6e0910aa6
7
- data.tar.gz: b0572e50952ce49e44b280e94764d0d204d482e8e89cc2124e9a3025c1bd4cf7debe5625ad141337e5f81e6b6816286ee81f931f6b782f0a8cffcbfa0ed12dc3
6
+ metadata.gz: 1bc665321d8aed75d579a638d11d9254f2f3b50a4b2ce16b03e2db0ad2742fce155352c792cdeed1b244945430598d170eae546ab1e0cd4d4beedf5f0d9ea2c2
7
+ data.tar.gz: 6f54a08d03c458e7cf3dced5df532fbb5f5e2df2c9e8669c52a9bb3ebb0da231246717384653add9e585cbd2a068c00ab5be0fd12505fd04fccb7d0ca970faa3
data/History.md CHANGED
@@ -2,6 +2,9 @@
2
2
 
3
3
  ### [Unreleased]
4
4
 
5
+ ### 4.1.0
6
+ - Implement Fallback selector and configurable selector class (#782)
7
+
5
8
  ### 4.0.11
6
9
  - Add custom and time placeholders combination testcase for ILM (#781)
7
10
  - Really support ILM on `logstash_format` enabled environment (#779)
data/README.md CHANGED
@@ -78,6 +78,7 @@ Current maintainers: @cosmo0920
78
78
  + [Hash flattening](#hash-flattening)
79
79
  + [Generate Hash ID](#generate-hash-id)
80
80
  + [sniffer_class_name](#sniffer-class-name)
81
+ + [selector_class_name](#selector-class-name)
81
82
  + [reload_after](#reload-after)
82
83
  + [validate_client_version](#validate-client-version)
83
84
  + [unrecoverable_error_types](#unrecoverable-error-types)
@@ -987,6 +988,18 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
987
988
  reload_after 100
988
989
  ```
989
990
 
991
+ ### Selector Class Name
992
+
993
+ The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should round robin and random selector cases. This doesn't work well when Fluentd should fallback behavior.
994
+ The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
995
+
996
+ The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
997
+
998
+ ```
999
+ hosts exhausted-host:9201,normal-host:9200
1000
+ selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
1001
+ ```
1002
+
990
1003
  #### Tips
991
1004
 
992
1005
  The included sniffer class does not required `out_elasticsearch`.
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '4.0.11'
6
+ s.version = '4.1.0'
7
7
  s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
9
9
  s.description = %q{Elasticsearch output plugin for Fluent event collector}
@@ -0,0 +1,9 @@
1
+ require 'elasticsearch/transport/transport/connections/selector'
2
+
3
+ class Fluent::Plugin::ElasticseatchFallbackSelector
4
+ include Elasticsearch::Transport::Transport::Connections::Selector::Base
5
+
6
+ def select(options={})
7
+ connections.first
8
+ end
9
+ end
@@ -25,6 +25,7 @@ require_relative 'elasticsearch_error_handler'
25
25
  require_relative 'elasticsearch_index_template'
26
26
  require_relative 'elasticsearch_index_lifecycle_management'
27
27
  require_relative 'elasticsearch_tls'
28
+ require_relative 'elasticsearch_fallback_selector'
28
29
  begin
29
30
  require_relative 'oj_serializer'
30
31
  rescue LoadError
@@ -137,6 +138,7 @@ EOC
137
138
  config_param :with_transporter_log, :bool, :default => false
138
139
  config_param :emit_error_for_missing_id, :bool, :default => false
139
140
  config_param :sniffer_class_name, :string, :default => nil
141
+ config_param :selector_class_name, :string, :default => nil
140
142
  config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
141
143
  config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
142
144
  :deprecated => <<EOC
@@ -300,6 +302,13 @@ EOC
300
302
  raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
301
303
  end
302
304
 
305
+ @selector_class = nil
306
+ begin
307
+ @selector_class = Object.const_get(@selector_class_name) if @selector_class_name
308
+ rescue Exception => ex
309
+ raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
310
+ end
311
+
303
312
  @last_seen_major_version = if major_version = handle_last_seen_es_major_version
304
313
  major_version
305
314
  else
@@ -566,6 +575,7 @@ EOC
566
575
  },
567
576
  sniffer_class: @sniffer_class,
568
577
  serializer_class: @serializer_class,
578
+ selector_class: @selector_class,
569
579
  compression: compress_connection,
570
580
  }), &adapter_conf)
571
581
  Elasticsearch::Client.new transport: transport
@@ -0,0 +1,73 @@
1
+ require_relative '../helper'
2
+ require 'fluent/test/driver/output'
3
+ require 'fluent/plugin/out_elasticsearch'
4
+
5
+ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
6
+ attr_accessor :index_cmds
7
+
8
+ def setup
9
+ Fluent::Test.setup
10
+ @driver = nil
11
+ log = Fluent::Engine.log
12
+ log.out.logs.slice!(0, log.out.logs.length)
13
+ end
14
+
15
+ def stub_elastic(url="http://localhost:9200/_bulk")
16
+ stub_request(:post, url).with do |req|
17
+ @index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
18
+ end
19
+ end
20
+
21
+ def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
22
+ body ="{\"version\":{\"number\":\"#{version}\"}}"
23
+ stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
24
+ end
25
+
26
+ def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
27
+ stub_request(:get, url).to_return(:status => [404, "Not Found"])
28
+ end
29
+
30
+ def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
31
+ stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
32
+ end
33
+
34
+ def sample_record(content={})
35
+ {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
36
+ end
37
+
38
+ def driver(conf='')
39
+ @driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
40
+ # v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
41
+ if !defined?(Fluent::Plugin::Output)
42
+ def format(tag, time, record)
43
+ [time, record].to_msgpack
44
+ end
45
+ end
46
+ }.configure(conf)
47
+ end
48
+
49
+ def test_fallback_on_info
50
+ stub_elastic_info_not_found("http://localhost:9202/")
51
+ stub_elastic_info_unavailable("http://localhost:9201/")
52
+ stub_elastic_info
53
+ stub_elastic
54
+ config = %[
55
+ hosts localhost:9202,localhost:9201,localhost:9200
56
+ selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
57
+ @log_level debug
58
+ with_transporter_log true
59
+ reload_connections true
60
+ reload_after 10
61
+ ]
62
+ assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
63
+ driver(config)
64
+ end
65
+ driver.run(default_tag: 'test') do
66
+ driver.feed(sample_record)
67
+ end
68
+ assert_equal(2, index_cmds.length)
69
+ assert_equal("fluentd", index_cmds.first['index']['_index'])
70
+ end
71
+
72
+ # TODO: on feed phase test case
73
+ end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.0.11
4
+ version: 4.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2020-07-09 00:00:00.000000000 Z
13
+ date: 2020-07-10 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -160,6 +160,7 @@ files:
160
160
  - lib/fluent/plugin/elasticsearch_constants.rb
161
161
  - lib/fluent/plugin/elasticsearch_error.rb
162
162
  - lib/fluent/plugin/elasticsearch_error_handler.rb
163
+ - lib/fluent/plugin/elasticsearch_fallback_selector.rb
163
164
  - lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
164
165
  - lib/fluent/plugin/elasticsearch_index_template.rb
165
166
  - lib/fluent/plugin/elasticsearch_simple_sniffer.rb
@@ -172,6 +173,7 @@ files:
172
173
  - test/helper.rb
173
174
  - test/plugin/test_alias_template.json
174
175
  - test/plugin/test_elasticsearch_error_handler.rb
176
+ - test/plugin/test_elasticsearch_fallback_selector.rb
175
177
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
176
178
  - test/plugin/test_elasticsearch_tls.rb
177
179
  - test/plugin/test_filter_elasticsearch_genid.rb
@@ -209,6 +211,7 @@ test_files:
209
211
  - test/helper.rb
210
212
  - test/plugin/test_alias_template.json
211
213
  - test/plugin/test_elasticsearch_error_handler.rb
214
+ - test/plugin/test_elasticsearch_fallback_selector.rb
212
215
  - test/plugin/test_elasticsearch_index_lifecycle_management.rb
213
216
  - test/plugin/test_elasticsearch_tls.rb
214
217
  - test/plugin/test_filter_elasticsearch_genid.rb