fluent-plugin-elasticsearch 4.0.11 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +3 -0
- data/README.md +13 -0
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_fallback_selector.rb +9 -0
- data/lib/fluent/plugin/out_elasticsearch.rb +10 -0
- data/test/plugin/test_elasticsearch_fallback_selector.rb +73 -0
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 12f88bef6f61ff3acbbd654c63a6402f7374678e78cf438f2026551246e9e0fc
|
4
|
+
data.tar.gz: 7c8ffbd9626bf8204820326ed7a3bb9157f0e3e6b406dbba51f2cb66c899b6ca
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1bc665321d8aed75d579a638d11d9254f2f3b50a4b2ce16b03e2db0ad2742fce155352c792cdeed1b244945430598d170eae546ab1e0cd4d4beedf5f0d9ea2c2
|
7
|
+
data.tar.gz: 6f54a08d03c458e7cf3dced5df532fbb5f5e2df2c9e8669c52a9bb3ebb0da231246717384653add9e585cbd2a068c00ab5be0fd12505fd04fccb7d0ca970faa3
|
data/History.md
CHANGED
data/README.md
CHANGED
@@ -78,6 +78,7 @@ Current maintainers: @cosmo0920
|
|
78
78
|
+ [Hash flattening](#hash-flattening)
|
79
79
|
+ [Generate Hash ID](#generate-hash-id)
|
80
80
|
+ [sniffer_class_name](#sniffer-class-name)
|
81
|
+
+ [selector_class_name](#selector-class-name)
|
81
82
|
+ [reload_after](#reload-after)
|
82
83
|
+ [validate_client_version](#validate-client-version)
|
83
84
|
+ [unrecoverable_error_types](#unrecoverable-error-types)
|
@@ -987,6 +988,18 @@ sniffer_class_name Fluent::Plugin::ElasticsearchSimpleSniffer
|
|
987
988
|
reload_after 100
|
988
989
|
```
|
989
990
|
|
991
|
+
### Selector Class Name
|
992
|
+
|
993
|
+
The default selector used by the `Elasticsearch::Transport` class works well when Fluentd should round robin and random selector cases. This doesn't work well when Fluentd should fallback behavior.
|
994
|
+
The parameter `selector_class_name` gives you the ability to provide your own Selector class to implement whatever selection nodes logic you require.
|
995
|
+
|
996
|
+
The below configuration is using plugin built-in `ElasticseatchFallbackSelector`:
|
997
|
+
|
998
|
+
```
|
999
|
+
hosts exhausted-host:9201,normal-host:9200
|
1000
|
+
selector_class_name "Fluent::Plugin::ElasticseatchFallbackSelector"
|
1001
|
+
```
|
1002
|
+
|
990
1003
|
#### Tips
|
991
1004
|
|
992
1005
|
The included sniffer class does not required `out_elasticsearch`.
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '4.0
|
6
|
+
s.version = '4.1.0'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -25,6 +25,7 @@ require_relative 'elasticsearch_error_handler'
|
|
25
25
|
require_relative 'elasticsearch_index_template'
|
26
26
|
require_relative 'elasticsearch_index_lifecycle_management'
|
27
27
|
require_relative 'elasticsearch_tls'
|
28
|
+
require_relative 'elasticsearch_fallback_selector'
|
28
29
|
begin
|
29
30
|
require_relative 'oj_serializer'
|
30
31
|
rescue LoadError
|
@@ -137,6 +138,7 @@ EOC
|
|
137
138
|
config_param :with_transporter_log, :bool, :default => false
|
138
139
|
config_param :emit_error_for_missing_id, :bool, :default => false
|
139
140
|
config_param :sniffer_class_name, :string, :default => nil
|
141
|
+
config_param :selector_class_name, :string, :default => nil
|
140
142
|
config_param :reload_after, :integer, :default => DEFAULT_RELOAD_AFTER
|
141
143
|
config_param :content_type, :enum, list: [:"application/json", :"application/x-ndjson"], :default => :"application/json",
|
142
144
|
:deprecated => <<EOC
|
@@ -300,6 +302,13 @@ EOC
|
|
300
302
|
raise Fluent::ConfigError, "Could not load sniffer class #{@sniffer_class_name}: #{ex}"
|
301
303
|
end
|
302
304
|
|
305
|
+
@selector_class = nil
|
306
|
+
begin
|
307
|
+
@selector_class = Object.const_get(@selector_class_name) if @selector_class_name
|
308
|
+
rescue Exception => ex
|
309
|
+
raise Fluent::ConfigError, "Could not load selector class #{@selector_class_name}: #{ex}"
|
310
|
+
end
|
311
|
+
|
303
312
|
@last_seen_major_version = if major_version = handle_last_seen_es_major_version
|
304
313
|
major_version
|
305
314
|
else
|
@@ -566,6 +575,7 @@ EOC
|
|
566
575
|
},
|
567
576
|
sniffer_class: @sniffer_class,
|
568
577
|
serializer_class: @serializer_class,
|
578
|
+
selector_class: @selector_class,
|
569
579
|
compression: compress_connection,
|
570
580
|
}), &adapter_conf)
|
571
581
|
Elasticsearch::Client.new transport: transport
|
@@ -0,0 +1,73 @@
|
|
1
|
+
require_relative '../helper'
|
2
|
+
require 'fluent/test/driver/output'
|
3
|
+
require 'fluent/plugin/out_elasticsearch'
|
4
|
+
|
5
|
+
class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
6
|
+
attr_accessor :index_cmds
|
7
|
+
|
8
|
+
def setup
|
9
|
+
Fluent::Test.setup
|
10
|
+
@driver = nil
|
11
|
+
log = Fluent::Engine.log
|
12
|
+
log.out.logs.slice!(0, log.out.logs.length)
|
13
|
+
end
|
14
|
+
|
15
|
+
def stub_elastic(url="http://localhost:9200/_bulk")
|
16
|
+
stub_request(:post, url).with do |req|
|
17
|
+
@index_cmds = req.body.split("\n").map {|r| JSON.parse(r) }
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
|
22
|
+
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
23
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
24
|
+
end
|
25
|
+
|
26
|
+
def stub_elastic_info_not_found(url="http://localhost:9200/", version="6.4.2")
|
27
|
+
stub_request(:get, url).to_return(:status => [404, "Not Found"])
|
28
|
+
end
|
29
|
+
|
30
|
+
def stub_elastic_info_unavailable(url="http://localhost:9200/", version="6.4.2")
|
31
|
+
stub_request(:get, url).to_return(:status => [503, "Service Unavailable"])
|
32
|
+
end
|
33
|
+
|
34
|
+
def sample_record(content={})
|
35
|
+
{'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'routing_id' => 'routing'}.merge(content)
|
36
|
+
end
|
37
|
+
|
38
|
+
def driver(conf='')
|
39
|
+
@driver ||= Fluent::Test::Driver::Output.new(Fluent::Plugin::ElasticsearchOutput) {
|
40
|
+
# v0.12's test driver assume format definition. This simulates ObjectBufferedOutput format
|
41
|
+
if !defined?(Fluent::Plugin::Output)
|
42
|
+
def format(tag, time, record)
|
43
|
+
[time, record].to_msgpack
|
44
|
+
end
|
45
|
+
end
|
46
|
+
}.configure(conf)
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_fallback_on_info
|
50
|
+
stub_elastic_info_not_found("http://localhost:9202/")
|
51
|
+
stub_elastic_info_unavailable("http://localhost:9201/")
|
52
|
+
stub_elastic_info
|
53
|
+
stub_elastic
|
54
|
+
config = %[
|
55
|
+
hosts localhost:9202,localhost:9201,localhost:9200
|
56
|
+
selector_class_name Fluent::Plugin::ElasticseatchFallbackSelector
|
57
|
+
@log_level debug
|
58
|
+
with_transporter_log true
|
59
|
+
reload_connections true
|
60
|
+
reload_after 10
|
61
|
+
]
|
62
|
+
assert_raise(Elasticsearch::Transport::Transport::Errors::NotFound) do
|
63
|
+
driver(config)
|
64
|
+
end
|
65
|
+
driver.run(default_tag: 'test') do
|
66
|
+
driver.feed(sample_record)
|
67
|
+
end
|
68
|
+
assert_equal(2, index_cmds.length)
|
69
|
+
assert_equal("fluentd", index_cmds.first['index']['_index'])
|
70
|
+
end
|
71
|
+
|
72
|
+
# TODO: on feed phase test case
|
73
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.0
|
4
|
+
version: 4.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2020-07-
|
13
|
+
date: 2020-07-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: fluentd
|
@@ -160,6 +160,7 @@ files:
|
|
160
160
|
- lib/fluent/plugin/elasticsearch_constants.rb
|
161
161
|
- lib/fluent/plugin/elasticsearch_error.rb
|
162
162
|
- lib/fluent/plugin/elasticsearch_error_handler.rb
|
163
|
+
- lib/fluent/plugin/elasticsearch_fallback_selector.rb
|
163
164
|
- lib/fluent/plugin/elasticsearch_index_lifecycle_management.rb
|
164
165
|
- lib/fluent/plugin/elasticsearch_index_template.rb
|
165
166
|
- lib/fluent/plugin/elasticsearch_simple_sniffer.rb
|
@@ -172,6 +173,7 @@ files:
|
|
172
173
|
- test/helper.rb
|
173
174
|
- test/plugin/test_alias_template.json
|
174
175
|
- test/plugin/test_elasticsearch_error_handler.rb
|
176
|
+
- test/plugin/test_elasticsearch_fallback_selector.rb
|
175
177
|
- test/plugin/test_elasticsearch_index_lifecycle_management.rb
|
176
178
|
- test/plugin/test_elasticsearch_tls.rb
|
177
179
|
- test/plugin/test_filter_elasticsearch_genid.rb
|
@@ -209,6 +211,7 @@ test_files:
|
|
209
211
|
- test/helper.rb
|
210
212
|
- test/plugin/test_alias_template.json
|
211
213
|
- test/plugin/test_elasticsearch_error_handler.rb
|
214
|
+
- test/plugin/test_elasticsearch_fallback_selector.rb
|
212
215
|
- test/plugin/test_elasticsearch_index_lifecycle_management.rb
|
213
216
|
- test/plugin/test_elasticsearch_tls.rb
|
214
217
|
- test/plugin/test_filter_elasticsearch_genid.rb
|