fluent-plugin-elasticsearch 5.0.5 → 5.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +15 -0
- data/README.md +14 -2
- data/fluent-plugin-elasticsearch.gemspec +1 -1
- data/lib/fluent/plugin/elasticsearch_index_template.rb +13 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +2 -2
- data/lib/fluent/plugin/out_elasticsearch_data_stream.rb +84 -49
- data/test/plugin/test_elasticsearch_fallback_selector.rb +1 -1
- data/test/plugin/test_elasticsearch_index_lifecycle_management.rb +10 -0
- data/test/plugin/test_in_elasticsearch.rb +12 -0
- data/test/plugin/test_out_elasticsearch.rb +224 -17
- data/test/plugin/test_out_elasticsearch_data_stream.rb +423 -98
- data/test/plugin/test_out_elasticsearch_dynamic.rb +100 -5
- metadata +3 -5
- data/.travis.yml +0 -40
- data/appveyor.yml +0 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6ad91e234f95a4d2536ac1de2630c6af572e249f5b1ff9a40a83d427f8901413
|
4
|
+
data.tar.gz: 91fbf4c3d8fe17e05094f5fa991ec08ee5491a528d39d3d048678adfc30f1192
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 531ba1b0b8824894925a502eb310ea7b9288314dbe940bcee1190f60abea5f9d398c1b815ad24f47cc213342e93d87e1ddc74172f7a8ca931ea59da5527f5060
|
7
|
+
data.tar.gz: c8820ca2ae83a2b9082b35f806669cd5783b3ee6cb3e4baa960a937cb2a2b5554b6665bfe9dea666bfd9b5b6b2f462fc7147d9f28ccf4d2281ba4620de0f4acf
|
data/History.md
CHANGED
@@ -2,6 +2,21 @@
|
|
2
2
|
|
3
3
|
### [Unreleased]
|
4
4
|
|
5
|
+
### 5.1.3
|
6
|
+
- fixing execution order for dynamic data stream creation (#928)
|
7
|
+
|
8
|
+
### 5.1.2
|
9
|
+
- Fix default values of datastream parameters (#926)
|
10
|
+
|
11
|
+
### 5.1.1
|
12
|
+
- Report appropriate error for data_stream parameters (#922)
|
13
|
+
- Add ILM and template parameters for data streams (#920)
|
14
|
+
- Support Buffer in Data Stream Output (#917)
|
15
|
+
|
16
|
+
### 5.1.0
|
17
|
+
- Correct default target bytes value (#914)
|
18
|
+
- Handle elasticsearch-ruby 7.14 properly (#913)
|
19
|
+
|
5
20
|
### 5.0.5
|
6
21
|
- Drop json_parse_exception messages for bulk failures (#900)
|
7
22
|
- GitHub Actions: Drop Ruby 2.5 due to EOL (#894)
|
data/README.md
CHANGED
@@ -11,7 +11,7 @@ Send your logs to Elasticsearch (and search them with Kibana maybe?)
|
|
11
11
|
|
12
12
|
Note: For Amazon Elasticsearch Service please consider using [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service)
|
13
13
|
|
14
|
-
Current maintainers: @cosmo0920
|
14
|
+
Current maintainers: [Hiroshi Hatake | @cosmo0920](https://github.com/cosmo0920), [Kentaro Hayashi | @kenhys](https://github.com/kenhys)
|
15
15
|
|
16
16
|
* [Installation](#installation)
|
17
17
|
* [Usage](#usage)
|
@@ -1521,7 +1521,7 @@ You can enable this feature by specifying `@type elasticsearch_data_stream`.
|
|
1521
1521
|
data_stream_name test
|
1522
1522
|
```
|
1523
1523
|
|
1524
|
-
When `@type elasticsearch_data_stream` is used, ILM default policy is set to the specified data stream.
|
1524
|
+
When `@type elasticsearch_data_stream` is used, unless specified with `data_stream_ilm_name` and `data_stream_template_name`, ILM default policy is set to the specified data stream.
|
1525
1525
|
Then, the matching index template is also created automatically.
|
1526
1526
|
|
1527
1527
|
### data_stream_name
|
@@ -1529,6 +1529,18 @@ Then, the matching index template is also created automatically.
|
|
1529
1529
|
You can specify Elasticsearch data stream name by this parameter.
|
1530
1530
|
This parameter is mandatory for `elasticsearch_data_stream`.
|
1531
1531
|
|
1532
|
+
### data_stream_template_name
|
1533
|
+
|
1534
|
+
You can specify an existing matching index template for the data stream. If not present, it creates a new matching index template.
|
1535
|
+
|
1536
|
+
Default value is `data_stream_name`.
|
1537
|
+
|
1538
|
+
### data_stream_ilm_name
|
1539
|
+
|
1540
|
+
You can specify the name of an existing ILM policy, which will be applied to the data stream. If not present, it creates a new ILM default policy (unless `data_stream_template_name` is defined, in that case the ILM will be set to the one specified in the matching index template).
|
1541
|
+
|
1542
|
+
Default value is `data_stream_name`.
|
1543
|
+
|
1532
1544
|
There are some limitations about naming rule.
|
1533
1545
|
|
1534
1546
|
In more detail, please refer to the [Path parameters](https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html#indices-create-data-stream-api-path-params).
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '5.
|
6
|
+
s.version = '5.1.3'
|
7
7
|
s.authors = ['diogo', 'pitr', 'Hiroshi Hatake']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com', 'cosmo0920.wp@gmail.com']
|
9
9
|
s.description = %q{Elasticsearch output plugin for Fluent event collector}
|
@@ -32,13 +32,25 @@ module Fluent::ElasticsearchIndexTemplate
|
|
32
32
|
return false
|
33
33
|
end
|
34
34
|
|
35
|
+
def host_unreachable_exceptions
|
36
|
+
if Gem::Version.new(::Elasticsearch::Transport::VERSION) >= Gem::Version.new("7.14.0")
|
37
|
+
# elasticsearch-ruby 7.14.0's elasticsearch-transport does not extends
|
38
|
+
# Elasticsearch class on Transport.
|
39
|
+
# This is why #host_unreachable_exceptions is not callable directly
|
40
|
+
# via transport (not transport's transport instance accessor) any more.
|
41
|
+
client.transport.transport.host_unreachable_exceptions
|
42
|
+
else
|
43
|
+
client.transport.host_unreachable_exceptions
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
35
47
|
def retry_operate(max_retries, fail_on_retry_exceed = true, catch_trasport_exceptions = true)
|
36
48
|
return unless block_given?
|
37
49
|
retries = 0
|
38
50
|
transport_errors = Elasticsearch::Transport::Transport::Errors.constants.map{ |c| Elasticsearch::Transport::Transport::Errors.const_get c } if catch_trasport_exceptions
|
39
51
|
begin
|
40
52
|
yield
|
41
|
-
rescue *
|
53
|
+
rescue *host_unreachable_exceptions, *transport_errors, Timeout::Error => e
|
42
54
|
@_es = nil
|
43
55
|
@_es_info = nil
|
44
56
|
if retries < max_retries
|
@@ -72,7 +72,7 @@ module Fluent::Plugin
|
|
72
72
|
DEFAULT_TYPE_NAME_ES_7x = "_doc".freeze
|
73
73
|
DEFAULT_TYPE_NAME = "fluentd".freeze
|
74
74
|
DEFAULT_RELOAD_AFTER = -1
|
75
|
-
|
75
|
+
DEFAULT_TARGET_BULK_BYTES = -1
|
76
76
|
DEFAULT_POLICY_ID = "logstash-policy"
|
77
77
|
|
78
78
|
config_param :host, :string, :default => 'localhost'
|
@@ -166,7 +166,7 @@ EOC
|
|
166
166
|
config_param :suppress_doc_wrap, :bool, :default => false
|
167
167
|
config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
|
168
168
|
config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
|
169
|
-
config_param :bulk_message_request_threshold, :size, :default =>
|
169
|
+
config_param :bulk_message_request_threshold, :size, :default => DEFAULT_TARGET_BULK_BYTES
|
170
170
|
config_param :compression_level, :enum, list: [:no_compression, :best_speed, :best_compression, :default_compression], :default => :no_compression
|
171
171
|
config_param :enable_ilm, :bool, :default => false
|
172
172
|
config_param :ilm_policy_id, :string, :default => DEFAULT_POLICY_ID
|
@@ -1,3 +1,4 @@
|
|
1
|
+
|
1
2
|
require_relative 'out_elasticsearch'
|
2
3
|
|
3
4
|
module Fluent::Plugin
|
@@ -8,6 +9,8 @@ module Fluent::Plugin
|
|
8
9
|
helpers :event_emitter
|
9
10
|
|
10
11
|
config_param :data_stream_name, :string
|
12
|
+
config_param :data_stream_ilm_name, :string, :default => nil
|
13
|
+
config_param :data_stream_template_name, :string, :default => nil
|
11
14
|
# Elasticsearch 7.9 or later always support new style of index template.
|
12
15
|
config_set_default :use_legacy_template, false
|
13
16
|
|
@@ -24,9 +27,12 @@ module Fluent::Plugin
|
|
24
27
|
raise Fluent::ConfigError, "'elasticsearch/api', 'elasticsearch/xpack' are required for <@elasticsearch_data_stream>."
|
25
28
|
end
|
26
29
|
|
30
|
+
@data_stream_ilm_name = "#{@data_stream_name}_policy" if @data_stream_ilm_name.nil?
|
31
|
+
@data_stream_template_name = "#{@data_stream_name}_template" if @data_stream_template_name.nil?
|
32
|
+
|
27
33
|
# ref. https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-data-stream.html
|
28
34
|
unless placeholder?(:data_stream_name_placeholder, @data_stream_name)
|
29
|
-
|
35
|
+
validate_data_stream_parameters
|
30
36
|
else
|
31
37
|
@use_placeholder = true
|
32
38
|
@data_stream_names = []
|
@@ -36,8 +42,8 @@ module Fluent::Plugin
|
|
36
42
|
unless @use_placeholder
|
37
43
|
begin
|
38
44
|
@data_stream_names = [@data_stream_name]
|
39
|
-
create_ilm_policy(@data_stream_name)
|
40
|
-
create_index_template(@data_stream_name)
|
45
|
+
create_ilm_policy(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
|
46
|
+
create_index_template(@data_stream_name, @data_stream_template_name, @data_stream_ilm_name, @host)
|
41
47
|
create_data_stream(@data_stream_name)
|
42
48
|
rescue => e
|
43
49
|
raise Fluent::ConfigError, "Failed to create data stream: <#{@data_stream_name}> #{e.message}"
|
@@ -45,31 +51,35 @@ module Fluent::Plugin
|
|
45
51
|
end
|
46
52
|
end
|
47
53
|
|
48
|
-
def
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
54
|
+
def validate_data_stream_parameters
|
55
|
+
{"data_stream_name" => @data_stream_name,
|
56
|
+
"data_stream_template_name"=> @data_stream_template_name,
|
57
|
+
"data_stream_ilm_name" => @data_stream_ilm_name}.each do |parameter, value|
|
58
|
+
unless valid_data_stream_parameters?(value)
|
59
|
+
unless start_with_valid_characters?(value)
|
60
|
+
if not_dots?(value)
|
61
|
+
raise Fluent::ConfigError, "'#{parameter}' must not start with #{INVALID_START_CHRACTERS.join(",")}: <#{value}>"
|
62
|
+
else
|
63
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be . or ..: <#{value}>"
|
64
|
+
end
|
65
|
+
end
|
66
|
+
unless valid_characters?(value)
|
67
|
+
raise Fluent::ConfigError, "'#{parameter}' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{value}>"
|
68
|
+
end
|
69
|
+
unless lowercase_only?(value)
|
70
|
+
raise Fluent::ConfigError, "'#{parameter}' must be lowercase only: <#{value}>"
|
71
|
+
end
|
72
|
+
if value.bytes.size > 255
|
73
|
+
raise Fluent::ConfigError, "'#{parameter}' must not be longer than 255 bytes: <#{value}>"
|
55
74
|
end
|
56
|
-
end
|
57
|
-
unless valid_characters?
|
58
|
-
raise Fluent::ConfigError, "'data_stream_name' must not contain invalid characters #{INVALID_CHARACTERS.join(",")}: <#{@data_stream_name}>"
|
59
|
-
end
|
60
|
-
unless lowercase_only?
|
61
|
-
raise Fluent::ConfigError, "'data_stream_name' must be lowercase only: <#{@data_stream_name}>"
|
62
|
-
end
|
63
|
-
if @data_stream_name.bytes.size > 255
|
64
|
-
raise Fluent::ConfigError, "'data_stream_name' must not be longer than 255 bytes: <#{@data_stream_name}>"
|
65
75
|
end
|
66
76
|
end
|
67
77
|
end
|
68
78
|
|
69
|
-
def create_ilm_policy(
|
70
|
-
return if data_stream_exist?(
|
79
|
+
def create_ilm_policy(datastream_name, template_name, ilm_name, host)
|
80
|
+
return if data_stream_exist?(datastream_name) or template_exists?(template_name, host) or ilm_policy_exists?(ilm_name)
|
71
81
|
params = {
|
72
|
-
policy_id: "#{
|
82
|
+
policy_id: "#{ilm_name}_policy",
|
73
83
|
body: File.read(File.join(File.dirname(__FILE__), "default-ilm-policy.json"))
|
74
84
|
}
|
75
85
|
retry_operate(@max_retry_putting_template,
|
@@ -79,19 +89,19 @@ module Fluent::Plugin
|
|
79
89
|
end
|
80
90
|
end
|
81
91
|
|
82
|
-
def create_index_template(
|
83
|
-
return if data_stream_exist?(
|
92
|
+
def create_index_template(datastream_name, template_name, ilm_name, host)
|
93
|
+
return if data_stream_exist?(datastream_name) or template_exists?(template_name, host)
|
84
94
|
body = {
|
85
|
-
"index_patterns" => ["#{
|
95
|
+
"index_patterns" => ["#{datastream_name}*"],
|
86
96
|
"data_stream" => {},
|
87
97
|
"template" => {
|
88
98
|
"settings" => {
|
89
|
-
"index.lifecycle.name" => "#{
|
99
|
+
"index.lifecycle.name" => "#{ilm_name}"
|
90
100
|
}
|
91
101
|
}
|
92
102
|
}
|
93
103
|
params = {
|
94
|
-
name:
|
104
|
+
name: template_name,
|
95
105
|
body: body
|
96
106
|
}
|
97
107
|
retry_operate(@max_retry_putting_template,
|
@@ -101,9 +111,9 @@ module Fluent::Plugin
|
|
101
111
|
end
|
102
112
|
end
|
103
113
|
|
104
|
-
def data_stream_exist?(
|
114
|
+
def data_stream_exist?(datastream_name)
|
105
115
|
params = {
|
106
|
-
|
116
|
+
name: datastream_name
|
107
117
|
}
|
108
118
|
begin
|
109
119
|
response = @client.indices.get_data_stream(params)
|
@@ -114,10 +124,10 @@ module Fluent::Plugin
|
|
114
124
|
end
|
115
125
|
end
|
116
126
|
|
117
|
-
def create_data_stream(
|
118
|
-
return if data_stream_exist?(
|
127
|
+
def create_data_stream(datastream_name)
|
128
|
+
return if data_stream_exist?(datastream_name)
|
119
129
|
params = {
|
120
|
-
|
130
|
+
name: datastream_name
|
121
131
|
}
|
122
132
|
retry_operate(@max_retry_putting_template,
|
123
133
|
@fail_on_putting_template_retry_exceed,
|
@@ -126,28 +136,48 @@ module Fluent::Plugin
|
|
126
136
|
end
|
127
137
|
end
|
128
138
|
|
129
|
-
def
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
139
|
+
def ilm_policy_exists?(policy_id)
|
140
|
+
begin
|
141
|
+
@client.ilm.get_policy(policy_id: policy_id)
|
142
|
+
true
|
143
|
+
rescue
|
144
|
+
false
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
def template_exists?(name, host = nil)
|
149
|
+
if @use_legacy_template
|
150
|
+
client(host).indices.get_template(:name => name)
|
151
|
+
else
|
152
|
+
client(host).indices.get_index_template(:name => name)
|
153
|
+
end
|
154
|
+
return true
|
155
|
+
rescue Elasticsearch::Transport::Transport::Errors::NotFound
|
156
|
+
return false
|
157
|
+
end
|
158
|
+
|
159
|
+
def valid_data_stream_parameters?(data_stream_parameter)
|
160
|
+
lowercase_only?(data_stream_parameter) and
|
161
|
+
valid_characters?(data_stream_parameter) and
|
162
|
+
start_with_valid_characters?(data_stream_parameter) and
|
163
|
+
not_dots?(data_stream_parameter) and
|
164
|
+
data_stream_parameter.bytes.size <= 255
|
135
165
|
end
|
136
166
|
|
137
|
-
def lowercase_only?
|
138
|
-
|
167
|
+
def lowercase_only?(data_stream_parameter)
|
168
|
+
data_stream_parameter.downcase == data_stream_parameter
|
139
169
|
end
|
140
170
|
|
141
|
-
def valid_characters?
|
142
|
-
not (INVALID_CHARACTERS.each.any? do |v|
|
171
|
+
def valid_characters?(data_stream_parameter)
|
172
|
+
not (INVALID_CHARACTERS.each.any? do |v| data_stream_parameter.include?(v) end)
|
143
173
|
end
|
144
174
|
|
145
|
-
def start_with_valid_characters?
|
146
|
-
not (INVALID_START_CHRACTERS.each.any? do |v|
|
175
|
+
def start_with_valid_characters?(data_stream_parameter)
|
176
|
+
not (INVALID_START_CHRACTERS.each.any? do |v| data_stream_parameter.start_with?(v) end)
|
147
177
|
end
|
148
178
|
|
149
|
-
def not_dots?
|
150
|
-
not (
|
179
|
+
def not_dots?(data_stream_parameter)
|
180
|
+
not (data_stream_parameter == "." or data_stream_parameter == "..")
|
151
181
|
end
|
152
182
|
|
153
183
|
def client_library_version
|
@@ -160,12 +190,17 @@ module Fluent::Plugin
|
|
160
190
|
|
161
191
|
def write(chunk)
|
162
192
|
data_stream_name = @data_stream_name
|
193
|
+
data_stream_template_name = @data_stream_template_name
|
194
|
+
data_stream_ilm_name = @data_stream_ilm_name
|
195
|
+
host = @host
|
163
196
|
if @use_placeholder
|
164
197
|
data_stream_name = extract_placeholders(@data_stream_name, chunk)
|
198
|
+
data_stream_template_name = extract_placeholders(@data_stream_template_name, chunk)
|
199
|
+
data_stream_ilm_name = extract_placeholders(@data_stream_ilm_name, chunk)
|
165
200
|
unless @data_stream_names.include?(data_stream_name)
|
166
201
|
begin
|
167
|
-
create_ilm_policy(data_stream_name)
|
168
|
-
create_index_template(data_stream_name)
|
202
|
+
create_ilm_policy(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
203
|
+
create_index_template(data_stream_name, data_stream_template_name, data_stream_ilm_name, host)
|
169
204
|
create_data_stream(data_stream_name)
|
170
205
|
@data_stream_names << data_stream_name
|
171
206
|
rescue => e
|
@@ -200,7 +235,7 @@ module Fluent::Plugin
|
|
200
235
|
log.error "Could not bulk insert to Data Stream: #{data_stream_name} #{response}"
|
201
236
|
end
|
202
237
|
rescue => e
|
203
|
-
|
238
|
+
raise RecoverableRequestFailure, "could not push logs to Elasticsearch cluster (#{data_stream_name}): #{e.message}"
|
204
239
|
end
|
205
240
|
end
|
206
241
|
|
@@ -19,7 +19,7 @@ class ElasticsearchFallbackSelectorTest < Test::Unit::TestCase
|
|
19
19
|
end
|
20
20
|
|
21
21
|
def stub_elastic_info(url="http://localhost:9200/", version="6.4.2")
|
22
|
-
body ="{\"version\":{\"number\":\"#{version}\"}}"
|
22
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
23
23
|
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
24
24
|
end
|
25
25
|
|
@@ -27,9 +27,15 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
|
|
27
27
|
CODE
|
28
28
|
end
|
29
29
|
|
30
|
+
def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
|
31
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
32
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
33
|
+
end
|
34
|
+
|
30
35
|
def test_xpack_info
|
31
36
|
stub_request(:get, "http://localhost:9200/_xpack").
|
32
37
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
38
|
+
stub_elastic_info
|
33
39
|
expected = {"features"=>{"ilm"=>{"available"=>true, "enabled"=>true}}}
|
34
40
|
assert_equal(expected, xpack_info)
|
35
41
|
end
|
@@ -37,18 +43,21 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
|
|
37
43
|
def test_verify_ilm_working
|
38
44
|
stub_request(:get, "http://localhost:9200/_xpack").
|
39
45
|
to_return(:status => 200, :body => '{"features":{"ilm":{"available":true,"enabled":true}}}', :headers => {"Content-Type"=> "application/json"})
|
46
|
+
stub_elastic_info
|
40
47
|
assert_nothing_raised { verify_ilm_working }
|
41
48
|
end
|
42
49
|
|
43
50
|
def test_ilm_policy_doesnt_exists
|
44
51
|
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluentd-policy%22%7D").
|
45
52
|
to_return(:status => 404, :body => "", :headers => {})
|
53
|
+
stub_elastic_info
|
46
54
|
assert_false(ilm_policy_exists?(policy_id: "fluentd-policy"))
|
47
55
|
end
|
48
56
|
|
49
57
|
def test_ilm_policy_exists
|
50
58
|
stub_request(:get, "http://localhost:9200/_ilm/policy/%7B:policy_id=%3E%22fluent-policy%22%7D").
|
51
59
|
to_return(:status => 200, :body => "", :headers => {})
|
60
|
+
stub_elastic_info
|
52
61
|
assert_true(ilm_policy_exists?(policy_id: "fluent-policy"))
|
53
62
|
end
|
54
63
|
|
@@ -59,6 +68,7 @@ class TestElasticsearchIndexLifecycleManagement < Test::Unit::TestCase
|
|
59
68
|
with(:body => "{\"policy\":{\"phases\":{\"hot\":{\"actions\":{\"rollover\":{\"max_size\":\"50gb\",\"max_age\":\"30d\"}}}}}}",
|
60
69
|
:headers => {'Content-Type'=>'application/json'}).
|
61
70
|
to_return(:status => 200, :body => "", :headers => {})
|
71
|
+
stub_elastic_info
|
62
72
|
create_ilm_policy("fluent-policy")
|
63
73
|
|
64
74
|
assert_requested(:put, "http://localhost:9200/_ilm/policy/fluent-policy", times: 1)
|
@@ -31,6 +31,11 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
31
31
|
@driver ||= Fluent::Test::Driver::Input.new(Fluent::Plugin::ElasticsearchInput).configure(conf)
|
32
32
|
end
|
33
33
|
|
34
|
+
def stub_elastic_info(url="http://localhost:9200/", version="7.9.0")
|
35
|
+
body ="{\"version\":{\"number\":\"#{version}\", \"build_flavor\":\"default\"},\"tagline\" : \"You Know, for Search\"}"
|
36
|
+
stub_request(:get, url).to_return({:status => 200, :body => body, :headers => { 'Content-Type' => 'json' } })
|
37
|
+
end
|
38
|
+
|
34
39
|
def sample_response(index_name="fluentd")
|
35
40
|
{
|
36
41
|
"took"=>4,
|
@@ -322,6 +327,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
322
327
|
with(body: "{\"sort\":[\"_doc\"]}").
|
323
328
|
to_return(status: 200, body: sample_response.to_s,
|
324
329
|
headers: {'Content-Type' => 'application/json'})
|
330
|
+
stub_elastic_info
|
325
331
|
|
326
332
|
driver(CONFIG)
|
327
333
|
driver.run(expect_emits: 1, timeout: 10)
|
@@ -337,6 +343,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
337
343
|
with(body: "{\"sort\":[\"_doc\"]}").
|
338
344
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
339
345
|
headers: {'Content-Type' => 'application/json'})
|
346
|
+
stub_elastic_info
|
340
347
|
|
341
348
|
driver(CONFIG + %[index_name #{index_name}])
|
342
349
|
driver.run(expect_emits: 1, timeout: 10)
|
@@ -352,6 +359,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
352
359
|
with(body: "{\"sort\":[\"_doc\"]}").
|
353
360
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
354
361
|
headers: {'Content-Type' => 'application/json'})
|
362
|
+
stub_elastic_info
|
355
363
|
|
356
364
|
driver(CONFIG + %[parse_timestamp])
|
357
365
|
driver.run(expect_emits: 1, timeout: 10)
|
@@ -370,6 +378,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
370
378
|
with(body: "{\"sort\":[\"_doc\"]}").
|
371
379
|
to_return(status: 200, body: sample_response(index_name).to_s,
|
372
380
|
headers: {'Content-Type' => 'application/json'})
|
381
|
+
stub_elastic_info
|
373
382
|
|
374
383
|
driver(CONFIG + %[parse_timestamp true
|
375
384
|
timestamp_key_format %Y-%m-%dT%H:%M:%S.%N%z
|
@@ -389,6 +398,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
389
398
|
with(body: "{\"sort\":[\"_doc\"]}").
|
390
399
|
to_return(status: 200, body: sample_response.to_s,
|
391
400
|
headers: {'Content-Type' => 'application/json'})
|
401
|
+
stub_elastic_info
|
392
402
|
|
393
403
|
driver(CONFIG + %[docinfo true])
|
394
404
|
driver.run(expect_emits: 1, timeout: 10)
|
@@ -412,6 +422,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
412
422
|
with(body: "{\"sort\":[\"_doc\"],\"slice\":{\"id\":1,\"max\":2}}").
|
413
423
|
to_return(status: 200, body: sample_response.to_s,
|
414
424
|
headers: {'Content-Type' => 'application/json'})
|
425
|
+
stub_elastic_info
|
415
426
|
|
416
427
|
driver(CONFIG + %[num_slices 2])
|
417
428
|
driver.run(expect_emits: 1, timeout: 10)
|
@@ -434,6 +445,7 @@ class ElasticsearchInputTest < Test::Unit::TestCase
|
|
434
445
|
body: "{\"scroll_id\":\"WomkoUKG0QPB679Ulo6TqQgh3pIGRUmrl9qXXGK3EeiQh9rbYNasTkspZQcJ01uz\"}") do
|
435
446
|
connection += 1
|
436
447
|
end
|
448
|
+
stub_elastic_info
|
437
449
|
scroll_request.to_return(lambda do |req|
|
438
450
|
if connection <= 1
|
439
451
|
{status: 200, body: sample_scroll_response_2.to_s,
|