fluent-plugin-elasticsearch 1.10.0 → 1.10.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/History.md +4 -0
- data/README.md +13 -0
- data/fluent-plugin-elasticsearch.gemspec +2 -1
- data/lib/fluent/plugin/out_elasticsearch.rb +20 -1
- data/lib/fluent/plugin/out_elasticsearch_dynamic.rb +17 -11
- data/test/plugin/test_out_elasticsearch.rb +45 -0
- data/test/plugin/test_out_elasticsearch_dynamic.rb +60 -2
- metadata +17 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 6924c1c7c90f7eb629011bbae3db822799584200
|
4
|
+
data.tar.gz: 52ab17e9b443275e215de285327be011881681b9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 85120102f59c4713273af0706b90e36a696e27329c63a3f748762c626633f02fe22ae60ebd6819c9f9819dcc4fa5d05d5df7ed58f2d4e43a3e2625be7599559d
|
7
|
+
data.tar.gz: 4f0a559e177c0fccaf0e1bb916decf0226508b49af463448556b76622c5c9ba064246052da1b92fb876d36ce77f0e3644f18144f3cd8b7e72fc7fae6ed70737d
|
data/History.md
CHANGED
@@ -4,6 +4,10 @@
|
|
4
4
|
- Log ES response errors (#230)
|
5
5
|
- Use latest elasticsearch-ruby (#240)
|
6
6
|
|
7
|
+
### 1.10.1
|
8
|
+
- backport escaping basic authentication user information placeholders (#309)
|
9
|
+
- backport handling dynamic config misconfiguration (#308)
|
10
|
+
|
7
11
|
### 1.10.0
|
8
12
|
- backport adding `logstash_prefix_separator` parameter fix
|
9
13
|
- backport making configuraable SSL/TLS version (#300)
|
data/README.md
CHANGED
@@ -94,6 +94,12 @@ You can specify multiple ElasticSearch hosts with separator ",".
|
|
94
94
|
|
95
95
|
If you specify multiple hosts, this plugin will load balance updates to ElasticSearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
|
96
96
|
|
97
|
+
And this plugin will escape required URL encoded characters within `%{}` placeholders.
|
98
|
+
|
99
|
+
```
|
100
|
+
hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
|
101
|
+
```
|
102
|
+
|
97
103
|
### user, password, path, scheme, ssl_verify
|
98
104
|
|
99
105
|
If you specify this option, host and port options are ignored.
|
@@ -107,6 +113,13 @@ scheme https
|
|
107
113
|
|
108
114
|
You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
|
109
115
|
|
116
|
+
And this plugin will escape required URL encoded characters within `%{}` placeholders.
|
117
|
+
|
118
|
+
```
|
119
|
+
user %{demo+}
|
120
|
+
password %{@secret}
|
121
|
+
```
|
122
|
+
|
110
123
|
Specify `ssl_verify false` to skip ssl verification (defaults to true)
|
111
124
|
|
112
125
|
### logstash_format
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-elasticsearch'
|
6
|
-
s.version = '1.10.
|
6
|
+
s.version = '1.10.1'
|
7
7
|
s.authors = ['diogo', 'pitr']
|
8
8
|
s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
|
9
9
|
s.description = %q{ElasticSearch output plugin for Fluent event collector}
|
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
|
|
27
27
|
s.add_development_dependency 'webmock', '~> 1'
|
28
28
|
s.add_development_dependency 'test-unit', '~> 3.1.0'
|
29
29
|
s.add_development_dependency 'minitest', '~> 5.8'
|
30
|
+
s.add_development_dependency 'flexmock', '~> 2.3.5'
|
30
31
|
end
|
@@ -105,6 +105,13 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
105
105
|
rescue LoadError
|
106
106
|
@dump_proc = Yajl.method(:dump)
|
107
107
|
end
|
108
|
+
|
109
|
+
if @user && m = @user.match(/%{(?<user>.*)}/)
|
110
|
+
@user = URI.encode_www_form_component(m["user"])
|
111
|
+
end
|
112
|
+
if @password && m = @password.match(/%{(?<password>.*)}/)
|
113
|
+
@password = URI.encode_www_form_component(m["password"])
|
114
|
+
end
|
108
115
|
end
|
109
116
|
|
110
117
|
def create_meta_config_map
|
@@ -172,6 +179,18 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
172
179
|
end
|
173
180
|
end
|
174
181
|
|
182
|
+
def get_escaped_userinfo(host_str)
|
183
|
+
if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
|
184
|
+
m["scheme"] +
|
185
|
+
URI.encode_www_form_component(m["user"]) +
|
186
|
+
':' +
|
187
|
+
URI.encode_www_form_component(m["password"]) +
|
188
|
+
m["path"]
|
189
|
+
else
|
190
|
+
host_str
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
175
194
|
def get_connection_options
|
176
195
|
raise "`password` must be present if `user` is present" if @user && !@password
|
177
196
|
|
@@ -186,7 +205,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
|
|
186
205
|
}
|
187
206
|
else
|
188
207
|
# New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
|
189
|
-
uri = URI(host_str)
|
208
|
+
uri = URI(get_escaped_userinfo(host_str))
|
190
209
|
%w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
|
191
210
|
hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
|
192
211
|
hash
|
@@ -80,7 +80,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
|
|
80
80
|
}
|
81
81
|
else
|
82
82
|
# New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
|
83
|
-
uri = URI(host_str)
|
83
|
+
uri = URI(get_escaped_userinfo(host_str))
|
84
84
|
%w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
|
85
85
|
hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
|
86
86
|
hash
|
@@ -121,17 +121,23 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
|
|
121
121
|
chunk.msgpack_each do |time, record|
|
122
122
|
next unless record.is_a? Hash
|
123
123
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
124
|
+
begin
|
125
|
+
# evaluate all configurations here
|
126
|
+
DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
|
127
|
+
k = DYNAMIC_PARAM_NAMES[i]
|
128
|
+
v = self.instance_variable_get(var)
|
129
|
+
# check here to determine if we should evaluate
|
130
|
+
if dynamic_conf[k] != v
|
131
|
+
value = expand_param(v, tag, time, record)
|
132
|
+
dynamic_conf[k] = value
|
133
|
+
end
|
134
|
+
}
|
134
135
|
# end eval all configs
|
136
|
+
rescue => e
|
137
|
+
# handle dynamic parameters misconfigurations
|
138
|
+
router.emit_error_event(tag, time, record, e)
|
139
|
+
next
|
140
|
+
end
|
135
141
|
|
136
142
|
if eval_or_val(dynamic_conf['logstash_format'])
|
137
143
|
if record.has_key?("@timestamp")
|
@@ -311,6 +311,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
311
311
|
assert_equal '/default_path', host2[:path]
|
312
312
|
end
|
313
313
|
|
314
|
+
def test_hosts_list_with_escape_placeholders
|
315
|
+
config = %{
|
316
|
+
hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
|
317
|
+
path /default_path
|
318
|
+
user default_user
|
319
|
+
password default_password
|
320
|
+
}
|
321
|
+
instance = driver('test', config).instance
|
322
|
+
|
323
|
+
assert_equal 2, instance.get_connection_options[:hosts].length
|
324
|
+
host1, host2 = instance.get_connection_options[:hosts]
|
325
|
+
|
326
|
+
assert_equal 'host1', host1[:host]
|
327
|
+
assert_equal 443, host1[:port]
|
328
|
+
assert_equal 'https', host1[:scheme]
|
329
|
+
assert_equal 'j%2Bhn', host1[:user]
|
330
|
+
assert_equal 'passw%40rd', host1[:password]
|
331
|
+
assert_equal '/elastic/', host1[:path]
|
332
|
+
|
333
|
+
assert_equal 'host2', host2[:host]
|
334
|
+
assert_equal 'http', host2[:scheme]
|
335
|
+
assert_equal 'default_user', host2[:user]
|
336
|
+
assert_equal 'default_password', host2[:password]
|
337
|
+
assert_equal '/default_path', host2[:path]
|
338
|
+
end
|
339
|
+
|
314
340
|
def test_single_host_params_and_defaults
|
315
341
|
config = %{
|
316
342
|
host logs.google.com
|
@@ -330,6 +356,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
|
|
330
356
|
assert_equal nil, host1[:path]
|
331
357
|
end
|
332
358
|
|
359
|
+
def test_single_host_params_and_defaults_with_escape_placeholders
|
360
|
+
config = %{
|
361
|
+
host logs.google.com
|
362
|
+
user %{j+hn}
|
363
|
+
password %{d@e}
|
364
|
+
}
|
365
|
+
instance = driver('test', config).instance
|
366
|
+
|
367
|
+
assert_equal 1, instance.get_connection_options[:hosts].length
|
368
|
+
host1 = instance.get_connection_options[:hosts][0]
|
369
|
+
|
370
|
+
assert_equal 'logs.google.com', host1[:host]
|
371
|
+
assert_equal 9200, host1[:port]
|
372
|
+
assert_equal 'http', host1[:scheme]
|
373
|
+
assert_equal 'j%2Bhn', host1[:user]
|
374
|
+
assert_equal 'd%40e', host1[:password]
|
375
|
+
assert_equal nil, host1[:path]
|
376
|
+
end
|
377
|
+
|
333
378
|
def test_content_type_header
|
334
379
|
stub_request(:head, "http://localhost:9200/").
|
335
380
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -1,7 +1,10 @@
|
|
1
1
|
require 'helper'
|
2
2
|
require 'date'
|
3
|
+
require 'flexmock/test_unit'
|
3
4
|
|
4
5
|
class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
6
|
+
include FlexMock::TestCase
|
7
|
+
|
5
8
|
attr_accessor :index_cmds, :index_command_counts
|
6
9
|
|
7
10
|
def setup
|
@@ -139,6 +142,32 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
139
142
|
assert_equal '/default_path', host2[:path]
|
140
143
|
end
|
141
144
|
|
145
|
+
def test_hosts_list_with_escape_placeholders
|
146
|
+
config = %{
|
147
|
+
hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
|
148
|
+
path /default_path
|
149
|
+
user default_user
|
150
|
+
password default_password
|
151
|
+
}
|
152
|
+
instance = driver('test', config).instance
|
153
|
+
|
154
|
+
assert_equal 2, instance.get_connection_options(nil)[:hosts].length
|
155
|
+
host1, host2 = instance.get_connection_options(nil)[:hosts]
|
156
|
+
|
157
|
+
assert_equal 'host1', host1[:host]
|
158
|
+
assert_equal 443, host1[:port]
|
159
|
+
assert_equal 'https', host1[:scheme]
|
160
|
+
assert_equal 'j%2Bhn', host1[:user]
|
161
|
+
assert_equal 'passw%40rd', host1[:password]
|
162
|
+
assert_equal '/elastic/', host1[:path]
|
163
|
+
|
164
|
+
assert_equal 'host2', host2[:host]
|
165
|
+
assert_equal 'http', host2[:scheme]
|
166
|
+
assert_equal 'default_user', host2[:user]
|
167
|
+
assert_equal 'default_password', host2[:password]
|
168
|
+
assert_equal '/default_path', host2[:path]
|
169
|
+
end
|
170
|
+
|
142
171
|
def test_single_host_params_and_defaults
|
143
172
|
config = %{
|
144
173
|
host logs.google.com
|
@@ -158,6 +187,25 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
158
187
|
assert_equal nil, host1[:path]
|
159
188
|
end
|
160
189
|
|
190
|
+
def test_single_host_params_and_defaults_with_escape_placeholders
|
191
|
+
config = %{
|
192
|
+
host logs.google.com
|
193
|
+
user %{j+hn}
|
194
|
+
password %{d@e}
|
195
|
+
}
|
196
|
+
instance = driver('test', config).instance
|
197
|
+
|
198
|
+
assert_equal 1, instance.get_connection_options(nil)[:hosts].length
|
199
|
+
host1 = instance.get_connection_options(nil)[:hosts][0]
|
200
|
+
|
201
|
+
assert_equal 'logs.google.com', host1[:host]
|
202
|
+
assert_equal 9200, host1[:port]
|
203
|
+
assert_equal 'http', host1[:scheme]
|
204
|
+
assert_equal 'j%2Bhn', host1[:user]
|
205
|
+
assert_equal 'd%40e', host1[:password]
|
206
|
+
assert_equal nil, host1[:path]
|
207
|
+
end
|
208
|
+
|
161
209
|
def test_content_type_header
|
162
210
|
stub_request(:head, "http://localhost:9200/").
|
163
211
|
to_return(:status => 200, :body => "", :headers => {})
|
@@ -537,6 +585,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
537
585
|
driver.run
|
538
586
|
end
|
539
587
|
|
588
|
+
def test_tag_parts_index_error_event
|
589
|
+
stub_elastic_ping
|
590
|
+
stub_elastic
|
591
|
+
driver.configure("logstash_prefix ${tag_parts[1]}\n")
|
592
|
+
flexmock(driver.instance.router).should_receive(:emit_error_event)
|
593
|
+
.with('test', Integer, Hash, TypeError).once
|
594
|
+
driver.emit(sample_record)
|
595
|
+
driver.run
|
596
|
+
end
|
597
|
+
|
540
598
|
def test_connection_failed_retry
|
541
599
|
connection_resets = 0
|
542
600
|
|
@@ -566,7 +624,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
566
624
|
stub_request(:post, "http://localhost:9200/_bulk").with do |req|
|
567
625
|
raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
|
568
626
|
end
|
569
|
-
|
627
|
+
|
570
628
|
driver.configure("reconnect_on_error true\n")
|
571
629
|
driver.emit(sample_record)
|
572
630
|
|
@@ -590,7 +648,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
|
|
590
648
|
stub_request(:post, "http://localhost:9200/_bulk").with do |req|
|
591
649
|
raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
|
592
650
|
end
|
593
|
-
|
651
|
+
|
594
652
|
driver.configure("reconnect_on_error false\n")
|
595
653
|
driver.emit(sample_record)
|
596
654
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.10.
|
4
|
+
version: 1.10.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- diogo
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-10-16 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: fluentd
|
@@ -109,6 +109,20 @@ dependencies:
|
|
109
109
|
- - "~>"
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: '5.8'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: flexmock
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - "~>"
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: 2.3.5
|
119
|
+
type: :development
|
120
|
+
prerelease: false
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - "~>"
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: 2.3.5
|
112
126
|
description: ElasticSearch output plugin for Fluent event collector
|
113
127
|
email:
|
114
128
|
- pitr.vern@gmail.com
|
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
157
171
|
version: '0'
|
158
172
|
requirements: []
|
159
173
|
rubyforge_project:
|
160
|
-
rubygems_version: 2.6.
|
174
|
+
rubygems_version: 2.6.13
|
161
175
|
signing_key:
|
162
176
|
specification_version: 4
|
163
177
|
summary: ElasticSearch output plugin for Fluent event collector
|