fluent-plugin-elasticsearch 1.10.0 → 1.10.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d7f9583b7923380fe06d399cc24006cf75a17528
4
- data.tar.gz: ffb1c9ed360ef6c48cfa003ee81022d179f0b9bf
3
+ metadata.gz: 6924c1c7c90f7eb629011bbae3db822799584200
4
+ data.tar.gz: 52ab17e9b443275e215de285327be011881681b9
5
5
  SHA512:
6
- metadata.gz: 74eaadfe9d121623014befb052b8437cdb092a9857153c521039ee463ef51a20877f7f1e63dea4ad9c5f3a8f70ab33b0ea1c77cf0aa93fac0c33b0da63e15af6
7
- data.tar.gz: 4a84606298bf3e2874485c6d5d10212f065d89f68ab7bc44e0731cfa3908a2cefce18aa9eef51f1b2fb09248e9a98d8610ebc5bd3496b9223cbb4245e00d6b04
6
+ metadata.gz: 85120102f59c4713273af0706b90e36a696e27329c63a3f748762c626633f02fe22ae60ebd6819c9f9819dcc4fa5d05d5df7ed58f2d4e43a3e2625be7599559d
7
+ data.tar.gz: 4f0a559e177c0fccaf0e1bb916decf0226508b49af463448556b76622c5c9ba064246052da1b92fb876d36ce77f0e3644f18144f3cd8b7e72fc7fae6ed70737d
data/History.md CHANGED
@@ -4,6 +4,10 @@
4
4
  - Log ES response errors (#230)
5
5
  - Use latest elasticsearch-ruby (#240)
6
6
 
7
+ ### 1.10.1
8
+ - backport escaping basic authentication user information placeholders (#309)
9
+ - backport handling dynamic config misconfiguration (#308)
10
+
7
11
  ### 1.10.0
8
12
  - backport adding `logstash_prefix_separator` parameter fix
9
13
  - backport making configuraable SSL/TLS version (#300)
data/README.md CHANGED
@@ -94,6 +94,12 @@ You can specify multiple ElasticSearch hosts with separator ",".
94
94
 
95
95
  If you specify multiple hosts, this plugin will load balance updates to ElasticSearch. This is an [elasticsearch-ruby](https://github.com/elasticsearch/elasticsearch-ruby) feature, the default strategy is round-robin.
96
96
 
97
+ And this plugin will escape required URL encoded characters within `%{}` placeholders.
98
+
99
+ ```
100
+ hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
101
+ ```
102
+
97
103
  ### user, password, path, scheme, ssl_verify
98
104
 
99
105
  If you specify this option, host and port options are ignored.
@@ -107,6 +113,13 @@ scheme https
107
113
 
108
114
  You can specify user and password for HTTP basic auth. If used in conjunction with a hosts list, then these options will be used by default i.e. if you do not provide any of these options within the hosts listed.
109
115
 
116
+ And this plugin will escape required URL encoded characters within `%{}` placeholders.
117
+
118
+ ```
119
+ user %{demo+}
120
+ password %{@secret}
121
+ ```
122
+
110
123
  Specify `ssl_verify false` to skip ssl verification (defaults to true)
111
124
 
112
125
  ### logstash_format
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = 'fluent-plugin-elasticsearch'
6
- s.version = '1.10.0'
6
+ s.version = '1.10.1'
7
7
  s.authors = ['diogo', 'pitr']
8
8
  s.email = ['pitr.vern@gmail.com', 'me@diogoterror.com']
9
9
  s.description = %q{ElasticSearch output plugin for Fluent event collector}
@@ -27,4 +27,5 @@ Gem::Specification.new do |s|
27
27
  s.add_development_dependency 'webmock', '~> 1'
28
28
  s.add_development_dependency 'test-unit', '~> 3.1.0'
29
29
  s.add_development_dependency 'minitest', '~> 5.8'
30
+ s.add_development_dependency 'flexmock', '~> 2.3.5'
30
31
  end
@@ -105,6 +105,13 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
105
105
  rescue LoadError
106
106
  @dump_proc = Yajl.method(:dump)
107
107
  end
108
+
109
+ if @user && m = @user.match(/%{(?<user>.*)}/)
110
+ @user = URI.encode_www_form_component(m["user"])
111
+ end
112
+ if @password && m = @password.match(/%{(?<password>.*)}/)
113
+ @password = URI.encode_www_form_component(m["password"])
114
+ end
108
115
  end
109
116
 
110
117
  def create_meta_config_map
@@ -172,6 +179,18 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
172
179
  end
173
180
  end
174
181
 
182
+ def get_escaped_userinfo(host_str)
183
+ if m = host_str.match(/(?<scheme>.*)%{(?<user>.*)}:%{(?<password>.*)}(?<path>@.*)/)
184
+ m["scheme"] +
185
+ URI.encode_www_form_component(m["user"]) +
186
+ ':' +
187
+ URI.encode_www_form_component(m["password"]) +
188
+ m["path"]
189
+ else
190
+ host_str
191
+ end
192
+ end
193
+
175
194
  def get_connection_options
176
195
  raise "`password` must be present if `user` is present" if @user && !@password
177
196
 
@@ -186,7 +205,7 @@ class Fluent::ElasticsearchOutput < Fluent::ObjectBufferedOutput
186
205
  }
187
206
  else
188
207
  # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
189
- uri = URI(host_str)
208
+ uri = URI(get_escaped_userinfo(host_str))
190
209
  %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
191
210
  hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
192
211
  hash
@@ -80,7 +80,7 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
80
80
  }
81
81
  else
82
82
  # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic
83
- uri = URI(host_str)
83
+ uri = URI(get_escaped_userinfo(host_str))
84
84
  %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key|
85
85
  hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == ''
86
86
  hash
@@ -121,17 +121,23 @@ class Fluent::ElasticsearchOutputDynamic < Fluent::ElasticsearchOutput
121
121
  chunk.msgpack_each do |time, record|
122
122
  next unless record.is_a? Hash
123
123
 
124
- # evaluate all configurations here
125
- DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
126
- k = DYNAMIC_PARAM_NAMES[i]
127
- v = self.instance_variable_get(var)
128
- # check here to determine if we should evaluate
129
- if dynamic_conf[k] != v
130
- value = expand_param(v, tag, time, record)
131
- dynamic_conf[k] = value
132
- end
133
- }
124
+ begin
125
+ # evaluate all configurations here
126
+ DYNAMIC_PARAM_SYMBOLS.each_with_index { |var, i|
127
+ k = DYNAMIC_PARAM_NAMES[i]
128
+ v = self.instance_variable_get(var)
129
+ # check here to determine if we should evaluate
130
+ if dynamic_conf[k] != v
131
+ value = expand_param(v, tag, time, record)
132
+ dynamic_conf[k] = value
133
+ end
134
+ }
134
135
  # end eval all configs
136
+ rescue => e
137
+ # handle dynamic parameters misconfigurations
138
+ router.emit_error_event(tag, time, record, e)
139
+ next
140
+ end
135
141
 
136
142
  if eval_or_val(dynamic_conf['logstash_format'])
137
143
  if record.has_key?("@timestamp")
@@ -311,6 +311,32 @@ class ElasticsearchOutput < Test::Unit::TestCase
311
311
  assert_equal '/default_path', host2[:path]
312
312
  end
313
313
 
314
+ def test_hosts_list_with_escape_placeholders
315
+ config = %{
316
+ hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
317
+ path /default_path
318
+ user default_user
319
+ password default_password
320
+ }
321
+ instance = driver('test', config).instance
322
+
323
+ assert_equal 2, instance.get_connection_options[:hosts].length
324
+ host1, host2 = instance.get_connection_options[:hosts]
325
+
326
+ assert_equal 'host1', host1[:host]
327
+ assert_equal 443, host1[:port]
328
+ assert_equal 'https', host1[:scheme]
329
+ assert_equal 'j%2Bhn', host1[:user]
330
+ assert_equal 'passw%40rd', host1[:password]
331
+ assert_equal '/elastic/', host1[:path]
332
+
333
+ assert_equal 'host2', host2[:host]
334
+ assert_equal 'http', host2[:scheme]
335
+ assert_equal 'default_user', host2[:user]
336
+ assert_equal 'default_password', host2[:password]
337
+ assert_equal '/default_path', host2[:path]
338
+ end
339
+
314
340
  def test_single_host_params_and_defaults
315
341
  config = %{
316
342
  host logs.google.com
@@ -330,6 +356,25 @@ class ElasticsearchOutput < Test::Unit::TestCase
330
356
  assert_equal nil, host1[:path]
331
357
  end
332
358
 
359
+ def test_single_host_params_and_defaults_with_escape_placeholders
360
+ config = %{
361
+ host logs.google.com
362
+ user %{j+hn}
363
+ password %{d@e}
364
+ }
365
+ instance = driver('test', config).instance
366
+
367
+ assert_equal 1, instance.get_connection_options[:hosts].length
368
+ host1 = instance.get_connection_options[:hosts][0]
369
+
370
+ assert_equal 'logs.google.com', host1[:host]
371
+ assert_equal 9200, host1[:port]
372
+ assert_equal 'http', host1[:scheme]
373
+ assert_equal 'j%2Bhn', host1[:user]
374
+ assert_equal 'd%40e', host1[:password]
375
+ assert_equal nil, host1[:path]
376
+ end
377
+
333
378
  def test_content_type_header
334
379
  stub_request(:head, "http://localhost:9200/").
335
380
  to_return(:status => 200, :body => "", :headers => {})
@@ -1,7 +1,10 @@
1
1
  require 'helper'
2
2
  require 'date'
3
+ require 'flexmock/test_unit'
3
4
 
4
5
  class ElasticsearchOutputDynamic < Test::Unit::TestCase
6
+ include FlexMock::TestCase
7
+
5
8
  attr_accessor :index_cmds, :index_command_counts
6
9
 
7
10
  def setup
@@ -139,6 +142,32 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
139
142
  assert_equal '/default_path', host2[:path]
140
143
  end
141
144
 
145
+ def test_hosts_list_with_escape_placeholders
146
+ config = %{
147
+ hosts https://%{j+hn}:%{passw@rd}@host1:443/elastic/,http://host2
148
+ path /default_path
149
+ user default_user
150
+ password default_password
151
+ }
152
+ instance = driver('test', config).instance
153
+
154
+ assert_equal 2, instance.get_connection_options(nil)[:hosts].length
155
+ host1, host2 = instance.get_connection_options(nil)[:hosts]
156
+
157
+ assert_equal 'host1', host1[:host]
158
+ assert_equal 443, host1[:port]
159
+ assert_equal 'https', host1[:scheme]
160
+ assert_equal 'j%2Bhn', host1[:user]
161
+ assert_equal 'passw%40rd', host1[:password]
162
+ assert_equal '/elastic/', host1[:path]
163
+
164
+ assert_equal 'host2', host2[:host]
165
+ assert_equal 'http', host2[:scheme]
166
+ assert_equal 'default_user', host2[:user]
167
+ assert_equal 'default_password', host2[:password]
168
+ assert_equal '/default_path', host2[:path]
169
+ end
170
+
142
171
  def test_single_host_params_and_defaults
143
172
  config = %{
144
173
  host logs.google.com
@@ -158,6 +187,25 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
158
187
  assert_equal nil, host1[:path]
159
188
  end
160
189
 
190
+ def test_single_host_params_and_defaults_with_escape_placeholders
191
+ config = %{
192
+ host logs.google.com
193
+ user %{j+hn}
194
+ password %{d@e}
195
+ }
196
+ instance = driver('test', config).instance
197
+
198
+ assert_equal 1, instance.get_connection_options(nil)[:hosts].length
199
+ host1 = instance.get_connection_options(nil)[:hosts][0]
200
+
201
+ assert_equal 'logs.google.com', host1[:host]
202
+ assert_equal 9200, host1[:port]
203
+ assert_equal 'http', host1[:scheme]
204
+ assert_equal 'j%2Bhn', host1[:user]
205
+ assert_equal 'd%40e', host1[:password]
206
+ assert_equal nil, host1[:path]
207
+ end
208
+
161
209
  def test_content_type_header
162
210
  stub_request(:head, "http://localhost:9200/").
163
211
  to_return(:status => 200, :body => "", :headers => {})
@@ -537,6 +585,16 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
537
585
  driver.run
538
586
  end
539
587
 
588
+ def test_tag_parts_index_error_event
589
+ stub_elastic_ping
590
+ stub_elastic
591
+ driver.configure("logstash_prefix ${tag_parts[1]}\n")
592
+ flexmock(driver.instance.router).should_receive(:emit_error_event)
593
+ .with('test', Integer, Hash, TypeError).once
594
+ driver.emit(sample_record)
595
+ driver.run
596
+ end
597
+
540
598
  def test_connection_failed_retry
541
599
  connection_resets = 0
542
600
 
@@ -566,7 +624,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
566
624
  stub_request(:post, "http://localhost:9200/_bulk").with do |req|
567
625
  raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
568
626
  end
569
-
627
+
570
628
  driver.configure("reconnect_on_error true\n")
571
629
  driver.emit(sample_record)
572
630
 
@@ -590,7 +648,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
590
648
  stub_request(:post, "http://localhost:9200/_bulk").with do |req|
591
649
  raise ZeroDivisionError, "any not host_unreachable_exceptions exception"
592
650
  end
593
-
651
+
594
652
  driver.configure("reconnect_on_error false\n")
595
653
  driver.emit(sample_record)
596
654
 
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.10.0
4
+ version: 1.10.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - diogo
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2017-09-25 00:00:00.000000000 Z
12
+ date: 2017-10-16 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: fluentd
@@ -109,6 +109,20 @@ dependencies:
109
109
  - - "~>"
110
110
  - !ruby/object:Gem::Version
111
111
  version: '5.8'
112
+ - !ruby/object:Gem::Dependency
113
+ name: flexmock
114
+ requirement: !ruby/object:Gem::Requirement
115
+ requirements:
116
+ - - "~>"
117
+ - !ruby/object:Gem::Version
118
+ version: 2.3.5
119
+ type: :development
120
+ prerelease: false
121
+ version_requirements: !ruby/object:Gem::Requirement
122
+ requirements:
123
+ - - "~>"
124
+ - !ruby/object:Gem::Version
125
+ version: 2.3.5
112
126
  description: ElasticSearch output plugin for Fluent event collector
113
127
  email:
114
128
  - pitr.vern@gmail.com
@@ -157,7 +171,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
157
171
  version: '0'
158
172
  requirements: []
159
173
  rubyforge_project:
160
- rubygems_version: 2.6.11
174
+ rubygems_version: 2.6.13
161
175
  signing_key:
162
176
  specification_version: 4
163
177
  summary: ElasticSearch output plugin for Fluent event collector