logstash-output-elasticsearch 5.4.1-java → 6.0.0-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -0
- data/lib/logstash/outputs/elasticsearch.rb +6 -5
- data/lib/logstash/outputs/elasticsearch/common.rb +3 -4
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +3 -1
- data/lib/logstash/outputs/elasticsearch/http_client.rb +151 -110
- data/lib/logstash/outputs/elasticsearch/http_client/manticore_adapter.rb +43 -12
- data/lib/logstash/outputs/elasticsearch/http_client/pool.rb +26 -47
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +4 -20
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +1 -1
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/sniffer_spec.rb +5 -1
- data/spec/unit/outputs/elasticsearch/http_client/manticore_adapter_spec.rb +25 -2
- data/spec/unit/outputs/elasticsearch/http_client/pool_spec.rb +24 -23
- data/spec/unit/outputs/elasticsearch/http_client_spec.rb +111 -44
- data/spec/unit/outputs/elasticsearch_proxy_spec.rb +33 -33
- data/spec/unit/outputs/elasticsearch_spec.rb +74 -16
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +23 -4
- metadata +2 -5
- data/lib/logstash/outputs/elasticsearch/safe_url.rb +0 -16
- data/spec/unit/safe_url_spec.rb +0 -46
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9c739e5dde0b8370dcc47e86b0dc2f92efb6bc6e
|
4
|
+
data.tar.gz: efe8aebc49fb32623f0c8ed65f6427325357da1d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3cdc83c314948cdc3ddf5323755a4e4ff44ffc9df004ca5deaa66fa34e1f6c86532cb368c426f68d27f910963563e0dcceef6baa625985e60259e5a681180f25
|
7
|
+
data.tar.gz: dc6950d258a89c4b5dbc96150527de923606b1011e248f6ce145c703fc817614329cc964f21375a1f6ad69a46e35c463366209d58350d7257006065d29d7239b
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 6.0.0
|
2
|
+
- Proxies requiring auth now always work when a URL is specified
|
3
|
+
- It is no longer possible to specify a proxy as a hash due to security reasons
|
4
|
+
- Fix URL normalization logic to correctly apply all settings to sniffed hosts
|
5
|
+
- Proxies requiring auth now always work when a URL is specified
|
6
|
+
- Switch internals to new LogStash::Util::SafeURI type for more defensive approach to logging credentials
|
7
|
+
|
1
8
|
## 5.4.1
|
2
9
|
- Correctly sniff against ES 5.x clusters
|
3
10
|
|
@@ -8,6 +8,7 @@ require "stud/buffer"
|
|
8
8
|
require "socket" # for Socket.gethostname
|
9
9
|
require "thread" # for safe queueing
|
10
10
|
require "uri" # for escaping user input
|
11
|
+
require "forwardable"
|
11
12
|
|
12
13
|
# This plugin is the recommended method of storing logs in Elasticsearch.
|
13
14
|
# If you plan on using the Kibana web interface, you'll want to use this output.
|
@@ -145,10 +146,9 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
145
146
|
config :sniffing_delay, :validate => :number, :default => 5
|
146
147
|
|
147
148
|
# Set the address of a forward HTTP proxy.
|
148
|
-
#
|
149
|
-
# of
|
150
|
-
|
151
|
-
config :proxy
|
149
|
+
# This used to accept hashes as arguments but now only accepts
|
150
|
+
# arguments of the URI type to prevent leaking credentials.
|
151
|
+
config :proxy, :validate => :uri
|
152
152
|
|
153
153
|
# Set the timeout, in seconds, for network operations and requests sent Elasticsearch. If
|
154
154
|
# a timeout occurs, the request will be retried.
|
@@ -193,11 +193,12 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
193
193
|
config :validate_after_inactivity, :validate => :number, :default => 10000
|
194
194
|
|
195
195
|
def build_client
|
196
|
-
@client
|
196
|
+
@client ||= ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
|
197
197
|
end
|
198
198
|
|
199
199
|
def close
|
200
200
|
@stopping.make_true
|
201
|
+
@client.close if @client
|
201
202
|
end
|
202
203
|
|
203
204
|
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|
@@ -14,7 +14,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
14
14
|
install_template
|
15
15
|
check_action_validity
|
16
16
|
|
17
|
-
@logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts)
|
17
|
+
@logger.info("New Elasticsearch output", :class => self.class.name, :hosts => @hosts.map(&:sanitized))
|
18
18
|
end
|
19
19
|
|
20
20
|
# Receive an array of events and immediately attempt to index them (no buffering)
|
@@ -212,15 +212,14 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
212
212
|
retry unless @stopping.true?
|
213
213
|
rescue ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError => e
|
214
214
|
if RETRYABLE_CODES.include?(e.response_code)
|
215
|
-
|
216
|
-
log_hash = {:code => e.response_code, :url => safe_url}
|
215
|
+
log_hash = {:code => e.response_code, :url => e.url.sanitized}
|
217
216
|
log_hash[:body] = e.body if @logger.debug? # Generally this is too verbose
|
218
217
|
@logger.error("Attempted to send a bulk request to elasticsearch but received a bad HTTP response code!", log_hash)
|
219
218
|
|
220
219
|
sleep_interval = sleep_for_interval(sleep_interval)
|
221
220
|
retry unless @stopping.true?
|
222
221
|
else
|
223
|
-
@logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code, :body => e.
|
222
|
+
@logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code, :body => e.body)
|
224
223
|
end
|
225
224
|
rescue => e
|
226
225
|
# Stuff that should never happen
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'forwardable' # Needed for logstash core SafeURI. We need to patch this in core: https://github.com/elastic/logstash/pull/5978
|
2
|
+
|
1
3
|
module LogStash; module Outputs; class ElasticSearch
|
2
4
|
module CommonConfigs
|
3
5
|
def self.included(mod)
|
@@ -76,7 +78,7 @@ module LogStash; module Outputs; class ElasticSearch
|
|
76
78
|
# `["https://127.0.0.1:9200/mypath"]` (If using a proxy on a subpath)
|
77
79
|
# It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
|
78
80
|
# to prevent LS from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes in Elasticsearch.
|
79
|
-
mod.config :hosts, :validate => :
|
81
|
+
mod.config :hosts, :validate => :uri, :default => [::LogStash::Util::SafeURI.new("//127.0.0.1")], :list => true
|
80
82
|
|
81
83
|
# This plugin uses the bulk index API for improved indexing performance.
|
82
84
|
# This setting defines the maximum sized bulk request Logstash will make.
|
@@ -3,6 +3,7 @@ require "cabin"
|
|
3
3
|
require "base64"
|
4
4
|
require 'logstash/outputs/elasticsearch/http_client/pool'
|
5
5
|
require 'logstash/outputs/elasticsearch/http_client/manticore_adapter'
|
6
|
+
require 'cgi'
|
6
7
|
|
7
8
|
module LogStash; module Outputs; class ElasticSearch;
|
8
9
|
# This is a constant instead of a config option because
|
@@ -27,15 +28,48 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
27
28
|
# :setting => value
|
28
29
|
# }
|
29
30
|
|
31
|
+
#
|
32
|
+
# The `options` is a hash where the following symbol keys have meaning:
|
33
|
+
#
|
34
|
+
# * `:hosts` - array of String. Set a list of hosts to use for communication.
|
35
|
+
# * `:port` - number. set the port to use to communicate with Elasticsearch
|
36
|
+
# * `:user` - String. The user to use for authentication.
|
37
|
+
# * `:password` - String. The password to use for authentication.
|
38
|
+
# * `:timeout` - Float. A duration value, in seconds, after which a socket
|
39
|
+
# operation or request will be aborted if not yet successfull
|
40
|
+
# * `:client_settings` - a hash; see below for keys.
|
41
|
+
#
|
42
|
+
# The `client_settings` key is a has that can contain other settings:
|
43
|
+
#
|
44
|
+
# * `:ssl` - Boolean. Enable or disable SSL/TLS.
|
45
|
+
# * `:proxy` - String. Choose a HTTP HTTProxy to use.
|
46
|
+
# * `:path` - String. The leading path for prefixing Elasticsearch
|
47
|
+
# requests. This is sometimes used if you are proxying Elasticsearch access
|
48
|
+
# through a special http path, such as using mod_rewrite.
|
30
49
|
def initialize(options={})
|
31
50
|
@logger = options[:logger]
|
51
|
+
|
32
52
|
# Again, in case we use DEFAULT_OPTIONS in the future, uncomment this.
|
33
53
|
# @options = DEFAULT_OPTIONS.merge(options)
|
34
54
|
@options = options
|
55
|
+
|
56
|
+
@url_template = build_url_template
|
57
|
+
|
35
58
|
@pool = build_pool(@options)
|
36
59
|
# mutex to prevent requests and sniffing to access the
|
37
60
|
# connection pool at the same time
|
38
61
|
end
|
62
|
+
|
63
|
+
def build_url_template
|
64
|
+
{
|
65
|
+
:scheme => self.scheme,
|
66
|
+
:user => self.user,
|
67
|
+
:password => self.password,
|
68
|
+
:host => "URLTEMPLATE",
|
69
|
+
:port => self.port,
|
70
|
+
:path => self.path
|
71
|
+
}
|
72
|
+
end
|
39
73
|
|
40
74
|
def template_install(name, template, force=false)
|
41
75
|
if template_exists?(name) && !force
|
@@ -56,7 +90,6 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
56
90
|
|
57
91
|
return if actions.empty?
|
58
92
|
|
59
|
-
|
60
93
|
bulk_actions = actions.collect do |action, args, source|
|
61
94
|
args, source = update_action_builder(args, source) if action == 'update'
|
62
95
|
|
@@ -91,7 +124,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
91
124
|
def join_bulk_responses(bulk_responses)
|
92
125
|
{
|
93
126
|
"errors" => bulk_responses.any? {|r| r["errors"] == true},
|
94
|
-
"items" => bulk_responses.reduce([]) {|m,r| m.concat(r
|
127
|
+
"items" => bulk_responses.reduce([]) {|m,r| m.concat(r.fetch("items", []))}
|
95
128
|
}
|
96
129
|
end
|
97
130
|
|
@@ -105,34 +138,86 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
105
138
|
@pool.close
|
106
139
|
end
|
107
140
|
|
108
|
-
|
141
|
+
|
142
|
+
def calculate_property(uris, property, default, sniff_check)
|
143
|
+
values = uris.map(&property).uniq
|
109
144
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
hosts = options[:hosts] || ["127.0.0.1"]
|
130
|
-
client_settings = options[:client_settings] || {}
|
131
|
-
timeout = options[:timeout] || 0
|
145
|
+
if sniff_check && values.size > 1
|
146
|
+
raise LogStash::ConfigurationError, "Cannot have multiple values for #{property} in hosts when sniffing is enabled!"
|
147
|
+
end
|
148
|
+
|
149
|
+
uri_value = values.first
|
150
|
+
|
151
|
+
default = nil if default.is_a?(String) && default.empty? # Blanks are as good as nil
|
152
|
+
uri_value = nil if uri_value.is_a?(String) && uri_value.empty?
|
153
|
+
|
154
|
+
if default && uri_value && (default != uri_value)
|
155
|
+
raise LogStash::ConfigurationError, "Explicit value for '#{property}' was declared, but it is different in one of the URLs given! Please make sure your URLs are inline with explicit values. The URLs have the property set to '#{uri_value}', but it was also set to '#{default}' explicitly"
|
156
|
+
end
|
157
|
+
|
158
|
+
uri_value || default
|
159
|
+
end
|
160
|
+
|
161
|
+
def sniffing
|
162
|
+
@options[:sniffing]
|
163
|
+
end
|
132
164
|
|
133
|
-
|
134
|
-
|
165
|
+
def user
|
166
|
+
calculate_property(uris, :user, @options[:user], sniffing)
|
167
|
+
end
|
168
|
+
|
169
|
+
def password
|
170
|
+
calculate_property(uris, :password, @options[:password], sniffing)
|
171
|
+
end
|
172
|
+
|
173
|
+
def path
|
174
|
+
calculated = calculate_property(uris, :path, client_settings[:path], sniffing)
|
175
|
+
calculated = "/#{calculated}" if calculated && !calculated.start_with?("/")
|
176
|
+
calculated
|
177
|
+
end
|
178
|
+
|
179
|
+
def scheme
|
180
|
+
explicit_scheme = if ssl_options && ssl_options.has_key?(:enabled)
|
181
|
+
ssl_options[:enabled] ? 'https' : 'http'
|
182
|
+
else
|
183
|
+
nil
|
184
|
+
end
|
185
|
+
|
186
|
+
calculated_scheme = calculate_property(uris, :scheme, explicit_scheme, sniffing)
|
187
|
+
|
188
|
+
if calculated_scheme && calculated_scheme !~ /https?/
|
189
|
+
raise LogStash::ConfigurationError, "Bad scheme '#{calculated_scheme}' found should be one of http/https"
|
190
|
+
end
|
191
|
+
|
192
|
+
if calculated_scheme && explicit_scheme && calculated_scheme != explicit_scheme
|
193
|
+
raise LogStash::ConfigurationError, "SSL option was explicitly set to #{ssl_options[:enabled]} but a URL was also declared with a scheme of '#{explicit_scheme}'. Please reconcile this"
|
194
|
+
end
|
135
195
|
|
196
|
+
calculated_scheme # May be nil if explicit_scheme is nil!
|
197
|
+
end
|
198
|
+
|
199
|
+
def port
|
200
|
+
# We don't set the 'default' here because the default is what the user
|
201
|
+
# indicated, so we use an || outside of calculate_property. This lets people
|
202
|
+
# Enter things like foo:123, bar and wind up with foo:123, bar:9200
|
203
|
+
calculate_property(uris, :port, nil, sniffing) || 9200
|
204
|
+
end
|
205
|
+
|
206
|
+
def uris
|
207
|
+
@options[:hosts]
|
208
|
+
end
|
209
|
+
|
210
|
+
def client_settings
|
211
|
+
@options[:client_settings] || {}
|
212
|
+
end
|
213
|
+
|
214
|
+
def ssl_options
|
215
|
+
client_settings.fetch(:ssl, {})
|
216
|
+
end
|
217
|
+
|
218
|
+
def build_adapter(options)
|
219
|
+
timeout = options[:timeout] || 0
|
220
|
+
|
136
221
|
adapter_options = {
|
137
222
|
:socket_timeout => timeout,
|
138
223
|
:request_timeout => timeout,
|
@@ -152,102 +237,58 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
152
237
|
adapter_options[:pool_max_per_route] = client_settings[:pool_max_per_route]
|
153
238
|
end
|
154
239
|
|
155
|
-
adapter_options[:ssl] =
|
156
|
-
|
157
|
-
if options[:user]
|
158
|
-
adapter_options[:auth] = {
|
159
|
-
:user => options[:user],
|
160
|
-
:password => options[:password],
|
161
|
-
:eager => true
|
162
|
-
}
|
163
|
-
end
|
164
|
-
|
240
|
+
adapter_options[:ssl] = ssl_options if self.scheme == 'https'
|
241
|
+
|
165
242
|
adapter_class = ::LogStash::Outputs::ElasticSearch::HttpClient::ManticoreAdapter
|
166
243
|
adapter = adapter_class.new(@logger, adapter_options)
|
244
|
+
end
|
245
|
+
|
246
|
+
def build_pool(options)
|
247
|
+
adapter = build_adapter(options)
|
167
248
|
|
168
249
|
pool_options = {
|
169
|
-
:sniffing =>
|
250
|
+
:sniffing => sniffing,
|
170
251
|
:sniffer_delay => options[:sniffer_delay],
|
171
252
|
:healthcheck_path => options[:healthcheck_path],
|
172
|
-
:resurrect_delay => options[:resurrect_delay]
|
253
|
+
:resurrect_delay => options[:resurrect_delay],
|
254
|
+
:url_normalizer => self.method(:host_to_url)
|
173
255
|
}
|
174
|
-
|
175
|
-
ssl_options = options[:client_settings] ? options[:client_settings][:ssl] : {}
|
176
|
-
pool_options[:scheme] = ssl_options && ssl_options[:enabled] ? 'https' : 'http'
|
177
|
-
|
178
|
-
if options[:user]
|
179
|
-
pool_options[:auth] = {:user => options[:user], :password => options[:password]}
|
180
|
-
end
|
256
|
+
pool_options[:scheme] = self.scheme if self.scheme
|
181
257
|
|
182
258
|
pool_class = ::LogStash::Outputs::ElasticSearch::HttpClient::Pool
|
183
|
-
|
259
|
+
full_urls = @options[:hosts].map {|h| host_to_url(h) }
|
260
|
+
pool = pool_class.new(@logger, adapter, full_urls, pool_options)
|
261
|
+
pool.start
|
262
|
+
pool
|
184
263
|
end
|
185
264
|
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
# Please note that the ssl == nil case is different! If you didn't make an explicit
|
211
|
-
# choice we don't complain!
|
212
|
-
if url.scheme == "http" && ssl == true
|
213
|
-
raise LogStash::ConfigurationError, "You specified a plain 'http' URL '#{host}' but set 'ssl' to true! Aborting!"
|
214
|
-
elsif url.scheme == "https" && ssl == false
|
215
|
-
raise LogStash::ConfigurationError, "You have explicitly disabled SSL but passed in an https URL '#{host}'! Aborting!"
|
216
|
-
end
|
217
|
-
|
218
|
-
url.scheme = explicit_scheme if explicit_scheme
|
219
|
-
elsif (match_results = HOSTNAME_PORT_REGEX.match(host))
|
220
|
-
hostname = match_results["hostname"]
|
221
|
-
port = match_results["port"] || 9200
|
222
|
-
url = URI.parse("#{explicit_scheme || 'http'}://#{hostname}:#{port}")
|
223
|
-
else
|
224
|
-
raise LogStash::ConfigurationError, "Host '#{host}' was specified, but is not valid! Use either a full URL or a hostname:port string!"
|
225
|
-
end
|
226
|
-
|
227
|
-
if path && url.path && url.path != "/" && url.path != ''
|
228
|
-
safe_url = ::LogStash::Outputs::ElasticSearch::SafeURL.without_credentials(url)
|
229
|
-
raise LogStash::ConfigurationError, "A path '#{url.path}' has been explicitly specified in the url '#{safe_url}', but you also specified a path of '#{path}'. This is probably a mistake, please remove one setting."
|
230
|
-
end
|
231
|
-
|
232
|
-
if path
|
233
|
-
url.path = path # The URI library cannot stringify if it holds a nil
|
234
|
-
end
|
235
|
-
|
236
|
-
if parameters
|
237
|
-
query_string = parameters.map { |k,v| "#{k}=#{v}" }.join("&")
|
238
|
-
if query = url.query
|
239
|
-
url.query = "#{query}&#{query_string}"
|
240
|
-
else
|
241
|
-
url.query = query_string
|
242
|
-
end
|
243
|
-
end
|
244
|
-
|
245
|
-
if url.password || url.user
|
246
|
-
raise LogStash::ConfigurationError, "We do not support setting the user password in the URL directly as " +
|
247
|
-
"this may be logged to disk thus leaking credentials. Use the 'user' and 'password' options respectively"
|
265
|
+
def host_to_url(h)
|
266
|
+
# Build a naked URI class to be wrapped in a SafeURI before returning
|
267
|
+
# do NOT log this! It could leak passwords
|
268
|
+
uri_klass = @url_template[:scheme] == 'https' ? URI::HTTPS : URI::HTTP
|
269
|
+
uri = uri_klass.build(@url_template)
|
270
|
+
|
271
|
+
uri.user = h.user || user
|
272
|
+
uri.password = h.password || password
|
273
|
+
uri.host = h.host if h.host
|
274
|
+
uri.port = h.port if h.port
|
275
|
+
uri.path = h.path if !h.path.nil? && !h.path.empty? && h.path != "/"
|
276
|
+
uri.query = h.query
|
277
|
+
|
278
|
+
parameters = client_settings[:parameters]
|
279
|
+
if parameters && !parameters.empty?
|
280
|
+
combined = uri.query ?
|
281
|
+
Hash[URI::decode_www_form(uri.query)].merge(parameters) :
|
282
|
+
parameters
|
283
|
+
query_str = combined.flat_map {|k,v|
|
284
|
+
values = Array(v)
|
285
|
+
values.map {|av| "#{k}=#{av}"}
|
286
|
+
}.join("&")
|
287
|
+
|
288
|
+
uri.query = query_str
|
248
289
|
end
|
249
290
|
|
250
|
-
|
291
|
+
::LogStash::Util::SafeURI.new(uri.normalize)
|
251
292
|
end
|
252
293
|
|
253
294
|
def template_exists?(name)
|
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'manticore'
|
2
|
-
require "logstash/outputs/elasticsearch/safe_url"
|
3
2
|
|
4
3
|
module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
5
4
|
class ManticoreAdapter
|
@@ -7,16 +6,32 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
7
6
|
|
8
7
|
def initialize(logger, options={})
|
9
8
|
@logger = logger
|
10
|
-
|
11
|
-
|
9
|
+
options = options.clone || {}
|
10
|
+
options[:ssl] = options[:ssl] || {}
|
12
11
|
|
13
12
|
# We manage our own retries directly, so let's disable them here
|
14
|
-
|
13
|
+
options[:automatic_retries] = 0
|
15
14
|
# We definitely don't need cookies
|
16
|
-
|
15
|
+
options[:cookies] = false
|
17
16
|
|
18
|
-
@request_options =
|
19
|
-
|
17
|
+
@request_options = options[:headers] ? {:headers => @options[:headers]} : {}
|
18
|
+
|
19
|
+
if options[:proxy]
|
20
|
+
options[:proxy] = manticore_proxy_hash(options[:proxy])
|
21
|
+
end
|
22
|
+
|
23
|
+
@manticore = ::Manticore::Client.new(options)
|
24
|
+
end
|
25
|
+
|
26
|
+
# Transform the proxy option to a hash. Manticore's support for non-hash
|
27
|
+
# proxy options is broken. This was fixed in https://github.com/cheald/manticore/commit/34a00cee57a56148629ed0a47c329181e7319af5
|
28
|
+
# but this is not yet released
|
29
|
+
def manticore_proxy_hash(proxy_uri)
|
30
|
+
[:scheme, :port, :user, :password, :path].reduce(:host => proxy_uri.host) do |acc,opt|
|
31
|
+
value = proxy_uri.send(opt)
|
32
|
+
acc[opt] = value unless value.nil? || (value.is_a?(String) && value.empty?)
|
33
|
+
acc
|
34
|
+
end
|
20
35
|
end
|
21
36
|
|
22
37
|
def client
|
@@ -31,10 +46,27 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
31
46
|
def perform_request(url, method, path, params={}, body=nil)
|
32
47
|
params = (params || {}).merge @request_options
|
33
48
|
params[:body] = body if body
|
34
|
-
|
35
|
-
|
49
|
+
|
50
|
+
request_uri = if path
|
51
|
+
# Combine the paths using the minimal # of /s
|
52
|
+
# First, we make sure the path is relative so URI.join does
|
53
|
+
# the right thing
|
54
|
+
relative_path = path && path.start_with?("/") ? path[1..-1] : path
|
55
|
+
# Wrap this with a safe URI defensively against careless handling later
|
56
|
+
::LogStash::Util::SafeURI.new(URI.join(url.uri, relative_path))
|
57
|
+
else
|
58
|
+
::LogStash::Util::SafeURI.new(url.uri.clone)
|
59
|
+
end
|
60
|
+
|
61
|
+
# We excise auth info from the URL in case manticore itself tries to stick
|
62
|
+
# sensitive data in a thrown exception or log data
|
63
|
+
if request_uri.user
|
64
|
+
params[:auth] = { :user => request_uri.user, :password => request_uri.password, :eager => true }
|
65
|
+
request_uri.user = nil
|
66
|
+
request_uri.password = nil
|
67
|
+
end
|
36
68
|
|
37
|
-
resp = @manticore.send(method.downcase,
|
69
|
+
resp = @manticore.send(method.downcase, request_uri.to_s, params)
|
38
70
|
|
39
71
|
# Manticore returns lazy responses by default
|
40
72
|
# We want to block for our usage, this will wait for the repsonse
|
@@ -45,8 +77,7 @@ module LogStash; module Outputs; class ElasticSearch; class HttpClient;
|
|
45
77
|
# template installation. We might need a better story around this later
|
46
78
|
# but for our current purposes this is correct
|
47
79
|
if resp.code < 200 || resp.code > 299 && resp.code != 404
|
48
|
-
|
49
|
-
raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, safe_url + path, resp.body)
|
80
|
+
raise ::LogStash::Outputs::ElasticSearch::HttpClient::Pool::BadResponseCodeError.new(resp.code, request_uri, body)
|
50
81
|
end
|
51
82
|
|
52
83
|
resp
|