logstash-output-elasticsearch 2.1.2-java → 2.1.4-java
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -3
- data/Gemfile +1 -1
- data/README.md +2 -0
- data/lib/logstash/outputs/elasticsearch.rb +24 -414
- data/lib/logstash/outputs/elasticsearch/buffer.rb +124 -0
- data/lib/logstash/outputs/elasticsearch/common.rb +173 -0
- data/lib/logstash/outputs/elasticsearch/common_configs.rb +113 -0
- data/lib/logstash/outputs/elasticsearch/elasticsearch-template.json +85 -31
- data/lib/logstash/outputs/elasticsearch/http_client.rb +3 -2
- data/lib/logstash/outputs/elasticsearch/http_client_builder.rb +92 -0
- data/lib/logstash/outputs/elasticsearch/template_manager.rb +35 -0
- data/logstash-output-elasticsearch.gemspec +2 -2
- data/spec/es_spec_helper.rb +4 -4
- data/spec/integration/outputs/create_spec.rb +3 -4
- data/spec/integration/outputs/index_spec.rb +3 -5
- data/spec/integration/outputs/retry_spec.rb +23 -16
- data/spec/integration/outputs/routing_spec.rb +3 -5
- data/spec/integration/outputs/secure_spec.rb +4 -4
- data/spec/integration/outputs/templates_spec.rb +2 -3
- data/spec/integration/outputs/update_spec.rb +5 -6
- data/spec/unit/buffer_spec.rb +118 -0
- data/spec/unit/http_client_builder_spec.rb +27 -0
- data/spec/unit/outputs/elasticsearch/protocol_spec.rb +2 -2
- data/spec/unit/outputs/elasticsearch_spec.rb +46 -12
- data/spec/unit/outputs/elasticsearch_ssl_spec.rb +0 -1
- metadata +68 -59
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 83b2c74b37b6266cc97c1801321c858302faf85d
|
4
|
+
data.tar.gz: f47adea8d8172e63fd433e2e58135dd6657bff07
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0db5b0f53f6611fd0b7a5587d4d4a2764d849c02a984dea4dcf735c4f72986085d03acc0dc5ccd08fdc4817c01509aacc2dc98f52ddbb0dbfd74d297bf158d23
|
7
|
+
data.tar.gz: 07cc0da34006d0c84e756d54ea4938df4fbaa822c7c5a284770b50ad040e8270f5fe8ea021a743777fac00e83f0a9c918114a5ab0b145161a2a753b2d7eaa834
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,16 @@
|
|
1
|
+
## 2.1.4
|
2
|
+
- Improved the default template to disable fielddata on analyzed string fields. #309
|
3
|
+
- Dependend on logstash-core 2.0.0 released version, rather than RC1
|
4
|
+
|
5
|
+
## 2.1.3
|
6
|
+
- Improved the default template to use doc_values wherever possible.
|
7
|
+
- Template contains example mappings for every numeric type. You must map your
|
8
|
+
own fields to make use of anything other than long and double.
|
9
|
+
|
10
|
+
## 2.1.2
|
11
|
+
- Fixed dependencies (#280)
|
12
|
+
- Fixed an RSpec test (#281)
|
13
|
+
|
1
14
|
## 2.1.1
|
2
15
|
- Made host config obsolete.
|
3
16
|
|
@@ -7,7 +20,7 @@
|
|
7
20
|
firewall, or load balancer issues.
|
8
21
|
|
9
22
|
## 2.0.0
|
10
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
23
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
11
24
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
12
25
|
- Dependency on logstash-core update to 2.0
|
13
26
|
|
@@ -20,7 +33,7 @@
|
|
20
33
|
## 2.0.0-beta
|
21
34
|
- Only support HTTP Protocol
|
22
35
|
- Removed support for node and transport protocols (now in logstash-output-elasticsearch_java)
|
23
|
-
|
36
|
+
|
24
37
|
## 1.0.7
|
25
38
|
- Add update API support
|
26
39
|
|
@@ -31,7 +44,7 @@
|
|
31
44
|
- Update to Elasticsearch 1.7
|
32
45
|
|
33
46
|
## 1.0.3
|
34
|
-
- Add HTTP proxy support
|
47
|
+
- Add HTTP proxy support
|
35
48
|
|
36
49
|
## 1.0.2
|
37
50
|
- Upgrade Manticore HTTP Client
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,5 +1,7 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
ATTN: This plugin is undergoing a [major refactor](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/291)! Any refactors started now may need to be rebased/refactored before they are accepted!
|
4
|
+
|
3
5
|
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/)
|
5
7
|
|
@@ -8,7 +8,6 @@ require "stud/buffer"
|
|
8
8
|
require "socket" # for Socket.gethostname
|
9
9
|
require "thread" # for safe queueing
|
10
10
|
require "uri" # for escaping user input
|
11
|
-
require "logstash/outputs/elasticsearch/http_client"
|
12
11
|
|
13
12
|
# This plugin is the recommended method of storing logs in Elasticsearch.
|
14
13
|
# If you plan on using the Kibana web interface, you'll want to use this output.
|
@@ -38,104 +37,30 @@ require "logstash/outputs/elasticsearch/http_client"
|
|
38
37
|
# - Events from the retry queue are submitted again when the queue reaches its max size or when
|
39
38
|
# the max interval time is reached. The max interval time is configurable via :retry_max_interval.
|
40
39
|
# - Events which are not retryable or have reached their max retry count are logged to stderr.
|
40
|
+
#
|
41
|
+
# ==== DNS Caching
|
42
|
+
#
|
43
|
+
# This plugin uses the JVM to lookup DNS entries and is subject to the value of https://docs.oracle.com/javase/7/docs/technotes/guides/net/properties.html[networkaddress.cache.ttl],
|
44
|
+
# a global setting for the JVM.
|
45
|
+
#
|
46
|
+
# As an example, to set your DNS TTL to 1 second you would set
|
47
|
+
# the `LS_JAVA_OPTS` environment variable to `-Dnetwordaddress.cache.ttl=1`.
|
48
|
+
#
|
49
|
+
# Keep in mind that a connection with keepalive enabled will
|
50
|
+
# not reevaluate its DNS value while the keepalive is in effect.
|
41
51
|
class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
SUCCESS_CODES = [200, 201]
|
47
|
-
|
48
|
-
config_name "elasticsearch"
|
49
|
-
|
50
|
-
# The index to write events to. This can be dynamic using the `%{foo}` syntax.
|
51
|
-
# The default value will partition your indices by day so you can more easily
|
52
|
-
# delete old data or only search specific date ranges.
|
53
|
-
# Indexes may not contain uppercase characters.
|
54
|
-
# For weekly indexes ISO 8601 format is recommended, eg. logstash-%{+xxxx.ww}
|
55
|
-
config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
|
56
|
-
|
57
|
-
# The index type to write events to. Generally you should try to write only
|
58
|
-
# similar events to the same 'type'. String expansion `%{foo}` works here.
|
59
|
-
#
|
60
|
-
# Deprecated in favor of `document_type` field.
|
61
|
-
config :index_type, :validate => :string, :deprecated => "Please use the 'document_type' setting instead. It has the same effect, but is more appropriately named."
|
62
|
-
|
63
|
-
# The document type to write events to. Generally you should try to write only
|
64
|
-
# similar events to the same 'type'. String expansion `%{foo}` works here.
|
65
|
-
# Unless you set 'document_type', the event 'type' will be used if it exists
|
66
|
-
# otherwise the document type will be assigned the value of 'logs'
|
67
|
-
config :document_type, :validate => :string
|
68
|
-
|
69
|
-
# Starting in Logstash 1.3 (unless you set option `manage_template` to false)
|
70
|
-
# a default mapping template for Elasticsearch will be applied, if you do not
|
71
|
-
# already have one set to match the index pattern defined (default of
|
72
|
-
# `logstash-%{+YYYY.MM.dd}`), minus any variables. For example, in this case
|
73
|
-
# the template will be applied to all indices starting with `logstash-*`
|
74
|
-
#
|
75
|
-
# If you have dynamic templating (e.g. creating indices based on field names)
|
76
|
-
# then you should set `manage_template` to false and use the REST API to upload
|
77
|
-
# your templates manually.
|
78
|
-
config :manage_template, :validate => :boolean, :default => true
|
79
|
-
|
80
|
-
# This configuration option defines how the template is named inside Elasticsearch.
|
81
|
-
# Note that if you have used the template management features and subsequently
|
82
|
-
# change this, you will need to prune the old template manually, e.g.
|
83
|
-
#
|
84
|
-
# `curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>`
|
85
|
-
#
|
86
|
-
# where `OldTemplateName` is whatever the former setting was.
|
87
|
-
config :template_name, :validate => :string, :default => "logstash"
|
88
|
-
|
89
|
-
# You can set the path to your own template here, if you so desire.
|
90
|
-
# If not set, the included template will be used.
|
91
|
-
config :template, :validate => :path
|
92
|
-
|
93
|
-
# Overwrite the current template with whatever is configured
|
94
|
-
# in the `template` and `template_name` directives.
|
95
|
-
config :template_overwrite, :validate => :boolean, :default => false
|
96
|
-
|
97
|
-
# The document ID for the index. Useful for overwriting existing entries in
|
98
|
-
# Elasticsearch with the same ID.
|
99
|
-
config :document_id, :validate => :string
|
100
|
-
|
101
|
-
# A routing override to be applied to all processed events.
|
102
|
-
# This can be dynamic using the `%{foo}` syntax.
|
103
|
-
config :routing, :validate => :string
|
104
|
-
|
105
|
-
# Sets the host(s) of the remote instance. If given an array it will load balance requests across the hosts specified in the `hosts` parameter.
|
106
|
-
# Remember the `http` protocol uses the http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-http.html#modules-http[http] address (eg. 9200, not 9300).
|
107
|
-
# `"127.0.0.1"`
|
108
|
-
# `["127.0.0.1:9200","127.0.0.2:9200"]`
|
109
|
-
# It is important to exclude http://www.elastic.co/guide/en/elasticsearch/reference/current/modules-node.html[dedicated master nodes] from the `hosts` list
|
110
|
-
# to prevent LS from sending bulk requests to the master nodes. So this parameter should only reference either data or client nodes in Elasticsearch.
|
111
|
-
|
112
|
-
config :hosts, :validate => :array
|
52
|
+
require "logstash/outputs/elasticsearch/http_client"
|
53
|
+
require "logstash/outputs/elasticsearch/http_client_builder"
|
54
|
+
require "logstash/outputs/elasticsearch/common_configs"
|
55
|
+
require "logstash/outputs/elasticsearch/common"
|
113
56
|
|
114
|
-
|
57
|
+
# Protocol agnostic (i.e. non-http, non-java specific) configs go here
|
58
|
+
include(LogStash::Outputs::ElasticSearch::CommonConfigs)
|
115
59
|
|
116
|
-
#
|
117
|
-
|
118
|
-
config :port, :obsolete => "Please use the 'hosts' setting instead. Hosts entries can be in 'host:port' format."
|
60
|
+
# Protocol agnostic methods
|
61
|
+
include(LogStash::Outputs::ElasticSearch::Common)
|
119
62
|
|
120
|
-
|
121
|
-
# To make efficient bulk API calls, we will buffer a certain number of
|
122
|
-
# events before flushing that out to Elasticsearch. This setting
|
123
|
-
# controls how many events will be buffered before sending a batch
|
124
|
-
# of events. Increasing the `flush_size` has an effect on Logstash's heap size.
|
125
|
-
# Remember to also increase the heap size using `LS_HEAP_SIZE` if you are sending big documents
|
126
|
-
# or have increased the `flush_size` to a higher value.
|
127
|
-
config :flush_size, :validate => :number, :default => 500
|
128
|
-
|
129
|
-
# The amount of time since last flush before a flush is forced.
|
130
|
-
#
|
131
|
-
# This setting helps ensure slow event rates don't get stuck in Logstash.
|
132
|
-
# For example, if your `flush_size` is 100, and you have received 10 events,
|
133
|
-
# and it has been more than `idle_flush_time` seconds since the last flush,
|
134
|
-
# Logstash will flush those 10 events automatically.
|
135
|
-
#
|
136
|
-
# This helps keep both fast and slow log streams moving along in
|
137
|
-
# near-real-time.
|
138
|
-
config :idle_flush_time, :validate => :number, :default => 1
|
63
|
+
config_name "elasticsearch"
|
139
64
|
|
140
65
|
# The Elasticsearch action to perform. Valid actions are:
|
141
66
|
#
|
@@ -192,339 +117,24 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
192
117
|
# How long to wait, in seconds, between sniffing attempts
|
193
118
|
config :sniffing_delay, :validate => :number, :default => 5
|
194
119
|
|
195
|
-
# Set max retry for each event
|
196
|
-
config :max_retries, :validate => :number, :default => 3
|
197
|
-
|
198
|
-
# Set retry policy for events that failed to send
|
199
|
-
config :retry_max_items, :validate => :number, :default => 5000
|
200
|
-
|
201
|
-
# Set max interval between bulk retries
|
202
|
-
config :retry_max_interval, :validate => :number, :default => 5
|
203
|
-
|
204
120
|
# Set the address of a forward HTTP proxy.
|
205
121
|
# Can be either a string, such as `http://localhost:123` or a hash in the form
|
206
122
|
# of `{host: 'proxy.org' port: 80 scheme: 'http'}`.
|
207
123
|
# Note, this is NOT a SOCKS proxy, but a plain HTTP proxy
|
208
124
|
config :proxy
|
209
125
|
|
210
|
-
# Enable `doc_as_upsert` for update mode.
|
211
|
-
# Create a new document with source if `document_id` doesn't exist in Elasticsearch
|
212
|
-
config :doc_as_upsert, :validate => :boolean, :default => false
|
213
|
-
|
214
|
-
# Set upsert content for update mode.
|
215
|
-
# Create a new document with this parameter as json string if `document_id` doesn't exists
|
216
|
-
config :upsert, :validate => :string, :default => ""
|
217
|
-
|
218
126
|
# Set the timeout for network operations and requests sent Elasticsearch. If
|
219
127
|
# a timeout occurs, the request will be retried.
|
220
128
|
config :timeout, :validate => :number
|
221
129
|
|
222
|
-
|
223
|
-
|
224
|
-
@hosts = Array(@hosts)
|
225
|
-
# retry-specific variables
|
226
|
-
@retry_flush_mutex = Mutex.new
|
227
|
-
@retry_close_requested = Concurrent::AtomicBoolean.new(false)
|
228
|
-
# needs flushing when interval
|
229
|
-
@retry_queue_needs_flushing = ConditionVariable.new
|
230
|
-
@retry_queue_not_full = ConditionVariable.new
|
231
|
-
@retry_queue = Queue.new
|
232
|
-
@submit_mutex = Mutex.new
|
233
|
-
|
234
|
-
client_settings = {}
|
235
|
-
common_options = {
|
236
|
-
:client_settings => client_settings,
|
237
|
-
:sniffing => @sniffing,
|
238
|
-
:sniffing_delay => @sniffing_delay
|
239
|
-
}
|
240
|
-
|
241
|
-
common_options[:timeout] = @timeout if @timeout
|
242
|
-
client_settings[:path] = "/#{@path}/".gsub(/\/+/, "/") # Normalize slashes
|
243
|
-
@logger.debug? && @logger.debug("Normalizing http path", :path => @path, :normalized => client_settings[:path])
|
244
|
-
|
245
|
-
if @hosts.nil? || @hosts.empty?
|
246
|
-
@logger.info("No 'host' set in elasticsearch output. Defaulting to localhost")
|
247
|
-
@hosts = ["localhost"]
|
248
|
-
end
|
249
|
-
|
250
|
-
client_settings.merge! setup_ssl()
|
251
|
-
client_settings.merge! setup_proxy()
|
252
|
-
common_options.merge! setup_basic_auth()
|
253
|
-
|
254
|
-
# Update API setup
|
255
|
-
update_options = {
|
256
|
-
:upsert => @upsert,
|
257
|
-
:doc_as_upsert => @doc_as_upsert
|
258
|
-
}
|
259
|
-
common_options.merge! update_options if @action == 'update'
|
260
|
-
|
261
|
-
@client = LogStash::Outputs::Elasticsearch::HttpClient.new(
|
262
|
-
common_options.merge(:hosts => @hosts, :logger => @logger)
|
263
|
-
)
|
264
|
-
|
265
|
-
if @manage_template
|
266
|
-
begin
|
267
|
-
@logger.info("Automatic template management enabled", :manage_template => @manage_template.to_s)
|
268
|
-
@client.template_install(@template_name, get_template, @template_overwrite)
|
269
|
-
rescue => e
|
270
|
-
@logger.error("Failed to install template: #{e.message}")
|
271
|
-
end
|
272
|
-
end
|
273
|
-
|
274
|
-
@logger.info("New Elasticsearch output", :hosts => @hosts)
|
275
|
-
|
276
|
-
@client_idx = 0
|
277
|
-
|
278
|
-
buffer_initialize(
|
279
|
-
:max_items => @flush_size,
|
280
|
-
:max_interval => @idle_flush_time,
|
281
|
-
:logger => @logger
|
282
|
-
)
|
283
|
-
|
284
|
-
@retry_timer_thread = Thread.new do
|
285
|
-
loop do
|
286
|
-
sleep(@retry_max_interval)
|
287
|
-
@retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
|
288
|
-
end
|
289
|
-
end
|
290
|
-
|
291
|
-
@retry_thread = Thread.new do
|
292
|
-
while @retry_close_requested.false?
|
293
|
-
@retry_flush_mutex.synchronize { @retry_queue_needs_flushing.wait(@retry_flush_mutex) }
|
294
|
-
retry_flush
|
295
|
-
end
|
296
|
-
end
|
297
|
-
end # def register
|
298
|
-
|
299
|
-
public
|
300
|
-
def get_template
|
301
|
-
if @template.nil?
|
302
|
-
@template = ::File.expand_path('elasticsearch/elasticsearch-template.json', ::File.dirname(__FILE__))
|
303
|
-
if !File.exists?(@template)
|
304
|
-
raise "You must specify 'template => ...' in your elasticsearch output (I looked for '#{@template}')"
|
305
|
-
end
|
306
|
-
end
|
307
|
-
template_json = IO.read(@template).gsub(/\n/,'')
|
308
|
-
template = LogStash::Json.load(template_json)
|
309
|
-
@logger.info("Using mapping template", :template => template)
|
310
|
-
return template
|
311
|
-
end # def get_template
|
312
|
-
|
313
|
-
public
|
314
|
-
def receive(event)
|
315
|
-
|
316
|
-
# block until we have not maxed out our
|
317
|
-
# retry queue. This is applying back-pressure
|
318
|
-
# to slow down the receive-rate
|
319
|
-
@retry_flush_mutex.synchronize {
|
320
|
-
@retry_queue_not_full.wait(@retry_flush_mutex) while @retry_queue.size > @retry_max_items
|
321
|
-
}
|
322
|
-
|
323
|
-
event['@metadata']['retry_count'] = 0
|
324
|
-
|
325
|
-
# Set the 'type' value for the index.
|
326
|
-
type = if @document_type
|
327
|
-
event.sprintf(@document_type)
|
328
|
-
elsif @index_type # deprecated
|
329
|
-
event.sprintf(@index_type)
|
330
|
-
else
|
331
|
-
event["type"] || "logs"
|
332
|
-
end
|
333
|
-
|
334
|
-
params = {
|
335
|
-
:_id => @document_id ? event.sprintf(@document_id) : nil,
|
336
|
-
:_index => event.sprintf(@index),
|
337
|
-
:_type => type,
|
338
|
-
:_routing => @routing ? event.sprintf(@routing) : nil
|
339
|
-
}
|
340
|
-
|
341
|
-
params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @action == 'update' && @upsert != ""
|
342
|
-
|
343
|
-
buffer_receive([event.sprintf(@action), params, event])
|
344
|
-
end # def receive
|
345
|
-
|
346
|
-
public
|
347
|
-
# The submit method can be called from both the
|
348
|
-
# Stud::Buffer flush thread and from our own retry thread.
|
349
|
-
def submit(actions)
|
350
|
-
@submit_mutex.synchronize do
|
351
|
-
es_actions = actions.map { |a, doc, event| [a, doc, event.to_hash] }
|
352
|
-
|
353
|
-
bulk_response = @client.bulk(es_actions)
|
354
|
-
|
355
|
-
next unless bulk_response["errors"]
|
356
|
-
|
357
|
-
actions_to_retry = []
|
358
|
-
|
359
|
-
bulk_response["items"].each_with_index do |resp,idx|
|
360
|
-
action_type, action_props = resp.first
|
361
|
-
|
362
|
-
status = action_props["status"]
|
363
|
-
action = actions[idx]
|
364
|
-
|
365
|
-
if RETRYABLE_CODES.include?(status)
|
366
|
-
@logger.warn "retrying failed action with response code: #{status}"
|
367
|
-
actions_to_retry << action
|
368
|
-
elsif not SUCCESS_CODES.include?(status)
|
369
|
-
@logger.warn "Failed action. ", status: status, action: action, response: resp
|
370
|
-
end
|
371
|
-
end
|
372
|
-
|
373
|
-
retry_push(actions_to_retry) unless actions_to_retry.empty?
|
374
|
-
end
|
130
|
+
def build_client
|
131
|
+
@client = ::LogStash::Outputs::ElasticSearch::HttpClientBuilder.build(@logger, @hosts, params)
|
375
132
|
end
|
376
133
|
|
377
|
-
# When there are exceptions raised upon submission, we raise an exception so that
|
378
|
-
# Stud::Buffer will retry to flush
|
379
|
-
public
|
380
|
-
def flush(actions, close = false)
|
381
|
-
begin
|
382
|
-
submit(actions)
|
383
|
-
rescue Manticore::SocketException => e
|
384
|
-
# If we can't even connect to the server let's just print out the URL (:hosts is actually a URL)
|
385
|
-
# and let the user sort it out from there
|
386
|
-
@logger.error(
|
387
|
-
"Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}',"+
|
388
|
-
" but Elasticsearch appears to be unreachable or down!",
|
389
|
-
:client_config => @client.client_options,
|
390
|
-
:error_message => e.message
|
391
|
-
)
|
392
|
-
@logger.debug("Failed actions for last bad bulk request!", :actions => actions)
|
393
|
-
rescue => e
|
394
|
-
# For all other errors print out full connection issues
|
395
|
-
@logger.error(
|
396
|
-
"Attempted to send a bulk request to Elasticsearch configured at '#{@client.client_options[:hosts]}'," +
|
397
|
-
" but an error occurred and it failed! Are you sure you can reach elasticsearch from this machine using " +
|
398
|
-
"the configuration provided?",
|
399
|
-
:client_config => @client.client_options,
|
400
|
-
:error_message => e.message,
|
401
|
-
:error_class => e.class.name,
|
402
|
-
:backtrace => e.backtrace
|
403
|
-
)
|
404
|
-
|
405
|
-
@logger.debug("Failed actions for last bad bulk request!", :actions => actions)
|
406
|
-
|
407
|
-
raise e
|
408
|
-
end
|
409
|
-
end # def flush
|
410
|
-
|
411
|
-
public
|
412
134
|
def close
|
135
|
+
@stopping.make_true
|
413
136
|
@client.stop_sniffing!
|
414
|
-
|
415
|
-
@retry_close_requested.make_true
|
416
|
-
# First, make sure retry_timer_thread is stopped
|
417
|
-
# to ensure we do not signal a retry based on
|
418
|
-
# the retry interval.
|
419
|
-
Thread.kill(@retry_timer_thread)
|
420
|
-
@retry_timer_thread.join
|
421
|
-
# Signal flushing in the case that #retry_flush is in
|
422
|
-
# the process of waiting for a signal.
|
423
|
-
@retry_flush_mutex.synchronize { @retry_queue_needs_flushing.signal }
|
424
|
-
# Now, #retry_flush is ensured to not be in a state of
|
425
|
-
# waiting and can be safely joined into the main thread
|
426
|
-
# for further final execution of an in-process remaining call.
|
427
|
-
@retry_thread.join
|
428
|
-
|
429
|
-
# execute any final actions along with a proceeding retry for any
|
430
|
-
# final actions that did not succeed.
|
431
|
-
buffer_flush(:final => true)
|
432
|
-
retry_flush
|
433
|
-
end
|
434
|
-
|
435
|
-
private
|
436
|
-
def setup_proxy
|
437
|
-
return {} unless @proxy
|
438
|
-
|
439
|
-
# Symbolize keys
|
440
|
-
proxy = if @proxy.is_a?(Hash)
|
441
|
-
Hash[@proxy.map {|k,v| [k.to_sym, v]}]
|
442
|
-
elsif @proxy.is_a?(String)
|
443
|
-
@proxy
|
444
|
-
else
|
445
|
-
raise LogStash::ConfigurationError, "Expected 'proxy' to be a string or hash, not '#{@proxy}''!"
|
446
|
-
end
|
447
|
-
|
448
|
-
return {:proxy => proxy}
|
449
|
-
end
|
450
|
-
|
451
|
-
private
|
452
|
-
def setup_ssl
|
453
|
-
return {} unless @ssl
|
454
|
-
|
455
|
-
if @cacert && @truststore
|
456
|
-
raise(LogStash::ConfigurationError, "Use either \"cacert\" or \"truststore\" when configuring the CA certificate") if @truststore
|
457
|
-
end
|
458
|
-
|
459
|
-
ssl_options = {}
|
460
|
-
|
461
|
-
if @cacert
|
462
|
-
ssl_options[:ca_file] = @cacert
|
463
|
-
elsif @truststore
|
464
|
-
ssl_options[:truststore_password] = @truststore_password.value if @truststore_password
|
465
|
-
end
|
466
|
-
|
467
|
-
ssl_options[:truststore] = @truststore if @truststore
|
468
|
-
if @keystore
|
469
|
-
ssl_options[:keystore] = @keystore
|
470
|
-
ssl_options[:keystore_password] = @keystore_password.value if @keystore_password
|
471
|
-
end
|
472
|
-
if @ssl_certificate_verification == false
|
473
|
-
@logger.warn [
|
474
|
-
"** WARNING ** Detected UNSAFE options in elasticsearch output configuration!",
|
475
|
-
"** WARNING ** You have enabled encryption but DISABLED certificate verification.",
|
476
|
-
"** WARNING ** To make sure your data is secure change :ssl_certificate_verification to true"
|
477
|
-
].join("\n")
|
478
|
-
ssl_options[:verify] = false
|
479
|
-
end
|
480
|
-
{ ssl: ssl_options }
|
481
|
-
end
|
482
|
-
|
483
|
-
private
|
484
|
-
def setup_basic_auth
|
485
|
-
return {} unless @user && @password
|
486
|
-
|
487
|
-
{
|
488
|
-
:user => ::URI.escape(@user, "@:"),
|
489
|
-
:password => ::URI.escape(@password.value, "@:")
|
490
|
-
}
|
491
|
-
end
|
492
|
-
|
493
|
-
private
|
494
|
-
# in charge of submitting any actions in @retry_queue that need to be
|
495
|
-
# retried
|
496
|
-
#
|
497
|
-
# This method is not called concurrently. It is only called by @retry_thread
|
498
|
-
# and once that thread is ended during the close process, a final call
|
499
|
-
# to this method is done upon close in the main thread.
|
500
|
-
def retry_flush()
|
501
|
-
unless @retry_queue.empty?
|
502
|
-
buffer = @retry_queue.size.times.map do
|
503
|
-
next_action, next_doc, next_event = @retry_queue.pop
|
504
|
-
next_event['@metadata']['retry_count'] += 1
|
505
|
-
|
506
|
-
if next_event['@metadata']['retry_count'] > @max_retries
|
507
|
-
@logger.error "too many attempts at sending event. dropping: #{next_event}"
|
508
|
-
nil
|
509
|
-
else
|
510
|
-
[next_action, next_doc, next_event]
|
511
|
-
end
|
512
|
-
end.compact
|
513
|
-
|
514
|
-
submit(buffer) unless buffer.empty?
|
515
|
-
end
|
516
|
-
|
517
|
-
@retry_flush_mutex.synchronize {
|
518
|
-
@retry_queue_not_full.signal if @retry_queue.size < @retry_max_items
|
519
|
-
}
|
520
|
-
end
|
521
|
-
|
522
|
-
private
|
523
|
-
def retry_push(actions)
|
524
|
-
Array(actions).each{|action| @retry_queue << action}
|
525
|
-
@retry_flush_mutex.synchronize {
|
526
|
-
@retry_queue_needs_flushing.signal if @retry_queue.size >= @retry_max_items
|
527
|
-
}
|
137
|
+
@buffer.stop
|
528
138
|
end
|
529
139
|
|
530
140
|
@@plugins = Gem::Specification.find_all{|spec| spec.name =~ /logstash-output-elasticsearch-/ }
|