logstash-output-elasticsearch 4.1.1-java → 4.1.2-java
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/CONTRIBUTORS +1 -0
- data/lib/logstash/outputs/elasticsearch.rb +7 -1
- data/lib/logstash/outputs/elasticsearch/common.rb +6 -5
- data/logstash-output-elasticsearch.gemspec +1 -1
- data/spec/integration/outputs/routing_spec.rb +2 -1
- data/spec/integration/outputs/templates_spec.rb +1 -3
- data/spec/integration/outputs/update_spec.rb +2 -2
- data/spec/unit/outputs/error_whitelist_spec.rb +52 -0
- metadata +4 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d34b7350d64567ba86294e3497d0d37216df93b0
|
4
|
+
data.tar.gz: 681dbbb922820129b82c483b2d108da395191f5a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a3bb076147070c2f3e95da532f5203647e3b40ab3f25af3887cd36a2844634de890319d711bd9edc8167dcdb6aa996242e49ea5f50bf76512c608437087d1fc6
|
7
|
+
data.tar.gz: 50072bff0779e28344c7e7a5b686d53eac151b68403a6f6061b7d99b8964f10879c215578c0492488aa61ff78043b8c6676d72572bbd9f78f8af0fd7c4aeed5f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
-
## 4.1.
|
1
|
+
## 4.1.2
|
2
|
+
|
3
|
+
- Added a configuration called failure_type_logging_whitelist which takes a list of strings, that are error types from elasticsearch, so we prevent logging WARN if elasticsearch fails with that action. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/423
|
2
4
|
|
5
|
+
## 4.1.1
|
3
6
|
- Fix bug where setting credentials would cause fatal errors. See https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/441
|
4
7
|
|
5
8
|
## 4.1.0
|
data/CONTRIBUTORS
CHANGED
@@ -25,6 +25,7 @@ Contributors:
|
|
25
25
|
* Tal Levy (talevy)
|
26
26
|
* Tom Hodder (tolland)
|
27
27
|
* jimmyjones2
|
28
|
+
* Gabriel Moskovicz (gmoskovicz)
|
28
29
|
|
29
30
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
30
31
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -67,7 +67,8 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
67
67
|
# - delete: deletes a document by id (An id is required for this action)
|
68
68
|
# - create: indexes a document, fails if a document by that id already exists in the index.
|
69
69
|
# - update: updates a document by id. Update has a special case where you can upsert -- update a
|
70
|
-
# document if not already present. See the `upsert` option
|
70
|
+
# document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
|
71
|
+
# in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
|
71
72
|
# - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
|
72
73
|
# would use the foo field for the action
|
73
74
|
#
|
@@ -132,6 +133,11 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
|
|
132
133
|
# a timeout occurs, the request will be retried.
|
133
134
|
config :timeout, :validate => :number
|
134
135
|
|
136
|
+
# Set the Elasticsearch errors in the whitelist that you don't want to log.
|
137
|
+
# A useful example is when you want to skip all 409 errors
|
138
|
+
# which are `document_already_exists_exception`.
|
139
|
+
config :failure_type_logging_whitelist, :validate => :array, :default => []
|
140
|
+
|
135
141
|
# While the output tries to reuse connections efficiently we have a maximum.
|
136
142
|
# This sets the maximum number of open connections the output will create.
|
137
143
|
# Setting this too low may mean frequently closing / opening connections
|
@@ -112,17 +112,18 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
112
112
|
actions_to_retry = []
|
113
113
|
bulk_response["items"].each_with_index do |response,idx|
|
114
114
|
action_type, action_props = response.first
|
115
|
+
|
115
116
|
status = action_props["status"]
|
116
|
-
|
117
|
+
failure = action_props["error"]
|
117
118
|
action = actions[idx]
|
118
119
|
|
119
120
|
if SUCCESS_CODES.include?(status)
|
120
121
|
next
|
121
122
|
elsif RETRYABLE_CODES.include?(status)
|
122
|
-
@logger.info "retrying failed action with response code: #{status} (#{
|
123
|
+
@logger.info "retrying failed action with response code: #{status} (#{failure})"
|
123
124
|
actions_to_retry << action
|
124
|
-
|
125
|
-
@logger.warn "Failed action.
|
125
|
+
elsif !failure_type_logging_whitelist.include?(failure["type"])
|
126
|
+
@logger.warn "Failed action.", status: status, action: action, response: response
|
126
127
|
end
|
127
128
|
end
|
128
129
|
|
@@ -214,7 +215,7 @@ module LogStash; module Outputs; class ElasticSearch;
|
|
214
215
|
sleep_interval = sleep_for_interval(sleep_interval)
|
215
216
|
retry unless @stopping.true?
|
216
217
|
else
|
217
|
-
@logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.
|
218
|
+
@logger.error("Got a bad response code from server, but this code is not considered retryable. Request will be dropped", :code => e.response_code)
|
218
219
|
end
|
219
220
|
rescue => e
|
220
221
|
# Stuff that should never happen
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-output-elasticsearch'
|
4
|
-
s.version = '4.1.
|
4
|
+
s.version = '4.1.2'
|
5
5
|
s.licenses = ['apache-2.0']
|
6
6
|
s.summary = "Logstash Output to Elasticsearch"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -12,7 +12,7 @@ shared_examples "a routing indexer" do
|
|
12
12
|
before do
|
13
13
|
subject.register
|
14
14
|
event_count.times do
|
15
|
-
subject.multi_receive([LogStash::Event.new("message" => "
|
15
|
+
subject.multi_receive([LogStash::Event.new("message" => "test", "type" => type)])
|
16
16
|
end
|
17
17
|
end
|
18
18
|
|
@@ -26,6 +26,7 @@ shared_examples "a routing indexer" do
|
|
26
26
|
# Wait until all events are available.
|
27
27
|
Stud::try(10.times) do
|
28
28
|
data = ""
|
29
|
+
|
29
30
|
response = ftw.get!("#{index_url}/_count?q=*&routing=#{routing}")
|
30
31
|
response.read_body { |chunk| data << chunk }
|
31
32
|
result = LogStash::Json.load(data)
|
@@ -75,9 +75,7 @@ describe "index template expected behavior", :integration => true do
|
|
75
75
|
end
|
76
76
|
|
77
77
|
it "make [geoip][location] a geo_point" do
|
78
|
-
|
79
|
-
insist { results["hits"]["total"] } == 1
|
80
|
-
insist { results["hits"]["hits"][0]["_source"]["geoip"]["location"] } == [ 0.0, 0.0 ]
|
78
|
+
expect(@es.indices.get_template(name: "logstash")["logstash"]["mappings"]["_default_"]["properties"]["geoip"]["properties"]["location"]["type"]).to eq("geo_point")
|
81
79
|
end
|
82
80
|
|
83
81
|
it "aggregate .raw results correctly " do
|
@@ -1,6 +1,6 @@
|
|
1
1
|
require_relative "../../../spec/es_spec_helper"
|
2
2
|
|
3
|
-
describe "Update actions", :integration => true do
|
3
|
+
describe "Update actions", :integration => true, :version_2x_plus => true do
|
4
4
|
require "logstash/outputs/elasticsearch"
|
5
5
|
|
6
6
|
def get_es_output( options={} )
|
@@ -102,7 +102,7 @@ describe "Update actions", :integration => true do
|
|
102
102
|
'script_type' => 'inline'
|
103
103
|
})
|
104
104
|
subject.register
|
105
|
-
subject.multi_receive([LogStash::Event.new("
|
105
|
+
subject.multi_receive([LogStash::Event.new("counter" => 3 )])
|
106
106
|
r = @es.get(:index => 'logstash-update', :type => 'logs', :id => "123", :refresh => true)
|
107
107
|
insist { r["_source"]["counter"] } == 4
|
108
108
|
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require "logstash/outputs/elasticsearch"
|
2
|
+
require_relative "../../../spec/es_spec_helper"
|
3
|
+
|
4
|
+
describe "whitelisting error types in expected behavior" do
|
5
|
+
let(:template) { '{"template" : "not important, will be updated by :index"}' }
|
6
|
+
let(:event1) { LogStash::Event.new("somevalue" => 100, "@timestamp" => "2014-11-17T20:37:17.223Z") }
|
7
|
+
let(:action1) { ["index", {:_id=>1, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event1] }
|
8
|
+
let(:settings) { {"manage_template" => true, "index" => "logstash-2014.11.17", "template_overwrite" => true, "hosts" => get_host_port() } }
|
9
|
+
|
10
|
+
subject { LogStash::Outputs::ElasticSearch.new(settings) }
|
11
|
+
|
12
|
+
before :each do
|
13
|
+
allow(subject.logger).to receive(:warn)
|
14
|
+
|
15
|
+
subject.register
|
16
|
+
|
17
|
+
allow(subject.client).to receive(:bulk).and_return(
|
18
|
+
{
|
19
|
+
"errors" => true,
|
20
|
+
"items" => [{
|
21
|
+
"create" => {
|
22
|
+
"status" => 409,
|
23
|
+
"error" => {
|
24
|
+
"type" => "document_already_exists_exception",
|
25
|
+
"reason" => "[shard] document already exists"
|
26
|
+
}
|
27
|
+
}
|
28
|
+
}]
|
29
|
+
})
|
30
|
+
|
31
|
+
subject.multi_receive([event1])
|
32
|
+
end
|
33
|
+
|
34
|
+
after :each do
|
35
|
+
subject.close
|
36
|
+
end
|
37
|
+
|
38
|
+
describe "when failure logging is enabled for everything" do
|
39
|
+
it "should log a failure on the action" do
|
40
|
+
expect(subject.logger).to have_received(:warn).with("Failed action.", anything)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
describe "when failure logging is disabled for docuemnt exists error" do
|
45
|
+
let(:settings) { super.merge("failure_type_logging_whitelist" => ["document_already_exists_exception"]) }
|
46
|
+
|
47
|
+
it "should log a failure on the action" do
|
48
|
+
expect(subject.logger).not_to have_received(:warn).with("Failed action.", anything)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-elasticsearch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.1.
|
4
|
+
version: 4.1.2
|
5
5
|
platform: java
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-07-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -205,6 +205,7 @@ files:
|
|
205
205
|
- spec/unit/outputs/elasticsearch_proxy_spec.rb
|
206
206
|
- spec/unit/outputs/elasticsearch_spec.rb
|
207
207
|
- spec/unit/outputs/elasticsearch_ssl_spec.rb
|
208
|
+
- spec/unit/outputs/error_whitelist_spec.rb
|
208
209
|
homepage: http://logstash.net/
|
209
210
|
licenses:
|
210
211
|
- apache-2.0
|
@@ -253,3 +254,4 @@ test_files:
|
|
253
254
|
- spec/unit/outputs/elasticsearch_proxy_spec.rb
|
254
255
|
- spec/unit/outputs/elasticsearch_spec.rb
|
255
256
|
- spec/unit/outputs/elasticsearch_ssl_spec.rb
|
257
|
+
- spec/unit/outputs/error_whitelist_spec.rb
|