logstash-output-elasticsearch 2.2.0-java → 2.3.0-java

Sign up to get free protection for your applications and to get access to all the features.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -1
  3. data/README.md +0 -5
  4. data/lib/logstash/outputs/elasticsearch/common.rb +3 -6
  5. data/lib/logstash/outputs/elasticsearch/common_configs.rb +1 -2
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +1 -1
  7. data/lib/logstash/outputs/elasticsearch.rb +10 -13
  8. data/logstash-output-elasticsearch.gemspec +1 -1
  9. data/spec/integration/outputs/retry_spec.rb +3 -5
  10. metadata +48 -48
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0bacdce326400468748dde5d68f0d93612c481d0
4
- data.tar.gz: ed36edd4dc675f9c5940f34314dc511134e3b0db
3
+ metadata.gz: b69485c1e33ca13de8409921d2edfd4e431d16bb
4
+ data.tar.gz: 55105ccbadd6593f97b2bf720833dac0d530615e
5
5
  SHA512:
6
- metadata.gz: c1eefd941d00fcef32a5194f0bb8b457f8e8bb690cec751c4c347a89c655ea0ea4028addd7211d36e49fccda107fcbe569ed64fb5ea1869f93e0cd4738612946
7
- data.tar.gz: 13c2da8b12fa91911e720080d75c3d0afbc984a21ce4db3b4ce150159289db63189e4455eda63005e5c170c30e59c4f6bcb5ba7611a77d465be8319f054ffb63
6
+ metadata.gz: d1234fef6c6a6498b29d58999080ff44927cce98d31806daacdd0ac79b17f4f0c34be4ee70f0cec2c945bf1882baeb84beca8580d079a3b0ce1f32c4b503e047
7
+ data.tar.gz: 8abf03dad06ca162c7d57ed38dc613019781644923db28adacdbc05718ffa4af29923e4067754868bd42ad84b42de188709105235231a5a55c38b750584da961
data/CHANGELOG.md CHANGED
@@ -1,4 +1,9 @@
1
- ## 2.1.6
1
+ ## 2.3.0
2
+ - Now retry too busy and service unavailable errors infinitely.
3
+ - Never retry conflict errors
4
+ - Fix broken delete verb that would fail due to sending body with verb
5
+
6
+ ## 2.2.0
2
7
  - Serialize access to the connection pool in es-ruby client
3
8
  - Add support for parent relationship
4
9
 
data/README.md CHANGED
@@ -1,10 +1,5 @@
1
1
  # Logstash Plugin
2
2
 
3
- ATTN: This plugin is undergoing a [major refactor](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/291)! Any refactors started now may need to be rebased/refactored before they are accepted!
4
-
5
- [![Build
6
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/)
7
-
8
3
  [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-output-elasticsearch)
9
4
 
10
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
data/lib/logstash/outputs/elasticsearch/common.rb CHANGED
@@ -5,7 +5,7 @@ module LogStash; module Outputs; class ElasticSearch;
5
5
  module Common
6
6
  attr_reader :client, :hosts
7
7
 
8
- RETRYABLE_CODES = [409, 429, 503]
8
+ RETRYABLE_CODES = [429, 503]
9
9
  SUCCESS_CODES = [200, 201]
10
10
 
11
11
  def register
@@ -74,23 +74,20 @@ module LogStash; module Outputs; class ElasticSearch;
74
74
  end
75
75
 
76
76
  def retrying_submit(actions)
77
- retries_left = @max_retries+1 # +1 for the first attempt
78
-
79
77
  # Initially we submit the full list of actions
80
78
  submit_actions = actions
81
79
 
82
- while submit_actions && submit_actions.length > 0 && retries_left > 0
80
+ while submit_actions && submit_actions.length
83
81
  return if !submit_actions || submit_actions.empty? # If everything's a success we move along
84
82
  # We retry with whatever is didn't succeed
85
83
  begin
86
84
  submit_actions = submit(submit_actions)
87
85
  rescue => e
88
- @logger.warn("Encountered an unexpected error submitting a bulk request!",
86
+ @logger.warn("Encountered an unexpected error submitting a bulk request! Will retry.",
89
87
  :message => e.message,
90
88
  :class => e.class.name,
91
89
  :backtrace => e.backtrace)
92
90
  end
93
- retries_left -= 1
94
91
  end
95
92
  end
96
93
 
data/lib/logstash/outputs/elasticsearch/common_configs.rb CHANGED
@@ -102,8 +102,7 @@ module LogStash; module Outputs; class ElasticSearch
102
102
  # Create a new document with source if `document_id` doesn't exist in Elasticsearch
103
103
  mod.config :doc_as_upsert, :validate => :boolean, :default => false
104
104
 
105
- # Set max retry for each event. The total time spent blocked on retries will be
106
- # (max_retries * retry_max_interval). This may vary a bit if Elasticsearch is very slow to respond
105
+ # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
107
106
  mod.config :max_retries, :validate => :number, :default => 3
108
107
 
109
108
  # Set max interval between bulk retries.
data/lib/logstash/outputs/elasticsearch/http_client.rb CHANGED
@@ -56,7 +56,7 @@ module LogStash; module Outputs; class ElasticSearch;
56
56
 
57
57
  args.delete(:_upsert)
58
58
 
59
- if source
59
+ if source && action != 'delete'
60
60
  next [ { action => args }, source ]
61
61
  else
62
62
  next { action => args }
data/lib/logstash/outputs/elasticsearch.rb CHANGED
@@ -22,21 +22,18 @@ require "uri" # for escaping user input
22
22
  #
23
23
  # ==== Retry Policy
24
24
  #
25
+ # The retry policy has changed significantly in the 2.2.0 release.
25
26
  # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
26
- # either partial or total failures. Events are retried if they fail due to either a network error or the status codes
27
- # 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
27
+ # either partial or total failures.
28
28
  #
29
- # The retry policy's logic can be described as follows:
29
+ # The following errors are retried infinitely:
30
30
  #
31
- # - Block and retry all events in the bulk response that experience transient network exceptions until
32
- # a successful submission is received by Elasticsearch.
33
- # - Retry the subset of sent events which resulted in ES errors of a retryable nature.
34
- # - Events which returned retryable error codes will be pushed onto a separate queue for
35
- # retrying events. Events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries).
36
- # The size of this queue is capped by the value set in :retry_max_items.
37
- # - Events from the retry queue are submitted again when the queue reaches its max size or when
38
- # the max interval time is reached. The max interval time is configurable via :retry_max_interval.
39
- # - Events which are not retryable or have reached their max retry count are logged to stderr.
31
+ # - Network errors (inability to connect)
32
+ # - 429 (Too many requests) and
33
+ # - 503 (Service unavailable) errors
34
+ #
35
+ # NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
36
+ # It is more performant for Elasticsearch to retry these exceptions than this plugin.
40
37
  #
41
38
  # ==== DNS Caching
42
39
  #
@@ -113,7 +110,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
113
110
  # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
114
111
  # this with master nodes, you probably want to disable HTTP on them by setting
115
112
  # `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
116
- # manually enter multiple Elasticsearch hosts using the `hosts` paramater.
113
+ # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
117
114
  config :sniffing, :validate => :boolean, :default => false
118
115
 
119
116
  # How long to wait, in seconds, between sniffing attempts
data/logstash-output-elasticsearch.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '2.2.0'
4
+ s.version = '2.3.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "Output events to elasticsearch"
data/spec/integration/outputs/retry_spec.rb CHANGED
@@ -8,7 +8,6 @@ describe "failures in bulk class expected behavior", :integration => true do
8
8
  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
9
9
  let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
10
10
  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
11
- let(:max_retries) { 3 }
12
11
 
13
12
  def mock_actions_with_response(*resp)
14
13
  expanded_responses = resp.map do |resp|
@@ -33,7 +32,6 @@ describe "failures in bulk class expected behavior", :integration => true do
33
32
  "hosts" => get_host_port(),
34
33
  "retry_max_items" => 10,
35
34
  "retry_max_interval" => 1,
36
- "max_retries" => max_retries
37
35
  }
38
36
  next LogStash::Outputs::ElasticSearch.new(settings)
39
37
  end
@@ -108,14 +106,14 @@ describe "failures in bulk class expected behavior", :integration => true do
108
106
  sleep(3)
109
107
  end
110
108
 
111
- it "should retry an event until max_retries reached" do
109
+ it "should retry an event infinitely until a non retryable status occurs" do
112
110
  mock_actions_with_response({"errors" => true, "statuses" => [429]},
113
111
  {"errors" => true, "statuses" => [429]},
114
112
  {"errors" => true, "statuses" => [429]},
115
113
  {"errors" => true, "statuses" => [429]},
116
114
  {"errors" => true, "statuses" => [429]},
117
- {"errors" => true, "statuses" => [429]})
118
- expect(subject).to receive(:submit).with([action1]).exactly(max_retries+1).times.and_call_original
115
+ {"errors" => true, "statuses" => [500]})
116
+ expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
119
117
  subject.register
120
118
  subject.receive(event1)
121
119
  subject.flush
metadata CHANGED
@@ -1,30 +1,39 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 2.3.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-07 00:00:00.000000000 Z
11
+ date: 2015-12-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: concurrent-ruby
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
14
20
  requirement: !ruby/object:Gem::Requirement
15
21
  requirements:
16
22
  - - '>='
17
23
  - !ruby/object:Gem::Version
18
24
  version: '0'
19
- name: concurrent-ruby
20
25
  prerelease: false
21
26
  type: :runtime
27
+ - !ruby/object:Gem::Dependency
28
+ name: elasticsearch
22
29
  version_requirements: !ruby/object:Gem::Requirement
23
30
  requirements:
24
31
  - - '>='
25
32
  - !ruby/object:Gem::Version
26
- version: '0'
27
- - !ruby/object:Gem::Dependency
33
+ version: 1.0.13
34
+ - - ~>
35
+ - !ruby/object:Gem::Version
36
+ version: '1.0'
28
37
  requirement: !ruby/object:Gem::Requirement
29
38
  requirements:
30
39
  - - '>='
@@ -33,18 +42,18 @@ dependencies:
33
42
  - - ~>
34
43
  - !ruby/object:Gem::Version
35
44
  version: '1.0'
36
- name: elasticsearch
37
45
  prerelease: false
38
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: stud
39
49
  version_requirements: !ruby/object:Gem::Requirement
40
50
  requirements:
41
51
  - - '>='
42
52
  - !ruby/object:Gem::Version
43
- version: 1.0.13
53
+ version: 0.0.17
44
54
  - - ~>
45
55
  - !ruby/object:Gem::Version
46
- version: '1.0'
47
- - !ruby/object:Gem::Dependency
56
+ version: '0.0'
48
57
  requirement: !ruby/object:Gem::Requirement
49
58
  requirements:
50
59
  - - '>='
@@ -53,33 +62,25 @@ dependencies:
53
62
  - - ~>
54
63
  - !ruby/object:Gem::Version
55
64
  version: '0.0'
56
- name: stud
57
65
  prerelease: false
58
66
  type: :runtime
67
+ - !ruby/object:Gem::Dependency
68
+ name: cabin
59
69
  version_requirements: !ruby/object:Gem::Requirement
60
70
  requirements:
61
- - - '>='
62
- - !ruby/object:Gem::Version
63
- version: 0.0.17
64
71
  - - ~>
65
72
  - !ruby/object:Gem::Version
66
- version: '0.0'
67
- - !ruby/object:Gem::Dependency
73
+ version: '0.6'
68
74
  requirement: !ruby/object:Gem::Requirement
69
75
  requirements:
70
76
  - - ~>
71
77
  - !ruby/object:Gem::Version
72
78
  version: '0.6'
73
- name: cabin
74
79
  prerelease: false
75
80
  type: :runtime
76
- version_requirements: !ruby/object:Gem::Requirement
77
- requirements:
78
- - - ~>
79
- - !ruby/object:Gem::Version
80
- version: '0.6'
81
81
  - !ruby/object:Gem::Dependency
82
- requirement: !ruby/object:Gem::Requirement
82
+ name: logstash-core
83
+ version_requirements: !ruby/object:Gem::Requirement
83
84
  requirements:
84
85
  - - '>='
85
86
  - !ruby/object:Gem::Version
@@ -87,10 +88,7 @@ dependencies:
87
88
  - - <
88
89
  - !ruby/object:Gem::Version
89
90
  version: 3.0.0
90
- name: logstash-core
91
- prerelease: false
92
- type: :runtime
93
- version_requirements: !ruby/object:Gem::Requirement
91
+ requirement: !ruby/object:Gem::Requirement
94
92
  requirements:
95
93
  - - '>='
96
94
  - !ruby/object:Gem::Version
@@ -98,90 +96,92 @@ dependencies:
98
96
  - - <
99
97
  - !ruby/object:Gem::Version
100
98
  version: 3.0.0
99
+ prerelease: false
100
+ type: :runtime
101
101
  - !ruby/object:Gem::Dependency
102
+ name: ftw
103
+ version_requirements: !ruby/object:Gem::Requirement
104
+ requirements:
105
+ - - ~>
106
+ - !ruby/object:Gem::Version
107
+ version: 0.0.42
102
108
  requirement: !ruby/object:Gem::Requirement
103
109
  requirements:
104
110
  - - ~>
105
111
  - !ruby/object:Gem::Version
106
112
  version: 0.0.42
107
- name: ftw
108
113
  prerelease: false
109
114
  type: :development
115
+ - !ruby/object:Gem::Dependency
116
+ name: logstash-codec-plain
110
117
  version_requirements: !ruby/object:Gem::Requirement
111
118
  requirements:
112
- - - ~>
119
+ - - '>='
113
120
  - !ruby/object:Gem::Version
114
- version: 0.0.42
115
- - !ruby/object:Gem::Dependency
121
+ version: '0'
116
122
  requirement: !ruby/object:Gem::Requirement
117
123
  requirements:
118
124
  - - '>='
119
125
  - !ruby/object:Gem::Version
120
126
  version: '0'
121
- name: logstash-codec-plain
122
127
  prerelease: false
123
128
  type: :development
129
+ - !ruby/object:Gem::Dependency
130
+ name: manticore
124
131
  version_requirements: !ruby/object:Gem::Requirement
125
132
  requirements:
126
- - - '>='
133
+ - - ~>
127
134
  - !ruby/object:Gem::Version
128
- version: '0'
129
- - !ruby/object:Gem::Dependency
135
+ version: 0.4.2
130
136
  requirement: !ruby/object:Gem::Requirement
131
137
  requirements:
132
138
  - - ~>
133
139
  - !ruby/object:Gem::Version
134
140
  version: 0.4.2
135
- name: manticore
136
141
  prerelease: false
137
142
  type: :runtime
143
+ - !ruby/object:Gem::Dependency
144
+ name: logstash-devutils
138
145
  version_requirements: !ruby/object:Gem::Requirement
139
146
  requirements:
140
- - - ~>
147
+ - - '>='
141
148
  - !ruby/object:Gem::Version
142
- version: 0.4.2
143
- - !ruby/object:Gem::Dependency
149
+ version: '0'
144
150
  requirement: !ruby/object:Gem::Requirement
145
151
  requirements:
146
152
  - - '>='
147
153
  - !ruby/object:Gem::Version
148
154
  version: '0'
149
- name: logstash-devutils
150
155
  prerelease: false
151
156
  type: :development
157
+ - !ruby/object:Gem::Dependency
158
+ name: longshoreman
152
159
  version_requirements: !ruby/object:Gem::Requirement
153
160
  requirements:
154
161
  - - '>='
155
162
  - !ruby/object:Gem::Version
156
163
  version: '0'
157
- - !ruby/object:Gem::Dependency
158
164
  requirement: !ruby/object:Gem::Requirement
159
165
  requirements:
160
166
  - - '>='
161
167
  - !ruby/object:Gem::Version
162
168
  version: '0'
163
- name: longshoreman
164
169
  prerelease: false
165
170
  type: :development
171
+ - !ruby/object:Gem::Dependency
172
+ name: flores
166
173
  version_requirements: !ruby/object:Gem::Requirement
167
174
  requirements:
168
175
  - - '>='
169
176
  - !ruby/object:Gem::Version
170
177
  version: '0'
171
- - !ruby/object:Gem::Dependency
172
178
  requirement: !ruby/object:Gem::Requirement
173
179
  requirements:
174
180
  - - '>='
175
181
  - !ruby/object:Gem::Version
176
182
  version: '0'
177
- name: flores
178
183
  prerelease: false
179
184
  type: :development
180
- version_requirements: !ruby/object:Gem::Requirement
181
- requirements:
182
- - - '>='
183
- - !ruby/object:Gem::Version
184
- version: '0'
185
185
  description: Output events to elasticsearch
186
186
  email: info@elastic.co
187
187
  executables: []