logstash-output-elasticsearch 2.2.0-java → 2.3.0-java

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (10) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +6 -1
  3. data/README.md +0 -5
  4. data/lib/logstash/outputs/elasticsearch/common.rb +3 -6
  5. data/lib/logstash/outputs/elasticsearch/common_configs.rb +1 -2
  6. data/lib/logstash/outputs/elasticsearch/http_client.rb +1 -1
  7. data/lib/logstash/outputs/elasticsearch.rb +10 -13
  8. data/logstash-output-elasticsearch.gemspec +1 -1
  9. data/spec/integration/outputs/retry_spec.rb +3 -5
  10. metadata +48 -48
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 0bacdce326400468748dde5d68f0d93612c481d0
4
- data.tar.gz: ed36edd4dc675f9c5940f34314dc511134e3b0db
3
+ metadata.gz: b69485c1e33ca13de8409921d2edfd4e431d16bb
4
+ data.tar.gz: 55105ccbadd6593f97b2bf720833dac0d530615e
5
5
  SHA512:
6
- metadata.gz: c1eefd941d00fcef32a5194f0bb8b457f8e8bb690cec751c4c347a89c655ea0ea4028addd7211d36e49fccda107fcbe569ed64fb5ea1869f93e0cd4738612946
7
- data.tar.gz: 13c2da8b12fa91911e720080d75c3d0afbc984a21ce4db3b4ce150159289db63189e4455eda63005e5c170c30e59c4f6bcb5ba7611a77d465be8319f054ffb63
6
+ metadata.gz: d1234fef6c6a6498b29d58999080ff44927cce98d31806daacdd0ac79b17f4f0c34be4ee70f0cec2c945bf1882baeb84beca8580d079a3b0ce1f32c4b503e047
7
+ data.tar.gz: 8abf03dad06ca162c7d57ed38dc613019781644923db28adacdbc05718ffa4af29923e4067754868bd42ad84b42de188709105235231a5a55c38b750584da961
data/CHANGELOG.md CHANGED
@@ -1,4 +1,9 @@
1
- ## 2.1.6
1
+ ## 2.3.0
2
+ - Now retry too busy and service unavailable errors infinitely.
3
+ - Never retry conflict errors
4
+ - Fix broken delete verb that would fail due to sending body with verb
5
+
6
+ ## 2.2.0
2
7
  - Serialize access to the connection pool in es-ruby client
3
8
  - Add support for parent relationship
4
9
 
data/README.md CHANGED
@@ -1,10 +1,5 @@
1
1
  # Logstash Plugin
2
2
 
3
- ATTN: This plugin is undergoing a [major refactor](https://github.com/logstash-plugins/logstash-output-elasticsearch/issues/291)! Any refactors started now may need to be rebased/refactored before they are accepted!
4
-
5
- [![Build
6
- Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Outputs/job/logstash-plugin-output-elasticsearch-unit/)
7
-
8
3
  [![Travis Build Status](https://travis-ci.org/logstash-plugins/logstash-output-elasticsearch.svg)](https://travis-ci.org/logstash-plugins/logstash-output-elasticsearch)
9
4
 
10
5
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
data/lib/logstash/outputs/elasticsearch/common.rb CHANGED
@@ -5,7 +5,7 @@ module LogStash; module Outputs; class ElasticSearch;
5
5
  module Common
6
6
  attr_reader :client, :hosts
7
7
 
8
- RETRYABLE_CODES = [409, 429, 503]
8
+ RETRYABLE_CODES = [429, 503]
9
9
  SUCCESS_CODES = [200, 201]
10
10
 
11
11
  def register
@@ -74,23 +74,20 @@ module LogStash; module Outputs; class ElasticSearch;
74
74
  end
75
75
 
76
76
  def retrying_submit(actions)
77
- retries_left = @max_retries+1 # +1 for the first attempt
78
-
79
77
  # Initially we submit the full list of actions
80
78
  submit_actions = actions
81
79
 
82
- while submit_actions && submit_actions.length > 0 && retries_left > 0
80
+ while submit_actions && submit_actions.length
83
81
  return if !submit_actions || submit_actions.empty? # If everything's a success we move along
84
82
  # We retry with whatever is didn't succeed
85
83
  begin
86
84
  submit_actions = submit(submit_actions)
87
85
  rescue => e
88
- @logger.warn("Encountered an unexpected error submitting a bulk request!",
86
+ @logger.warn("Encountered an unexpected error submitting a bulk request! Will retry.",
89
87
  :message => e.message,
90
88
  :class => e.class.name,
91
89
  :backtrace => e.backtrace)
92
90
  end
93
- retries_left -= 1
94
91
  end
95
92
  end
96
93
 
data/lib/logstash/outputs/elasticsearch/common_configs.rb CHANGED
@@ -102,8 +102,7 @@ module LogStash; module Outputs; class ElasticSearch
102
102
  # Create a new document with source if `document_id` doesn't exist in Elasticsearch
103
103
  mod.config :doc_as_upsert, :validate => :boolean, :default => false
104
104
 
105
- # Set max retry for each event. The total time spent blocked on retries will be
106
- # (max_retries * retry_max_interval). This may vary a bit if Elasticsearch is very slow to respond
105
+ # DEPRECATED This setting no longer does anything. It will be marked obsolete in a future version.
107
106
  mod.config :max_retries, :validate => :number, :default => 3
108
107
 
109
108
  # Set max interval between bulk retries.
data/lib/logstash/outputs/elasticsearch/http_client.rb CHANGED
@@ -56,7 +56,7 @@ module LogStash; module Outputs; class ElasticSearch;
56
56
 
57
57
  args.delete(:_upsert)
58
58
 
59
- if source
59
+ if source && action != 'delete'
60
60
  next [ { action => args }, source ]
61
61
  else
62
62
  next { action => args }
data/lib/logstash/outputs/elasticsearch.rb CHANGED
@@ -22,21 +22,18 @@ require "uri" # for escaping user input
22
22
  #
23
23
  # ==== Retry Policy
24
24
  #
25
+ # The retry policy has changed significantly in the 2.2.0 release.
25
26
  # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
26
- # either partial or total failures. Events are retried if they fail due to either a network error or the status codes
27
- # 429 (the server is busy), 409 (Version Conflict), or 503 (temporary overloading/maintenance).
27
+ # either partial or total failures.
28
28
  #
29
- # The retry policy's logic can be described as follows:
29
+ # The following errors are retried infinitely:
30
30
  #
31
- # - Block and retry all events in the bulk response that experience transient network exceptions until
32
- # a successful submission is received by Elasticsearch.
33
- # - Retry the subset of sent events which resulted in ES errors of a retryable nature.
34
- # - Events which returned retryable error codes will be pushed onto a separate queue for
35
- # retrying events. Events in this queue will be retried a maximum of 5 times by default (configurable through :max_retries).
36
- # The size of this queue is capped by the value set in :retry_max_items.
37
- # - Events from the retry queue are submitted again when the queue reaches its max size or when
38
- # the max interval time is reached. The max interval time is configurable via :retry_max_interval.
39
- # - Events which are not retryable or have reached their max retry count are logged to stderr.
31
+ # - Network errors (inability to connect)
32
+ # - 429 (Too many requests) and
33
+ # - 503 (Service unavailable) errors
34
+ #
35
+ # NOTE: 409 exceptions are no longer retried. Please set a higher `retry_on_conflict` value if you experience 409 exceptions.
36
+ # It is more performant for Elasticsearch to retry these exceptions than this plugin.
40
37
  #
41
38
  # ==== DNS Caching
42
39
  #
@@ -113,7 +110,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
113
110
  # Note: This will return ALL nodes with HTTP enabled (including master nodes!). If you use
114
111
  # this with master nodes, you probably want to disable HTTP on them by setting
115
112
  # `http.enabled` to false in their elasticsearch.yml. You can either use the `sniffing` option or
116
- # manually enter multiple Elasticsearch hosts using the `hosts` paramater.
113
+ # manually enter multiple Elasticsearch hosts using the `hosts` parameter.
117
114
  config :sniffing, :validate => :boolean, :default => false
118
115
 
119
116
  # How long to wait, in seconds, between sniffing attempts
data/logstash-output-elasticsearch.gemspec CHANGED
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-output-elasticsearch'
4
- s.version = '2.2.0'
4
+ s.version = '2.3.0'
5
5
  s.licenses = ['apache-2.0']
6
6
  s.summary = "Logstash Output to Elasticsearch"
7
7
  s.description = "Output events to elasticsearch"
data/spec/integration/outputs/retry_spec.rb CHANGED
@@ -8,7 +8,6 @@ describe "failures in bulk class expected behavior", :integration => true do
8
8
  let(:event2) { LogStash::Event.new("geoip" => { "location" => [ 0.0, 0.0] }, "@timestamp" => "2014-11-17T20:37:17.223Z", "@metadata" => {"retry_count" => 0}) }
9
9
  let(:action2) { ["index", {:_id=>nil, :_routing=>nil, :_index=>"logstash-2014.11.17", :_type=>"logs"}, event2] }
10
10
  let(:invalid_event) { LogStash::Event.new("geoip" => { "location" => "notlatlon" }, "@timestamp" => "2014-11-17T20:37:17.223Z") }
11
- let(:max_retries) { 3 }
12
11
 
13
12
  def mock_actions_with_response(*resp)
14
13
  expanded_responses = resp.map do |resp|
@@ -33,7 +32,6 @@ describe "failures in bulk class expected behavior", :integration => true do
33
32
  "hosts" => get_host_port(),
34
33
  "retry_max_items" => 10,
35
34
  "retry_max_interval" => 1,
36
- "max_retries" => max_retries
37
35
  }
38
36
  next LogStash::Outputs::ElasticSearch.new(settings)
39
37
  end
@@ -108,14 +106,14 @@ describe "failures in bulk class expected behavior", :integration => true do
108
106
  sleep(3)
109
107
  end
110
108
 
111
- it "should retry an event until max_retries reached" do
109
+ it "should retry an event infinitely until a non retryable status occurs" do
112
110
  mock_actions_with_response({"errors" => true, "statuses" => [429]},
113
111
  {"errors" => true, "statuses" => [429]},
114
112
  {"errors" => true, "statuses" => [429]},
115
113
  {"errors" => true, "statuses" => [429]},
116
114
  {"errors" => true, "statuses" => [429]},
117
- {"errors" => true, "statuses" => [429]})
118
- expect(subject).to receive(:submit).with([action1]).exactly(max_retries+1).times.and_call_original
115
+ {"errors" => true, "statuses" => [500]})
116
+ expect(subject).to receive(:submit).with([action1]).exactly(6).times.and_call_original
119
117
  subject.register
120
118
  subject.receive(event1)
121
119
  subject.flush
metadata CHANGED
@@ -1,30 +1,39 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-output-elasticsearch
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.2.0
4
+ version: 2.3.0
5
5
  platform: java
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-12-07 00:00:00.000000000 Z
11
+ date: 2015-12-17 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
+ name: concurrent-ruby
15
+ version_requirements: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '>='
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
14
20
  requirement: !ruby/object:Gem::Requirement
15
21
  requirements:
16
22
  - - '>='
17
23
  - !ruby/object:Gem::Version
18
24
  version: '0'
19
- name: concurrent-ruby
20
25
  prerelease: false
21
26
  type: :runtime
27
+ - !ruby/object:Gem::Dependency
28
+ name: elasticsearch
22
29
  version_requirements: !ruby/object:Gem::Requirement
23
30
  requirements:
24
31
  - - '>='
25
32
  - !ruby/object:Gem::Version
26
- version: '0'
27
- - !ruby/object:Gem::Dependency
33
+ version: 1.0.13
34
+ - - ~>
35
+ - !ruby/object:Gem::Version
36
+ version: '1.0'
28
37
  requirement: !ruby/object:Gem::Requirement
29
38
  requirements:
30
39
  - - '>='
@@ -33,18 +42,18 @@ dependencies:
33
42
  - - ~>
34
43
  - !ruby/object:Gem::Version
35
44
  version: '1.0'
36
- name: elasticsearch
37
45
  prerelease: false
38
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: stud
39
49
  version_requirements: !ruby/object:Gem::Requirement
40
50
  requirements:
41
51
  - - '>='
42
52
  - !ruby/object:Gem::Version
43
- version: 1.0.13
53
+ version: 0.0.17
44
54
  - - ~>
45
55
  - !ruby/object:Gem::Version
46
- version: '1.0'
47
- - !ruby/object:Gem::Dependency
56
+ version: '0.0'
48
57
  requirement: !ruby/object:Gem::Requirement
49
58
  requirements:
50
59
  - - '>='
@@ -53,33 +62,25 @@ dependencies:
53
62
  - - ~>
54
63
  - !ruby/object:Gem::Version
55
64
  version: '0.0'
56
- name: stud
57
65
  prerelease: false
58
66
  type: :runtime
67
+ - !ruby/object:Gem::Dependency
68
+ name: cabin
59
69
  version_requirements: !ruby/object:Gem::Requirement
60
70
  requirements:
61
- - - '>='
62
- - !ruby/object:Gem::Version
63
- version: 0.0.17
64
71
  - - ~>
65
72
  - !ruby/object:Gem::Version
66
- version: '0.0'
67
- - !ruby/object:Gem::Dependency
73
+ version: '0.6'
68
74
  requirement: !ruby/object:Gem::Requirement
69
75
  requirements:
70
76
  - - ~>
71
77
  - !ruby/object:Gem::Version
72
78
  version: '0.6'
73
- name: cabin
74
79
  prerelease: false
75
80
  type: :runtime
76
- version_requirements: !ruby/object:Gem::Requirement
77
- requirements:
78
- - - ~>
79
- - !ruby/object:Gem::Version
80
- version: '0.6'
81
81
  - !ruby/object:Gem::Dependency
82
- requirement: !ruby/object:Gem::Requirement
82
+ name: logstash-core
83
+ version_requirements: !ruby/object:Gem::Requirement
83
84
  requirements:
84
85
  - - '>='
85
86
  - !ruby/object:Gem::Version
@@ -87,10 +88,7 @@ dependencies:
87
88
  - - <
88
89
  - !ruby/object:Gem::Version
89
90
  version: 3.0.0
90
- name: logstash-core
91
- prerelease: false
92
- type: :runtime
93
- version_requirements: !ruby/object:Gem::Requirement
91
+ requirement: !ruby/object:Gem::Requirement
94
92
  requirements:
95
93
  - - '>='
96
94
  - !ruby/object:Gem::Version
@@ -98,90 +96,92 @@ dependencies:
98
96
  - - <
99
97
  - !ruby/object:Gem::Version
100
98
  version: 3.0.0
99
+ prerelease: false
100
+ type: :runtime
101
101
  - !ruby/object:Gem::Dependency
102
+ name: ftw
103
+ version_requirements: !ruby/object:Gem::Requirement
104
+ requirements:
105
+ - - ~>
106
+ - !ruby/object:Gem::Version
107
+ version: 0.0.42
102
108
  requirement: !ruby/object:Gem::Requirement
103
109
  requirements:
104
110
  - - ~>
105
111
  - !ruby/object:Gem::Version
106
112
  version: 0.0.42
107
- name: ftw
108
113
  prerelease: false
109
114
  type: :development
115
+ - !ruby/object:Gem::Dependency
116
+ name: logstash-codec-plain
110
117
  version_requirements: !ruby/object:Gem::Requirement
111
118
  requirements:
112
- - - ~>
119
+ - - '>='
113
120
  - !ruby/object:Gem::Version
114
- version: 0.0.42
115
- - !ruby/object:Gem::Dependency
121
+ version: '0'
116
122
  requirement: !ruby/object:Gem::Requirement
117
123
  requirements:
118
124
  - - '>='
119
125
  - !ruby/object:Gem::Version
120
126
  version: '0'
121
- name: logstash-codec-plain
122
127
  prerelease: false
123
128
  type: :development
129
+ - !ruby/object:Gem::Dependency
130
+ name: manticore
124
131
  version_requirements: !ruby/object:Gem::Requirement
125
132
  requirements:
126
- - - '>='
133
+ - - ~>
127
134
  - !ruby/object:Gem::Version
128
- version: '0'
129
- - !ruby/object:Gem::Dependency
135
+ version: 0.4.2
130
136
  requirement: !ruby/object:Gem::Requirement
131
137
  requirements:
132
138
  - - ~>
133
139
  - !ruby/object:Gem::Version
134
140
  version: 0.4.2
135
- name: manticore
136
141
  prerelease: false
137
142
  type: :runtime
143
+ - !ruby/object:Gem::Dependency
144
+ name: logstash-devutils
138
145
  version_requirements: !ruby/object:Gem::Requirement
139
146
  requirements:
140
- - - ~>
147
+ - - '>='
141
148
  - !ruby/object:Gem::Version
142
- version: 0.4.2
143
- - !ruby/object:Gem::Dependency
149
+ version: '0'
144
150
  requirement: !ruby/object:Gem::Requirement
145
151
  requirements:
146
152
  - - '>='
147
153
  - !ruby/object:Gem::Version
148
154
  version: '0'
149
- name: logstash-devutils
150
155
  prerelease: false
151
156
  type: :development
157
+ - !ruby/object:Gem::Dependency
158
+ name: longshoreman
152
159
  version_requirements: !ruby/object:Gem::Requirement
153
160
  requirements:
154
161
  - - '>='
155
162
  - !ruby/object:Gem::Version
156
163
  version: '0'
157
- - !ruby/object:Gem::Dependency
158
164
  requirement: !ruby/object:Gem::Requirement
159
165
  requirements:
160
166
  - - '>='
161
167
  - !ruby/object:Gem::Version
162
168
  version: '0'
163
- name: longshoreman
164
169
  prerelease: false
165
170
  type: :development
171
+ - !ruby/object:Gem::Dependency
172
+ name: flores
166
173
  version_requirements: !ruby/object:Gem::Requirement
167
174
  requirements:
168
175
  - - '>='
169
176
  - !ruby/object:Gem::Version
170
177
  version: '0'
171
- - !ruby/object:Gem::Dependency
172
178
  requirement: !ruby/object:Gem::Requirement
173
179
  requirements:
174
180
  - - '>='
175
181
  - !ruby/object:Gem::Version
176
182
  version: '0'
177
- name: flores
178
183
  prerelease: false
179
184
  type: :development
180
- version_requirements: !ruby/object:Gem::Requirement
181
- requirements:
182
- - - '>='
183
- - !ruby/object:Gem::Version
184
- version: '0'
185
185
  description: Output events to elasticsearch
186
186
  email: info@elastic.co
187
187
  executables: []