logstash-output-scalyr 0.1.11.beta → 0.1.12
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/README.md +1 -1
- data/lib/logstash/outputs/scalyr.rb +36 -3
- data/lib/scalyr/common/util.rb +7 -0
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +1 -1
- data/spec/logstash/outputs/scalyr_integration_spec.rb +31 -2
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8e13b3be77498d8ff0613a34733b477a5c648d48af1070c4ed083d3cd178b69c
|
4
|
+
data.tar.gz: 2873fd90e70e8ac42bfdda09041fc2863eda1a0760efca7203bdc8e63bcbaa5d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 192c043d86b0b2b7e12532e16e0c94654789d8a6e959994b317b699b02f318b36a220f1935c5a214174d67f46a5d3dea77b01a3299d290246c59d9ed652de150
|
7
|
+
data.tar.gz: 7b9b18c86a9abe7141c6b0a74ec99a37ee2cc301dcaabf4e2c25bff02673a2388912a3f155af2534a22658165c5998278ba194851a8b96317b02ed3ff127b94f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,9 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.1.12
|
4
|
+
- Add logging of successful request retries after an error for additional clarity.
|
5
|
+
- Add debug level logging of request body on error.
|
6
|
+
|
3
7
|
## 0.1.11.beta
|
4
8
|
- Fixes to retry mechanisms.
|
5
9
|
- More thorough catching of events, preferring to retry requests rather than crashing the plugin.
|
data/README.md
CHANGED
@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
|
|
10
10
|
# Quick start
|
11
11
|
|
12
12
|
1. Build the gem, run `gem build logstash-output-scalyr.gemspec`
|
13
|
-
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.
|
13
|
+
2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.12.gem` or follow the latest official instructions on working with plugins from Logstash.
|
14
14
|
3. Configure the output plugin (e.g. add it to a pipeline .conf)
|
15
15
|
4. Restart Logstash
|
16
16
|
|
@@ -292,6 +292,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
292
292
|
|
293
293
|
while !multi_event_request_array.to_a.empty?
|
294
294
|
multi_event_request = multi_event_request_array.pop
|
295
|
+
# Variables to hold information about exceptions we run into, and our handling of retries for this request. We
|
296
|
+
# track this to log it when the retries succeed so we can be sure logs are going through.
|
297
|
+
# General exception info we log in the error
|
298
|
+
exc_data = nil
|
299
|
+
# Whether the exception is commonly retried or not, for determining log level
|
300
|
+
exc_commonly_retried = false
|
301
|
+
# Count of retries attempted for this request
|
302
|
+
exc_retries = 0
|
303
|
+
# Total time spent sleeping while retrying this request due to backoff
|
304
|
+
exc_sleep = 0
|
295
305
|
begin
|
296
306
|
# For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
|
297
307
|
# ignore these.
|
@@ -305,6 +315,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
305
315
|
|
306
316
|
rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
|
307
317
|
sleep_interval = sleep_for(sleep_interval)
|
318
|
+
exc_sleep += sleep_interval
|
319
|
+
exc_retries += 1
|
308
320
|
message = "Error uploading to Scalyr (will backoff-retry)"
|
309
321
|
exc_data = {
|
310
322
|
:url => e.url.to_s,
|
@@ -316,16 +328,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
316
328
|
:will_retry_in_seconds => sleep_interval,
|
317
329
|
}
|
318
330
|
exc_data[:code] = e.code if e.code
|
319
|
-
|
331
|
+
if @logger.debug? and e.body
|
332
|
+
exc_data[:body] = e.body
|
333
|
+
elsif e.message == "Invalid JSON response from server" and e.body
|
334
|
+
exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
|
335
|
+
end
|
320
336
|
exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
|
321
337
|
if e.is_commonly_retried?
|
322
338
|
# well-known retriable errors should be debug
|
323
339
|
@logger.debug(message, exc_data)
|
340
|
+
exc_commonly_retried = true
|
324
341
|
else
|
325
342
|
# all other failed uploads should be errors
|
326
343
|
@logger.error(message, exc_data)
|
344
|
+
exc_commonly_retried = false
|
327
345
|
end
|
328
|
-
sleep_interval *= 2
|
329
346
|
retry if @running
|
330
347
|
|
331
348
|
rescue => e
|
@@ -338,9 +355,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
338
355
|
)
|
339
356
|
@logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
|
340
357
|
sleep_interval = sleep_for(sleep_interval)
|
341
|
-
|
358
|
+
exc_data = {
|
359
|
+
:error_message => e.message,
|
360
|
+
:error_class => e.class.name,
|
361
|
+
:backtrace => e.backtrace,
|
362
|
+
:multi_event_request => multi_event_request
|
363
|
+
}
|
364
|
+
exc_sleep += sleep_interval
|
365
|
+
exc_retries += 1
|
342
366
|
retry if @running
|
343
367
|
end
|
368
|
+
|
369
|
+
if !exc_data.nil?
|
370
|
+
message = "Retry successful after error."
|
371
|
+
if exc_commonly_retried
|
372
|
+
@logger.debug(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
|
373
|
+
else
|
374
|
+
@logger.info(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
|
375
|
+
end
|
376
|
+
end
|
344
377
|
end
|
345
378
|
|
346
379
|
if records_count > 0
|
data/lib/scalyr/common/util.rb
CHANGED
data/lib/scalyr/constants.rb
CHANGED
@@ -1,2 +1,2 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
PLUGIN_VERSION = "v0.1.
|
2
|
+
PLUGIN_VERSION = "v0.1.12"
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.1.
|
3
|
+
s.version = '0.1.12'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -152,7 +152,8 @@ describe LogStash::Outputs::Scalyr do
|
|
152
152
|
:record_count=>3,
|
153
153
|
:total_batches=>1,
|
154
154
|
:url=>"https://agent.scalyr.com/addEvents",
|
155
|
-
:will_retry_in_seconds=>2
|
155
|
+
:will_retry_in_seconds=>2,
|
156
|
+
:body=>"stubbed response"
|
156
157
|
}
|
157
158
|
)
|
158
159
|
end
|
@@ -178,7 +179,35 @@ describe LogStash::Outputs::Scalyr do
|
|
178
179
|
:record_count=>3,
|
179
180
|
:total_batches=>1,
|
180
181
|
:url=>"https://agent.scalyr.com/addEvents",
|
181
|
-
:will_retry_in_seconds=>2
|
182
|
+
:will_retry_in_seconds=>2,
|
183
|
+
:body=>"stubbed response"
|
184
|
+
}
|
185
|
+
)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
context "when receiving a long non-json response" do
|
190
|
+
it "don't throw an error but do log one to error" do
|
191
|
+
stub_request(:post, "https://agent.scalyr.com/addEvents").
|
192
|
+
to_return(status: 500, body: "0123456789" * 52, headers: {})
|
193
|
+
|
194
|
+
plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
|
195
|
+
plugin.register
|
196
|
+
plugin.instance_variable_set(:@running, false)
|
197
|
+
|
198
|
+
allow(plugin.instance_variable_get(:@logger)).to receive(:error)
|
199
|
+
plugin.multi_receive(sample_events)
|
200
|
+
expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
|
201
|
+
{
|
202
|
+
:batch_num=>1,
|
203
|
+
:code=>500,
|
204
|
+
:message=>"Invalid JSON response from server",
|
205
|
+
:payload_size=>781,
|
206
|
+
:record_count=>3,
|
207
|
+
:total_batches=>1,
|
208
|
+
:url=>"https://agent.scalyr.com/addEvents",
|
209
|
+
:will_retry_in_seconds=>2,
|
210
|
+
:body=>("0123456789" * 50) + "012345678..."
|
182
211
|
}
|
183
212
|
)
|
184
213
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-output-scalyr
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Edward Chee
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-06-
|
11
|
+
date: 2021-06-15 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -4052,9 +4052,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
4052
4052
|
version: '0'
|
4053
4053
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
4054
4054
|
requirements:
|
4055
|
-
- - "
|
4055
|
+
- - ">="
|
4056
4056
|
- !ruby/object:Gem::Version
|
4057
|
-
version:
|
4057
|
+
version: '0'
|
4058
4058
|
requirements: []
|
4059
4059
|
rubyforge_project:
|
4060
4060
|
rubygems_version: 2.7.10
|