logstash-output-scalyr 0.2.7.beta → 0.2.9.beta
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +32 -0
- data/Gemfile +5 -2
- data/README.md +2 -4
- data/lib/logstash/outputs/scalyr.rb +192 -48
- data/lib/scalyr/common/client.rb +40 -4
- data/lib/scalyr/common/util.rb +5 -5
- data/lib/scalyr/constants.rb +1 -1
- data/logstash-output-scalyr.gemspec +2 -2
- metadata +3 -23
- data/spec/benchmarks/bignum_fixing.rb +0 -87
- data/spec/benchmarks/flattening_and_serialization.rb +0 -100
- data/spec/benchmarks/json_serialization.rb +0 -85
- data/spec/benchmarks/metrics_overhead.rb +0 -48
- data/spec/benchmarks/set_session_level_serverhost_on_events.rb +0 -107
- data/spec/benchmarks/util.rb +0 -24
- data/spec/logstash/outputs/fixtures/example_com.pem +0 -41
- data/spec/logstash/outputs/scalyr_integration_spec.rb +0 -318
- data/spec/logstash/outputs/scalyr_spec.rb +0 -1262
- data/spec/scalyr/common/util_spec.rb +0 -543
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a0d3f1dbfffae370fea3b6f119ce439c66358c5ba8f3bec03fec386a48f034c5
|
4
|
+
data.tar.gz: 66c5224a53503e11e3c48ce8bf7768eb8d3284a56c205f6505254c74f4655bdc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: fc609b65672aaa0d32888d81c2f7e811b73a91bf6419ef44225c05e6945f3e6a6bc76eccc5b831db95c119fbe11fe2ccc318876b06cbf0893ebc10f286a60f9e
|
7
|
+
data.tar.gz: 4b4130bd8cdeb85ae5122f4272a8d708ab4442dfb88a87e010bfc7f1cf428b687ea96b153fbc4458eb4a5ae9bddb3454daea1f6171368a71dca84930a71494ac
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,37 @@
|
|
1
1
|
# Beta
|
2
2
|
|
3
|
+
## 0.2.9.beta
|
4
|
+
|
5
|
+
* Introduce new plugin config options which allows failed HTTP request retry options to be
|
6
|
+
configured differently for a different set of errors.
|
7
|
+
|
8
|
+
Those options should be left as-is, unless instructed differently by the DataSet support team.
|
9
|
+
|
10
|
+
* Use longer retry delays where we don't want to retry as soon as possible (e.g. deploy related
|
11
|
+
errors or client being throttled by the server).
|
12
|
+
|
13
|
+
* Update context which is logged with errors which represent HTTP requests which are retried
|
14
|
+
to also include ``total_retries_so_far`` and ``total_sleep_time_so_far`` attribute.
|
15
|
+
|
16
|
+
* Add new ``retry_backoff_factor`` config option with which user can change a default value of 2
|
17
|
+
for the retry backoff factor (exponential delay).
|
18
|
+
|
19
|
+
## 0.2.8.beta
|
20
|
+
|
21
|
+
* Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
|
22
|
+
fixtures with the actual production gem file.
|
23
|
+
|
24
|
+
* Do not retry requests that will never be accepted by the server.
|
25
|
+
Specifically, any request that returns HTTP Status code 413 is too large, and
|
26
|
+
will never be accepted. Instead of simply retrying for 10 minutes before
|
27
|
+
sending the request to the DLQ, skip the retries go directly to sending the
|
28
|
+
request to the DLQ.
|
29
|
+
|
30
|
+
To be notified when an event fails to be ingested for whatever reason, create
|
31
|
+
an alert using the query: ``parser='logstash_plugin_metrics'
|
32
|
+
failed_events_processed > 0``. Instructions on how to create an alert can be
|
33
|
+
found in our docs here: https://scalyr.com/help/alerts
|
34
|
+
|
3
35
|
## 0.2.7.beta
|
4
36
|
|
5
37
|
* SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
|
data/Gemfile
CHANGED
@@ -11,10 +11,13 @@ if Dir.exist?(logstash_path) && use_logstash_source
|
|
11
11
|
end
|
12
12
|
|
13
13
|
group :test do
|
14
|
-
gem "webmock"
|
14
|
+
gem "webmock", "~> 3.18.1"
|
15
|
+
|
16
|
+
# Newer versions depend on Ruby >= 2.6
|
17
|
+
gem "rubocop", "~> 1.28.2"
|
15
18
|
|
16
19
|
# Require the specific version of `json` used in logstash while testing
|
17
|
-
gem 'json', '
|
20
|
+
gem 'json', '2.6.2'
|
18
21
|
end
|
19
22
|
|
20
23
|
gem 'pry'
|
data/README.md
CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
|
|
505
505
|
|
506
506
|
```
|
507
507
|
rm -rf vendor/
|
508
|
-
bundle
|
508
|
+
bundle install
|
509
509
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
510
510
|
chmod 0600 ~/.gem/credentials
|
511
|
-
bundle exec rake vendor
|
512
511
|
bundle exec rspec
|
513
512
|
bundle exec rake publish_gem
|
514
513
|
```
|
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
|
|
518
517
|
|
519
518
|
```
|
520
519
|
rm -rf vendor/
|
521
|
-
bundle
|
520
|
+
bundle install
|
522
521
|
curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
|
523
522
|
chmod 0600 ~/.gem/credentials
|
524
|
-
bundle exec rake vendor
|
525
523
|
bundle exec rspec
|
526
524
|
rvm use jruby
|
527
525
|
bundle exec gem build logstash-output-scalyr.gemspec
|
@@ -4,7 +4,9 @@ require "logstash/namespace"
|
|
4
4
|
require "concurrent"
|
5
5
|
require "stud/buffer"
|
6
6
|
require "socket" # for Socket.gethostname
|
7
|
+
# rubocop:disable Lint/RedundantRequireStatement
|
7
8
|
require "thread" # for safe queueing
|
9
|
+
# rubocop:enable Lint/RedundantRequireStatement
|
8
10
|
require "uri" # for escaping user input
|
9
11
|
require 'json' # for converting event object to JSON for upload
|
10
12
|
|
@@ -118,18 +120,50 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
118
120
|
config :flat_tag_prefix, :validate => :string, :default => 'tag_'
|
119
121
|
config :flat_tag_value, :default => 1
|
120
122
|
|
121
|
-
|
123
|
+
#####
|
124
|
+
## Retry settings for non deploy and non throttling related errors
|
125
|
+
####
|
126
|
+
|
127
|
+
# Initial interval in seconds between bulk retries. Doubled (by default, can be overriden using
|
128
|
+
# retry_backoff_factor config option) on each retry up to `retry_max_interval`
|
122
129
|
config :retry_initial_interval, :validate => :number, :default => 1
|
130
|
+
|
123
131
|
# How many times to retry sending an event before giving up on it
|
124
132
|
# This will result in a total of around 12 minutes of retrying / sleeping with a default value
|
125
133
|
# for retry_max_interval
|
126
134
|
config :max_retries, :validate => :number, :default => 15
|
127
|
-
# Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
|
128
|
-
config :send_to_dlq, :validate => :boolean, :default => true
|
129
135
|
|
130
136
|
# Set max interval in seconds between bulk retries.
|
131
137
|
config :retry_max_interval, :validate => :number, :default => 64
|
132
138
|
|
139
|
+
# Back off factor for retries. We default to 2 (exponential retry delay).
|
140
|
+
config :retry_backoff_factor, :validate => :number, :default => 2
|
141
|
+
|
142
|
+
#####
|
143
|
+
## Retry settings for deploy related errors
|
144
|
+
####
|
145
|
+
|
146
|
+
config :retry_initial_interval_deploy_errors, :validate => :number, :default => 30
|
147
|
+
config :max_retries_deploy_errors, :validate => :number, :default => 5
|
148
|
+
config :retry_max_interval_deploy_errors, :validate => :number, :default => 64
|
149
|
+
config :retry_backoff_factor_deploy_errors, :validate => :number, :default => 1.5
|
150
|
+
|
151
|
+
#####
|
152
|
+
## Retry settings for throttling related errors
|
153
|
+
####
|
154
|
+
|
155
|
+
config :retry_initial_interval_throttling_errors, :validate => :number, :default => 20
|
156
|
+
config :max_retries_throttling_errors, :validate => :number, :default => 6
|
157
|
+
config :retry_max_interval_throttling_errors, :validate => :number, :default => 64
|
158
|
+
config :retry_backoff_factor_throttling_errors, :validate => :number, :default => 1.5
|
159
|
+
|
160
|
+
#####
|
161
|
+
## Common retry related settings
|
162
|
+
#####
|
163
|
+
|
164
|
+
# Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
|
165
|
+
config :send_to_dlq, :validate => :boolean, :default => true
|
166
|
+
|
133
167
|
# Whether or not to verify the connection to Scalyr, only set to false for debugging.
|
134
168
|
config :ssl_verify_peer, :validate => :boolean, :default => true
|
135
169
|
|
@@ -224,7 +258,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
224
258
|
@client_session.close if @client_session
|
225
259
|
end
|
226
260
|
|
227
|
-
public
|
228
261
|
def register
|
229
262
|
# This prng is used exclusively to determine when to sample statistics and no security related purpose, for this
|
230
263
|
# reason we do not ensure thread safety for it.
|
@@ -349,6 +382,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
349
382
|
:successful_events_processed => 0,
|
350
383
|
:failed_events_processed => 0,
|
351
384
|
:total_retry_count => 0,
|
385
|
+
:total_retry_duration_secs => 0,
|
352
386
|
:total_java_class_cast_errors => 0
|
353
387
|
}
|
354
388
|
@plugin_metrics = get_new_metrics
|
@@ -455,7 +489,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
455
489
|
# Also note that event uploads are broken up into batches such that each batch is less than max_request_buffer.
|
456
490
|
# Increasing max_request_buffer beyond 3MB will lead to failed requests.
|
457
491
|
#
|
458
|
-
public
|
459
492
|
def multi_receive(events)
|
460
493
|
# Just return and pretend we did something if running in noop mode
|
461
494
|
return events if @noop_mode
|
@@ -471,7 +504,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
471
504
|
build_multi_duration_secs = Time.now.to_f - start_time
|
472
505
|
|
473
506
|
# Loop over all array of multi-event requests, sending each multi-event to Scalyr
|
474
|
-
sleep_interval = @retry_initial_interval
|
475
507
|
batch_num = 1
|
476
508
|
total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
|
477
509
|
|
@@ -485,28 +517,51 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
485
517
|
exc_data = nil
|
486
518
|
# Whether the exception is commonly retried or not, for determining log level
|
487
519
|
exc_commonly_retried = false
|
488
|
-
|
489
|
-
|
490
|
-
#
|
491
|
-
|
520
|
+
|
521
|
+
# We use new and clean retry state object for each request
|
522
|
+
# Since @running is only available directly on the output plugin instance and we don't
|
523
|
+
# want to create a cyclic reference between output and state tracker instance we pass
|
524
|
+
# this lambda method to the state tracker
|
525
|
+
is_plugin_running = lambda { @running }
|
526
|
+
|
527
|
+
retry_state = RetryStateTracker.new(@config, is_plugin_running)
|
528
|
+
|
492
529
|
begin
|
493
530
|
# For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
|
494
531
|
# ignore these.
|
495
532
|
if !multi_event_request.nil?
|
496
533
|
@client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
|
497
|
-
|
498
|
-
sleep_interval = @retry_initial_interval
|
499
534
|
batch_num += 1
|
500
535
|
result.push(multi_event_request)
|
501
536
|
end
|
502
537
|
|
538
|
+
rescue Scalyr::Common::Client::PayloadTooLargeError => e
|
539
|
+
# if the payload is too large, we do not retry. we send to DLQ or drop it.
|
540
|
+
exc_data = {
|
541
|
+
:error_class => e.e_class,
|
542
|
+
:url => e.url.to_s,
|
543
|
+
:message => e.message,
|
544
|
+
:batch_num => batch_num,
|
545
|
+
:total_batches => total_batches,
|
546
|
+
:record_count => multi_event_request[:record_count],
|
547
|
+
:payload_size => multi_event_request[:body].bytesize,
|
548
|
+
}
|
549
|
+
exc_data[:code] = e.code if e.code
|
550
|
+
if defined?(e.body) and e.body
|
551
|
+
exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
|
552
|
+
end
|
553
|
+
exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
|
554
|
+
log_retry_failure(multi_event_request, exc_data, 0, 0)
|
555
|
+
next
|
503
556
|
rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
|
504
|
-
|
505
|
-
|
506
|
-
|
557
|
+
previous_state = retry_state.get_state_for_error(e)
|
558
|
+
updated_state = retry_state.sleep_for_error_and_update_state(e)
|
559
|
+
|
507
560
|
@stats_lock.synchronize do
|
508
561
|
@multi_receive_statistics[:total_retry_count] += 1
|
562
|
+
@multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
|
509
563
|
end
|
564
|
+
|
510
565
|
message = "Error uploading to Scalyr (will backoff-retry)"
|
511
566
|
exc_data = {
|
512
567
|
:error_class => e.e_class,
|
@@ -516,7 +571,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
516
571
|
:total_batches => total_batches,
|
517
572
|
:record_count => multi_event_request[:record_count],
|
518
573
|
:payload_size => multi_event_request[:body].bytesize,
|
519
|
-
|
574
|
+
# retry related values
|
575
|
+
:max_retries => updated_state[:options][:max_retries],
|
576
|
+
:retry_backoff_factor => updated_state[:options][:retry_backoff_factor],
|
577
|
+
:retry_max_interval => updated_state[:options][:retry_max_interval],
|
578
|
+
:will_retry_in_seconds => updated_state[:sleep_interval],
|
579
|
+
# to get values which include this next retry, you need to add +1
|
580
|
+
# to :total_retries_so_far and +:sleep_interval to :total_sleep_time_so_far
|
581
|
+
:total_retries_so_far => previous_state[:retries],
|
582
|
+
:total_sleep_time_so_far => previous_state[:sleep],
|
520
583
|
}
|
521
584
|
exc_data[:code] = e.code if e.code
|
522
585
|
if @logger.debug? and defined?(e.body) and e.body
|
@@ -534,8 +597,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
534
597
|
@logger.warn(message, exc_data)
|
535
598
|
exc_commonly_retried = false
|
536
599
|
end
|
537
|
-
|
538
|
-
|
600
|
+
|
601
|
+
retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
|
602
|
+
log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
|
539
603
|
next
|
540
604
|
|
541
605
|
rescue => e
|
@@ -547,20 +611,22 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
547
611
|
:backtrace => e.backtrace
|
548
612
|
)
|
549
613
|
@logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
|
550
|
-
|
614
|
+
|
615
|
+
updated_state = retry_state.sleep_for_error_and_update_state(e)
|
551
616
|
exc_data = {
|
552
617
|
:error_message => e.message,
|
553
618
|
:error_class => e.class.name,
|
554
619
|
:backtrace => e.backtrace,
|
555
620
|
:multi_event_request => multi_event_request
|
556
621
|
}
|
557
|
-
|
558
|
-
exc_retries += 1
|
622
|
+
|
559
623
|
@stats_lock.synchronize do
|
560
624
|
@multi_receive_statistics[:total_retry_count] += 1
|
625
|
+
@multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
|
561
626
|
end
|
562
|
-
|
563
|
-
|
627
|
+
|
628
|
+
retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
|
629
|
+
log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
|
564
630
|
next
|
565
631
|
end
|
566
632
|
|
@@ -572,9 +638,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
572
638
|
if !exc_data.nil?
|
573
639
|
message = "Retry successful after error."
|
574
640
|
if exc_commonly_retried
|
575
|
-
@logger.debug(message, :error_data => exc_data, :retries =>
|
641
|
+
@logger.debug(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
|
576
642
|
else
|
577
|
-
@logger.info(message, :error_data => exc_data, :retries =>
|
643
|
+
@logger.info(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
|
578
644
|
end
|
579
645
|
end
|
580
646
|
end
|
@@ -634,18 +700,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
634
700
|
@multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
|
635
701
|
@multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
|
636
702
|
end
|
637
|
-
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
638
703
|
sample_events = Array.new
|
639
704
|
multi_event_request[:logstash_events][0,5].each {|l_event|
|
640
705
|
sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
|
641
706
|
}
|
642
|
-
|
707
|
+
if exc_data[:code] == 413
|
708
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
|
709
|
+
# For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
|
710
|
+
# to include redundant sample Logstash event objects
|
711
|
+
@logger.error(message, :error_data => exc_data)
|
712
|
+
else
|
713
|
+
message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
|
714
|
+
@logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
|
715
|
+
end
|
643
716
|
if @dlq_writer
|
644
717
|
multi_event_request[:logstash_events].each {|l_event|
|
645
718
|
@dlq_writer.write(l_event, "#{exc_data[:message]}")
|
646
719
|
}
|
647
720
|
else
|
648
|
-
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events
|
721
|
+
@logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
|
649
722
|
end
|
650
723
|
end
|
651
724
|
|
@@ -839,7 +912,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
839
912
|
if @flatten_nested_values
|
840
913
|
start_time = Time.now.to_f
|
841
914
|
begin
|
842
|
-
record = Scalyr::Common::Util.flatten(record,
|
915
|
+
record = Scalyr::Common::Util.flatten(record, @flatten_nested_values_delimiter, @flatten_nested_arrays, @fix_deep_flattening_delimiters, @flattening_max_key_count)
|
843
916
|
rescue Scalyr::Common::Util::MaxKeyCountError => e
|
844
917
|
@logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
|
845
918
|
end
|
@@ -913,7 +986,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
913
986
|
).force_encoding('UTF-8')
|
914
987
|
end
|
915
988
|
event_json = self.json_encode(scalyr_event)
|
916
|
-
rescue Java::JavaLang::ClassCastException
|
989
|
+
rescue Java::JavaLang::ClassCastException
|
917
990
|
# Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
|
918
991
|
# Because of the version of jruby logstash works with we don't have the option to just update this away,
|
919
992
|
# so if we run into it we convert bignums into strings so we can get the data in at least.
|
@@ -1017,7 +1090,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1017
1090
|
# build the scalyr thread logs object
|
1018
1091
|
if current_logs
|
1019
1092
|
logs = Array.new
|
1020
|
-
current_logs.each do |
|
1093
|
+
current_logs.each do |_identifier, log|
|
1021
1094
|
logs << log
|
1022
1095
|
end
|
1023
1096
|
body[:logs] = logs
|
@@ -1036,7 +1109,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1036
1109
|
start_time = Time.now.to_f
|
1037
1110
|
begin
|
1038
1111
|
serialized_body = self.json_encode(body)
|
1039
|
-
rescue Java::JavaLang::ClassCastException
|
1112
|
+
rescue Java::JavaLang::ClassCastException
|
1040
1113
|
@logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
|
1041
1114
|
@stats_lock.synchronize do
|
1042
1115
|
@multi_receive_statistics[:total_java_class_cast_errors] += 1
|
@@ -1050,7 +1123,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1050
1123
|
serialized_request_size = serialized_body.bytesize
|
1051
1124
|
|
1052
1125
|
# We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
|
1053
|
-
if
|
1126
|
+
if serialized_request_size >= @max_request_buffer + 5000
|
1054
1127
|
# TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
|
1055
1128
|
@logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
|
1056
1129
|
end
|
@@ -1136,14 +1209,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1136
1209
|
val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
|
1137
1210
|
val = val.nil? ? 0 : val
|
1138
1211
|
msg << ' ' if cnt > 0
|
1139
|
-
msg << "#{k
|
1212
|
+
msg << "#{k}=#{val}"
|
1140
1213
|
cnt += 1
|
1141
1214
|
end
|
1142
1215
|
get_stats.each do |k, v|
|
1143
1216
|
val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
|
1144
1217
|
val = val.nil? ? 0 : val
|
1145
1218
|
msg << ' ' if cnt > 0
|
1146
|
-
msg << "#{k
|
1219
|
+
msg << "#{k}=#{val}"
|
1147
1220
|
cnt += 1
|
1148
1221
|
end
|
1149
1222
|
status_event[:attrs]['message'] = msg
|
@@ -1198,26 +1271,97 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
|
|
1198
1271
|
end
|
1199
1272
|
end
|
1200
1273
|
|
1274
|
+
# Helper method to check if the dead-letter queue is enabled
|
1275
|
+
def dlq_enabled?
|
1276
|
+
# echee TODO submit to DLQ
|
1277
|
+
respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
|
1278
|
+
!execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
|
1279
|
+
end
|
1280
|
+
end
|
1201
1281
|
|
1202
|
-
|
1203
|
-
|
1204
|
-
|
1205
|
-
|
1282
|
+
# Class which allows us to track retry related settings and state for different type of errors for
|
1283
|
+
# which we use different retry settings (e.g. general errors vs errors during deploy windows vs
|
1284
|
+
# client throttled errors).
|
1285
|
+
class RetryStateTracker
|
1286
|
+
|
1287
|
+
def initialize(plugin_config, is_plugin_running_method)
|
1288
|
+
# :retries - stores number of times we have retried so far
|
1289
|
+
# :sleep - stores total duration (in seconds) we have slept / waited so far
|
1290
|
+
# :sleep_interval stores - sleep interval / delay (in seconds) for the next retry
|
1291
|
+
@STATE = {
|
1292
|
+
:deploy_errors => {
|
1293
|
+
:retries => 0,
|
1294
|
+
:sleep => 0,
|
1295
|
+
:sleep_interval => plugin_config["retry_initial_interval_deploy_errors"],
|
1296
|
+
:options => {
|
1297
|
+
:retry_initial_interval => plugin_config["retry_initial_interval_deploy_errors"],
|
1298
|
+
:max_retries => plugin_config["max_retries_deploy_errors"],
|
1299
|
+
:retry_max_interval => plugin_config["retry_max_interval_deploy_errors"],
|
1300
|
+
:retry_backoff_factor => plugin_config["retry_backoff_factor_deploy_errors"],
|
1301
|
+
}
|
1302
|
+
},
|
1303
|
+
:throttling_errors => {
|
1304
|
+
:retries => 0,
|
1305
|
+
:sleep => 0,
|
1306
|
+
:sleep_interval => plugin_config["retry_initial_interval_throttling_errors"],
|
1307
|
+
:options => {
|
1308
|
+
:retry_initial_interval => plugin_config["retry_initial_interval_throttling_errors"],
|
1309
|
+
:max_retries => plugin_config["max_retries_throttling_errors"],
|
1310
|
+
:retry_max_interval => plugin_config["retry_max_interval_throttling_errors"],
|
1311
|
+
:retry_backoff_factor => plugin_config["retry_backoff_factor_throttling_errors"],
|
1312
|
+
}
|
1313
|
+
},
|
1314
|
+
:other_errors => {
|
1315
|
+
:retries => 0,
|
1316
|
+
:sleep => 0,
|
1317
|
+
:sleep_interval => plugin_config["retry_initial_interval"],
|
1318
|
+
:options => {
|
1319
|
+
:retry_initial_interval => plugin_config["retry_initial_interval"],
|
1320
|
+
:max_retries => plugin_config["max_retries"],
|
1321
|
+
:retry_max_interval => plugin_config["retry_max_interval"],
|
1322
|
+
:retry_backoff_factor => plugin_config["retry_backoff_factor"],
|
1323
|
+
}
|
1324
|
+
},
|
1325
|
+
}
|
1326
|
+
|
1327
|
+
@is_plugin_running_method = is_plugin_running_method
|
1206
1328
|
end
|
1207
1329
|
|
1330
|
+
# Return state hash for a specific error
|
1331
|
+
def get_state_for_error(error)
|
1332
|
+
if error.instance_of?(Scalyr::Common::Client::ClientThrottledError)
|
1333
|
+
return @STATE[:throttling_errors]
|
1334
|
+
elsif error.instance_of?(Scalyr::Common::Client::DeployWindowError)
|
1335
|
+
return @STATE[:deploy_errors]
|
1336
|
+
else
|
1337
|
+
return @STATE[:other_errors]
|
1338
|
+
end
|
1339
|
+
end
|
1208
1340
|
|
1209
|
-
|
1210
|
-
|
1211
|
-
doubled = current_interval * 2
|
1212
|
-
doubled > @retry_max_interval ? @retry_max_interval : doubled
|
1341
|
+
def get_state()
|
1342
|
+
@STATE
|
1213
1343
|
end
|
1214
1344
|
|
1345
|
+
# Helper method that performs synchronous sleep for a certain time interval for a specific
|
1346
|
+
# error and updates internal error specific state. It also returns updated internal state
|
1347
|
+
# specific to that error.
|
1348
|
+
def sleep_for_error_and_update_state(error)
|
1349
|
+
# Sleep for a specific duration
|
1350
|
+
state = get_state_for_error(error)
|
1215
1351
|
|
1216
|
-
|
1217
|
-
|
1218
|
-
|
1219
|
-
|
1220
|
-
|
1352
|
+
current_interval = state[:sleep_interval]
|
1353
|
+
|
1354
|
+
Stud.stoppable_sleep(current_interval) { !@is_plugin_running_method.call }
|
1355
|
+
|
1356
|
+
# Update internal state + sleep interval for the next retry
|
1357
|
+
updated_interval = current_interval * state[:options][:retry_backoff_factor]
|
1358
|
+
updated_interval = updated_interval > state[:options][:retry_max_interval] ? state[:options][:retry_max_interval] : updated_interval
|
1359
|
+
|
1360
|
+
state[:retries] += 1
|
1361
|
+
state[:sleep] += current_interval
|
1362
|
+
state[:sleep_interval] = updated_interval
|
1363
|
+
|
1364
|
+
state
|
1221
1365
|
end
|
1222
1366
|
end
|
1223
1367
|
|
data/lib/scalyr/common/client.rb
CHANGED
@@ -27,6 +27,36 @@ end
|
|
27
27
|
# An exception that signifies the Scalyr server received the upload request but dropped it
|
28
28
|
#---------------------------------------------------------------------------------------------------------------------
|
29
29
|
class RequestDroppedError < ServerError;
|
30
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
|
31
|
+
super(msg, code, url, body, e_class)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
#---------------------------------------------------------------------------------------------------------------------
|
36
|
+
# An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
|
37
|
+
#---------------------------------------------------------------------------------------------------------------------
|
38
|
+
class PayloadTooLargeError < ServerError;
|
39
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
|
40
|
+
super(msg, code, url, body, e_class)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
#---------------------------------------------------------------------------------------------------------------------
|
45
|
+
# An exception that signifies an error which occured during Scalyr deploy window
|
46
|
+
#---------------------------------------------------------------------------------------------------------------------
|
47
|
+
class DeployWindowError < ServerError;
|
48
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::DeployWindowError")
|
49
|
+
super(msg, code, url, body, e_class)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
#---------------------------------------------------------------------------------------------------------------------
|
54
|
+
# An exception that signifies that the client has been throttled by the server
|
55
|
+
#---------------------------------------------------------------------------------------------------------------------
|
56
|
+
class ClientThrottledError < ServerError;
|
57
|
+
def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ClientThrottledError")
|
58
|
+
super(msg, code, url, body, e_class)
|
59
|
+
end
|
30
60
|
end
|
31
61
|
|
32
62
|
#---------------------------------------------------------------------------------------------------------------------
|
@@ -175,7 +205,7 @@ class ClientSession
|
|
175
205
|
# Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
|
176
206
|
# and verify that the API key is valid.
|
177
207
|
def send_ping(body)
|
178
|
-
post_body, post_headers,
|
208
|
+
post_body, post_headers, _ = prepare_post_object @add_events_uri.path, body
|
179
209
|
response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
|
180
210
|
handle_response(response)
|
181
211
|
|
@@ -236,7 +266,7 @@ class ClientSession
|
|
236
266
|
|
237
267
|
# Prepare a post object to be sent, compressing it if necessary
|
238
268
|
private
|
239
|
-
def prepare_post_object(
|
269
|
+
def prepare_post_object(_uri_path, body)
|
240
270
|
# use compression if enabled
|
241
271
|
encoding = nil
|
242
272
|
compression_duration = 0
|
@@ -308,15 +338,21 @@ class ClientSession
|
|
308
338
|
end
|
309
339
|
|
310
340
|
status = response_hash["status"]
|
341
|
+
code = response.code.to_s.strip.to_i
|
311
342
|
|
312
343
|
if status != "success"
|
313
|
-
if
|
344
|
+
if code == 413
|
345
|
+
raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
|
346
|
+
elsif [530, 500].include?(code)
|
347
|
+
raise DeployWindowError.new(status, response.code, @add_events_uri, response.body)
|
348
|
+
elsif code == 429
|
349
|
+
raise ClientThrottledError.new(status, response.code, @add_events_uri, response.body)
|
350
|
+
elsif status =~ /discardBuffer/
|
314
351
|
raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
|
315
352
|
else
|
316
353
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
317
354
|
end
|
318
355
|
else
|
319
|
-
code = response.code.to_s.strip.to_i
|
320
356
|
if code < 200 or code > 299
|
321
357
|
raise ServerError.new(status, response.code, @add_events_uri, response.body)
|
322
358
|
end
|
data/lib/scalyr/common/util.rb
CHANGED
@@ -4,6 +4,7 @@ class MaxKeyCountError < StandardError
|
|
4
4
|
attr_reader :message, :sample_keys
|
5
5
|
|
6
6
|
def initialize(message, sample_keys)
|
7
|
+
super(message)
|
7
8
|
@message = message
|
8
9
|
@sample_keys = sample_keys
|
9
10
|
end
|
@@ -32,7 +33,6 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
|
|
32
33
|
key_list = []
|
33
34
|
key_list_width = []
|
34
35
|
result = Hash.new
|
35
|
-
test_key = 0
|
36
36
|
#Debugging
|
37
37
|
#require 'pry'
|
38
38
|
#binding.pry
|
@@ -81,10 +81,10 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
|
|
81
81
|
)
|
82
82
|
end
|
83
83
|
|
84
|
-
|
84
|
+
key_list.pop
|
85
85
|
until key_list_width.empty? or key_list_width[-1] > 1
|
86
|
-
|
87
|
-
|
86
|
+
key_list_width.pop
|
87
|
+
key_list.pop
|
88
88
|
end
|
89
89
|
if not key_list_width.empty?
|
90
90
|
key_list_width[-1] -= 1
|
@@ -116,7 +116,7 @@ def self.convert_bignums(obj)
|
|
116
116
|
obj[index] = convert_bignums(value)
|
117
117
|
end
|
118
118
|
|
119
|
-
elsif obj.is_a?
|
119
|
+
elsif obj.is_a? Integer
|
120
120
|
return obj.to_s
|
121
121
|
|
122
122
|
else
|
data/lib/scalyr/constants.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-output-scalyr'
|
3
|
-
s.version = '0.2.
|
3
|
+
s.version = '0.2.9.beta'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = "Scalyr output plugin for Logstash"
|
6
6
|
s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
|
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
|
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
13
|
-
s.files = Dir['lib/**/*','
|
13
|
+
s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
|
14
14
|
# Tests
|
15
15
|
s.test_files = s.files.grep(%r{^(test|spec|features)/})
|
16
16
|
|