logstash-output-scalyr 0.2.7.beta → 0.2.9.beta

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: '098686f8c748dbe1d02cec87e92aa49df5db05049ff0d87ccd6fbb6271ccd797'
4
- data.tar.gz: '0073297594f936703190252584559ae60cd81a7b3807f1b2685b6966fece5610'
3
+ metadata.gz: a0d3f1dbfffae370fea3b6f119ce439c66358c5ba8f3bec03fec386a48f034c5
4
+ data.tar.gz: 66c5224a53503e11e3c48ce8bf7768eb8d3284a56c205f6505254c74f4655bdc
5
5
  SHA512:
6
- metadata.gz: 507ddc7c825ae4612907c4ad451597bf516732b389b5a628598f8eec12e8cdc1f7c4b5b71533803665d1fd18b3fce43a0aa3989729192eea26b2948273f9d267
7
- data.tar.gz: 310a452e0df0b74081a061030322b3f2a013bad233ba99b424fcea428af336f86370cb76bda62c06c5edcbee81731e5a498e137daf4f7d42a843ef988a8664df
6
+ metadata.gz: fc609b65672aaa0d32888d81c2f7e811b73a91bf6419ef44225c05e6945f3e6a6bc76eccc5b831db95c119fbe11fe2ccc318876b06cbf0893ebc10f286a60f9e
7
+ data.tar.gz: 4b4130bd8cdeb85ae5122f4272a8d708ab4442dfb88a87e010bfc7f1cf428b687ea96b153fbc4458eb4a5ae9bddb3454daea1f6171368a71dca84930a71494ac
data/CHANGELOG.md CHANGED
@@ -1,5 +1,37 @@
1
1
  # Beta
2
2
 
3
+ ## 0.2.9.beta
4
+
5
+ * Introduce new plugin config options which allows failed HTTP request retry options to be
6
+ configured differently for a different set of errors.
7
+
8
+ Those options should be left as-is, unless instructed differently by the DataSet support team.
9
+
10
+ * Use longer retry delays where we don't want to retry as soon as possible (e.g. deploy related
11
+ errors or client being throttled by the server).
12
+
13
+ * Update context which is logged with errors which represent HTTP requests which are retried
14
+ to also include ``total_retries_so_far`` and ``total_sleep_time_so_far`` attribute.
15
+
16
+ * Add new ``retry_backoff_factor`` config option with which user can change a default value of 2
17
+ for the retry backoff factor (exponential delay).
18
+
19
+ ## 0.2.8.beta
20
+
21
+ * Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
22
+ fixtures with the actual production gem file.
23
+
24
+ * Do not retry requests that will never be accepted by the server.
25
+ Specifically, any request that returns HTTP Status code 413 is too large, and
26
+ will never be accepted. Instead of simply retrying for 10 minutes before
27
+ sending the request to the DLQ, skip the retries go directly to sending the
28
+ request to the DLQ.
29
+
30
+ To be notified when an event fails to be ingested for whatever reason, create
31
+ an alert using the query: ``parser='logstash_plugin_metrics'
32
+ failed_events_processed > 0``. Instructions on how to create an alert can be
33
+ found in our docs here: https://scalyr.com/help/alerts
34
+
3
35
  ## 0.2.7.beta
4
36
 
5
37
  * SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option
data/Gemfile CHANGED
@@ -11,10 +11,13 @@ if Dir.exist?(logstash_path) && use_logstash_source
11
11
  end
12
12
 
13
13
  group :test do
14
- gem "webmock"
14
+ gem "webmock", "~> 3.18.1"
15
+
16
+ # Newer versions depend on Ruby >= 2.6
17
+ gem "rubocop", "~> 1.28.2"
15
18
 
16
19
  # Require the specific version of `json` used in logstash while testing
17
- gem 'json', '1.8.6'
20
+ gem 'json', '2.6.2'
18
21
  end
19
22
 
20
23
  gem 'pry'
data/README.md CHANGED
@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
505
505
 
506
506
  ```
507
507
  rm -rf vendor/
508
- bundle check --path vendor/bundle || bundle install --deployment
508
+ bundle install
509
509
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
510
510
  chmod 0600 ~/.gem/credentials
511
- bundle exec rake vendor
512
511
  bundle exec rspec
513
512
  bundle exec rake publish_gem
514
513
  ```
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
518
517
 
519
518
  ```
520
519
  rm -rf vendor/
521
- bundle check --path vendor/bundle || bundle install --deployment
520
+ bundle install
522
521
  curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
523
522
  chmod 0600 ~/.gem/credentials
524
- bundle exec rake vendor
525
523
  bundle exec rspec
526
524
  rvm use jruby
527
525
  bundle exec gem build logstash-output-scalyr.gemspec
@@ -4,7 +4,9 @@ require "logstash/namespace"
4
4
  require "concurrent"
5
5
  require "stud/buffer"
6
6
  require "socket" # for Socket.gethostname
7
+ # rubocop:disable Lint/RedundantRequireStatement
7
8
  require "thread" # for safe queueing
9
+ # rubocop:enable Lint/RedundantRequireStatement
8
10
  require "uri" # for escaping user input
9
11
  require 'json' # for converting event object to JSON for upload
10
12
 
@@ -118,18 +120,50 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
118
120
  config :flat_tag_prefix, :validate => :string, :default => 'tag_'
119
121
  config :flat_tag_value, :default => 1
120
122
 
121
- # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
123
+ #####
124
+ ## Retry settings for non deploy and non throttling related errors
125
+ ####
126
+
127
+ # Initial interval in seconds between bulk retries. Doubled (by default, can be overriden using
128
+ # retry_backoff_factor config option) on each retry up to `retry_max_interval`
122
129
  config :retry_initial_interval, :validate => :number, :default => 1
130
+
123
131
  # How many times to retry sending an event before giving up on it
124
132
  # This will result in a total of around 12 minutes of retrying / sleeping with a default value
125
133
  # for retry_max_interval
126
134
  config :max_retries, :validate => :number, :default => 15
127
- # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
128
- config :send_to_dlq, :validate => :boolean, :default => true
129
135
 
130
136
  # Set max interval in seconds between bulk retries.
131
137
  config :retry_max_interval, :validate => :number, :default => 64
132
138
 
139
+ # Back off factor for retries. We default to 2 (exponential retry delay).
140
+ config :retry_backoff_factor, :validate => :number, :default => 2
141
+
142
+ #####
143
+ ## Retry settings for deploy related errors
144
+ ####
145
+
146
+ config :retry_initial_interval_deploy_errors, :validate => :number, :default => 30
147
+ config :max_retries_deploy_errors, :validate => :number, :default => 5
148
+ config :retry_max_interval_deploy_errors, :validate => :number, :default => 64
149
+ config :retry_backoff_factor_deploy_errors, :validate => :number, :default => 1.5
150
+
151
+ #####
152
+ ## Retry settings for throttling related errors
153
+ ####
154
+
155
+ config :retry_initial_interval_throttling_errors, :validate => :number, :default => 20
156
+ config :max_retries_throttling_errors, :validate => :number, :default => 6
157
+ config :retry_max_interval_throttling_errors, :validate => :number, :default => 64
158
+ config :retry_backoff_factor_throttling_errors, :validate => :number, :default => 1.5
159
+
160
+ #####
161
+ ## Common retry related settings
162
+ #####
163
+
164
+ # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
165
+ config :send_to_dlq, :validate => :boolean, :default => true
166
+
133
167
  # Whether or not to verify the connection to Scalyr, only set to false for debugging.
134
168
  config :ssl_verify_peer, :validate => :boolean, :default => true
135
169
 
@@ -224,7 +258,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
224
258
  @client_session.close if @client_session
225
259
  end
226
260
 
227
- public
228
261
  def register
229
262
  # This prng is used exclusively to determine when to sample statistics and no security related purpose, for this
230
263
  # reason we do not ensure thread safety for it.
@@ -349,6 +382,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
349
382
  :successful_events_processed => 0,
350
383
  :failed_events_processed => 0,
351
384
  :total_retry_count => 0,
385
+ :total_retry_duration_secs => 0,
352
386
  :total_java_class_cast_errors => 0
353
387
  }
354
388
  @plugin_metrics = get_new_metrics
@@ -455,7 +489,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
455
489
  # Also note that event uploads are broken up into batches such that each batch is less than max_request_buffer.
456
490
  # Increasing max_request_buffer beyond 3MB will lead to failed requests.
457
491
  #
458
- public
459
492
  def multi_receive(events)
460
493
  # Just return and pretend we did something if running in noop mode
461
494
  return events if @noop_mode
@@ -471,7 +504,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
471
504
  build_multi_duration_secs = Time.now.to_f - start_time
472
505
 
473
506
  # Loop over all array of multi-event requests, sending each multi-event to Scalyr
474
- sleep_interval = @retry_initial_interval
475
507
  batch_num = 1
476
508
  total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
477
509
 
@@ -485,28 +517,51 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
485
517
  exc_data = nil
486
518
  # Whether the exception is commonly retried or not, for determining log level
487
519
  exc_commonly_retried = false
488
- # Count of retries attempted for this request
489
- exc_retries = 0
490
- # Total time spent sleeping while retrying this request due to backoff
491
- exc_sleep = 0
520
+
521
+ # We use new and clean retry state object for each request
522
+ # Since @running is only available directly on the output plugin instance and we don't
523
+ # want to create a cyclic reference between output and state tracker instance we pass
524
+ # this lambda method to the state tracker
525
+ is_plugin_running = lambda { @running }
526
+
527
+ retry_state = RetryStateTracker.new(@config, is_plugin_running)
528
+
492
529
  begin
493
530
  # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
494
531
  # ignore these.
495
532
  if !multi_event_request.nil?
496
533
  @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
497
-
498
- sleep_interval = @retry_initial_interval
499
534
  batch_num += 1
500
535
  result.push(multi_event_request)
501
536
  end
502
537
 
538
+ rescue Scalyr::Common::Client::PayloadTooLargeError => e
539
+ # if the payload is too large, we do not retry. we send to DLQ or drop it.
540
+ exc_data = {
541
+ :error_class => e.e_class,
542
+ :url => e.url.to_s,
543
+ :message => e.message,
544
+ :batch_num => batch_num,
545
+ :total_batches => total_batches,
546
+ :record_count => multi_event_request[:record_count],
547
+ :payload_size => multi_event_request[:body].bytesize,
548
+ }
549
+ exc_data[:code] = e.code if e.code
550
+ if defined?(e.body) and e.body
551
+ exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
552
+ end
553
+ exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
554
+ log_retry_failure(multi_event_request, exc_data, 0, 0)
555
+ next
503
556
  rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
504
- sleep_interval = sleep_for(sleep_interval)
505
- exc_sleep += sleep_interval
506
- exc_retries += 1
557
+ previous_state = retry_state.get_state_for_error(e)
558
+ updated_state = retry_state.sleep_for_error_and_update_state(e)
559
+
507
560
  @stats_lock.synchronize do
508
561
  @multi_receive_statistics[:total_retry_count] += 1
562
+ @multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
509
563
  end
564
+
510
565
  message = "Error uploading to Scalyr (will backoff-retry)"
511
566
  exc_data = {
512
567
  :error_class => e.e_class,
@@ -516,7 +571,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
516
571
  :total_batches => total_batches,
517
572
  :record_count => multi_event_request[:record_count],
518
573
  :payload_size => multi_event_request[:body].bytesize,
519
- :will_retry_in_seconds => sleep_interval,
574
+ # retry related values
575
+ :max_retries => updated_state[:options][:max_retries],
576
+ :retry_backoff_factor => updated_state[:options][:retry_backoff_factor],
577
+ :retry_max_interval => updated_state[:options][:retry_max_interval],
578
+ :will_retry_in_seconds => updated_state[:sleep_interval],
579
+ # to get values which include this next retry, you need to add +1
580
+ # to :total_retries_so_far and +:sleep_interval to :total_sleep_time_so_far
581
+ :total_retries_so_far => previous_state[:retries],
582
+ :total_sleep_time_so_far => previous_state[:sleep],
520
583
  }
521
584
  exc_data[:code] = e.code if e.code
522
585
  if @logger.debug? and defined?(e.body) and e.body
@@ -534,8 +597,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
534
597
  @logger.warn(message, exc_data)
535
598
  exc_commonly_retried = false
536
599
  end
537
- retry if @running and exc_retries < @max_retries
538
- log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
600
+
601
+ retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
602
+ log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
539
603
  next
540
604
 
541
605
  rescue => e
@@ -547,20 +611,22 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
547
611
  :backtrace => e.backtrace
548
612
  )
549
613
  @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
550
- sleep_interval = sleep_for(sleep_interval)
614
+
615
+ updated_state = retry_state.sleep_for_error_and_update_state(e)
551
616
  exc_data = {
552
617
  :error_message => e.message,
553
618
  :error_class => e.class.name,
554
619
  :backtrace => e.backtrace,
555
620
  :multi_event_request => multi_event_request
556
621
  }
557
- exc_sleep += sleep_interval
558
- exc_retries += 1
622
+
559
623
  @stats_lock.synchronize do
560
624
  @multi_receive_statistics[:total_retry_count] += 1
625
+ @multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
561
626
  end
562
- retry if @running and exc_retries < @max_retries
563
- log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
627
+
628
+ retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
629
+ log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
564
630
  next
565
631
  end
566
632
 
@@ -572,9 +638,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
572
638
  if !exc_data.nil?
573
639
  message = "Retry successful after error."
574
640
  if exc_commonly_retried
575
- @logger.debug(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
641
+ @logger.debug(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
576
642
  else
577
- @logger.info(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
643
+ @logger.info(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
578
644
  end
579
645
  end
580
646
  end
@@ -634,18 +700,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
634
700
  @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
635
701
  @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
636
702
  end
637
- message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
638
703
  sample_events = Array.new
639
704
  multi_event_request[:logstash_events][0,5].each {|l_event|
640
705
  sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
641
706
  }
642
- @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
707
+ if exc_data[:code] == 413
708
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
709
+ # For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
710
+ # to include redundant sample Logstash event objects
711
+ @logger.error(message, :error_data => exc_data)
712
+ else
713
+ message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
714
+ @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
715
+ end
643
716
  if @dlq_writer
644
717
  multi_event_request[:logstash_events].each {|l_event|
645
718
  @dlq_writer.write(l_event, "#{exc_data[:message]}")
646
719
  }
647
720
  else
648
- @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
721
+ @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
649
722
  end
650
723
  end
651
724
 
@@ -839,7 +912,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
839
912
  if @flatten_nested_values
840
913
  start_time = Time.now.to_f
841
914
  begin
842
- record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
915
+ record = Scalyr::Common::Util.flatten(record, @flatten_nested_values_delimiter, @flatten_nested_arrays, @fix_deep_flattening_delimiters, @flattening_max_key_count)
843
916
  rescue Scalyr::Common::Util::MaxKeyCountError => e
844
917
  @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
845
918
  end
@@ -913,7 +986,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
913
986
  ).force_encoding('UTF-8')
914
987
  end
915
988
  event_json = self.json_encode(scalyr_event)
916
- rescue Java::JavaLang::ClassCastException => e
989
+ rescue Java::JavaLang::ClassCastException
917
990
  # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
918
991
  # Because of the version of jruby logstash works with we don't have the option to just update this away,
919
992
  # so if we run into it we convert bignums into strings so we can get the data in at least.
@@ -1017,7 +1090,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1017
1090
  # build the scalyr thread logs object
1018
1091
  if current_logs
1019
1092
  logs = Array.new
1020
- current_logs.each do |identifier, log|
1093
+ current_logs.each do |_identifier, log|
1021
1094
  logs << log
1022
1095
  end
1023
1096
  body[:logs] = logs
@@ -1036,7 +1109,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1036
1109
  start_time = Time.now.to_f
1037
1110
  begin
1038
1111
  serialized_body = self.json_encode(body)
1039
- rescue Java::JavaLang::ClassCastException => e
1112
+ rescue Java::JavaLang::ClassCastException
1040
1113
  @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
1041
1114
  @stats_lock.synchronize do
1042
1115
  @multi_receive_statistics[:total_java_class_cast_errors] += 1
@@ -1050,7 +1123,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1050
1123
  serialized_request_size = serialized_body.bytesize
1051
1124
 
1052
1125
  # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
1053
- if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
1126
+ if serialized_request_size >= @max_request_buffer + 5000
1054
1127
  # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
1055
1128
  @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
1056
1129
  end
@@ -1136,14 +1209,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1136
1209
  val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
1137
1210
  val = val.nil? ? 0 : val
1138
1211
  msg << ' ' if cnt > 0
1139
- msg << "#{k.to_s}=#{val}"
1212
+ msg << "#{k}=#{val}"
1140
1213
  cnt += 1
1141
1214
  end
1142
1215
  get_stats.each do |k, v|
1143
1216
  val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
1144
1217
  val = val.nil? ? 0 : val
1145
1218
  msg << ' ' if cnt > 0
1146
- msg << "#{k.to_s}=#{val}"
1219
+ msg << "#{k}=#{val}"
1147
1220
  cnt += 1
1148
1221
  end
1149
1222
  status_event[:attrs]['message'] = msg
@@ -1198,26 +1271,97 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
1198
1271
  end
1199
1272
  end
1200
1273
 
1274
+ # Helper method to check if the dead-letter queue is enabled
1275
+ def dlq_enabled?
1276
+ # echee TODO submit to DLQ
1277
+ respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
1278
+ !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
1279
+ end
1280
+ end
1201
1281
 
1202
- # Helper method that performs synchronous sleep for a certain time interval
1203
- def sleep_for(sleep_interval)
1204
- Stud.stoppable_sleep(sleep_interval) { !@running }
1205
- get_sleep_sec(sleep_interval)
1282
+ # Class which allows us to track retry related settings and state for different type of errors for
1283
+ # which we use different retry settings (e.g. general errors vs errors during deploy windows vs
1284
+ # client throttled errors).
1285
+ class RetryStateTracker
1286
+
1287
+ def initialize(plugin_config, is_plugin_running_method)
1288
+ # :retries - stores number of times we have retried so far
1289
+ # :sleep - stores total duration (in seconds) we have slept / waited so far
1290
+ # :sleep_interval stores - sleep interval / delay (in seconds) for the next retry
1291
+ @STATE = {
1292
+ :deploy_errors => {
1293
+ :retries => 0,
1294
+ :sleep => 0,
1295
+ :sleep_interval => plugin_config["retry_initial_interval_deploy_errors"],
1296
+ :options => {
1297
+ :retry_initial_interval => plugin_config["retry_initial_interval_deploy_errors"],
1298
+ :max_retries => plugin_config["max_retries_deploy_errors"],
1299
+ :retry_max_interval => plugin_config["retry_max_interval_deploy_errors"],
1300
+ :retry_backoff_factor => plugin_config["retry_backoff_factor_deploy_errors"],
1301
+ }
1302
+ },
1303
+ :throttling_errors => {
1304
+ :retries => 0,
1305
+ :sleep => 0,
1306
+ :sleep_interval => plugin_config["retry_initial_interval_throttling_errors"],
1307
+ :options => {
1308
+ :retry_initial_interval => plugin_config["retry_initial_interval_throttling_errors"],
1309
+ :max_retries => plugin_config["max_retries_throttling_errors"],
1310
+ :retry_max_interval => plugin_config["retry_max_interval_throttling_errors"],
1311
+ :retry_backoff_factor => plugin_config["retry_backoff_factor_throttling_errors"],
1312
+ }
1313
+ },
1314
+ :other_errors => {
1315
+ :retries => 0,
1316
+ :sleep => 0,
1317
+ :sleep_interval => plugin_config["retry_initial_interval"],
1318
+ :options => {
1319
+ :retry_initial_interval => plugin_config["retry_initial_interval"],
1320
+ :max_retries => plugin_config["max_retries"],
1321
+ :retry_max_interval => plugin_config["retry_max_interval"],
1322
+ :retry_backoff_factor => plugin_config["retry_backoff_factor"],
1323
+ }
1324
+ },
1325
+ }
1326
+
1327
+ @is_plugin_running_method = is_plugin_running_method
1206
1328
  end
1207
1329
 
1330
+ # Return state hash for a specific error
1331
+ def get_state_for_error(error)
1332
+ if error.instance_of?(Scalyr::Common::Client::ClientThrottledError)
1333
+ return @STATE[:throttling_errors]
1334
+ elsif error.instance_of?(Scalyr::Common::Client::DeployWindowError)
1335
+ return @STATE[:deploy_errors]
1336
+ else
1337
+ return @STATE[:other_errors]
1338
+ end
1339
+ end
1208
1340
 
1209
- # Helper method that gets the next sleep time for exponential backoff, capped at a defined maximum
1210
- def get_sleep_sec(current_interval)
1211
- doubled = current_interval * 2
1212
- doubled > @retry_max_interval ? @retry_max_interval : doubled
1341
+ def get_state()
1342
+ @STATE
1213
1343
  end
1214
1344
 
1345
+ # Helper method that performs synchronous sleep for a certain time interval for a specific
1346
+ # error and updates internal error specific state. It also returns updated internal state
1347
+ # specific to that error.
1348
+ def sleep_for_error_and_update_state(error)
1349
+ # Sleep for a specific duration
1350
+ state = get_state_for_error(error)
1215
1351
 
1216
- # Helper method to check if the dead-letter queue is enabled
1217
- def dlq_enabled?
1218
- # echee TODO submit to DLQ
1219
- respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
1220
- !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
1352
+ current_interval = state[:sleep_interval]
1353
+
1354
+ Stud.stoppable_sleep(current_interval) { !@is_plugin_running_method.call }
1355
+
1356
+ # Update internal state + sleep interval for the next retry
1357
+ updated_interval = current_interval * state[:options][:retry_backoff_factor]
1358
+ updated_interval = updated_interval > state[:options][:retry_max_interval] ? state[:options][:retry_max_interval] : updated_interval
1359
+
1360
+ state[:retries] += 1
1361
+ state[:sleep] += current_interval
1362
+ state[:sleep_interval] = updated_interval
1363
+
1364
+ state
1221
1365
  end
1222
1366
  end
1223
1367
 
@@ -27,6 +27,36 @@ end
27
27
  # An exception that signifies the Scalyr server received the upload request but dropped it
28
28
  #---------------------------------------------------------------------------------------------------------------------
29
29
  class RequestDroppedError < ServerError;
30
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
31
+ super(msg, code, url, body, e_class)
32
+ end
33
+ end
34
+
35
+ #---------------------------------------------------------------------------------------------------------------------
36
+ # An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
37
+ #---------------------------------------------------------------------------------------------------------------------
38
+ class PayloadTooLargeError < ServerError;
39
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
40
+ super(msg, code, url, body, e_class)
41
+ end
42
+ end
43
+
44
+ #---------------------------------------------------------------------------------------------------------------------
45
+ # An exception that signifies an error which occured during Scalyr deploy window
46
+ #---------------------------------------------------------------------------------------------------------------------
47
+ class DeployWindowError < ServerError;
48
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::DeployWindowError")
49
+ super(msg, code, url, body, e_class)
50
+ end
51
+ end
52
+
53
+ #---------------------------------------------------------------------------------------------------------------------
54
+ # An exception that signifies that the client has been throttled by the server
55
+ #---------------------------------------------------------------------------------------------------------------------
56
+ class ClientThrottledError < ServerError;
57
+ def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ClientThrottledError")
58
+ super(msg, code, url, body, e_class)
59
+ end
30
60
  end
31
61
 
32
62
  #---------------------------------------------------------------------------------------------------------------------
@@ -175,7 +205,7 @@ class ClientSession
175
205
  # Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
176
206
  # and verify that the API key is valid.
177
207
  def send_ping(body)
178
- post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
208
+ post_body, post_headers, _ = prepare_post_object @add_events_uri.path, body
179
209
  response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
180
210
  handle_response(response)
181
211
 
@@ -236,7 +266,7 @@ class ClientSession
236
266
 
237
267
  # Prepare a post object to be sent, compressing it if necessary
238
268
  private
239
- def prepare_post_object(uri_path, body)
269
+ def prepare_post_object(_uri_path, body)
240
270
  # use compression if enabled
241
271
  encoding = nil
242
272
  compression_duration = 0
@@ -308,15 +338,21 @@ class ClientSession
308
338
  end
309
339
 
310
340
  status = response_hash["status"]
341
+ code = response.code.to_s.strip.to_i
311
342
 
312
343
  if status != "success"
313
- if status =~ /discardBuffer/
344
+ if code == 413
345
+ raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
346
+ elsif [530, 500].include?(code)
347
+ raise DeployWindowError.new(status, response.code, @add_events_uri, response.body)
348
+ elsif code == 429
349
+ raise ClientThrottledError.new(status, response.code, @add_events_uri, response.body)
350
+ elsif status =~ /discardBuffer/
314
351
  raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
315
352
  else
316
353
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
317
354
  end
318
355
  else
319
- code = response.code.to_s.strip.to_i
320
356
  if code < 200 or code > 299
321
357
  raise ServerError.new(status, response.code, @add_events_uri, response.body)
322
358
  end
@@ -4,6 +4,7 @@ class MaxKeyCountError < StandardError
4
4
  attr_reader :message, :sample_keys
5
5
 
6
6
  def initialize(message, sample_keys)
7
+ super(message)
7
8
  @message = message
8
9
  @sample_keys = sample_keys
9
10
  end
@@ -32,7 +33,6 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
32
33
  key_list = []
33
34
  key_list_width = []
34
35
  result = Hash.new
35
- test_key = 0
36
36
  #Debugging
37
37
  #require 'pry'
38
38
  #binding.pry
@@ -81,10 +81,10 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
81
81
  )
82
82
  end
83
83
 
84
- throw_away = key_list.pop
84
+ key_list.pop
85
85
  until key_list_width.empty? or key_list_width[-1] > 1
86
- throw_away = key_list_width.pop
87
- throw_away = key_list.pop
86
+ key_list_width.pop
87
+ key_list.pop
88
88
  end
89
89
  if not key_list_width.empty?
90
90
  key_list_width[-1] -= 1
@@ -116,7 +116,7 @@ def self.convert_bignums(obj)
116
116
  obj[index] = convert_bignums(value)
117
117
  end
118
118
 
119
- elsif obj.is_a? Bignum
119
+ elsif obj.is_a? Integer
120
120
  return obj.to_s
121
121
 
122
122
  else
@@ -1,6 +1,6 @@
1
1
  # encoding: utf-8
2
2
 
3
- PLUGIN_VERSION = "v0.2.7.beta"
3
+ PLUGIN_VERSION = "v0.2.9.beta"
4
4
 
5
5
  # Special event level attribute name which can be used for setting event level serverHost attribute
6
6
  EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-output-scalyr'
3
- s.version = '0.2.7.beta'
3
+ s.version = '0.2.9.beta'
4
4
  s.licenses = ['Apache-2.0']
5
5
  s.summary = "Scalyr output plugin for Logstash"
6
6
  s.description = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
13
- s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
13
+ s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
14
14
  # Tests
15
15
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
16
16