logstash-output-scalyr 0.2.7.beta → 0.2.9.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '098686f8c748dbe1d02cec87e92aa49df5db05049ff0d87ccd6fbb6271ccd797'
-  data.tar.gz: '0073297594f936703190252584559ae60cd81a7b3807f1b2685b6966fece5610'
+  metadata.gz: a0d3f1dbfffae370fea3b6f119ce439c66358c5ba8f3bec03fec386a48f034c5
+  data.tar.gz: 66c5224a53503e11e3c48ce8bf7768eb8d3284a56c205f6505254c74f4655bdc
 SHA512:
-  metadata.gz: 507ddc7c825ae4612907c4ad451597bf516732b389b5a628598f8eec12e8cdc1f7c4b5b71533803665d1fd18b3fce43a0aa3989729192eea26b2948273f9d267
-  data.tar.gz: 310a452e0df0b74081a061030322b3f2a013bad233ba99b424fcea428af336f86370cb76bda62c06c5edcbee81731e5a498e137daf4f7d42a843ef988a8664df
+  metadata.gz: fc609b65672aaa0d32888d81c2f7e811b73a91bf6419ef44225c05e6945f3e6a6bc76eccc5b831db95c119fbe11fe2ccc318876b06cbf0893ebc10f286a60f9e
+  data.tar.gz: 4b4130bd8cdeb85ae5122f4272a8d708ab4442dfb88a87e010bfc7f1cf428b687ea96b153fbc4458eb4a5ae9bddb3454daea1f6171368a71dca84930a71494ac

data/CHANGELOG.md

@@ -1,5 +1,37 @@
 # Beta
 
+## 0.2.9.beta
+
+* Introduce new plugin config options which allows failed HTTP request retry options to be
+  configured differently for a different set of errors.
+
+  Those options should be left as-is, unless instructed differently by the DataSet support team.
+
+* Use longer retry delays where we don't want to retry as soon as possible (e.g. deploy related
+  errors or client being throttled by the server).
+
+* Update context which is logged with errors which represent HTTP requests which are retried
+  to also include ``total_retries_so_far`` and ``total_sleep_time_so_far`` attribute.
+
+* Add new ``retry_backoff_factor`` config option with which user can change a default value of 2
+  for the retry backoff factor (exponential delay).
+
+## 0.2.8.beta
+
+* Update ``.gemspec`` gem metadata to not include ``spec/`` directory with the tests and tests
+  fixtures with the actual production gem file.
+
+* Do not retry requests that will never be accepted by the server.
+  Specifically, any request that returns HTTP Status code 413 is too large, and
+  will never be accepted. Instead of simply retrying for 10 minutes before
+  sending the request to the DLQ, skip the retries go directly to sending the
+  request to the DLQ.
+
+  To be notified when an event fails to be ingested for whatever reason, create
+  an alert using the query: ``parser='logstash_plugin_metrics'
+  failed_events_processed > 0``. Instructions on how to create an alert can be
+  found in our docs here: https://scalyr.com/help/alerts
+
 ## 0.2.7.beta
 
 * SSL cert validation code has been simplified. Now ``ssl_ca_bundle_path`` config option

data/Gemfile

@@ -11,10 +11,13 @@ if Dir.exist?(logstash_path) && use_logstash_source
 end
 
 group :test do
-  gem "webmock"
+  gem "webmock", "~> 3.18.1"
+
+  # Newer versions depend on Ruby >= 2.6
+  gem "rubocop", "~> 1.28.2"
 
   # Require the specific version of `json` used in logstash while testing
-  gem 'json', '1.8.6'
+  gem 'json', '2.6.2'
 end
 
 gem 'pry'

data/README.md

@@ -505,10 +505,9 @@ To deploy the current code on your machine run these commands:
 
 ```
 rm -rf vendor/
-bundle check --path vendor/bundle || bundle install --deployment
+bundle install
 curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
 chmod 0600 ~/.gem/credentials
-bundle exec rake vendor
 bundle exec rspec
 bundle exec rake publish_gem
 ```
@@ -518,10 +517,9 @@ Or as an alternative if ``rake publish_gem`` task doesn't appear to work for wha
 
 ```
 rm -rf vendor/
-bundle check --path vendor/bundle || bundle install --deployment
+bundle install
 curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
 chmod 0600 ~/.gem/credentials
-bundle exec rake vendor
 bundle exec rspec
 rvm use jruby
 bundle exec gem build logstash-output-scalyr.gemspec

data/lib/logstash/outputs/scalyr.rb

@@ -4,7 +4,9 @@ require "logstash/namespace"
 require "concurrent"
 require "stud/buffer"
 require "socket" # for Socket.gethostname
+# rubocop:disable Lint/RedundantRequireStatement
 require "thread" # for safe queueing
+# rubocop:enable Lint/RedundantRequireStatement
 require "uri" # for escaping user input
 require 'json' # for converting event object to JSON for upload
 
@@ -118,18 +120,50 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   config :flat_tag_prefix, :validate => :string, :default => 'tag_'
   config :flat_tag_value, :default => 1
 
-  # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
+  #####
+  ## Retry settings for non deploy and non throttling related errors
+  ####
+
+  # Initial interval in seconds between bulk retries. Doubled (by default, can be overriden using
+  # retry_backoff_factor config option) on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
+
   # How many times to retry sending an event before giving up on it
   # This will result in a total of around 12 minutes of retrying / sleeping with a default value
   # for retry_max_interval
   config :max_retries, :validate => :number, :default => 15
-  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
-  config :send_to_dlq, :validate => :boolean, :default => true
 
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
 
+  # Back off factor for retries. We default to 2 (exponential retry delay).
+  config :retry_backoff_factor, :validate => :number, :default => 2
+
+  #####
+  ## Retry settings for deploy related errors
+  ####
+
+  config :retry_initial_interval_deploy_errors, :validate => :number, :default => 30
+  config :max_retries_deploy_errors, :validate => :number, :default => 5
+  config :retry_max_interval_deploy_errors, :validate => :number, :default => 64
+  config :retry_backoff_factor_deploy_errors, :validate => :number, :default => 1.5
+
+  #####
+  ## Retry settings for throttling related errors
+  ####
+
+  config :retry_initial_interval_throttling_errors, :validate => :number, :default => 20
+  config :max_retries_throttling_errors, :validate => :number, :default => 6
+  config :retry_max_interval_throttling_errors, :validate => :number, :default => 64
+  config :retry_backoff_factor_throttling_errors, :validate => :number, :default => 1.5
+
+  #####
+  ## Common retry related settings
+  #####
+
+  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
+  config :send_to_dlq, :validate => :boolean, :default => true
+
   # Whether or not to verify the connection to Scalyr, only set to false for debugging.
   config :ssl_verify_peer, :validate => :boolean, :default => true
 
@@ -224,7 +258,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @client_session.close if @client_session
   end
 
-  public
   def register
     # This prng is used exclusively to determine when to sample statistics and no security related purpose, for this
     # reason we do not ensure thread safety for it.
@@ -349,6 +382,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       :successful_events_processed => 0,
       :failed_events_processed => 0,
       :total_retry_count => 0,
+      :total_retry_duration_secs => 0,
       :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
@@ -455,7 +489,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Also note that event uploads are broken up into batches such that each batch is less than max_request_buffer.
   # Increasing max_request_buffer beyond 3MB will lead to failed requests.
   #
-  public
   def multi_receive(events)
     # Just return and pretend we did something if running in noop mode
     return events if @noop_mode
@@ -471,7 +504,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       build_multi_duration_secs = Time.now.to_f - start_time
 
       # Loop over all array of multi-event requests, sending each multi-event to Scalyr
-      sleep_interval = @retry_initial_interval
       batch_num = 1
       total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
@@ -485,28 +517,51 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         exc_data = nil
         # Whether the exception is commonly retried or not, for determining log level
         exc_commonly_retried = false
-        # Count of retries attempted for this request
-        exc_retries = 0
-        # Total time spent sleeping while retrying this request due to backoff
-        exc_sleep = 0
+
+        # We use new and clean retry state object for each request
+        # Since @running is only available directly on the output plugin instance and we don't
+        # want to create a cyclic reference between output and state tracker instance we pass
+        # this lambda method to the state tracker
+        is_plugin_running = lambda { @running }
+
+        retry_state = RetryStateTracker.new(@config, is_plugin_running)
+
         begin
           # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
           # ignore these.
           if !multi_event_request.nil?
             @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
-
-            sleep_interval = @retry_initial_interval
             batch_num += 1
             result.push(multi_event_request)
           end
 
+        rescue Scalyr::Common::Client::PayloadTooLargeError => e
+          # if the payload is too large, we do not retry. we send to DLQ or drop it.
+          exc_data = {
+              :error_class => e.e_class,
+              :url => e.url.to_s,
+              :message => e.message,
+              :batch_num => batch_num,
+              :total_batches => total_batches,
+              :record_count => multi_event_request[:record_count],
+              :payload_size => multi_event_request[:body].bytesize,
+          }
+          exc_data[:code] = e.code if e.code
+          if defined?(e.body) and e.body
+            exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
+          end
+          exc_data[:payload] = "\tSample payload: #{multi_event_request[:body][0,1024]}..."
+          log_retry_failure(multi_event_request, exc_data, 0, 0)
+          next
         rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
-          sleep_interval = sleep_for(sleep_interval)
-          exc_sleep += sleep_interval
-          exc_retries += 1
+          previous_state = retry_state.get_state_for_error(e)
+          updated_state = retry_state.sleep_for_error_and_update_state(e)
+
           @stats_lock.synchronize do
             @multi_receive_statistics[:total_retry_count] += 1
+            @multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
           end
+
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
               :error_class => e.e_class,
@@ -516,7 +571,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
               :total_batches => total_batches,
               :record_count => multi_event_request[:record_count],
               :payload_size => multi_event_request[:body].bytesize,
-              :will_retry_in_seconds => sleep_interval,
+              # retry related values
+              :max_retries => updated_state[:options][:max_retries],
+              :retry_backoff_factor => updated_state[:options][:retry_backoff_factor],
+              :retry_max_interval => updated_state[:options][:retry_max_interval],
+              :will_retry_in_seconds => updated_state[:sleep_interval],
+              # to get values which include this next retry, you need to add +1
+              # to :total_retries_so_far and +:sleep_interval to :total_sleep_time_so_far
+              :total_retries_so_far => previous_state[:retries],
+              :total_sleep_time_so_far => previous_state[:sleep],
           }
           exc_data[:code] = e.code if e.code
           if @logger.debug? and defined?(e.body) and e.body
@@ -534,8 +597,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @logger.warn(message, exc_data)
             exc_commonly_retried = false
           end
-          retry if @running and exc_retries < @max_retries
-          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+
+          retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
+          log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
           next
 
         rescue => e
@@ -547,20 +611,22 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
               :backtrace => e.backtrace
           )
           @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
-          sleep_interval = sleep_for(sleep_interval)
+
+          updated_state = retry_state.sleep_for_error_and_update_state(e)
           exc_data = {
             :error_message => e.message,
             :error_class => e.class.name,
             :backtrace => e.backtrace,
             :multi_event_request => multi_event_request
           }
-          exc_sleep += sleep_interval
-          exc_retries += 1
+
           @stats_lock.synchronize do
             @multi_receive_statistics[:total_retry_count] += 1
+            @multi_receive_statistics[:total_retry_duration_secs] += updated_state[:sleep_interval]
           end
-          retry if @running and exc_retries < @max_retries
-          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+
+          retry if @running and updated_state[:retries] < updated_state[:options][:max_retries]
+          log_retry_failure(multi_event_request, exc_data, updated_state[:retries], updated_state[:sleep])
           next
         end
 
@@ -572,9 +638,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         if !exc_data.nil?
           message = "Retry successful after error."
           if exc_commonly_retried
-            @logger.debug(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+            @logger.debug(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
           else
-            @logger.info(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+            @logger.info(message, :error_data => exc_data, :retries => updated_state[:retries], :sleep_time => updated_state[:sleep_interval])
           end
         end
       end
@@ -634,18 +700,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
       @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
     end
-    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
     sample_events = Array.new
     multi_event_request[:logstash_events][0,5].each {|l_event|
       sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
     }
-    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if exc_data[:code] == 413
+      message = "Failed to send #{multi_event_request[:logstash_events].length} events due to exceeding maximum request size. Not retrying non-retriable request."
+      # For PayloadTooLargeError we already include sample Scalyr payload in exc_data so there is no need
+      # to include redundant sample Logstash event objects
+      @logger.error(message, :error_data => exc_data)
+    else
+      message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+      @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    end
     if @dlq_writer
       multi_event_request[:logstash_events].each {|l_event|
         @dlq_writer.write(l_event, "#{exc_data[:message]}")
       }
     else
-      @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+      @logger.warn("Dead letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events.", :sample_events => sample_events)
     end
   end
 
@@ -839,7 +912,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       if @flatten_nested_values
         start_time = Time.now.to_f
         begin
-          record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
+          record = Scalyr::Common::Util.flatten(record, @flatten_nested_values_delimiter, @flatten_nested_arrays, @fix_deep_flattening_delimiters, @flattening_max_key_count)
         rescue Scalyr::Common::Util::MaxKeyCountError => e
           @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
         end
@@ -913,7 +986,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             ).force_encoding('UTF-8')
           end
           event_json = self.json_encode(scalyr_event)
-        rescue Java::JavaLang::ClassCastException => e
+        rescue Java::JavaLang::ClassCastException
           # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
           # Because of the version of jruby logstash works with we don't have the option to just update this away,
           # so if we run into it we convert bignums into strings so we can get the data in at least.
@@ -1017,7 +1090,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # build the scalyr thread logs object
     if current_logs
       logs = Array.new
-      current_logs.each do |identifier, log|
+      current_logs.each do |_identifier, log|
         logs << log
       end
       body[:logs] = logs
@@ -1036,7 +1109,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     start_time = Time.now.to_f
     begin
       serialized_body = self.json_encode(body)
-    rescue Java::JavaLang::ClassCastException => e
+    rescue Java::JavaLang::ClassCastException
       @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
       @stats_lock.synchronize do
         @multi_receive_statistics[:total_java_class_cast_errors] += 1
@@ -1050,7 +1123,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_request_size = serialized_body.bytesize
 
     # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
-    if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
+    if serialized_request_size >= @max_request_buffer + 5000
       # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
       @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
     end
@@ -1136,14 +1209,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
-          msg << "#{k.to_s}=#{val}"
+          msg << "#{k}=#{val}"
           cnt += 1
         end
         get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
-          msg << "#{k.to_s}=#{val}"
+          msg << "#{k}=#{val}"
           cnt += 1
         end
         status_event[:attrs]['message'] = msg
@@ -1198,26 +1271,97 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
   end
 
+  # Helper method to check if the dead-letter queue is enabled
+  def dlq_enabled?
+    # echee TODO submit to DLQ
+    respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
+        !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
+  end
+end
 
-  # Helper method that performs synchronous sleep for a certain time interval
-  def sleep_for(sleep_interval)
-    Stud.stoppable_sleep(sleep_interval) { !@running }
-    get_sleep_sec(sleep_interval)
+# Class which allows us to track retry related settings and state for different type of errors for
+# which we use different retry settings (e.g. general errors vs errors during deploy windows vs
+# client throttled errors).
+class RetryStateTracker
+
+  def initialize(plugin_config, is_plugin_running_method)
+    # :retries - stores number of times we have retried so far
+    # :sleep - stores total duration (in seconds) we have slept / waited so far
+    # :sleep_interval stores - sleep interval / delay (in seconds) for the next retry
+    @STATE = {
+      :deploy_errors => {
+        :retries => 0,
+        :sleep => 0,
+        :sleep_interval => plugin_config["retry_initial_interval_deploy_errors"],
+        :options => {
+          :retry_initial_interval => plugin_config["retry_initial_interval_deploy_errors"],
+          :max_retries => plugin_config["max_retries_deploy_errors"],
+          :retry_max_interval => plugin_config["retry_max_interval_deploy_errors"],
+          :retry_backoff_factor => plugin_config["retry_backoff_factor_deploy_errors"],
+        }
+      },
+      :throttling_errors => {
+        :retries => 0,
+        :sleep => 0,
+        :sleep_interval => plugin_config["retry_initial_interval_throttling_errors"],
+        :options => {
+          :retry_initial_interval => plugin_config["retry_initial_interval_throttling_errors"],
+          :max_retries => plugin_config["max_retries_throttling_errors"],
+          :retry_max_interval => plugin_config["retry_max_interval_throttling_errors"],
+          :retry_backoff_factor => plugin_config["retry_backoff_factor_throttling_errors"],
+        }
+      },
+      :other_errors => {
+        :retries => 0,
+        :sleep => 0,
+        :sleep_interval => plugin_config["retry_initial_interval"],
+        :options => {
+          :retry_initial_interval => plugin_config["retry_initial_interval"],
+          :max_retries => plugin_config["max_retries"],
+          :retry_max_interval => plugin_config["retry_max_interval"],
+          :retry_backoff_factor => plugin_config["retry_backoff_factor"],
+        }
+      },
+    }
+
+    @is_plugin_running_method = is_plugin_running_method
   end
 
+  # Return state hash for a specific error
+  def get_state_for_error(error)
+    if error.instance_of?(Scalyr::Common::Client::ClientThrottledError)
+      return @STATE[:throttling_errors]
+    elsif error.instance_of?(Scalyr::Common::Client::DeployWindowError)
+      return @STATE[:deploy_errors]
+    else
+      return @STATE[:other_errors]
+    end
+  end
 
-  # Helper method that gets the next sleep time for exponential backoff, capped at a defined maximum
-  def get_sleep_sec(current_interval)
-    doubled = current_interval * 2
-    doubled > @retry_max_interval ? @retry_max_interval : doubled
+  def get_state()
+    @STATE
   end
 
+  # Helper method that performs synchronous sleep for a certain time interval for a specific
+  # error and updates internal error specific state. It also returns updated internal state
+  # specific to that error.
+  def sleep_for_error_and_update_state(error)
+    # Sleep for a specific duration
+    state = get_state_for_error(error)
 
-  # Helper method to check if the dead-letter queue is enabled
-  def dlq_enabled?
-    # echee TODO submit to DLQ
-    respond_to?(:execution_context) && execution_context.respond_to?(:dlq_writer) &&
-        !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
+    current_interval = state[:sleep_interval]
+
+    Stud.stoppable_sleep(current_interval) { !@is_plugin_running_method.call }
+
+    # Update internal state + sleep interval for the next retry
+    updated_interval = current_interval * state[:options][:retry_backoff_factor]
+    updated_interval = updated_interval > state[:options][:retry_max_interval] ? state[:options][:retry_max_interval] : updated_interval
+
+    state[:retries] += 1
+    state[:sleep] += current_interval
+    state[:sleep_interval] = updated_interval
+
+    state
   end
 end
 

data/lib/scalyr/common/client.rb

@@ -27,6 +27,36 @@ end
 # An exception that signifies the Scalyr server received the upload request but dropped it
 #---------------------------------------------------------------------------------------------------------------------
 class RequestDroppedError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::RequestDroppedError")
+    super(msg, code, url, body, e_class)
+  end
+end
+
+#---------------------------------------------------------------------------------------------------------------------
+# An exception that signifies the Scalyr server received the upload request but dropped it due to it being too large.
+#---------------------------------------------------------------------------------------------------------------------
+class PayloadTooLargeError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::PayloadTooLargeError")
+    super(msg, code, url, body, e_class)
+  end
+end
+
+#---------------------------------------------------------------------------------------------------------------------
+# An exception that signifies an error which occured during Scalyr deploy window
+#---------------------------------------------------------------------------------------------------------------------
+class DeployWindowError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::DeployWindowError")
+    super(msg, code, url, body, e_class)
+  end
+end
+
+#---------------------------------------------------------------------------------------------------------------------
+# An exception that signifies that the client has been throttled by the server
+#---------------------------------------------------------------------------------------------------------------------
+class ClientThrottledError < ServerError;
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ClientThrottledError")
+    super(msg, code, url, body, e_class)
+  end
 end
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -175,7 +205,7 @@ class ClientSession
   # Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
   # and verify that the API key is valid.
   def send_ping(body)
-    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
+    post_body, post_headers, _ = prepare_post_object @add_events_uri.path, body
     response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
     handle_response(response)
 
@@ -236,7 +266,7 @@ class ClientSession
 
   # Prepare a post object to be sent, compressing it if necessary
   private
-  def prepare_post_object(uri_path, body)
+  def prepare_post_object(_uri_path, body)
     # use compression if enabled
     encoding = nil
     compression_duration = 0
@@ -308,15 +338,21 @@ class ClientSession
     end
 
     status = response_hash["status"]
+    code = response.code.to_s.strip.to_i
 
     if status != "success"
-      if status =~ /discardBuffer/
+      if code == 413
+        raise PayloadTooLargeError.new(status, response.code, @add_events_uri, response.body)
+      elsif [530, 500].include?(code)
+        raise DeployWindowError.new(status, response.code, @add_events_uri, response.body)
+      elsif code == 429
+        raise ClientThrottledError.new(status, response.code, @add_events_uri, response.body)
+      elsif status =~ /discardBuffer/
         raise RequestDroppedError.new(status, response.code, @add_events_uri, response.body)
       else
         raise ServerError.new(status, response.code, @add_events_uri, response.body)
       end
     else
-      code = response.code.to_s.strip.to_i
       if code < 200 or code > 299
         raise ServerError.new(status, response.code, @add_events_uri, response.body)
       end

data/lib/scalyr/common/util.rb

@@ -4,6 +4,7 @@ class MaxKeyCountError < StandardError
   attr_reader :message, :sample_keys
 
   def initialize(message, sample_keys)
+    super(message)
     @message = message
     @sample_keys = sample_keys
   end
@@ -32,7 +33,6 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
   key_list = []
   key_list_width = []
   result = Hash.new
-  test_key = 0
   #Debugging
   #require 'pry'
   #binding.pry
@@ -81,10 +81,10 @@ def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flatteni
         )
       end
 
-      throw_away = key_list.pop
+      key_list.pop
       until key_list_width.empty? or key_list_width[-1] > 1
-        throw_away = key_list_width.pop
-        throw_away = key_list.pop
+        key_list_width.pop
+        key_list.pop
       end
       if not key_list_width.empty?
         key_list_width[-1] -= 1
@@ -116,7 +116,7 @@ def self.convert_bignums(obj)
       obj[index] = convert_bignums(value)
     end
 
-  elsif obj.is_a? Bignum
+  elsif obj.is_a? Integer
     return obj.to_s
 
   else

data/lib/scalyr/constants.rb

@@ -1,6 +1,6 @@
 # encoding: utf-8
 
-PLUGIN_VERSION = "v0.2.7.beta"
+PLUGIN_VERSION = "v0.2.9.beta"
 
 # Special event level attribute name which can be used for setting event level serverHost attribute
 EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.2.7.beta'
+  s.version         = '0.2.9.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir['lib/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})