logstash-output-scalyr 0.1.11.beta → 0.1.16.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d6df8debd88e7675c00715a8b4fa2888e14cea861436c0049c1cbb99b96f2d0c
-  data.tar.gz: f07b38d925f732664390d391da3e84b3930644b77197b871043cabbf28bfd218
+  metadata.gz: 16e88c3869fdb18e4132726af302e37c8bac2c10038ca3a078f85cebe67c4940
+  data.tar.gz: a42194413ec2969e31bbe44e72df7cf71d09911b61adae9db16b6041a97bc1b4
 SHA512:
-  metadata.gz: 542cacbdb0ecb8d9fbabfd955b3c88dcd826af141ded1924eeb9863b8a075fcb8c323448bf0dbf4ac06254309309cdc8a60b1899e5aaa9a51f167243ff1fb818
-  data.tar.gz: f19aeb46e95f7207e11dccd1b89aa6927e9bec88276cc1a5393ac144139b559c40ac2102da0a22624ac532c27dc753fca7b83f860b6b4763b9f28f67e4662555
+  metadata.gz: 66d4261da66deaa8f5227b0ecfcdfce51f170c561902397c6020d3baa8694bd7c92291b0faa84178e4dda180c71892ab4a30ba425dc1133f5f7b349c1b7e44fa
+  data.tar.gz: 6689e3c9fd5fa2a571ca8076a4ae24b5eb1bf5da9216030b8c759a0baff17f4577a478f3c3efced4ddb22ecf273ec675f8831733aded928fd2c92c7ce91d1983

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Beta
 
+## 0.1.16.beta
+- Fix race condition in ``register()`` method.
+
+## 0.1.15.beta
+- Only call ``send_status`` method at the end of ``multi_receive()`` if there is at least one
+  record in the batch when ``report_status_for_empty_batches`` config option is set to ``false``.
+- Update ``register()`` method to use a separate short-lived client session for sending initial
+  client status.
+
+## 0.1.14.beta
+- Add configurable max retries for requests when running into errors.
+- Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.
+- Log truncated error body for all errors to help with debugging.
+
+## 0.1.13
+- Fix synchronization of status message sending code to avoid duplicate logs.
+
+## 0.1.12
+- Add logging of successful request retries after an error for additional clarity.
+- Add debug level logging of request body on error.
+
 ## 0.1.11.beta
 - Fixes to retry mechanisms.
 - More thorough catching of events, preferring to retry requests rather than crashing the plugin.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.11.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -78,6 +78,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
+  # How many times to retry sending an event before giving up on it
+  config :max_retries, :validate => :number, :default => 5
+  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
+  config :send_to_dlq, :validate => :boolean, :default => true
 
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
@@ -105,6 +109,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # True to also call send_status when multi_receive() is called with no events.
+  # In some situations (e.g. when logstash is configured with multiple scalyr
+  # plugins conditionally where most are idle) you may want to set this to false
+  config :report_status_for_empty_batches, :validate => :boolean, :default => true
+
   # Set to true to also log status messages with various metrics to stdout in addition to sending
   # this data to Scalyr
   config :log_status_messages_to_stdout, :validate => :boolean, :default => false
@@ -247,7 +256,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
-    send_status
+    # We use a special separate short lived client session for sending the initial client status.
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # plugins configured with conditionals and majority of them are inactive (aka receive no data).
+    # This way we don't need to keep idle long running connection open.
+    initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
+        @logger, @add_events_uri,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
+    )
+    send_status(initial_send_status_client_session)
+    initial_send_status_client_session.close
+
+    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
+    # Here priming just means setting up the client parameters without opening any connections.
+    # Since client writes certs to a temporary file there could be a race in case we don't do that
+    # here since multi_receive() is multi threaded. An alternative would be to put a look around
+    # client init method (aka client_config())
+    @client_session.client
 
   end # def register
 
@@ -292,6 +319,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
       while !multi_event_request_array.to_a.empty?
         multi_event_request = multi_event_request_array.pop
+        # Variables to hold information about exceptions we run into, and our handling of retries for this request. We
+        # track this to log it when the retries succeed so we can be sure logs are going through.
+        # General exception info we log in the error
+        exc_data = nil
+        # Whether the exception is commonly retried or not, for determining log level
+        exc_commonly_retried = false
+        # Count of retries attempted for this request
+        exc_retries = 0
+        # Total time spent sleeping while retrying this request due to backoff
+        exc_sleep = 0
         begin
           # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
           # ignore these.
@@ -305,8 +342,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
         rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
           sleep_interval = sleep_for(sleep_interval)
+          exc_sleep += sleep_interval
+          exc_retries += 1
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
+              :error_class => e.e_class,
               :url => e.url.to_s,
               :message => e.message,
               :batch_num => batch_num,
@@ -316,17 +356,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
               :will_retry_in_seconds => sleep_interval,
           }
           exc_data[:code] = e.code if e.code
-          exc_data[:body] = e.body if @logger.debug? and e.body
+          if @logger.debug? and e.body
+            exc_data[:body] = e.body
+          elsif e.body
+            exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
+          end
           exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
           if e.is_commonly_retried?
             # well-known retriable errors should be debug
             @logger.debug(message, exc_data)
+            exc_commonly_retried = true
           else
             # all other failed uploads should be errors
             @logger.error(message, exc_data)
+            exc_commonly_retried = false
           end
-          sleep_interval *= 2
-          retry if @running
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
 
         rescue => e
           # Any unexpected errors should be fully logged
@@ -338,8 +385,26 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           )
           @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
           sleep_interval = sleep_for(sleep_interval)
-          sleep_interval *= 2
-          retry if @running
+          exc_data = {
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :backtrace => e.backtrace,
+            :multi_event_request => multi_event_request
+          }
+          exc_sleep += sleep_interval
+          exc_retries += 1
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
+        end
+
+        if !exc_data.nil?
+          message = "Retry successful after error."
+          if exc_commonly_retried
+            @logger.debug(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+          else
+            @logger.info(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+          end
         end
       end
 
@@ -352,7 +417,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      send_status
+      if @report_status_for_empty_batches or records_count > 0
+        send_status
+      end
+
       return result
 
     rescue => e
@@ -367,6 +435,23 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end  # def multi_receive
 
 
+  def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+    sample_events = Array.new
+    multi_event_request[:logstash_events][0,5].each {|l_event|
+      sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
+    }
+    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if @dlq_writer
+      multi_event_request[:logstash_events].each {|l_event|
+        @dlq_writer.write(l_event, "#{exc_data[:message]}")
+      }
+    else
+      @logger.warn("Deal letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+    end
+  end
+
+
   # Builds an array of multi-event requests from LogStash events
   # Each array element is a request that groups multiple events (to be posted to Scalyr's addEvents endpoint)
   #
@@ -395,6 +480,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     current_threads = Hash.new
     # Create a Scalyr event object for each record in the chunk
     scalyr_events = Array.new
+    # Track the logstash events in each chunk to send them to the dlq in case of an error
+    l_events = Array.new
 
     thread_ids = Hash.new
     next_id = 1 #incrementing thread id for the session
@@ -586,9 +673,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # make sure we always have at least one event
         if scalyr_events.size == 0
           scalyr_events << scalyr_event
+          l_events << l_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -596,19 +684,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         logs = Hash.new
         logs_ids = Hash.new
         scalyr_events = Array.new
+        l_events = Array.new
       end
 
       # if we haven't consumed the current event already
       # add it to the end of our array and keep track of the json bytesize
       if append_event
         scalyr_events << scalyr_event
+        l_events << l_event
         total_bytes += add_bytes
       end
 
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
@@ -626,7 +716,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs)
+  def create_multi_event_request(scalyr_events, logstash_events, current_threads, current_logs)
 
     body = {
       :session => @session_id + Thread.current.object_id.to_s,
@@ -662,7 +752,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_body = body.to_json
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
-    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration }
+    {
+      :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
+      :logstash_events => logstash_events
+    }
 
   end  # def create_multi_event_request
 
@@ -711,7 +804,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Finally, note that there could be multiple instances of this plugin (one per worker), in which case each worker
   # thread sends their own status updates.  This is intentional so that we know how much data each worker thread is
   # uploading to Scalyr over time.
-  def send_status
+  def send_status(client_session = nil)
+    client_session = @client_session if client_session.nil?
 
     status_event = {
       :ts => (Time.now.to_f * (10**9)).round,
@@ -730,7 +824,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # echee TODO: get instance stats from session and create a status log line
         msg = 'plugin_status: '
         cnt = 0
-        @client_session.get_stats.each do |k, v|
+        client_session.get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
@@ -748,19 +842,28 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         status_event[:attrs]['serverHost'] = @node_hostname
         status_event[:attrs]['parser'] = @status_parser
       end
+      multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
+      begin
+        client_session.post_add_events(multi_event_request[:body], true, 0)
+      rescue => e
+        if e.body
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :body => Scalyr::Common::Util.truncate(e.body, 512)
+          )
+        else
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name
+          )
+        end
+        return
+      end
+      @last_status_transmit_time = Time.now()
     end
-    multi_event_request = create_multi_event_request([status_event], nil, nil)
-    begin
-      @client_session.post_add_events(multi_event_request[:body], true, 0)
-    rescue => e
-      @logger.warn(
-        "Unexpected error occurred while uploading status to Scalyr",
-        :error_message => e.message,
-        :error_class => e.class.name
-      )
-      return
-    end
-    @last_status_transmit_time = Time.now()
 
     if @log_status_messages_to_stdout
       @logger.info msg

data/lib/scalyr/common/client.rb

@@ -7,13 +7,14 @@ module Scalyr; module Common; module Client
 #---------------------------------------------------------------------------------------------------------------------
 class ServerError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, code=nil, url=nil, body=nil)
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ServerError")
     super(msg)
     @code = code.to_i
     @url = url
     @body = body
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -33,13 +34,14 @@ end
 #---------------------------------------------------------------------------------------------------------------------
 class ClientError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, url=nil)
+  def initialize(msg=nil, url=nil, e_class="Scalyr::Common::Client::ClientError")
     super(msg)
     @code = nil  # currently no way to get this from Net::HTTP::Persistent::Error
     @url = url
     @body = nil
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -236,15 +238,10 @@ class ClientSession
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
-    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
-    # way to detect that it is from SSL.
-    rescue OpenSSL::SSL::SSLError => e
-      raise e
-
     rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
-      raise ClientError.new(e.message, @add_events_uri)
+      raise ClientError.new(e.message, @add_events_uri, e.class.name)
 
     ensure
       if @record_stats_for_status or !is_status
@@ -270,6 +267,7 @@ class ClientSession
 
 
   def close
+    @client.close if @client
   end  # def close
 
 

data/lib/scalyr/common/util.rb

@@ -45,5 +45,12 @@ def self.flatten(obj, delimiter='_')
   return result
 end
 
+def self.truncate(content, max)
+  if content.length > max
+    return "#{content[0...(max-3)]}..."
+  end
+  return content
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.11.beta"
+PLUGIN_VERSION = "v0.1.16.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.11.beta'
+  s.version         = '0.1.16.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -28,8 +28,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Scalyr::Common::Client::ServerError",
                   :batch_num=>1,
                   :code=>401,
                   :message=>"error/client/badParam",
@@ -37,10 +40,10 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>2,
+                  :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -49,8 +52,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -60,7 +66,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -73,8 +78,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -84,7 +92,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -110,8 +117,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
                   :payload_size=>781,
@@ -121,7 +131,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -129,6 +138,17 @@ describe LogStash::Outputs::Scalyr do
           end
         end
       end
+
+      context "when an error occurs with retries at 5" do
+        it "exits after 5 retries and emits a log" do
+              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              )
+        end
+      end
   end
 
   describe "response_handling_tests" do
@@ -145,6 +165,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>503,
             :message=>"Invalid JSON response from server",
@@ -152,7 +173,8 @@ describe LogStash::Outputs::Scalyr do
             :record_count=>3,
             :total_batches=>1,
             :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>2
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
           }
         )
       end
@@ -171,6 +193,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>500,
             :message=>"Invalid JSON response from server",
@@ -178,11 +201,56 @@ describe LogStash::Outputs::Scalyr do
             :record_count=>3,
             :total_batches=>1,
             :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>2
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
           }
         )
       end
     end
+
+    context "when receiving a long non-json response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "0123456789" * 52, headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :error_class=>"Scalyr::Common::Client::ServerError",
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>("0123456789" * 50) + "012345678..."
+          }
+        )
+      end
+    end
+
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      it 'should send the event to the DLQ' do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+        plugin.instance_variable_set('@dlq_writer', dlq_writer)
+
+        expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
+        plugin.multi_receive(sample_events)
+      end
+    end
   end
 
 end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Outputs::Scalyr do
       end
 
       it "it doesnt include flatten metrics if flattening is disabled" do
-          plugin1 = LogStash::Outputs::Scalyr.new({
+        plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
@@ -122,7 +122,7 @@ describe LogStash::Outputs::Scalyr do
         expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
-      it "send_stats is called when events list is empty, but otherwise noop" do
+      it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
@@ -137,6 +137,30 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive([])
       end
 
+      it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
+        plugin2 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                     'report_status_for_empty_batches' => false,
+                                                 })
+
+        mock_client_session = MockClientSession.new
+        quantile_estimator = Quantile::Estimator.new
+        plugin2.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin2.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin2).not_to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin2.multi_receive([])
+      end
+
       # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
       it "send_status only sends posts with is_status = true" do
         # 1. Initial send

data/vendor/bundle/jruby/2.5.0/bin/htmldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

data/vendor/bundle/jruby/2.5.0/bin/ldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.11.beta
+  version: 0.1.16.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-11 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement