logstash-output-scalyr 0.1.13 → 0.1.14.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89b457b34b9579d3e6c7b82c7aed5a5ae54026904deee677d31f0a0ef5ed3e22
-  data.tar.gz: 945e0414df96259ba39747e1f79ee8ed6d3c265d0c91aae0c15c91b9578988b9
+  metadata.gz: 1af6e37035c37c270036487e9b939f40e8454d39ddea9983a737f409957e0747
+  data.tar.gz: 95438909311aa595d2d4e771424765f3705cdc649225ca81c053e3b9c129b183
 SHA512:
-  metadata.gz: 411e2319e7fb06de8af26ea876776500327e8fb57002cef3f4a3b153294039b486fe574089f3acdd8761309335dd83b9e3f75dbbd23c5ad701d59702e7d7fe11
-  data.tar.gz: d17765c84763f27858548ede5afd45cc0d2ff11ef1462b586afa195fa0186f2cf0318a86463354285acae7d598aa45319305dea7ccc94581aa614a82e56dbe85
+  metadata.gz: d2a9f542cfaaf8bb9e84ebf976694c548e153e1e170417309644b9206bf4df2ba8844447cbb2b0afd4ba8d03922688f6348a6fb77a6c8489a3f2a49bb873fed9
+  data.tar.gz: 514fab3aa32f39e1ebe6e4bf164a9b9f648b82e7aa861d9cbc2edd6d44d9c6b4d9d3ea9b7c281ae26bf719e4373bde6059f220033dc660821a369b6186180798

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Beta
 
+## 0.1.14.beta
+- Add configurable max retries for requests when running into errors.
+- Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.
+- Log truncated error body for all errors to help with debugging.
+
 ## 0.1.13
 - Fix synchronization of status message sending code to avoid duplicate logs.
 

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.13.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -78,6 +78,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
+  # How many times to retry sending an event before giving up on it
+  config :max_retries, :validate => :number, :default => 5
+  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
+  config :send_to_dlq, :validate => :boolean, :default => true
 
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
@@ -319,6 +323,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           exc_retries += 1
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
+              :error_class => e.e_class,
               :url => e.url.to_s,
               :message => e.message,
               :batch_num => batch_num,
@@ -330,7 +335,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           exc_data[:code] = e.code if e.code
           if @logger.debug? and e.body
             exc_data[:body] = e.body
-          elsif e.message == "Invalid JSON response from server" and e.body
+          elsif e.body
             exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
           end
           exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
@@ -343,7 +348,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @logger.error(message, exc_data)
             exc_commonly_retried = false
           end
-          retry if @running
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
 
         rescue => e
           # Any unexpected errors should be fully logged
@@ -363,7 +370,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           }
           exc_sleep += sleep_interval
           exc_retries += 1
-          retry if @running
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
         end
 
         if !exc_data.nil?
@@ -400,6 +409,23 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end  # def multi_receive
 
 
+  def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+    sample_events = Array.new
+    multi_event_request[:logstash_events][0,5].each {|l_event|
+      sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
+    }
+    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if @dlq_writer
+      multi_event_request[:logstash_events].each {|l_event|
+        @dlq_writer.write(l_event, "#{exc_data[:message]}")
+      }
+    else
+      @logger.warn("Deal letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+    end
+  end
+
+
   # Builds an array of multi-event requests from LogStash events
   # Each array element is a request that groups multiple events (to be posted to Scalyr's addEvents endpoint)
   #
@@ -428,6 +454,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     current_threads = Hash.new
     # Create a Scalyr event object for each record in the chunk
     scalyr_events = Array.new
+    # Track the logstash events in each chunk to send them to the dlq in case of an error
+    l_events = Array.new
 
     thread_ids = Hash.new
     next_id = 1 #incrementing thread id for the session
@@ -619,9 +647,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # make sure we always have at least one event
         if scalyr_events.size == 0
           scalyr_events << scalyr_event
+          l_events << l_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -629,19 +658,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         logs = Hash.new
         logs_ids = Hash.new
         scalyr_events = Array.new
+        l_events = Array.new
       end
 
       # if we haven't consumed the current event already
       # add it to the end of our array and keep track of the json bytesize
       if append_event
         scalyr_events << scalyr_event
+        l_events << l_event
         total_bytes += add_bytes
       end
 
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
@@ -659,7 +690,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs)
+  def create_multi_event_request(scalyr_events, logstash_events, current_threads, current_logs)
 
     body = {
       :session => @session_id + Thread.current.object_id.to_s,
@@ -695,7 +726,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_body = body.to_json
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
-    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration }
+    {
+      :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
+      :logstash_events => logstash_events
+    }
 
   end  # def create_multi_event_request
 
@@ -781,15 +815,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         status_event[:attrs]['serverHost'] = @node_hostname
         status_event[:attrs]['parser'] = @status_parser
       end
-      multi_event_request = create_multi_event_request([status_event], nil, nil)
+      multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
       begin
         @client_session.post_add_events(multi_event_request[:body], true, 0)
       rescue => e
-        @logger.warn(
-          "Unexpected error occurred while uploading status to Scalyr",
-          :error_message => e.message,
-          :error_class => e.class.name
-        )
+        if e.body
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :body => Scalyr::Common::Util.truncate(e.body, 512)
+          )
+        else
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name
+          )
+        end
         return
       end
       @last_status_transmit_time = Time.now()

data/lib/scalyr/common/client.rb

@@ -7,13 +7,14 @@ module Scalyr; module Common; module Client
 #---------------------------------------------------------------------------------------------------------------------
 class ServerError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, code=nil, url=nil, body=nil)
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ServerError")
     super(msg)
     @code = code.to_i
     @url = url
     @body = body
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -33,13 +34,14 @@ end
 #---------------------------------------------------------------------------------------------------------------------
 class ClientError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, url=nil)
+  def initialize(msg=nil, url=nil, e_class="Scalyr::Common::Client::ClientError")
     super(msg)
     @code = nil  # currently no way to get this from Net::HTTP::Persistent::Error
     @url = url
     @body = nil
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -236,15 +238,10 @@ class ClientSession
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
-    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
-    # way to detect that it is from SSL.
-    rescue OpenSSL::SSL::SSLError => e
-      raise e
-
     rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
-      raise ClientError.new(e.message, @add_events_uri)
+      raise ClientError.new(e.message, @add_events_uri, e.class.name)
 
     ensure
       if @record_stats_for_status or !is_status

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.13"
+PLUGIN_VERSION = "v0.1.14.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.13'
+  s.version         = '0.1.14.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -28,8 +28,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Scalyr::Common::Client::ServerError",
                   :batch_num=>1,
                   :code=>401,
                   :message=>"error/client/badParam",
@@ -37,10 +40,10 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>2,
+                  :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -49,8 +52,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -60,7 +66,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -73,8 +78,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -84,7 +92,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -110,8 +117,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
                   :payload_size=>781,
@@ -121,7 +131,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -129,6 +138,17 @@ describe LogStash::Outputs::Scalyr do
           end
         end
       end
+
+      context "when an error occurs with retries at 5" do
+        it "exits after 5 retries and emits a log" do
+              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              )
+        end
+      end
   end
 
   describe "response_handling_tests" do
@@ -145,6 +165,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>503,
             :message=>"Invalid JSON response from server",
@@ -172,6 +193,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>500,
             :message=>"Invalid JSON response from server",
@@ -199,6 +221,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>500,
             :message=>"Invalid JSON response from server",
@@ -212,6 +235,22 @@ describe LogStash::Outputs::Scalyr do
         )
       end
     end
+
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      it 'should send the event to the DLQ' do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+        plugin.instance_variable_set('@dlq_writer', dlq_writer)
+
+        expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
+        plugin.multi_receive(sample_events)
+      end
+    end
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.14.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-19 00:00:00.000000000 Z
+date: 2021-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -4052,9 +4052,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10