logstash-output-scalyr 0.1.10.beta → 0.1.15.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 290310d36108a3311f55a47efbb3a73e01f19ccdf025a15590f86493459963da
-  data.tar.gz: a5bf9f872c496fa65670cab36e0955f1417541db5f151e11f0d914abe5fc9920
+  metadata.gz: b2e4aabe9bf971e1e0638899d713f5f16d7123e9be6f1dd98b82ab7ccfadb63f
+  data.tar.gz: 59ed216e6811e485f48ebed2a8fb3875f31c64cdf0f3a3b6e0a6112b2d8e11b6
 SHA512:
-  metadata.gz: 35d60d700b6bb6ccc43aea5151d07c306b29c5524ac8d80145ae792263ffe5e7264d22a59de3a21de9aa549c5963b4cd3c9b230b845f67c6ff9618faca863cad
-  data.tar.gz: 41cf50b968a9a11efff957cda768cf9d05e1ccf3c36e4f4d8f12c3d1aca9fb9da049647432bf047d51c2b6fe33f7b711484632d108dcd390ca852bc45aa7a00d
+  metadata.gz: 45f6f63a99a7d232ea7cedaaa15ba5a64893d26788701ccfd6140367035771748cf4829a7a856642b704a20524a3d41710c9d9f23cc1daf3fea5b0b50394a626
+  data.tar.gz: 9a24eb6f935f9accd942b6091c33d9c3aca3601e7b833f07516c17a07e8e47660715f3e024748605ae57c40c38f3fc2f507151fa5708f6ab99280d3021c93e07

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Beta
 
+## 0.1.15.beta
+- Only call ``send_status`` method at the end of ``multi_receive()`` if there is at least one
+  record in the batch when ``report_status_for_empty_batches`` config option is set to ``false``.
+- Update ``register()`` method to use a separate short-lived client session for sending initial
+  client status.
+
+## 0.1.14.beta
+- Add configurable max retries for requests when running into errors.
+- Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.
+- Log truncated error body for all errors to help with debugging.
+
+## 0.1.13
+- Fix synchronization of status message sending code to avoid duplicate logs.
+
+## 0.1.12
+- Add logging of successful request retries after an error for additional clarity.
+- Add debug level logging of request body on error.
+
+## 0.1.11.beta
+- Fixes to retry mechanisms.
+- More thorough catching of events, preferring to retry requests rather than crashing the plugin.
+
 ## 0.1.10.beta
 
 - Switch to shared concurrency to allow the use of multiple worker threads for increased

data/Gemfile

@@ -10,6 +10,10 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
 
+group :test do
+  gem "webmock"
+end
+
 gem 'pry'
 gem 'pry-nav'
 gem 'quantile'

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.9.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -369,7 +369,7 @@ If you want to change status reporting interval you can do that by changing the
 ## Updating version
 
 Currently references to the version need to be manually updated, files to look in for this are `logstash-putput-scalyr.gemspec`,
- `client.rb`, twice in the CircleCI `Dockerfile`, and under "Quick Start" in this `README.md`.
+ `lib/scalyr/constants.rb`, and under "Quick Start" in this `README.md`.
 
 The changelog should also be updated with the latest version and changes of note.
 

data/lib/logstash/outputs/scalyr.rb

@@ -16,6 +16,7 @@ require 'quantile'
 
 require 'scalyr/common/client'
 require "scalyr/common/util"
+require "scalyr/constants"
 
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -77,6 +78,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
+  # How many times to retry sending an event before giving up on it
+  config :max_retries, :validate => :number, :default => 5
+  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
+  config :send_to_dlq, :validate => :boolean, :default => true
 
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
@@ -104,6 +109,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # True to also call send_status when multi_receive() is called with no events.
+  # In some situations (e.g. when logstash is configured with multiple scalyr
+  # plugins conditionally where most are idle) you may want to set this to false
+  config :report_status_for_empty_batches, :validate => :boolean, :default => true
+
   # Set to true to also log status messages with various metrics to stdout in addition to sending
   # this data to Scalyr
   config :log_status_messages_to_stdout, :validate => :boolean, :default => false
@@ -243,10 +253,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
 
-    @logger.info("Started Scalyr output plugin", :class => self.class.name)
+    @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
-    send_status
+    # We use a special separate short lived client session for sending the initial client status.
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # plugins configured with conditionals and majority of them are inactive (aka receive no data).
+    # This way we don't need to keep idle long running connection open.
+    initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
+        @logger, @add_events_uri,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
+    )
+    send_status(initial_send_status_client_session)
+    initial_send_status_client_session.close
 
   end # def register
 
@@ -256,7 +277,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       :multi_receive_duration_secs => Quantile::Estimator.new,
       :multi_receive_event_count => Quantile::Estimator.new,
       :event_attributes_count => Quantile::Estimator.new,
-      :flatten_values_duration_secs => Quantile::Estimator.new
+      :flatten_values_duration_secs => Quantile::Estimator.new,
+      :batches_per_multi_receive => Quantile::Estimator.new
     }
   end
 
@@ -275,90 +297,154 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Just return and pretend we did something if running in noop mode
     return events if @noop_mode
 
-    start_time = Time.now.to_f
+    begin
+      start_time = Time.now.to_f
 
-    multi_event_request_array = build_multi_event_request_array(events)
-    # Loop over all array of multi-event requests, sending each multi-event to Scalyr
+      multi_event_request_array = build_multi_event_request_array(events)
+      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
 
-    sleep_interval = @retry_initial_interval
-    batch_num = 1
-    total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
+      sleep_interval = @retry_initial_interval
+      batch_num = 1
+      total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
-    result = []
-    records_count = events.to_a.length
+      result = []
+      records_count = events.to_a.length
 
-    while !multi_event_request_array.to_a.empty?
-      begin
+      while !multi_event_request_array.to_a.empty?
         multi_event_request = multi_event_request_array.pop
-        # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
-        # ignore these.
-        if !multi_event_request.nil?
-          @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
+        # Variables to hold information about exceptions we run into, and our handling of retries for this request. We
+        # track this to log it when the retries succeed so we can be sure logs are going through.
+        # General exception info we log in the error
+        exc_data = nil
+        # Whether the exception is commonly retried or not, for determining log level
+        exc_commonly_retried = false
+        # Count of retries attempted for this request
+        exc_retries = 0
+        # Total time spent sleeping while retrying this request due to backoff
+        exc_sleep = 0
+        begin
+          # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
+          # ignore these.
+          if !multi_event_request.nil?
+            @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
+
+            sleep_interval = @retry_initial_interval
+            batch_num += 1
+            result.push(multi_event_request)
+          end
 
-          sleep_interval = 0
-          result.push(multi_event_request)
+        rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
+          sleep_interval = sleep_for(sleep_interval)
+          exc_sleep += sleep_interval
+          exc_retries += 1
+          message = "Error uploading to Scalyr (will backoff-retry)"
+          exc_data = {
+              :error_class => e.e_class,
+              :url => e.url.to_s,
+              :message => e.message,
+              :batch_num => batch_num,
+              :total_batches => total_batches,
+              :record_count => multi_event_request[:record_count],
+              :payload_size => multi_event_request[:body].bytesize,
+              :will_retry_in_seconds => sleep_interval,
+          }
+          exc_data[:code] = e.code if e.code
+          if @logger.debug? and e.body
+            exc_data[:body] = e.body
+          elsif e.body
+            exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
+          end
+          exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
+          if e.is_commonly_retried?
+            # well-known retriable errors should be debug
+            @logger.debug(message, exc_data)
+            exc_commonly_retried = true
+          else
+            # all other failed uploads should be errors
+            @logger.error(message, exc_data)
+            exc_commonly_retried = false
+          end
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
+
+        rescue => e
+          # Any unexpected errors should be fully logged
+          @logger.error(
+              "Unexpected error occurred while uploading to Scalyr (will backoff-retry)",
+              :error_message => e.message,
+              :error_class => e.class.name,
+              :backtrace => e.backtrace
+          )
+          @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
+          sleep_interval = sleep_for(sleep_interval)
+          exc_data = {
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :backtrace => e.backtrace,
+            :multi_event_request => multi_event_request
+          }
+          exc_sleep += sleep_interval
+          exc_retries += 1
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
         end
 
-      rescue OpenSSL::SSL::SSLError => e
-        # cannot rely on exception message, so we always log the following warning
-        @logger.error "SSL certificate verification failed.  " +
-        "Please make sure your certificate bundle is configured correctly and points to a valid file.  " +
-        "You can configure this with the ssl_ca_bundle_path configuration option.  " +
-        "The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
-        @logger.error e.message
-        @logger.error "Discarding buffer chunk without retrying."
-
-      rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
-        sleep_interval = sleep_for(sleep_interval)
-        message = "Error uploading to Scalyr (will backoff-retry)"
-        exc_data = {
-            :url => e.url.to_s,
-            :message => e.message,
-            :batch_num => batch_num,
-            :total_batches => total_batches,
-            :record_count => multi_event_request[:record_count],
-            :payload_size => multi_event_request[:body].bytesize,
-            :will_retry_in_seconds => sleep_interval,
-        }
-        exc_data[:code] = e.response_code if e.code
-        exc_data[:body] = e.response_body if @logger.debug? and e.body
-        exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
-        if e.is_commonly_retried?
-          # well-known retriable errors should be debug
-          @logger.error(message, exc_data)
-        else
-          # all other failed uploads should be errors
-          @logger.error(message, exc_data)
+        if !exc_data.nil?
+          message = "Retry successful after error."
+          if exc_commonly_retried
+            @logger.debug(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+          else
+            @logger.info(message, :error_data => exc_data, :retries => exc_retries, :sleep_time => exc_sleep)
+          end
         end
-        retry if @running
+      end
 
-      rescue => e
-        # Any unexpected errors should be fully logged
-        @logger.error(
-            "Unexpected error occurred while uploading to Scalyr (will backoff-retry)",
-            :error_message => e.message,
-            :error_class => e.class.name,
-            :backtrace => e.backtrace
-        )
-        @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
-        sleep_interval = sleep_for(sleep_interval)
-        retry if @running
+      if records_count > 0
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+          @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
+          @plugin_metrics[:multi_receive_event_count].observe(records_count)
+          @plugin_metrics[:batches_per_multi_receive].observe(total_batches)
+        end
       end
-    end
 
-    if records_count > 0
-      @stats_lock.synchronize do
-        @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
-        @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
-        @plugin_metrics[:multi_receive_event_count].observe(records_count)
+      if @report_status_for_empty_batches or records_count > 0
+        send_status
       end
-    end
 
-    send_status
-    return result
+      return result
+
+    rescue => e
+      # Any unexpected errors should be fully logged
+      @logger.error(
+          "Unexpected error occurred while executing multi_receive.",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+      )
+    end
   end  # def multi_receive
 
 
+  def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+    sample_events = Array.new
+    multi_event_request[:logstash_events][0,5].each {|l_event|
+      sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
+    }
+    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if @dlq_writer
+      multi_event_request[:logstash_events].each {|l_event|
+        @dlq_writer.write(l_event, "#{exc_data[:message]}")
+      }
+    else
+      @logger.warn("Deal letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+    end
+  end
+
+
   # Builds an array of multi-event requests from LogStash events
   # Each array element is a request that groups multiple events (to be posted to Scalyr's addEvents endpoint)
   #
@@ -387,6 +473,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     current_threads = Hash.new
     # Create a Scalyr event object for each record in the chunk
     scalyr_events = Array.new
+    # Track the logstash events in each chunk to send them to the dlq in case of an error
+    l_events = Array.new
 
     thread_ids = Hash.new
     next_id = 1 #incrementing thread id for the session
@@ -578,9 +666,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # make sure we always have at least one event
         if scalyr_events.size == 0
           scalyr_events << scalyr_event
+          l_events << l_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -588,19 +677,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         logs = Hash.new
         logs_ids = Hash.new
         scalyr_events = Array.new
+        l_events = Array.new
       end
 
       # if we haven't consumed the current event already
       # add it to the end of our array and keep track of the json bytesize
       if append_event
         scalyr_events << scalyr_event
+        l_events << l_event
         total_bytes += add_bytes
       end
 
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
@@ -618,7 +709,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs)
+  def create_multi_event_request(scalyr_events, logstash_events, current_threads, current_logs)
 
     body = {
       :session => @session_id + Thread.current.object_id.to_s,
@@ -654,7 +745,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     serialized_body = body.to_json
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
-    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration }
+    {
+      :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
+      :logstash_events => logstash_events
+    }
 
   end  # def create_multi_event_request
 
@@ -676,6 +770,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
       current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
 
+      current_stats[:batches_per_multi_receive_p50] = @plugin_metrics[:batches_per_multi_receive].query(0.5)
+      current_stats[:batches_per_multi_receive_p90] = @plugin_metrics[:batches_per_multi_receive].query(0.9)
+      current_stats[:batches_per_multi_receive_p99] = @plugin_metrics[:batches_per_multi_receive].query(0.99)
+
       if @flatten_nested_values
         # We only return those metrics in case flattening is enabled
         current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
@@ -699,7 +797,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Finally, note that there could be multiple instances of this plugin (one per worker), in which case each worker
   # thread sends their own status updates.  This is intentional so that we know how much data each worker thread is
   # uploading to Scalyr over time.
-  def send_status
+  def send_status(client_session = nil)
+    client_session = @client_session if client_session.nil?
 
     status_event = {
       :ts => (Time.now.to_f * (10**9)).round,
@@ -710,7 +809,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     }
     @send_stats.synchronize do
       if !@last_status_transmit_time
-        status_event[:attrs]['message'] = "Started Scalyr LogStash output plugin."
+        status_event[:attrs]['message'] = sprintf("Started Scalyr LogStash output plugin (%s)." % [PLUGIN_VERSION])
         status_event[:attrs]['serverHost'] = @node_hostname
       else
         cur_time = Time.now()
@@ -718,7 +817,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # echee TODO: get instance stats from session and create a status log line
         msg = 'plugin_status: '
         cnt = 0
-        @client_session.get_stats.each do |k, v|
+        client_session.get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
@@ -736,10 +835,28 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         status_event[:attrs]['serverHost'] = @node_hostname
         status_event[:attrs]['parser'] = @status_parser
       end
+      multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
+      begin
+        client_session.post_add_events(multi_event_request[:body], true, 0)
+      rescue => e
+        if e.body
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :body => Scalyr::Common::Util.truncate(e.body, 512)
+          )
+        else
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name
+          )
+        end
+        return
+      end
+      @last_status_transmit_time = Time.now()
     end
-    multi_event_request = create_multi_event_request([status_event], nil, nil)
-    @client_session.post_add_events(multi_event_request[:body], true, 0)
-    @last_status_transmit_time = Time.now()
 
     if @log_status_messages_to_stdout
       @logger.info msg