logstash-output-scalyr 0.1.13 → 0.1.18.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89b457b34b9579d3e6c7b82c7aed5a5ae54026904deee677d31f0a0ef5ed3e22
-  data.tar.gz: 945e0414df96259ba39747e1f79ee8ed6d3c265d0c91aae0c15c91b9578988b9
+  metadata.gz: 9099fcf20201bf8e3905273393a38bb1addefb219e1874a45a439bbfd2dcddbd
+  data.tar.gz: a031faa8657951b66a09c172f6be69b774a2192e661ef99370813abe62250cbb
 SHA512:
-  metadata.gz: 411e2319e7fb06de8af26ea876776500327e8fb57002cef3f4a3b153294039b486fe574089f3acdd8761309335dd83b9e3f75dbbd23c5ad701d59702e7d7fe11
-  data.tar.gz: d17765c84763f27858548ede5afd45cc0d2ff11ef1462b586afa195fa0186f2cf0318a86463354285acae7d598aa45319305dea7ccc94581aa614a82e56dbe85
+  metadata.gz: c25cad81473d0224fb5c75558e302f6383d69361d3263aeaf0fe619b8c2a3a6866bf0eaf811588b9235edb4e15e11a2d0b95ecb99e2a3045ee6c6e66998e1627
+  data.tar.gz: fed7e02e32c70ce199dbd7ec5841711ff77c65d3a56ebe53eeada8c165d5b8fa603fc15b3c00e0e397682a50899dd141bfdcacdcbb97244e2b26205c92802238

data/CHANGELOG.md

@@ -1,5 +1,27 @@
 # Beta
 
+## 0.1.18.beta
+- Add metrics for successfully sent and failed logstash events, and retries.
+- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
+
+## 0.1.17.beta
+- Catch errors relating to Bignum conversions present in the ``json`` library and manually convert to string as
+a workaround.
+
+## 0.1.16.beta
+- Fix race condition in ``register()`` method.
+
+## 0.1.15.beta
+- Only call ``send_status`` method at the end of ``multi_receive()`` if there is at least one
+  record in the batch when ``report_status_for_empty_batches`` config option is set to ``false``.
+- Update ``register()`` method to use a separate short-lived client session for sending initial
+  client status.
+
+## 0.1.14.beta
+- Add configurable max retries for requests when running into errors.
+- Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.
+- Log truncated error body for all errors to help with debugging.
+
 ## 0.1.13
 - Fix synchronization of status message sending code to avoid duplicate logs.
 

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.13.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.18.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -68,7 +68,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If true, nested values will be flattened (which changes keys to delimiter-separated concatenation of all
   # nested keys).
   config :flatten_nested_values, :validate => :boolean, :default => false
-  config :flatten_nested_values_delimiter, :validate => :string, :default => "_" 
+  config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
+  config :flatten_nested_arrays, :validate => :boolean, :default => true
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -78,6 +79,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
+  # How many times to retry sending an event before giving up on it
+  config :max_retries, :validate => :number, :default => 5
+  # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
+  config :send_to_dlq, :validate => :boolean, :default => true
 
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
@@ -105,6 +110,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # True to also call send_status when multi_receive() is called with no events.
+  # In some situations (e.g. when logstash is configured with multiple scalyr
+  # plugins conditionally where most are idle) you may want to set this to false
+  config :report_status_for_empty_batches, :validate => :boolean, :default => true
+
   # Set to true to also log status messages with various metrics to stdout in addition to sending
   # this data to Scalyr
   config :log_status_messages_to_stdout, :validate => :boolean, :default => false
@@ -231,7 +241,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Plugin level (either per batch or event level metrics). Other request
     # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
-      :total_multi_receive_secs => 0
+      :total_multi_receive_secs => 0,
+      :total_events_processed => 0,
+      :successful_events_processed => 0,
+      :failed_events_processed => 0,
+      :total_retry_count => 0,
+      :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
 
@@ -247,7 +262,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
-    send_status
+    # We use a special separate short lived client session for sending the initial client status.
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # plugins configured with conditionals and majority of them are inactive (aka receive no data).
+    # This way we don't need to keep idle long running connection open.
+    initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
+        @logger, @add_events_uri,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
+    )
+    send_status(initial_send_status_client_session)
+    initial_send_status_client_session.close
+
+    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
+    # Here priming just means setting up the client parameters without opening any connections.
+    # Since client writes certs to a temporary file there could be a race in case we don't do that
+    # here since multi_receive() is multi threaded. An alternative would be to put a look around
+    # client init method (aka client_config())
+    @client_session.client
 
   end # def register
 
@@ -317,8 +350,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           sleep_interval = sleep_for(sleep_interval)
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
+              :error_class => e.e_class,
               :url => e.url.to_s,
               :message => e.message,
               :batch_num => batch_num,
@@ -330,7 +367,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           exc_data[:code] = e.code if e.code
           if @logger.debug? and e.body
             exc_data[:body] = e.body
-          elsif e.message == "Invalid JSON response from server" and e.body
+          elsif e.body
             exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
           end
           exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
@@ -343,7 +380,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @logger.error(message, exc_data)
             exc_commonly_retried = false
           end
-          retry if @running
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
 
         rescue => e
           # Any unexpected errors should be fully logged
@@ -363,7 +402,17 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           }
           exc_sleep += sleep_interval
           exc_retries += 1
-          retry if @running
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
+          retry if @running and exc_retries < @max_retries
+          log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+          next
+        end
+
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+          @multi_receive_statistics[:successful_events_processed] += multi_event_request[:logstash_events].length
         end
 
         if !exc_data.nil?
@@ -385,7 +434,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      send_status
+      if @report_status_for_empty_batches or records_count > 0
+        send_status
+      end
+
       return result
 
     rescue => e
@@ -400,6 +452,27 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   end  # def multi_receive
 
 
+  def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    @stats_lock.synchronize do
+      @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+      @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
+    end
+    message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
+    sample_events = Array.new
+    multi_event_request[:logstash_events][0,5].each {|l_event|
+      sample_events << Scalyr::Common::Util.truncate(l_event.to_hash.to_json, 256)
+    }
+    @logger.error(message, :error_data => exc_data, :sample_events => sample_events, :retries => exc_retries, :sleep_time => exc_sleep)
+    if @dlq_writer
+      multi_event_request[:logstash_events].each {|l_event|
+        @dlq_writer.write(l_event, "#{exc_data[:message]}")
+      }
+    else
+      @logger.warn("Deal letter queue not configured, dropping #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries.", :sample_events => sample_events)
+    end
+  end
+
+
   # Builds an array of multi-event requests from LogStash events
   # Each array element is a request that groups multiple events (to be posted to Scalyr's addEvents endpoint)
   #
@@ -428,6 +501,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     current_threads = Hash.new
     # Create a Scalyr event object for each record in the chunk
     scalyr_events = Array.new
+    # Track the logstash events in each chunk to send them to the dlq in case of an error
+    l_events = Array.new
 
     thread_ids = Hash.new
     next_id = 1 #incrementing thread id for the session
@@ -557,7 +632,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter)
+        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays)
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
@@ -607,6 +682,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           ).force_encoding('UTF-8')
         end
         event_json = scalyr_event.to_json
+      rescue Java::JavaLang::ClassCastException => e
+        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+        # Because of the version of jruby logstash works with we don't have the option to just update this away,
+        # so if we run into it we convert bignums into strings so we can get the data in at least.
+        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+        end
+        Scalyr::Common::Util.convert_bignums(scalyr_event)
+        event_json = scalyr_event.to_json
+        log_json = nil
+        if add_log
+          log_json = logs[log_identifier].to_json
+        end
       end
 
       # generate new request if json size of events in the array exceed maximum request buffer size
@@ -619,9 +709,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # make sure we always have at least one event
         if scalyr_events.size == 0
           scalyr_events << scalyr_event
+          l_events << l_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -629,19 +720,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         logs = Hash.new
         logs_ids = Hash.new
         scalyr_events = Array.new
+        l_events = Array.new
       end
 
       # if we haven't consumed the current event already
       # add it to the end of our array and keep track of the json bytesize
       if append_event
         scalyr_events << scalyr_event
+        l_events << l_event
         total_bytes += add_bytes
       end
 
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
@@ -659,7 +752,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs)
+  def create_multi_event_request(scalyr_events, logstash_events, current_threads, current_logs)
 
     body = {
       :session => @session_id + Thread.current.object_id.to_s,
@@ -692,10 +785,22 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
-    serialized_body = body.to_json
+    begin
+      serialized_body = body.to_json
+    rescue Java::JavaLang::ClassCastException => e
+      @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+      @stats_lock.synchronize do
+        @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      end
+      Scalyr::Common::Util.convert_bignums(body)
+      serialized_body = body.to_json
+    end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
-    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration }
+    {
+      :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
+      :logstash_events => logstash_events
+    }
 
   end  # def create_multi_event_request
 
@@ -744,7 +849,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Finally, note that there could be multiple instances of this plugin (one per worker), in which case each worker
   # thread sends their own status updates.  This is intentional so that we know how much data each worker thread is
   # uploading to Scalyr over time.
-  def send_status
+  def send_status(client_session = nil)
+    client_session = @client_session if client_session.nil?
 
     status_event = {
       :ts => (Time.now.to_f * (10**9)).round,
@@ -763,7 +869,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # echee TODO: get instance stats from session and create a status log line
         msg = 'plugin_status: '
         cnt = 0
-        @client_session.get_stats.each do |k, v|
+        client_session.get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
@@ -781,15 +887,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         status_event[:attrs]['serverHost'] = @node_hostname
         status_event[:attrs]['parser'] = @status_parser
       end
-      multi_event_request = create_multi_event_request([status_event], nil, nil)
+      multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
       begin
-        @client_session.post_add_events(multi_event_request[:body], true, 0)
+        client_session.post_add_events(multi_event_request[:body], true, 0)
       rescue => e
-        @logger.warn(
-          "Unexpected error occurred while uploading status to Scalyr",
-          :error_message => e.message,
-          :error_class => e.class.name
-        )
+        if e.body
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name,
+            :body => Scalyr::Common::Util.truncate(e.body, 512)
+          )
+        else
+          @logger.warn(
+            "Unexpected error occurred while uploading status to Scalyr",
+            :error_message => e.message,
+            :error_class => e.class.name
+          )
+        end
         return
       end
       @last_status_transmit_time = Time.now()

data/lib/scalyr/common/client.rb

@@ -7,13 +7,14 @@ module Scalyr; module Common; module Client
 #---------------------------------------------------------------------------------------------------------------------
 class ServerError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, code=nil, url=nil, body=nil)
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ServerError")
     super(msg)
     @code = code.to_i
     @url = url
     @body = body
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -33,13 +34,14 @@ end
 #---------------------------------------------------------------------------------------------------------------------
 class ClientError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, url=nil)
+  def initialize(msg=nil, url=nil, e_class="Scalyr::Common::Client::ClientError")
     super(msg)
     @code = nil  # currently no way to get this from Net::HTTP::Persistent::Error
     @url = url
     @body = nil
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -236,15 +238,10 @@ class ClientSession
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
-    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
-    # way to detect that it is from SSL.
-    rescue OpenSSL::SSL::SSLError => e
-      raise e
-
     rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
-      raise ClientError.new(e.message, @add_events_uri)
+      raise ClientError.new(e.message, @add_events_uri, e.class.name)
 
     ensure
       if @record_stats_for_status or !is_status
@@ -270,6 +267,7 @@ class ClientSession
 
 
   def close
+    @client.close if @client
   end  # def close
 
 

data/lib/scalyr/common/util.rb

@@ -4,7 +4,7 @@ module Scalyr; module Common; module Util;
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
-def self.flatten(obj, delimiter='_')
+def self.flatten(obj, delimiter='_', flatten_arrays=true)
 
   # base case is input object is not enumerable, in which case simply return it
   if !obj.respond_to?(:each)
@@ -19,8 +19,8 @@ def self.flatten(obj, delimiter='_')
 
     # input object is a hash
     obj.each do |key, value|
-      if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
+        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
           result["#{key}#{delimiter}#{subkey}"] = subvalue
         end
       else
@@ -28,18 +28,23 @@ def self.flatten(obj, delimiter='_')
       end
     end
 
-  else
+  elsif flatten_arrays
 
     # input object is an array or set
     obj.each_with_index do |value, index|
       if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
           result["#{index}#{delimiter}#{subkey}"] = subvalue
         end
       else
         result["#{index}"] = value
       end
     end
+
+  else
+
+    result = obj
+
   end
 
   return result
@@ -52,5 +57,26 @@ def self.truncate(content, max)
   return content
 end
 
+def self.convert_bignums(obj)
+  if obj.respond_to?(:has_key?) and obj.respond_to?(:each)
+    # input object is a hash
+    obj.each do |key, value|
+      obj[key] = convert_bignums(value)
+    end
+
+  elsif obj.respond_to?(:each)
+    # input object is an array or set
+    obj.each_with_index do |value, index|
+      obj[index] = convert_bignums(value)
+    end
+
+  elsif obj.is_a? Bignum
+    return obj.to_s
+
+  else
+    return obj
+  end
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.13"
+PLUGIN_VERSION = "v0.1.18.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.13'
+  s.version         = '0.1.18.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/benchmarks/bignum_fixing.rb

@@ -0,0 +1,90 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
+
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def rand_bignum()
+  return 200004000020304050300 + rand(999999)
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_bignum()
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def convert_bignums(record)
+  Scalyr::Common::Util.convert_bignums(record)
+end
+
+puts "Util.convert_bignums()"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~200 keys in a hash (single level)
+data = generate_data_array_for_spec([200])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -28,8 +28,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Scalyr::Common::Client::ServerError",
                   :batch_num=>1,
                   :code=>401,
                   :message=>"error/client/badParam",
@@ -37,10 +40,10 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>2,
+                  :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -49,8 +52,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -60,7 +66,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
         end
       end
 
@@ -73,8 +78,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
@@ -84,7 +92,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -110,8 +117,11 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
                 {
+                  :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
                   :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
                   :payload_size=>781,
@@ -121,7 +131,6 @@ describe LogStash::Outputs::Scalyr do
                   :will_retry_in_seconds=>2
                 }
               )
-              plugin.multi_receive(sample_events)
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -129,6 +138,17 @@ describe LogStash::Outputs::Scalyr do
           end
         end
       end
+
+      context "when an error occurs with retries at 5" do
+        it "exits after 5 retries and emits a log" do
+              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              )
+        end
+      end
   end
 
   describe "response_handling_tests" do
@@ -145,6 +165,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>503,
             :message=>"Invalid JSON response from server",
@@ -172,6 +193,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>500,
             :message=>"Invalid JSON response from server",
@@ -199,6 +221,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive(sample_events)
         expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
           {
+            :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
             :code=>500,
             :message=>"Invalid JSON response from server",
@@ -212,6 +235,22 @@ describe LogStash::Outputs::Scalyr do
         )
       end
     end
+
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      it 'should send the event to the DLQ' do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+        plugin.instance_variable_set('@dlq_writer', dlq_writer)
+
+        expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
+        plugin.multi_receive(sample_events)
+      end
+    end
   end
 
 end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Outputs::Scalyr do
       end
 
       it "it doesnt include flatten metrics if flattening is disabled" do
-          plugin1 = LogStash::Outputs::Scalyr.new({
+        plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
@@ -122,7 +122,7 @@ describe LogStash::Outputs::Scalyr do
         expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
-      it "send_stats is called when events list is empty, but otherwise noop" do
+      it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
@@ -137,6 +137,30 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive([])
       end
 
+      it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
+        plugin2 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                     'report_status_for_empty_batches' => false,
+                                                 })
+
+        mock_client_session = MockClientSession.new
+        quantile_estimator = Quantile::Estimator.new
+        plugin2.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin2.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin2).not_to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin2.multi_receive([])
+      end
+
       # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
       it "send_status only sends posts with is_status = true" do
         # 1. Initial send
@@ -249,9 +273,41 @@ describe LogStash::Outputs::Scalyr do
         expect(body['events'].size).to eq(3)
         expect(body['events'][2]['attrs']).to eq({
                                                      "nested.a" => 1,
-                                                     "nested.b_0" => 3,
-                                                     "nested.b_1" => 4,
-                                                     "nested.b_2" => 5,
+                                                     "nested.b.0" => 3,
+                                                     "nested.b.1" => 4,
+                                                     "nested.b.2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
+    context "when configured to flatten values with custom delimiter, no array flattening" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_arrays' => false,
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b" => [3, 4, 5],
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',
@@ -318,5 +374,23 @@ describe LogStash::Outputs::Scalyr do
                                                  })
       end
     end
+
+    context "when receiving an event with Bignums" do
+      config = {
+          'api_write_token' => '1234',
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "doesn't throw an error" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 2000023030042002050202030320240)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+        expect(plugin.instance_variable_get(:@logger)).to_not receive(:error)
+      end
+    end
   end
 end

data/spec/scalyr/common/util_spec.rb

@@ -132,6 +132,70 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din)).to eq(dout)
   end
 
+  it "flattens a single-level array, no array flattening" do
+    din = [1, 2, 3]
+    dout = [1, 2, 3]
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a multi-level array, no array flattening" do
+    din = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    dout = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.13
+  version: 0.1.18.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-19 00:00:00.000000000 Z
+date: 2021-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ files:
 - lib/scalyr/common/util.rb
 - lib/scalyr/constants.rb
 - logstash-output-scalyr.gemspec
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb
@@ -4052,9 +4053,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10
@@ -4062,6 +4063,7 @@ signing_key:
 specification_version: 4
 summary: Scalyr output plugin for Logstash
 test_files:
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb