logstash-output-scalyr 0.1.19.beta → 0.1.23.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
-  data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
+  metadata.gz: 8e2757da74eb03ea09a88597ed713db01bc050e17c31c771558bdb16a80ca40e
+  data.tar.gz: 1bc8519d1b30aea135d66b8b9ecfb632f688fd62abd58cd3b91cc92c4ad8957f
 SHA512:
-  metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
-  data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c
+  metadata.gz: 327a3e2ba020f3166b0c435700bdfedf4f96598a352f1f66f45fe9e671a053ee684aa812d686a4d58b5dd9f319e8b0f87e2ea8290d876e504b8416a8010795ed
+  data.tar.gz: 1ba44520daddda5c17da810221063ce815af082645ae3fd12e76b031247ff8a13cb3e7dab4fbf69e84e5a876605290702785cc17639de69435f0b780a74691f9

data/CHANGELOG.md

@@ -1,5 +1,20 @@
 # Beta
 
+## 0.1.23.beta
+- Add testing support for disabling estimation of serialized event size for each event in the batch.
+
+## 0.1.22.beta
+- Add new plugin metric for tracking the duration of ``build_multi_event_request_array`` method.
+- Update internal dependencies (``manticore``) to latest stable version.
+
+## 0.1.21.beta
+- Fix issue with iterative flattening function when dealing with empty collections.
+
+## 0.1.20.beta
+- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
+- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
+It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
+
 ## 0.1.19.beta
 - Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
 by setting the `fix_deep_flattening_delimiters` configuration option to true.

data/Gemfile

@@ -16,5 +16,5 @@ end
 
 gem 'pry'
 gem 'pry-nav'
-gem 'quantile'
-gem 'manticore', platform: :jruby
+gem 'quantile', '~> 0.2.1'
+gem 'manticore', '~> 0.7.1', platform: :jruby

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.22.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -321,6 +321,12 @@ If you want to run just the unit tests, you can run the command displayed below.
 bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
 ```
 
+Or to run a single test function defined on line XXX
+
+```bash
+bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
+```
+
 ## Instrumentation and metrics
 
 By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains

data/lib/logstash/outputs/scalyr.rb

@@ -71,6 +71,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
   config :flatten_nested_arrays, :validate => :boolean, :default => true
   config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
+  config :flattening_max_key_count, :validate => :number, :default => -1
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -139,11 +140,17 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
   # these stas this might be wanted or not.
   config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
-  
+
   # Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
   # after no work being done.
   config :noop_mode, :validate => :boolean, :default => false
 
+  # Set to true to disable estimiating the size of each serialized event to make sure we don't go over the max request
+  # size (5.5) and split batch into multiple Scalyr requests, if needed. Since this estimation is not "free", especially
+  # for large batches, it may make sense to disable this option when logstash batch size is configured in a way that
+  # Scalyr single request limit won't be reached.
+  config :estimate_each_event_size, :validate => :boolean, :default => true
+
   # Manticore related options
   config :http_connect_timeout, :validate => :number, :default => 10
   config :http_socket_timeout, :validate => :number, :default => 10
@@ -288,6 +295,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Convenience method to create a fresh quantile estimator
   def get_new_metrics
     return {
+      :build_multi_duration_secs => Quantile::Estimator.new,
       :multi_receive_duration_secs => Quantile::Estimator.new,
       :multi_receive_event_count => Quantile::Estimator.new,
       :event_attributes_count => Quantile::Estimator.new,
@@ -312,17 +320,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     return events if @noop_mode
 
     begin
+      records_count = events.to_a.length
+
+      # We also time the duration of the build_multi_event_request_array method. To avoid locking twice,
+      # we store the duration value here and record metric at the end.
       start_time = Time.now.to_f
 
       multi_event_request_array = build_multi_event_request_array(events)
-      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
+      build_multi_duration_secs = Time.now.to_f - start_time
 
+      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
       sleep_interval = @retry_initial_interval
       batch_num = 1
       total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
       result = []
-      records_count = events.to_a.length
 
       while !multi_event_request_array.to_a.empty?
         multi_event_request = multi_event_request_array.pop
@@ -429,6 +441,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       if records_count > 0
         @stats_lock.synchronize do
           @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+          @plugin_metrics[:build_multi_duration_secs].observe(build_multi_duration_secs)
           @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
           @plugin_metrics[:multi_receive_event_count].observe(records_count)
           @plugin_metrics[:batches_per_multi_receive].observe(total_batches)
@@ -633,7 +646,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
+        begin
+          record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
+        rescue Scalyr::Common::Util::MaxKeyCountError => e
+          @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
+        end
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
@@ -661,67 +678,74 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         scalyr_event[:log] = logs_ids[log_identifier]
       end
 
-      # get json string of event to keep track of how many bytes we are sending
-      begin
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
-        end
-      rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
-        @logger.warn "#{e.class}: #{e.message}"
-
-        # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
-        # TODO
-        # atime = Fluent::EventTime.new( sec, nsec )
-        # router.emit_error_event(serverHost, time, record, e)
-
-        scalyr_event[:attrs].each do |key, value|
-          @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
-          scalyr_event[:attrs][key] = value.encode(
-              "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
-          ).force_encoding('UTF-8')
-        end
-        event_json = scalyr_event.to_json
-      rescue Java::JavaLang::ClassCastException => e
-        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
-        # Because of the version of jruby logstash works with we don't have the option to just update this away,
-        # so if we run into it we convert bignums into strings so we can get the data in at least.
-        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
-        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
-        @stats_lock.synchronize do
-          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      if @estimate_each_event_size
+        # get json string of event to keep track of how many bytes we are sending
+        begin
+          event_json = scalyr_event.to_json
+          log_json = nil
+          if add_log
+            log_json = logs[log_identifier].to_json
+          end
+        rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
+          @logger.warn "#{e.class}: #{e.message}"
+
+          # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
+          # TODO
+          # atime = Fluent::EventTime.new( sec, nsec )
+          # router.emit_error_event(serverHost, time, record, e)
+
+          scalyr_event[:attrs].each do |key, value|
+            @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
+            scalyr_event[:attrs][key] = value.encode(
+                "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
+            ).force_encoding('UTF-8')
+          end
+          event_json = scalyr_event.to_json
+        rescue Java::JavaLang::ClassCastException => e
+          # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+          # Because of the version of jruby logstash works with we don't have the option to just update this away,
+          # so if we run into it we convert bignums into strings so we can get the data in at least.
+          # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+          @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_java_class_cast_errors] += 1
+          end
+          Scalyr::Common::Util.convert_bignums(scalyr_event)
+          event_json = scalyr_event.to_json
+          log_json = nil
+          if add_log
+            log_json = logs[log_identifier].to_json
+          end
         end
-        Scalyr::Common::Util.convert_bignums(scalyr_event)
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
+
+        # generate new request if json size of events in the array exceed maximum request buffer size
+        append_event = true
+        add_bytes = event_json.bytesize
+        if log_json
+          add_bytes = add_bytes + log_json.bytesize
         end
-      end
 
-      # generate new request if json size of events in the array exceed maximum request buffer size
-      append_event = true
-      add_bytes =  event_json.bytesize
-      if log_json
-        add_bytes = add_bytes + log_json.bytesize
-      end
-      if total_bytes + add_bytes > @max_request_buffer
-        # make sure we always have at least one event
-        if scalyr_events.size == 0
-          scalyr_events << scalyr_event
-          l_events << l_event
-          append_event = false
+        if total_bytes + add_bytes > @max_request_buffer
+          # make sure we always have at least one event
+          if scalyr_events.size == 0
+            scalyr_events << scalyr_event
+            l_events << l_event
+            append_event = false
+          end
+
+          multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
+          multi_event_request_array << multi_event_request
+
+          total_bytes = 0
+          current_threads = Hash.new
+          logs = Hash.new
+          logs_ids = Hash.new
+          scalyr_events = Array.new
+          l_events = Array.new
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
-        multi_event_request_array << multi_event_request
-
-        total_bytes = 0
-        current_threads = Hash.new
-        logs = Hash.new
-        logs_ids = Hash.new
-        scalyr_events = Array.new
-        l_events = Array.new
+      else
+        # If size estimation is disabled we simply append the event and handle splitting later on (if needed)
+        append_event = true
       end
 
       # if we haven't consumed the current event already
@@ -798,6 +822,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
+
+    serialized_request_size = serialized_body.bytesize
+
+    if serialized_request_size >= @max_request_buffer
+      # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
+      @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
+    end
+
     {
       :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
       :logstash_events => logstash_events
@@ -811,6 +843,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @stats_lock.synchronize do
       current_stats = @multi_receive_statistics.clone
 
+      current_stats[:build_multi_duration_secs_p50] = @plugin_metrics[:build_multi_duration_secs].query(0.5)
+      current_stats[:build_multi_duration_secs_p90] = @plugin_metrics[:build_multi_duration_secs].query(0.9)
+      current_stats[:build_multi_duration_secs_p99] = @plugin_metrics[:build_multi_duration_secs].query(0.99)
+
       current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
       current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
       current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)

data/lib/scalyr/common/util.rb

@@ -1,52 +1,96 @@
 module Scalyr; module Common; module Util;
 
+class MaxKeyCountError < StandardError
+  attr_reader :message, :sample_keys
+
+  def initialize(message, sample_keys)
+    @message = message
+    @sample_keys = sample_keys
+  end
+end
 
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
 # Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
 # because some queries and dashboards already rely on the broken functionality.
-def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
+def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
 
   # base case is input object is not enumerable, in which case simply return it
-  if !obj.respond_to?(:each)
+  if !hash_obj.respond_to?(:each)
     raise TypeError.new('Input must be a hash or array')
   end
+  # case where we pass in a valid array, but don't want to flatten arrays
+  if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
+    return hash_obj
+  end
 
+  stack = []
+  stack << hash_obj
+  key_stack = []
+  key_stack << ""
+  key_list = []
+  key_list_width = []
   result = Hash.new
-  # require 'pry'
-  # binding.pry
-
-  if obj.respond_to?(:has_key?)
-
-    # input object is a hash
-    obj.each do |key, value|
-      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
-        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
-          result["#{key}#{delimiter}#{subkey}"] = subvalue
-        end
-      else
-        result["#{key}"] = value
+  test_key = 0
+  #Debugging
+  #require 'pry'
+  #binding.pry
+
+  until stack.empty?
+    obj = stack.pop
+    key_list << key_stack.pop
+
+    # Case when object is a hash
+    if obj.respond_to?(:has_key?) and obj.keys.count > 0
+      key_list_width << obj.keys.count
+      obj.each do |key, value|
+        key_stack << key
+        stack << value
       end
-    end
 
-  elsif flatten_arrays
+    # Case when object is an array we intend to flatten
+    elsif flatten_arrays and obj.respond_to?(:each) and obj.count > 0
+        key_list_width << obj.count
+        obj.each_with_index do |value, index|
+          key_stack << index
+          stack << value
+        end
 
-    # input object is an array or set
-    obj.each_with_index do |value, index|
-      if value.respond_to?(:each)
-        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
-          result["#{index}#{delimiter}#{subkey}"] = subvalue
+    else
+      result_key = ""
+      delim = delimiter
+      key_list.each_with_index do |key, index|
+        # We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
+        # for the first two keys
+        if index > 1
+          result_key += "#{delim}#{key}"
+          if not fix_deep_flattening_delimiters
+            delim = "_"
+          end
+        else
+          result_key += "#{key}"
         end
-      else
-        result["#{index}"] = value
       end
-    end
+      result[result_key] = obj
 
-  else
+      if max_key_count > -1 and result.keys.count > max_key_count
+        raise MaxKeyCountError.new(
+          "Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
+          result.keys[0..6]
+        )
+      end
 
-    result = obj
+      throw_away = key_list.pop
+      until key_list_width.empty? or key_list_width[-1] > 1
+        throw_away = key_list_width.pop
+        throw_away = key_list.pop
+      end
+      if not key_list_width.empty?
+        key_list_width[-1] -= 1
+      end
 
+    end
   end
 
   return result

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.19.beta"
+PLUGIN_VERSION = "v0.1.23.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.19.beta'
+  s.version         = '0.1.23.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -7,6 +7,12 @@ require "json"
 require 'webmock/rspec'
 WebMock.allow_net_connect!
 
+RSpec.configure do |rspec|
+  rspec.expect_with :rspec do |c|
+    c.max_formatted_output_length = nil
+  end
+end
+
 describe LogStash::Outputs::Scalyr do
   let(:sample_events) {
     events = []
@@ -58,7 +64,7 @@ describe LogStash::Outputs::Scalyr do
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
                   :record_count=>3,
                   :total_batches=>1,
@@ -84,7 +90,7 @@ describe LogStash::Outputs::Scalyr do
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
                   :record_count=>3,
                   :total_batches=>1,