logstash-output-scalyr 0.1.16.beta → 0.1.20.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16e88c3869fdb18e4132726af302e37c8bac2c10038ca3a078f85cebe67c4940
-  data.tar.gz: a42194413ec2969e31bbe44e72df7cf71d09911b61adae9db16b6041a97bc1b4
+  metadata.gz: 23ede2e07ed42483146d4a267c8517fed2f8639fb27998e3f37e8732bd50a45d
+  data.tar.gz: 4d3594dd94a908d5852d58fc6e5214982ae731ef8b3de3eb3707455bfa77bd3d
 SHA512:
-  metadata.gz: 66d4261da66deaa8f5227b0ecfcdfce51f170c561902397c6020d3baa8694bd7c92291b0faa84178e4dda180c71892ab4a30ba425dc1133f5f7b349c1b7e44fa
-  data.tar.gz: 6689e3c9fd5fa2a571ca8076a4ae24b5eb1bf5da9216030b8c759a0baff17f4577a478f3c3efced4ddb22ecf273ec675f8831733aded928fd2c92c7ce91d1983
+  metadata.gz: b11f8aaeea8530b316de904d2fd4b654b8b2a33c2c13ca10614ca3cb346a154e494d9ec7c831a1cf0a83449101ed2bb59c97cc5506870bd252c31cbf9dbc9052
+  data.tar.gz: dc6a23b5eae108b7ac8818cfd018c9a9670f69c31fb582b7c8568ebb33720fef19564b54420b2cfc4e7fc04e4a742836bdd2eefc013b15e240379726858fe15f

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Beta
 
+## 0.1.20.beta
+- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
+- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
+It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
+
+## 0.1.19.beta
+- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
+by setting the `fix_deep_flattening_delimiters` configuration option to true.
+
+## 0.1.18.beta
+- Add metrics for successfully sent and failed logstash events, and retries.
+- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
+
+## 0.1.17.beta
+- Catch errors relating to Bignum conversions present in the ``json`` library and manually convert to string as
+a workaround.
+
 ## 0.1.16.beta
 - Fix race condition in ``register()`` method.
 

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.20.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -68,7 +68,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If true, nested values will be flattened (which changes keys to delimiter-separated concatenation of all
   # nested keys).
   config :flatten_nested_values, :validate => :boolean, :default => false
-  config :flatten_nested_values_delimiter, :validate => :string, :default => "_" 
+  config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
+  config :flatten_nested_arrays, :validate => :boolean, :default => true
+  config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
+  config :flattening_max_key_count, :validate => :number, :default => -1
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -240,7 +243,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Plugin level (either per batch or event level metrics). Other request
     # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
-      :total_multi_receive_secs => 0
+      :total_multi_receive_secs => 0,
+      :total_events_processed => 0,
+      :successful_events_processed => 0,
+      :failed_events_processed => 0,
+      :total_retry_count => 0,
+      :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
 
@@ -344,6 +352,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           sleep_interval = sleep_for(sleep_interval)
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
               :error_class => e.e_class,
@@ -393,11 +404,19 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           }
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           retry if @running and exc_retries < @max_retries
           log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
           next
         end
 
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+          @multi_receive_statistics[:successful_events_processed] += multi_event_request[:logstash_events].length
+        end
+
         if !exc_data.nil?
           message = "Retry successful after error."
           if exc_commonly_retried
@@ -436,6 +455,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
 
   def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    @stats_lock.synchronize do
+      @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+      @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
+    end
     message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
     sample_events = Array.new
     multi_event_request[:logstash_events][0,5].each {|l_event|
@@ -611,7 +634,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter)
+        begin
+          record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
+        rescue Scalyr::Common::Util::MaxKeyCountError => e
+          @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
+        end
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
@@ -661,6 +688,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           ).force_encoding('UTF-8')
         end
         event_json = scalyr_event.to_json
+      rescue Java::JavaLang::ClassCastException => e
+        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+        # Because of the version of jruby logstash works with we don't have the option to just update this away,
+        # so if we run into it we convert bignums into strings so we can get the data in at least.
+        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+        end
+        Scalyr::Common::Util.convert_bignums(scalyr_event)
+        event_json = scalyr_event.to_json
+        log_json = nil
+        if add_log
+          log_json = logs[log_identifier].to_json
+        end
       end
 
       # generate new request if json size of events in the array exceed maximum request buffer size
@@ -749,7 +791,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
-    serialized_body = body.to_json
+    begin
+      serialized_body = body.to_json
+    rescue Java::JavaLang::ClassCastException => e
+      @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+      @stats_lock.synchronize do
+        @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      end
+      Scalyr::Common::Util.convert_bignums(body)
+      serialized_body = body.to_json
+    end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
     {

data/lib/scalyr/common/util.rb

@@ -1,44 +1,95 @@
 module Scalyr; module Common; module Util;
 
+class MaxKeyCountError < StandardError
+  attr_reader :message, :sample_keys
+
+  def initialize(message, sample_keys)
+    @message = message
+    @sample_keys = sample_keys
+  end
+end
 
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
-def self.flatten(obj, delimiter='_')
+# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
+# because some queries and dashboards already rely on the broken functionality.
+def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
 
   # base case is input object is not enumerable, in which case simply return it
-  if !obj.respond_to?(:each)
+  if !hash_obj.respond_to?(:each)
     raise TypeError.new('Input must be a hash or array')
   end
+  # case where we pass in a valid array, but don't want to flatten arrays
+  if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
+    return hash_obj
+  end
 
+  stack = []
+  stack << hash_obj
+  key_stack = []
+  key_stack << ""
+  key_list = []
+  key_list_width = []
   result = Hash.new
-  # require 'pry'
-  # binding.pry
+  test_key = 0
+  #Debugging
+  #require 'pry'
+  #binding.pry
 
-  if obj.respond_to?(:has_key?)
+  until stack.empty?
+    obj = stack.pop
+    key_list << key_stack.pop
 
-    # input object is a hash
-    obj.each do |key, value|
-      if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
-          result["#{key}#{delimiter}#{subkey}"] = subvalue
-        end
-      else
-        result["#{key}"] = value
+    # Case when object is a hash
+    if obj.respond_to?(:has_key?)
+      key_list_width << obj.keys.count
+      obj.each do |key, value|
+        key_stack << key
+        stack << value
       end
-    end
 
-  else
+    # Case when object is an array we intend to flatten
+    elsif flatten_arrays and obj.respond_to?(:each)
+      key_list_width << obj.count
+      obj.each_with_index do |value, index|
+        key_stack << index
+        stack << value
+      end
 
-    # input object is an array or set
-    obj.each_with_index do |value, index|
-      if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
-          result["#{index}#{delimiter}#{subkey}"] = subvalue
+    else
+      result_key = ""
+      delim = delimiter
+      key_list.each_with_index do |key, index|
+        # We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
+        # for the first two keys
+        if index > 1
+          result_key += "#{delim}#{key}"
+          if not fix_deep_flattening_delimiters
+            delim = "_"
+          end
+        else
+          result_key += "#{key}"
         end
-      else
-        result["#{index}"] = value
       end
+      result[result_key] = obj
+
+      if max_key_count > -1 and result.keys.count > max_key_count
+        raise MaxKeyCountError.new(
+          "Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
+          result.keys[0..6]
+        )
+      end
+
+      throw_away = key_list.pop
+      until key_list_width.empty? or key_list_width[-1] > 1
+        throw_away = key_list_width.pop
+        throw_away = key_list.pop
+      end
+      if not key_list_width.empty?
+        key_list_width[-1] -= 1
+      end
+
     end
   end
 
@@ -52,5 +103,26 @@ def self.truncate(content, max)
   return content
 end
 
+def self.convert_bignums(obj)
+  if obj.respond_to?(:has_key?) and obj.respond_to?(:each)
+    # input object is a hash
+    obj.each do |key, value|
+      obj[key] = convert_bignums(value)
+    end
+
+  elsif obj.respond_to?(:each)
+    # input object is an array or set
+    obj.each_with_index do |value, index|
+      obj[index] = convert_bignums(value)
+    end
+
+  elsif obj.is_a? Bignum
+    return obj.to_s
+
+  else
+    return obj
+  end
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.16.beta"
+PLUGIN_VERSION = "v0.1.20.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.16.beta'
+  s.version         = '0.1.20.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/benchmarks/bignum_fixing.rb

@@ -0,0 +1,90 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
+
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def rand_bignum()
+  return 200004000020304050300 + rand(999999)
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_bignum()
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def convert_bignums(record)
+  Scalyr::Common::Util.convert_bignums(record)
+end
+
+puts "Util.convert_bignums()"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~200 keys in a hash (single level)
+data = generate_data_array_for_spec([200])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))

data/spec/logstash/outputs/scalyr_spec.rb

@@ -288,6 +288,72 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "when configured to flatten values with custom delimiter and deep delimiter fix" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_values_delimiter' => ".",
+          'fix_deep_flattening_delimiters' => true,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b.0" => 3,
+                                                     "nested.b.1" => 4,
+                                                     "nested.b.2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
+    context "when configured to flatten values with custom delimiter, no array flattening" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_arrays' => false,
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b" => [3, 4, 5],
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
     context "when configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
@@ -342,5 +408,55 @@ describe LogStash::Outputs::Scalyr do
                                                  })
       end
     end
+
+    context "when configured to flatten with max keys configured to 3" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flattening_max_key_count' => 3,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "does not flatten" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested" => {'a'=>1, 'b'=>[3,4,5]},
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tags" => ["t1", "t2", "t3"],
+                                                     "parser" => "logstashParser",
+                                                 })
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error while flattening record",
+          {
+            :error_message=>"Resulting flattened object will contain more keys than the configured flattening_max_key_count of 3",
+            :sample_keys=>["serverHost", "parser", "tags_2", "tags_1"]
+          }
+        ).exactly(3).times
+      end
+    end
+
+    context "when receiving an event with Bignums" do
+      config = {
+          'api_write_token' => '1234',
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "doesn't throw an error" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 2000023030042002050202030320240)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+        expect(plugin.instance_variable_get(:@logger)).to_not receive(:error)
+      end
+    end
   end
 end

data/spec/scalyr/common/util_spec.rb

@@ -132,6 +132,70 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din)).to eq(dout)
   end
 
+  it "flattens a single-level array, no array flattening" do
+    din = [1, 2, 3]
+    dout = [1, 2, 3]
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a multi-level array, no array flattening" do
+    din = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    dout = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,
@@ -148,6 +212,42 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
   end
 
+  it "accepts custom delimiters with greater depth" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c_e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
+  end
+
+  it "accepts custom delimiters with greater depth and deep delimiters fix" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c:e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':', true, true)).to eq(dout)
+  end
+
   it "stringifies non-string keys" do
     din = {
         'a' => 1,
@@ -183,4 +283,23 @@ describe Scalyr::Common::Util do
   it "raises exception if a non-dict is provided" do
     expect {Scalyr::Common::Util.flatten(1)}.to raise_error(TypeError)
   end
+
+  it "flattens a hash 5000 layers deep" do
+    din = {
+        'a' => {},
+    }
+    hash = din
+    for i in 0...4999
+      hash = hash["a"]
+      hash["a"] = {}
+      if i == 4998
+        hash["a"] = "b"
+      end
+    end
+
+    dout = {
+        'a' + "_a" * 4999 => "b",
+    }
+    expect(Scalyr::Common::Util.flatten(din, '_')).to eq(dout)
+  end
 end

data/vendor/bundle/jruby/2.5.0/bin/htmldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

data/vendor/bundle/jruby/2.5.0/bin/ldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.16.beta
+  version: 0.1.20.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ files:
 - lib/scalyr/common/util.rb
 - lib/scalyr/constants.rb
 - logstash-output-scalyr.gemspec
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb
@@ -4062,6 +4063,7 @@ signing_key:
 specification_version: 4
 summary: Scalyr output plugin for Logstash
 test_files:
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb