logstash-output-scalyr 0.1.18.beta → 0.1.22.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9099fcf20201bf8e3905273393a38bb1addefb219e1874a45a439bbfd2dcddbd
-  data.tar.gz: a031faa8657951b66a09c172f6be69b774a2192e661ef99370813abe62250cbb
+  metadata.gz: 8ea703e55bc28d2bb00445c2a3e2d822bfe978bbe889292caafa6265dff1de2b
+  data.tar.gz: 3dc35cf36a356e5651d69e1a7d4623f296e5818a4b1c0fc8965b1a55b94a20ab
 SHA512:
-  metadata.gz: c25cad81473d0224fb5c75558e302f6383d69361d3263aeaf0fe619b8c2a3a6866bf0eaf811588b9235edb4e15e11a2d0b95ecb99e2a3045ee6c6e66998e1627
-  data.tar.gz: fed7e02e32c70ce199dbd7ec5841711ff77c65d3a56ebe53eeada8c165d5b8fa603fc15b3c00e0e397682a50899dd141bfdcacdcbb97244e2b26205c92802238
+  metadata.gz: 9306c619ce51c8a9cf83cb545229d4c73ac496a133e07e7e55b434301e1efc2f668187c3aa1584997ab1252ce3c7510ac9c4d719ba363933160bf2f874d0a137
+  data.tar.gz: 621b8f9b4095f16e51b88b8aec2844d795de75cba980ba27e0510a04f57ee9e4b34eb42513455d7509acaa8af8e085febb92cd248401e642129e09d70987a476

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Beta
 
+## 0.1.22.beta
+- Add new plugin metric for tracking the duration of ``build_multi_event_request_array`` method.
+- Update internal dependencies (``manticore``) to latest stable version.
+
+## 0.1.21.beta
+- Fix issue with iterative flattening function when dealing with empty collections.
+
+## 0.1.20.beta
+- Rewrite flattening function to no longer be recursive, to help avoid maxing out the stack.
+- Added a configurable value `flattening_max_key_count` to create a limit on how large of a record we can flatten.
+It limits the maximum amount of keys we can have in the final flattened record. Defaults to unlimited.
+
+## 0.1.19.beta
+- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
+by setting the `fix_deep_flattening_delimiters` configuration option to true.
+
 ## 0.1.18.beta
 - Add metrics for successfully sent and failed logstash events, and retries.
 - Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.

data/Gemfile

@@ -16,5 +16,5 @@ end
 
 gem 'pry'
 gem 'pry-nav'
-gem 'quantile'
-gem 'manticore', platform: :jruby
+gem 'quantile', '~> 0.2.1'
+gem 'manticore', '~> 0.7.1', platform: :jruby

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.18.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.22.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -321,6 +321,12 @@ If you want to run just the unit tests, you can run the command displayed below.
 bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
 ```
 
+Or to run a single test function defined on line XXX
+
+```bash
+bundle exec rspec spec/scalyr/common/util_spec.rb:XXX
+```
+
 ## Instrumentation and metrics
 
 By default, plugin logs a special line with metrics to Scalyr every 5 minutes. This line contains

data/lib/logstash/outputs/scalyr.rb

@@ -70,6 +70,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   config :flatten_nested_values, :validate => :boolean, :default => false
   config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
   config :flatten_nested_arrays, :validate => :boolean, :default => true
+  config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
+  config :flattening_max_key_count, :validate => :number, :default => -1
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -287,6 +289,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Convenience method to create a fresh quantile estimator
   def get_new_metrics
     return {
+      :build_multi_duration_secs => Quantile::Estimator.new,
       :multi_receive_duration_secs => Quantile::Estimator.new,
       :multi_receive_event_count => Quantile::Estimator.new,
       :event_attributes_count => Quantile::Estimator.new,
@@ -311,17 +314,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     return events if @noop_mode
 
     begin
+      records_count = events.to_a.length
+
+      # We also time the duration of the build_multi_event_request_array method
       start_time = Time.now.to_f
 
       multi_event_request_array = build_multi_event_request_array(events)
-      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
 
+      if records_count > 0
+        @stats_lock.synchronize do
+          @plugin_metrics[:build_multi_duration_secs].observe(Time.now.to_f - start_time)
+        end
+      end
+
+      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
       sleep_interval = @retry_initial_interval
       batch_num = 1
       total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
       result = []
-      records_count = events.to_a.length
 
       while !multi_event_request_array.to_a.empty?
         multi_event_request = multi_event_request_array.pop
@@ -632,7 +643,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays)
+        begin
+          record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters, max_key_count=@flattening_max_key_count)
+        rescue Scalyr::Common::Util::MaxKeyCountError => e
+          @logger.warn("Error while flattening record", :error_message => e.message, :sample_keys => e.sample_keys)
+        end
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
@@ -810,6 +825,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @stats_lock.synchronize do
       current_stats = @multi_receive_statistics.clone
 
+      current_stats[:build_multi_duration_secs_p50] = @plugin_metrics[:build_multi_duration_secs].query(0.5)
+      current_stats[:build_multi_duration_secs_p90] = @plugin_metrics[:build_multi_duration_secs].query(0.9)
+      current_stats[:build_multi_duration_secs_p99] = @plugin_metrics[:build_multi_duration_secs].query(0.99)
+
       current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
       current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
       current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)

data/lib/scalyr/common/util.rb

@@ -1,50 +1,96 @@
 module Scalyr; module Common; module Util;
 
+class MaxKeyCountError < StandardError
+  attr_reader :message, :sample_keys
+
+  def initialize(message, sample_keys)
+    @message = message
+    @sample_keys = sample_keys
+  end
+end
 
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
-def self.flatten(obj, delimiter='_', flatten_arrays=true)
+# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
+# because some queries and dashboards already rely on the broken functionality.
+def self.flatten(hash_obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false, max_key_count=-1)
 
   # base case is input object is not enumerable, in which case simply return it
-  if !obj.respond_to?(:each)
+  if !hash_obj.respond_to?(:each)
     raise TypeError.new('Input must be a hash or array')
   end
+  # case where we pass in a valid array, but don't want to flatten arrays
+  if !hash_obj.respond_to?(:has_key?) and !flatten_arrays
+    return hash_obj
+  end
 
+  stack = []
+  stack << hash_obj
+  key_stack = []
+  key_stack << ""
+  key_list = []
+  key_list_width = []
   result = Hash.new
-  # require 'pry'
-  # binding.pry
-
-  if obj.respond_to?(:has_key?)
-
-    # input object is a hash
-    obj.each do |key, value|
-      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
-        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
-          result["#{key}#{delimiter}#{subkey}"] = subvalue
-        end
-      else
-        result["#{key}"] = value
+  test_key = 0
+  #Debugging
+  #require 'pry'
+  #binding.pry
+
+  until stack.empty?
+    obj = stack.pop
+    key_list << key_stack.pop
+
+    # Case when object is a hash
+    if obj.respond_to?(:has_key?) and obj.keys.count > 0
+      key_list_width << obj.keys.count
+      obj.each do |key, value|
+        key_stack << key
+        stack << value
       end
-    end
 
-  elsif flatten_arrays
+    # Case when object is an array we intend to flatten
+    elsif flatten_arrays and obj.respond_to?(:each) and obj.count > 0
+        key_list_width << obj.count
+        obj.each_with_index do |value, index|
+          key_stack << index
+          stack << value
+        end
 
-    # input object is an array or set
-    obj.each_with_index do |value, index|
-      if value.respond_to?(:each)
-        flatten(value, delimiter, flatten_arrays).each do |subkey, subvalue|
-          result["#{index}#{delimiter}#{subkey}"] = subvalue
+    else
+      result_key = ""
+      delim = delimiter
+      key_list.each_with_index do |key, index|
+        # We have a blank key at the start of the key list to avoid issues with calling pop, so we ignore delimiter
+        # for the first two keys
+        if index > 1
+          result_key += "#{delim}#{key}"
+          if not fix_deep_flattening_delimiters
+            delim = "_"
+          end
+        else
+          result_key += "#{key}"
         end
-      else
-        result["#{index}"] = value
       end
-    end
+      result[result_key] = obj
 
-  else
+      if max_key_count > -1 and result.keys.count > max_key_count
+        raise MaxKeyCountError.new(
+          "Resulting flattened object will contain more keys than the configured flattening_max_key_count of #{max_key_count}",
+          result.keys[0..6]
+        )
+      end
 
-    result = obj
+      throw_away = key_list.pop
+      until key_list_width.empty? or key_list_width[-1] > 1
+        throw_away = key_list_width.pop
+        throw_away = key_list.pop
+      end
+      if not key_list_width.empty?
+        key_list_width[-1] -= 1
+      end
 
+    end
   end
 
   return result

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.18.beta"
+PLUGIN_VERSION = "v0.1.22.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.18.beta'
+  s.version         = '0.1.22.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -7,6 +7,12 @@ require "json"
 require 'webmock/rspec'
 WebMock.allow_net_connect!
 
+RSpec.configure do |rspec|
+  rspec.expect_with :rspec do |c|
+    c.max_formatted_output_length = nil
+  end
+end
+
 describe LogStash::Outputs::Scalyr do
   let(:sample_events) {
     events = []
@@ -58,7 +64,7 @@ describe LogStash::Outputs::Scalyr do
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
                   :record_count=>3,
                   :total_batches=>1,
@@ -84,7 +90,7 @@ describe LogStash::Outputs::Scalyr do
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
                   :payload_size=>781,
                   :record_count=>3,
                   :total_batches=>1,

data/spec/logstash/outputs/scalyr_spec.rb

@@ -80,6 +80,7 @@ describe LogStash::Outputs::Scalyr do
         plugin1.instance_variable_set(:@client_session, mock_client_session)
         plugin1.instance_variable_set(:@session_id, "some_session_id")
         plugin1.instance_variable_set(:@plugin_metrics, {
+          :build_multi_duration_secs => Quantile::Estimator.new,
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
@@ -87,10 +88,11 @@ describe LogStash::Outputs::Scalyr do
           :batches_per_multi_receive => Quantile::Estimator.new
         })
         plugin1.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(1)
+        plugin1.instance_variable_get(:@plugin_metrics)[:build_multi_duration_secs].observe(1)
         plugin1.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
 
         status_event = plugin1.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 build_multi_duration_secs_p50=1 build_multi_duration_secs_p90=1 build_multi_duration_secs_p99=1 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0")
       end
 
       it "returns and sends correct status event on send_stats on initial and subsequent send" do
@@ -106,6 +108,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.instance_variable_set(:@client_session, mock_client_session)
         # Setup one quantile calculation to make sure at least one of them calculates as expected
         plugin.instance_variable_set(:@plugin_metrics, {
+          :build_multi_duration_secs => Quantile::Estimator.new,
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
@@ -119,12 +122,13 @@ describe LogStash::Outputs::Scalyr do
 
         plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
         status_event = plugin.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 build_multi_duration_secs_p50=0 build_multi_duration_secs_p90=0 build_multi_duration_secs_p99=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
       it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
+          :build_multi_duration_secs => Quantile::Estimator.new,
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count => Quantile::Estimator.new,
@@ -149,6 +153,7 @@ describe LogStash::Outputs::Scalyr do
         mock_client_session = MockClientSession.new
         quantile_estimator = Quantile::Estimator.new
         plugin2.instance_variable_set(:@plugin_metrics, {
+          :build_multi_duration_secs => Quantile::Estimator.new,
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count => Quantile::Estimator.new,
@@ -174,6 +179,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.instance_variable_set(:@last_status_transmit_time, 100)
         plugin.instance_variable_set(:@client_session, mock_client_session)
         plugin.instance_variable_set(:@plugin_metrics, {
+          :build_multi_duration_secs => Quantile::Estimator.new,
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
@@ -265,6 +271,40 @@ describe LogStash::Outputs::Scalyr do
           'flatten_nested_values_delimiter' => ".",
       }
       plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b_0" => 3,
+                                                     "nested.b_1" => 4,
+                                                     "nested.b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
+    context "when configured to flatten values with custom delimiter and deep delimiter fix" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_values_delimiter' => ".",
+          'fix_deep_flattening_delimiters' => true,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
       it "flattens nested values with a period" do
         allow(plugin).to receive(:send_status).and_return(nil)
         plugin.register
@@ -375,6 +415,38 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "when configured to flatten with max keys configured to 3" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flattening_max_key_count' => 3,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "does not flatten" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested" => {'a'=>1, 'b'=>[3,4,5]},
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tags" => ["t1", "t2", "t3"],
+                                                     "parser" => "logstashParser",
+                                                 })
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error while flattening record",
+          {
+            :error_message=>"Resulting flattened object will contain more keys than the configured flattening_max_key_count of 3",
+            :sample_keys=>["serverHost", "parser", "tags_2", "tags_1"]
+          }
+        ).exactly(3).times
+      end
+    end
+
     context "when receiving an event with Bignums" do
       config = {
           'api_write_token' => '1234',

data/spec/scalyr/common/util_spec.rb

@@ -1,7 +1,113 @@
 # encoding: utf-8
 require "scalyr/common/util"
 
+LARGE_OBJECT_IN = {
+  "level": "info",
+  "ts": "2020-08-11T02:26:17.078Z",
+  "caller": "api/foo:480",
+  "msg": "assign active task foobar",
+  "accountId": 12345,
+  "cycleId": 6789,
+  "uuid": "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "task": {
+    "Id": 1211111181111111400,
+    "TaskTypes": [
+      4,
+      11,
+      10,
+      12,
+      17,
+      14
+    ],
+    "Ips": [
+      "127.0.0.1",
+      "127.0.0.2",
+      "127.0.0.3",
+      "127.0.0.4",
+      "127.0.0.5",
+    ],
+    "FooProps": {
+      "10": {
+        "TcpPorts": [
+          22,
+          23,
+          25,
+          80,
+          55,
+          8000,
+          8080,
+        ],
+        "UdpPorts": []
+      }
+    },
+    "Subnet": "127.0.0.0/24"
+  },
+  "relevance": 0,
+  "scannerIp": "10.0.0.2",
+  "gatewayIp": "10.0.0.1",
+  "gatewayMac": "fa:fa:fa:fa",
+  "wired": true,
+  "elapsed": 74.86664
+}
 
+LARGE_OBJECT_OUT = {
+  "accountId" => 12345,
+  "caller" => "api/foo:480",
+  "cycleId" => 6789,
+  "elapsed" => 74.86664,
+  "gatewayIp" => "10.0.0.1",
+  "gatewayMac" => "fa:fa:fa:fa",
+  "level" => "info",
+  "msg" => "assign active task foobar",
+  "relevance" => 0,
+  "scannerIp" => "10.0.0.2",
+  "task_FooProps_10_TcpPorts_0" => 22,
+  "task_FooProps_10_TcpPorts_1" => 23,
+  "task_FooProps_10_TcpPorts_2" => 25,
+  "task_FooProps_10_TcpPorts_3" => 80,
+  "task_FooProps_10_TcpPorts_4" => 55,
+  "task_FooProps_10_TcpPorts_5" => 8000,
+  "task_FooProps_10_TcpPorts_6" => 8080,
+  "task_FooProps_10_UdpPorts" => [],
+  "task_Id" => 1211111181111111400,
+  "task_Ips_0" => "127.0.0.1",
+  "task_Ips_1" => "127.0.0.2",
+  "task_Ips_2" => "127.0.0.3",
+  "task_Ips_3" => "127.0.0.4",
+  "task_Ips_4" => "127.0.0.5",
+  "task_Subnet" => "127.0.0.0/24",
+  "task_TaskTypes_0" => 4,
+  "task_TaskTypes_1" => 11,
+  "task_TaskTypes_2" => 10,
+  "task_TaskTypes_3" => 12,
+  "task_TaskTypes_4" => 17,
+  "task_TaskTypes_5" => 14,
+  "ts" => "2020-08-11T02:26:17.078Z",
+  "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "wired" => true,
+}
+
+LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS = {
+  "accountId" => 12345,
+  "caller" => "api/foo:480",
+  "cycleId" => 6789,
+  "elapsed" => 74.86664,
+  "gatewayIp" => "10.0.0.1",
+  "gatewayMac" => "fa:fa:fa:fa",
+  "level" => "info",
+  "msg" => "assign active task foobar",
+  "relevance" => 0,
+  "scannerIp" => "10.0.0.2",
+  "task_FooProps_10_TcpPorts" => [22, 23, 25, 80, 55, 8000, 8080],
+  "task_FooProps_10_UdpPorts" => [],
+  "task_Id" => 1211111181111111400,
+  "task_Ips" => ["127.0.0.1", "127.0.0.2", "127.0.0.3", "127.0.0.4", "127.0.0.5"],
+  "task_Subnet" => "127.0.0.0/24",
+  "task_TaskTypes" => [4, 11, 10, 12, 17, 14],
+  "ts" => "2020-08-11T02:26:17.078Z",
+  "uuid" => "a405a4b58810e3aaa078f751bd32baa8b60aaad1",
+  "wired" => true,
+}
 describe Scalyr::Common::Util do
   it "does not flatten an already-flat dict" do
     din = {
@@ -144,6 +250,20 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
   end
 
+  it "flattens a hash that contains an array with hashes, no array flattening" do
+    din = {
+        'a' => 1,
+        "b" => {"a": "a"},
+        'c' => { "f" => [100, 200, {"g" => 1}] }
+    }
+    dout = {
+        'a' => 1,
+        "b_a" => "a",
+        'c_f' => [100, 200, {"g" => 1}]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "flattens a hash that contains an array, no array flattening" do
     din = {
         'a' => 1,
@@ -196,6 +316,120 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
   end
 
+  it "flattens large hash correctly" do
+    expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=true)).to eq(LARGE_OBJECT_OUT)
+  end
+
+  it "flattens large hash correctly not flatten arrays" do
+    expect(Scalyr::Common::Util.flatten(LARGE_OBJECT_IN, "_", flatten_arrays=false)).to eq(LARGE_OBJECT_OUT_NO_FLATTEN_ARRAYS)
+  end
+
+  it "flattens hash containing empty list correctly" do
+    obj_in = {
+      "abc" => 123,
+      "array" => [],
+      "hash" => {
+        "value" => "abc123",
+        "another_array" => []
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "array" => [],
+      "hash_value" => "abc123",
+      "hash_another_array" => []
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing empty list correctly not flatten arrays" do
+    obj_in = {
+      "abc" => 123,
+      "array" => [],
+      "hash" => {
+        "value" => "abc123",
+        "another_array" => []
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "array" => [],
+      "hash_value" => "abc123",
+      "hash_another_array" => []
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=false)).to eq(obj_out)
+  end
+
+  it "flattens hash containing empty hash correctly" do
+    obj_in = {
+      "abc" => 123,
+      "empty_hash" => {},
+      "hash" => {
+        "value" => "abc123",
+        "another_hash" => {}
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "empty_hash" => {},
+      "hash_value" => "abc123",
+      "hash_another_hash" => {}
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested bignum correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => 2000023030042002050202030320240
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => 2000023030042002050202030320240
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested boolean correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => true,
+        "otherValue" => false
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => true,
+      "hash_otherValue" => false
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
+  it "flattens hash containing nested float correctly" do
+    obj_in = {
+      "abc" => 123,
+      "hash" => {
+        "value" => 321.12345,
+        "otherValue" => 0.0000003
+      }
+    }
+
+    obj_out = {
+      "abc" => 123,
+      "hash_value" => 321.12345,
+      "hash_otherValue" => 0.0000003
+    }
+    expect(Scalyr::Common::Util.flatten(obj_in, "_", flatten_arrays=true)).to eq(obj_out)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,
@@ -212,6 +446,42 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
   end
 
+  it "accepts custom delimiters with greater depth" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c_e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
+  end
+
+  it "accepts custom delimiters with greater depth and deep delimiters fix" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c:e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':', true, true)).to eq(dout)
+  end
+
   it "stringifies non-string keys" do
     din = {
         'a' => 1,
@@ -247,4 +517,23 @@ describe Scalyr::Common::Util do
   it "raises exception if a non-dict is provided" do
     expect {Scalyr::Common::Util.flatten(1)}.to raise_error(TypeError)
   end
+
+  it "flattens a hash 5000 layers deep" do
+    din = {
+        'a' => {},
+    }
+    hash = din
+    for i in 0...4999
+      hash = hash["a"]
+      hash["a"] = {}
+      if i == 4998
+        hash["a"] = "b"
+      end
+    end
+
+    dout = {
+        'a' + "_a" * 4999 => "b",
+    }
+    expect(Scalyr::Common::Util.flatten(din, '_')).to eq(dout)
+  end
 end