logstash-output-scalyr 0.1.14.beta → 0.1.19.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1af6e37035c37c270036487e9b939f40e8454d39ddea9983a737f409957e0747
-  data.tar.gz: 95438909311aa595d2d4e771424765f3705cdc649225ca81c053e3b9c129b183
+  metadata.gz: '084d41a09e7ef86214868f26a97639169df8982d9488eb827d337ef53f02fc07'
+  data.tar.gz: 5bd3b17804c60044901d2f40c40419c119e680f7ac4a4ffc543701a885cb0c2e
 SHA512:
-  metadata.gz: d2a9f542cfaaf8bb9e84ebf976694c548e153e1e170417309644b9206bf4df2ba8844447cbb2b0afd4ba8d03922688f6348a6fb77a6c8489a3f2a49bb873fed9
-  data.tar.gz: 514fab3aa32f39e1ebe6e4bf164a9b9f648b82e7aa861d9cbc2edd6d44d9c6b4d9d3ea9b7c281ae26bf719e4373bde6059f220033dc660821a369b6186180798
+  metadata.gz: 837bd2d4d142f31bcd30494175731818de9f60f9f39ebc13cf0ae92b36e58ad4b6d288ba2f92109323f096d30a1774334651907b07e1d1310f6b42dd24698b2e
+  data.tar.gz: 221e47feab585a99dc69b475f446f9a9b4f9d2bd91ffd60753886438c348e0139618afa46f3889180aba68582ed9532376ee5449fb0cf91bb05cbb796c11a08c

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Beta
 
+## 0.1.19.beta
+- Undo a change to nested value flattening functionality to keep existing formatting. This change can be re-enabled
+by setting the `fix_deep_flattening_delimiters` configuration option to true.
+
+## 0.1.18.beta
+- Add metrics for successfully sent and failed logstash events, and retries.
+- Make array flattening optional during nested value flattening with the `flatten_nested_arrays` configuration option.
+
+## 0.1.17.beta
+- Catch errors relating to Bignum conversions present in the ``json`` library and manually convert to string as
+a workaround.
+
+## 0.1.16.beta
+- Fix race condition in ``register()`` method.
+
+## 0.1.15.beta
+- Only call ``send_status`` method at the end of ``multi_receive()`` if there is at least one
+  record in the batch when ``report_status_for_empty_batches`` config option is set to ``false``.
+- Update ``register()`` method to use a separate short-lived client session for sending initial
+  client status.
+
 ## 0.1.14.beta
 - Add configurable max retries for requests when running into errors.
 - Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.19.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -68,7 +68,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If true, nested values will be flattened (which changes keys to delimiter-separated concatenation of all
   # nested keys).
   config :flatten_nested_values, :validate => :boolean, :default => false
-  config :flatten_nested_values_delimiter, :validate => :string, :default => "_" 
+  config :flatten_nested_values_delimiter, :validate => :string, :default => "_"
+  config :flatten_nested_arrays, :validate => :boolean, :default => true
+  config :fix_deep_flattening_delimiters, :validate => :boolean, :default => false
 
   # If true, the 'tags' field will be flattened into key-values where each key is a tag and each value is set to
   # :flat_tag_value
@@ -109,6 +111,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # True to also call send_status when multi_receive() is called with no events.
+  # In some situations (e.g. when logstash is configured with multiple scalyr
+  # plugins conditionally where most are idle) you may want to set this to false
+  config :report_status_for_empty_batches, :validate => :boolean, :default => true
+
   # Set to true to also log status messages with various metrics to stdout in addition to sending
   # this data to Scalyr
   config :log_status_messages_to_stdout, :validate => :boolean, :default => false
@@ -235,7 +242,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Plugin level (either per batch or event level metrics). Other request
     # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
-      :total_multi_receive_secs => 0
+      :total_multi_receive_secs => 0,
+      :total_events_processed => 0,
+      :successful_events_processed => 0,
+      :failed_events_processed => 0,
+      :total_retry_count => 0,
+      :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
 
@@ -251,7 +263,25 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
-    send_status
+    # We use a special separate short lived client session for sending the initial client status.
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # plugins configured with conditionals and majority of them are inactive (aka receive no data).
+    # This way we don't need to keep idle long running connection open.
+    initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
+        @logger, @add_events_uri,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
+    )
+    send_status(initial_send_status_client_session)
+    initial_send_status_client_session.close
+
+    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
+    # Here priming just means setting up the client parameters without opening any connections.
+    # Since client writes certs to a temporary file there could be a race in case we don't do that
+    # here since multi_receive() is multi threaded. An alternative would be to put a look around
+    # client init method (aka client_config())
+    @client_session.client
 
   end # def register
 
@@ -321,6 +351,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           sleep_interval = sleep_for(sleep_interval)
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           message = "Error uploading to Scalyr (will backoff-retry)"
           exc_data = {
               :error_class => e.e_class,
@@ -370,11 +403,19 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           }
           exc_sleep += sleep_interval
           exc_retries += 1
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_retry_count] += 1
+          end
           retry if @running and exc_retries < @max_retries
           log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
           next
         end
 
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+          @multi_receive_statistics[:successful_events_processed] += multi_event_request[:logstash_events].length
+        end
+
         if !exc_data.nil?
           message = "Retry successful after error."
           if exc_commonly_retried
@@ -394,7 +435,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      send_status
+      if @report_status_for_empty_batches or records_count > 0
+        send_status
+      end
+
       return result
 
     rescue => e
@@ -410,6 +454,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
 
   def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
+    @stats_lock.synchronize do
+      @multi_receive_statistics[:total_events_processed] += multi_event_request[:logstash_events].length
+      @multi_receive_statistics[:failed_events_processed] += multi_event_request[:logstash_events].length
+    end
     message = "Failed to send #{multi_event_request[:logstash_events].length} events after #{exc_retries} tries."
     sample_events = Array.new
     multi_event_request[:logstash_events][0,5].each {|l_event|
@@ -585,7 +633,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # flatten record
       if @flatten_nested_values
         start_time = Time.now.to_f
-        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter)
+        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter, flatten_arrays=@flatten_nested_arrays, fix_deep_flattening_delimiters=@fix_deep_flattening_delimiters)
         end_time = Time.now.to_f
         flatten_nested_values_duration = end_time - start_time
       end
@@ -635,6 +683,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           ).force_encoding('UTF-8')
         end
         event_json = scalyr_event.to_json
+      rescue Java::JavaLang::ClassCastException => e
+        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+        # Because of the version of jruby logstash works with we don't have the option to just update this away,
+        # so if we run into it we convert bignums into strings so we can get the data in at least.
+        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+        end
+        Scalyr::Common::Util.convert_bignums(scalyr_event)
+        event_json = scalyr_event.to_json
+        log_json = nil
+        if add_log
+          log_json = logs[log_identifier].to_json
+        end
       end
 
       # generate new request if json size of events in the array exceed maximum request buffer size
@@ -723,7 +786,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
-    serialized_body = body.to_json
+    begin
+      serialized_body = body.to_json
+    rescue Java::JavaLang::ClassCastException => e
+      @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+      @stats_lock.synchronize do
+        @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      end
+      Scalyr::Common::Util.convert_bignums(body)
+      serialized_body = body.to_json
+    end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
     {
@@ -778,7 +850,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Finally, note that there could be multiple instances of this plugin (one per worker), in which case each worker
   # thread sends their own status updates.  This is intentional so that we know how much data each worker thread is
   # uploading to Scalyr over time.
-  def send_status
+  def send_status(client_session = nil)
+    client_session = @client_session if client_session.nil?
 
     status_event = {
       :ts => (Time.now.to_f * (10**9)).round,
@@ -797,7 +870,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # echee TODO: get instance stats from session and create a status log line
         msg = 'plugin_status: '
         cnt = 0
-        @client_session.get_stats.each do |k, v|
+        client_session.get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
@@ -817,7 +890,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
       multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
       begin
-        @client_session.post_add_events(multi_event_request[:body], true, 0)
+        client_session.post_add_events(multi_event_request[:body], true, 0)
       rescue => e
         if e.body
           @logger.warn(

data/lib/scalyr/common/client.rb

@@ -267,6 +267,7 @@ class ClientSession
 
 
   def close
+    @client.close if @client
   end  # def close
 
 

data/lib/scalyr/common/util.rb

@@ -4,7 +4,9 @@ module Scalyr; module Common; module Util;
 # Flattens a hash or array, returning a hash where keys are a delimiter-separated string concatenation of all
 # nested keys.  Returned keys are always strings.  If a non-hash or array is provided, raises TypeError.
 # Please see rspec util_spec.rb for expected behavior.
-def self.flatten(obj, delimiter='_')
+# Includes a known bug where defined delimiter will not be used for nesting levels past the first, this is kept
+# because some queries and dashboards already rely on the broken functionality.
+def self.flatten(obj, delimiter='_', flatten_arrays=true, fix_deep_flattening_delimiters=false)
 
   # base case is input object is not enumerable, in which case simply return it
   if !obj.respond_to?(:each)
@@ -19,8 +21,8 @@ def self.flatten(obj, delimiter='_')
 
     # input object is a hash
     obj.each do |key, value|
-      if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+      if (flatten_arrays and value.respond_to?(:each)) or value.respond_to?(:has_key?)
+        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
           result["#{key}#{delimiter}#{subkey}"] = subvalue
         end
       else
@@ -28,18 +30,23 @@ def self.flatten(obj, delimiter='_')
       end
     end
 
-  else
+  elsif flatten_arrays
 
     # input object is an array or set
     obj.each_with_index do |value, index|
       if value.respond_to?(:each)
-        flatten(value).each do |subkey, subvalue|
+        flatten(value, fix_deep_flattening_delimiters ? delimiter : '_', flatten_arrays).each do |subkey, subvalue|
           result["#{index}#{delimiter}#{subkey}"] = subvalue
         end
       else
         result["#{index}"] = value
       end
     end
+
+  else
+
+    result = obj
+
   end
 
   return result
@@ -52,5 +59,26 @@ def self.truncate(content, max)
   return content
 end
 
+def self.convert_bignums(obj)
+  if obj.respond_to?(:has_key?) and obj.respond_to?(:each)
+    # input object is a hash
+    obj.each do |key, value|
+      obj[key] = convert_bignums(value)
+    end
+
+  elsif obj.respond_to?(:each)
+    # input object is an array or set
+    obj.each_with_index do |value, index|
+      obj[index] = convert_bignums(value)
+    end
+
+  elsif obj.is_a? Bignum
+    return obj.to_s
+
+  else
+    return obj
+  end
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.14.beta"
+PLUGIN_VERSION = "v0.1.19.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.14.beta'
+  s.version         = '0.1.19.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/benchmarks/bignum_fixing.rb

@@ -0,0 +1,90 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
+
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def rand_bignum()
+  return 200004000020304050300 + rand(999999)
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_bignum()
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def convert_bignums(record)
+  Scalyr::Common::Util.convert_bignums(record)
+end
+
+puts "Util.convert_bignums()"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~200 keys in a hash (single level)
+data = generate_data_array_for_spec([200])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))

data/spec/logstash/outputs/scalyr_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Outputs::Scalyr do
       end
 
       it "it doesnt include flatten metrics if flattening is disabled" do
-          plugin1 = LogStash::Outputs::Scalyr.new({
+        plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
@@ -122,7 +122,7 @@ describe LogStash::Outputs::Scalyr do
         expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
-      it "send_stats is called when events list is empty, but otherwise noop" do
+      it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
@@ -137,6 +137,30 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive([])
       end
 
+      it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
+        plugin2 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                     'report_status_for_empty_batches' => false,
+                                                 })
+
+        mock_client_session = MockClientSession.new
+        quantile_estimator = Quantile::Estimator.new
+        plugin2.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin2.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin2).not_to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin2.multi_receive([])
+      end
+
       # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
       it "send_status only sends posts with is_status = true" do
         # 1. Initial send
@@ -264,6 +288,72 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "when configured to flatten values with custom delimiter and deep delimiter fix" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_values_delimiter' => ".",
+          'fix_deep_flattening_delimiters' => true,
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b.0" => 3,
+                                                     "nested.b.1" => 4,
+                                                     "nested.b.2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
+    context "when configured to flatten values with custom delimiter, no array flattening" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_arrays' => false,
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b" => [3, 4, 5],
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
     context "when configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
@@ -318,5 +408,23 @@ describe LogStash::Outputs::Scalyr do
                                                  })
       end
     end
+
+    context "when receiving an event with Bignums" do
+      config = {
+          'api_write_token' => '1234',
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "doesn't throw an error" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 2000023030042002050202030320240)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+        expect(plugin.instance_variable_get(:@logger)).to_not receive(:error)
+      end
+    end
   end
 end

data/spec/scalyr/common/util_spec.rb

@@ -132,6 +132,70 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din)).to eq(dout)
   end
 
+  it "flattens a single-level array, no array flattening" do
+    din = [1, 2, 3]
+    dout = [1, 2, 3]
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a multi-level array, no array flattening" do
+    din = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    dout = ['a', 'b', ['c', ['d', 'e', 'f'], 'g'], 'h', 'i']
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [100, 200, 300]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
+  it "flattens a hash that contains an array that contains a hash that contains an array, no array flattening" do
+    din = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    dout = {
+        'a' => 1,
+        'c' => [
+            100,
+            {'d' => 1000, 'e' => 2000, 'f' => [4, 5, 6]},
+            300
+        ]
+    }
+    expect(Scalyr::Common::Util.flatten(din, "_", flatten_arrays=false)).to eq(dout)
+  end
+
   it "accepts custom delimiters" do
     din = {
         'a' => 1,
@@ -148,6 +212,42 @@ describe Scalyr::Common::Util do
     expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
   end
 
+  it "accepts custom delimiters with greater depth" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c_e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':')).to eq(dout)
+  end
+
+  it "accepts custom delimiters with greater depth and deep delimiters fix" do
+    din = {
+        'a' => 1,
+        'b' => {
+            'c' => {
+              'e' => 100
+            },
+            'd' => 200,
+        }
+    }
+    dout = {
+        'a' => 1,
+        'b:c:e' => 100,
+        'b:d' => 200,
+    }
+    expect(Scalyr::Common::Util.flatten(din, ':', true, true)).to eq(dout)
+  end
+
   it "stringifies non-string keys" do
     din = {
         'a' => 1,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.14.beta
+  version: 0.1.19.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-22 00:00:00.000000000 Z
+date: 2021-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ files:
 - lib/scalyr/common/util.rb
 - lib/scalyr/constants.rb
 - logstash-output-scalyr.gemspec
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb
@@ -4062,6 +4063,7 @@ signing_key:
 specification_version: 4
 summary: Scalyr output plugin for Logstash
 test_files:
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb