logstash-output-scalyr 0.1.16.beta → 0.1.17.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16e88c3869fdb18e4132726af302e37c8bac2c10038ca3a078f85cebe67c4940
-  data.tar.gz: a42194413ec2969e31bbe44e72df7cf71d09911b61adae9db16b6041a97bc1b4
+  metadata.gz: 96fd1b3adba21150a05f43d1e8d6510711197da866d3cd6ec8b57bae14e53391
+  data.tar.gz: 221aac42711955c49c878c324e2570ebb55050cad13f2d4911b99c4b3f1eb5a1
 SHA512:
-  metadata.gz: 66d4261da66deaa8f5227b0ecfcdfce51f170c561902397c6020d3baa8694bd7c92291b0faa84178e4dda180c71892ab4a30ba425dc1133f5f7b349c1b7e44fa
-  data.tar.gz: 6689e3c9fd5fa2a571ca8076a4ae24b5eb1bf5da9216030b8c759a0baff17f4577a478f3c3efced4ddb22ecf273ec675f8831733aded928fd2c92c7ce91d1983
+  metadata.gz: c8a678e7caf8778dbe9b44e8d8e48c809b016ac6b89424c4c29c29b63578fea61f236ae80a528dab7707c8ee534cfab0f20a252d688f44df53faccdd1ce907b0
+  data.tar.gz: c263ae0ecffacb454f98f112d384c544cfd68fb0e0e6a0d7339a6b84add2c94b4bb99afd20a815b587f5722e0125d80fc25f9285574187c9f2a0ff2af7f219ec

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Beta
 
+## 0.1.17.beta
+- Catch errors relating to Bignum conversions present in the ``json`` library and manually convert to string as
+a workaround.
+
 ## 0.1.16.beta
 - Fix race condition in ``register()`` method.
 

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.14.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.17.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -240,7 +240,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Plugin level (either per batch or event level metrics). Other request
     # level metrics are handled by the HTTP Client class.
     @multi_receive_statistics = {
-      :total_multi_receive_secs => 0
+      :total_multi_receive_secs => 0,
+      :total_java_class_cast_errors => 0
     }
     @plugin_metrics = get_new_metrics
 
@@ -661,6 +662,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           ).force_encoding('UTF-8')
         end
         event_json = scalyr_event.to_json
+      rescue Java::JavaLang::ClassCastException => e
+        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+        # Because of the version of jruby logstash works with we don't have the option to just update this away,
+        # so if we run into it we convert bignums into strings so we can get the data in at least.
+        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+        end
+        Scalyr::Common::Util.convert_bignums(scalyr_event)
+        event_json = scalyr_event.to_json
+        log_json = nil
+        if add_log
+          log_json = logs[log_identifier].to_json
+        end
       end
 
       # generate new request if json size of events in the array exceed maximum request buffer size
@@ -749,7 +765,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
-    serialized_body = body.to_json
+    begin
+      serialized_body = body.to_json
+    rescue Java::JavaLang::ClassCastException => e
+      @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+      @stats_lock.synchronize do
+        @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      end
+      Scalyr::Common::Util.convert_bignums(body)
+      serialized_body = body.to_json
+    end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
     {

data/lib/scalyr/common/util.rb

@@ -52,5 +52,26 @@ def self.truncate(content, max)
   return content
 end
 
+def self.convert_bignums(obj)
+  if obj.respond_to?(:has_key?) and obj.respond_to?(:each)
+    # input object is a hash
+    obj.each do |key, value|
+      obj[key] = convert_bignums(value)
+    end
+
+  elsif obj.respond_to?(:each)
+    # input object is an array or set
+    obj.each_with_index do |value, index|
+      obj[index] = convert_bignums(value)
+    end
+
+  elsif obj.is_a? Bignum
+    return obj.to_s
+
+  else
+    return obj
+  end
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.16.beta"
+PLUGIN_VERSION = "v0.1.17.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.16.beta'
+  s.version         = '0.1.17.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/benchmarks/bignum_fixing.rb

@@ -0,0 +1,90 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how long it takes to find all the Bignums in a record and convert them to strings
+
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def rand_bignum()
+  return 200004000020304050300 + rand(999999)
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_bignum()
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def convert_bignums(record)
+  Scalyr::Common::Util.convert_bignums(record)
+end
+
+puts "Util.convert_bignums()"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~200 keys in a hash (single level)
+data = generate_data_array_for_spec([200])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:convert_bignums))

data/spec/logstash/outputs/scalyr_spec.rb

@@ -342,5 +342,23 @@ describe LogStash::Outputs::Scalyr do
                                                  })
       end
     end
+
+    context "when receiving an event with Bignums" do
+      config = {
+          'api_write_token' => '1234',
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "doesn't throw an error" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 2000023030042002050202030320240)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(1)
+        expect(plugin.instance_variable_get(:@logger)).to_not receive(:error)
+      end
+    end
   end
 end

data/vendor/bundle/jruby/2.5.0/bin/htmldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

data/vendor/bundle/jruby/2.5.0/bin/ldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.16.beta
+  version: 0.1.17.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-23 00:00:00.000000000 Z
+date: 2021-07-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -132,6 +132,7 @@ files:
 - lib/scalyr/common/util.rb
 - lib/scalyr/constants.rb
 - logstash-output-scalyr.gemspec
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb
@@ -4062,6 +4063,7 @@ signing_key:
 specification_version: 4
 summary: Scalyr output plugin for Logstash
 test_files:
+- spec/benchmarks/bignum_fixing.rb
 - spec/benchmarks/flattening_and_serialization.rb
 - spec/benchmarks/metrics_overhead.rb
 - spec/logstash/outputs/scalyr_integration_spec.rb