logstash-output-scalyr 0.1.25.beta → 0.1.26.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5d84626fc9f17fb9cc549552722612e2620c10ead3700689cb5625715a5441bc
-  data.tar.gz: a49c2c03b3f97678df4a20f66ceec0ec2fdcf4fedcac732a130abe642a469057
+  metadata.gz: 5593e93dedad0ddb545f8911e7f1cd8cabde4043a6a76b8c5b78d99c0c4d12a5
+  data.tar.gz: 6c821768c3d780529d00bc1f58c4ec8841ecba2758a46c71ecf17201fb8ba3d1
 SHA512:
-  metadata.gz: 015e7216df91f4bcc696038ea305936275194d87eb68f5fe5d067e704932663ab5a0cdbb8440810682964c6bfa7a9f052774065ed9fb498fc175900b77eaf161
-  data.tar.gz: e08e06532da18aaf597937482fdee0561ea816b09f3b77c3eee259aeaaaaececcdded5e4dbd2101aacf02018167764c8e98faa7db1c9abe76e4f1094d7fc68ca
+  metadata.gz: 57901dc94f49aa844130f4b2a7ac084c95d473c1a948902b699cbaba8d63049d1e960b69b49b20cf8ffa8b2c1f5f68a53ad9687b72867799f113a384f490e01d
+  data.tar.gz: 329f985e8121d49bab836381861ca878c229f2b316e160792263461900a8fc4cf7f46bdc2aba889174a0f35b0c67f567b0e5a09d429f888948b788707ab3c0b7

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Beta
 
+## 0.1.26.beta
+- Add support for new ``json_library`` config option. Valid values are ``stdlib`` (default) are ``jrjackson``. The later may offer 2-4x faster JSON serialization.
+
+
 ## 0.1.23.beta
 - Add testing support for disabling estimation of serialized event size for each event in the batch.
 

data/Gemfile

@@ -18,3 +18,4 @@ gem 'pry'
 gem 'pry-nav'
 gem 'quantile', '~> 0.2.1'
 gem 'manticore', '~> 0.7.1', platform: :jruby
+gem 'jrjackson', '~> 0.4.14', platform: :jruby

data/README.md

@@ -1,4 +1,3 @@
-
 [![CircleCI](https://circleci.com/gh/scalyr/logstash-output-scalyr.svg?style=svg)](https://circleci.com/gh/scalyr/logstash-output-scalyr)
 
 # [Scalyr output plugin for Logstash (Beta release)]
@@ -10,7 +9,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.22.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.25.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -13,6 +13,7 @@ require 'rbzip2'
 require 'zlib'
 require 'stringio'
 require 'quantile'
+require 'jrjackson'
 
 require 'scalyr/common/client'
 require "scalyr/common/util"
@@ -151,6 +152,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Scalyr single request limit won't be reached.
   config :estimate_each_event_size, :validate => :boolean, :default => true
 
+  # Library to use for JSON serialization. Valid values are "stdlib" and "jrjackson". The later may offer 2-4 performance
+  # improvements on serialization.
+  config :json_library, :validate => :string, :default => "stdlib"
+
   # Manticore related options
   config :http_connect_timeout, :validate => :number, :default => 10
   config :http_socket_timeout, :validate => :number, :default => 10
@@ -194,6 +199,20 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       @logger.warn("estimate_each_event_size config option is false, this means very large batches may be rejected or partially processed by the server")
     end
 
+    if @json_library != "stdlib" and @json_library != "jrjackson"
+      raise LogStash::ConfigurationError, "json_library config option needs to be either stdlib or jrjackson"
+    end
+
+    if @json_library == "stdlib"
+      define_singleton_method "json_encode" do |data|
+        data.to_json
+      end
+    elsif @json_library == "jrjackson"
+      define_singleton_method "json_encode" do |data|
+        JrJackson::Json.dump(data)
+      end
+    end
+
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
     @message_encoding = nil
@@ -685,10 +704,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       if @estimate_each_event_size
         # get json string of event to keep track of how many bytes we are sending
         begin
-          event_json = scalyr_event.to_json
+          event_json = self.json_encode(scalyr_event)
           log_json = nil
           if add_log
-            log_json = logs[log_identifier].to_json
+            log_json = self.json_encode(logs[log_identifier])
           end
         rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
           @logger.warn "#{e.class}: #{e.message}"
@@ -704,7 +723,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
                 "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
             ).force_encoding('UTF-8')
           end
-          event_json = scalyr_event.to_json
+          event_json = self.json_encode(scalyr_event)
         rescue Java::JavaLang::ClassCastException => e
           # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
           # Because of the version of jruby logstash works with we don't have the option to just update this away,
@@ -715,10 +734,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @multi_receive_statistics[:total_java_class_cast_errors] += 1
           end
           Scalyr::Common::Util.convert_bignums(scalyr_event)
-          event_json = scalyr_event.to_json
+          event_json = self.json_encode(scalyr_event)
           log_json = nil
           if add_log
-            log_json = logs[log_identifier].to_json
+            log_json = self.json_encode(logs[log_identifier])
           end
         end
 
@@ -819,14 +838,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
     begin
-      serialized_body = body.to_json
+      serialized_body = self.json_encode(body)
     rescue Java::JavaLang::ClassCastException => e
       @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
       @stats_lock.synchronize do
         @multi_receive_statistics[:total_java_class_cast_errors] += 1
       end
       Scalyr::Common::Util.convert_bignums(body)
-      serialized_body = body.to_json
+      serialized_body = self.json_encode(body)
     end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.25.beta"
+PLUGIN_VERSION = "v0.1.26.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.25.beta'
+  s.version         = '0.1.26.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -18,11 +18,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  #
   s.add_runtime_dependency 'net-http-persistent'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'quantile'
+  s.add_runtime_dependency 'jrjackson'
+  s.add_runtime_dependency 'manticore'
   s.add_runtime_dependency 'ffi', '>= 1.9.18'
   s.add_runtime_dependency 'rbzip2', '0.3.0'
   s.add_development_dependency 'logstash-devutils'

data/spec/benchmarks/flattening_and_serialization.rb

@@ -2,6 +2,7 @@ require 'benchmark'
 require 'json'
 
 require_relative '../../lib/scalyr/common/util'
+require_relative './util'
 
 # NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
 # sense since it will just use JITed version of the code which will be very fast. If we
@@ -9,31 +10,6 @@ require_relative '../../lib/scalyr/common/util'
 # input data for each iteration.
 ITERATIONS = 500
 
-def rand_str(len)
-  return (0...len).map { (65 + rand(26)).chr }.join
-end
-
-def generate_hash(widths)
-  result = {}
-  if widths.empty?
-    return rand_str(20)
-  else
-    widths[0].times do
-      result[rand_str(9)] = generate_hash(widths[1..widths.length])
-    end
-    return result
-  end
-end
-
-def generate_data_array_for_spec(spec)
-  data = []
-  ITERATIONS.times do
-    data << generate_hash(spec)
-  end
-
-  data
-end
-
 def run_benchmark_and_print_results(data, run_benchmark_func)
   puts ""
   puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
@@ -70,7 +46,6 @@ DATASETS = {
   :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
 }
 
-
 puts "Using %s iterations" % [ITERATIONS]
 puts ""
 

data/spec/benchmarks/json_serialization.rb

@@ -0,0 +1,85 @@
+require 'benchmark'
+require 'json'
+require 'jrjackson'
+
+require_relative '../../lib/scalyr/common/util'
+require_relative './util'
+
+ITERATIONS = 500
+
+def json_serialize_data_native(data)
+  data.to_json
+end
+
+def json_serialize_data_jrjackson(data)
+  JrJackson::Json.dump(data)
+end
+
+DATASETS = {
+  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
+  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
+  :keys_200_flat => generate_data_array_for_spec([200]),
+  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
+  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
+  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
+}
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+puts "native"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+puts "jrjackson"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))

data/spec/benchmarks/util.rb

@@ -0,0 +1,24 @@
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_str(20)
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -60,6 +60,7 @@ describe LogStash::Outputs::Scalyr do
                                                      'log_constants' => ['tags'],
                                                      'flatten_nested_values' => true,
                                                  })
+      plugin.register
 
       mock_client_session = MockClientSession.new
 
@@ -75,6 +76,7 @@ describe LogStash::Outputs::Scalyr do
                                                      'log_constants' => ['tags'],
                                                      'flatten_nested_values' => false,
                                                  })
+        plugin1.register()
         mock_client_session = MockClientSession.new
         plugin1.instance_variable_set(:@last_status_transmit_time, 100)
         plugin1.instance_variable_set(:@client_session, mock_client_session)
@@ -582,5 +584,57 @@ describe LogStash::Outputs::Scalyr do
         expect(plugin.instance_variable_get(:@logger)).to_not receive(:error)
       end
     end
+
+    context "when using custom json library" do
+      it "stdlib (implicit)" do
+        config = {
+            'api_write_token' => '1234',
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 20)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(result[0][:body]).to include('{"monitor":"pluginLogstash"}')
+        expect(body['events'].size).to eq(1)
+      end
+
+      it "stdlib (explicit)" do
+        config = {
+            'api_write_token' => '1234',
+            'json_library' => 'stdlib'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 20)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(result[0][:body]).to include('{"monitor":"pluginLogstash"}')
+        expect(body['events'].size).to eq(1)
+      end
+
+      it "jrjackson" do
+        config = {
+            'api_write_token' => '1234',
+            'json_library' => 'jrjackson'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        e = LogStash::Event.new
+        e.set('bignumber', 20)
+        result = plugin.build_multi_event_request_array([e])
+        body = JSON.parse(result[0][:body])
+        expect(result[0][:body]).to include('{"monitor":"pluginLogstash"}')
+        expect(body['events'].size).to eq(1)
+      end
+    end
   end
 end

data/vendor/bundle/jruby/2.5.0/gems/{jrjackson-0.4.8-java → jrjackson-0.4.14-java}/Mavenfile

@@ -14,7 +14,7 @@ properties 'project.build.sourceEncoding' => 'UTF-8',
 
 jar 'junit:junit', '4.11', :scope => :test
 
-jar 'org.jruby:jruby', '9.2.4.0', :scope => :provided
+jar 'org.jruby:jruby', '9.2.13.0', :scope => :provided
 
 plugin :compiler, '3.1', :source => '1.8', :target => '1.8',
        :showDeprecation => false,
@@ -23,9 +23,3 @@ plugin :compiler, '3.1', :source => '1.8', :target => '1.8',
        :fork => true
 
 plugin :surefire, '2.17', :skipTests => true
-
-# since bundle install does not vendor our jars we need to it manually
-plugin 'org.torquebox.mojo:jruby9-exec-maven-plugin', '0.3.1' do
-  execute_goal :exec, :id => 'vendor-jars', :phase => 'prepare-package',
-               :script => "require 'jars/installer';Jars::Installer.vendor_jars!"
-end

data/vendor/bundle/jruby/2.5.0/gems/{jrjackson-0.4.8-java → jrjackson-0.4.14-java}/Rakefile

@@ -10,12 +10,19 @@ end
 
 desc "Run benchmarks"
 task :benchmark do
-  load 'benchmarking/benchmark.rb'
+  load 'benchmarking/benchmark_threaded.rb'
 end
 
 desc "Pack jar after compiling classes, use this to rebuild the pom.xml"
 task :compile do
   RubyMaven.exec('prepare-package')
+  # after packaging the jrjackson-x.y.z.jar vendor jar dependencies
+  Rake::Task['vendor_jars'].invoke
+end
+
+task :vendor_jars do
+  require 'jars/installer'
+  Jars::Installer.vendor_jars!
 end
 
 desc "Clean build"
@@ -30,4 +37,3 @@ Gem::PackageTask.new( eval File.read( 'jrjackson.gemspec' ) ) do
   desc 'Pack gem'
   task :package => [:compile]
 end
-