logstash-output-scalyr 0.1.22.beta → 0.1.26.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ea703e55bc28d2bb00445c2a3e2d822bfe978bbe889292caafa6265dff1de2b
-  data.tar.gz: 3dc35cf36a356e5651d69e1a7d4623f296e5818a4b1c0fc8965b1a55b94a20ab
+  metadata.gz: 5593e93dedad0ddb545f8911e7f1cd8cabde4043a6a76b8c5b78d99c0c4d12a5
+  data.tar.gz: 6c821768c3d780529d00bc1f58c4ec8841ecba2758a46c71ecf17201fb8ba3d1
 SHA512:
-  metadata.gz: 9306c619ce51c8a9cf83cb545229d4c73ac496a133e07e7e55b434301e1efc2f668187c3aa1584997ab1252ce3c7510ac9c4d719ba363933160bf2f874d0a137
-  data.tar.gz: 621b8f9b4095f16e51b88b8aec2844d795de75cba980ba27e0510a04f57ee9e4b34eb42513455d7509acaa8af8e085febb92cd248401e642129e09d70987a476
+  metadata.gz: 57901dc94f49aa844130f4b2a7ac084c95d473c1a948902b699cbaba8d63049d1e960b69b49b20cf8ffa8b2c1f5f68a53ad9687b72867799f113a384f490e01d
+  data.tar.gz: 329f985e8121d49bab836381861ca878c229f2b316e160792263461900a8fc4cf7f46bdc2aba889174a0f35b0c67f567b0e5a09d429f888948b788707ab3c0b7

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Beta
 
+## 0.1.26.beta
+- Add support for new ``json_library`` config option. Valid values are ``stdlib`` (default) are ``jrjackson``. The later may offer 2-4x faster JSON serialization.
+
+
+## 0.1.23.beta
+- Add testing support for disabling estimation of serialized event size for each event in the batch.
+
 ## 0.1.22.beta
 - Add new plugin metric for tracking the duration of ``build_multi_event_request_array`` method.
 - Update internal dependencies (``manticore``) to latest stable version.

data/Gemfile

@@ -18,3 +18,4 @@ gem 'pry'
 gem 'pry-nav'
 gem 'quantile', '~> 0.2.1'
 gem 'manticore', '~> 0.7.1', platform: :jruby
+gem 'jrjackson', '~> 0.4.14', platform: :jruby

data/README.md

@@ -1,4 +1,3 @@
-
 [![CircleCI](https://circleci.com/gh/scalyr/logstash-output-scalyr.svg?style=svg)](https://circleci.com/gh/scalyr/logstash-output-scalyr)
 
 # [Scalyr output plugin for Logstash (Beta release)]
@@ -10,7 +9,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.22.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.25.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 

data/lib/logstash/outputs/scalyr.rb

@@ -13,6 +13,7 @@ require 'rbzip2'
 require 'zlib'
 require 'stringio'
 require 'quantile'
+require 'jrjackson'
 
 require 'scalyr/common/client'
 require "scalyr/common/util"
@@ -140,11 +141,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
   # these stas this might be wanted or not.
   config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
-  
+
   # Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
   # after no work being done.
   config :noop_mode, :validate => :boolean, :default => false
 
+  # Set to true to disable estimiating the size of each serialized event to make sure we don't go over the max request
+  # size (5.5) and split batch into multiple Scalyr requests, if needed. Since this estimation is not "free", especially
+  # for large batches, it may make sense to disable this option when logstash batch size is configured in a way that
+  # Scalyr single request limit won't be reached.
+  config :estimate_each_event_size, :validate => :boolean, :default => true
+
+  # Library to use for JSON serialization. Valid values are "stdlib" and "jrjackson". The later may offer 2-4 performance
+  # improvements on serialization.
+  config :json_library, :validate => :string, :default => "stdlib"
+
   # Manticore related options
   config :http_connect_timeout, :validate => :number, :default => 10
   config :http_socket_timeout, :validate => :number, :default => 10
@@ -184,6 +195,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       @logger.warn "Maximum request buffer > 6 MB.  This may result in requests being rejected by Scalyr."
     end
 
+    if not @estimate_each_event_size
+      @logger.warn("estimate_each_event_size config option is false, this means very large batches may be rejected or partially processed by the server")
+    end
+
+    if @json_library != "stdlib" and @json_library != "jrjackson"
+      raise LogStash::ConfigurationError, "json_library config option needs to be either stdlib or jrjackson"
+    end
+
+    if @json_library == "stdlib"
+      define_singleton_method "json_encode" do |data|
+        data.to_json
+      end
+    elsif @json_library == "jrjackson"
+      define_singleton_method "json_encode" do |data|
+        JrJackson::Json.dump(data)
+      end
+    end
+
     @dlq_writer = dlq_enabled? ? execution_context.dlq_writer : nil
 
     @message_encoding = nil
@@ -316,16 +345,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     begin
       records_count = events.to_a.length
 
-      # We also time the duration of the build_multi_event_request_array method
+      # We also time the duration of the build_multi_event_request_array method. To avoid locking twice,
+      # we store the duration value here and record metric at the end.
       start_time = Time.now.to_f
 
       multi_event_request_array = build_multi_event_request_array(events)
-
-      if records_count > 0
-        @stats_lock.synchronize do
-          @plugin_metrics[:build_multi_duration_secs].observe(Time.now.to_f - start_time)
-        end
-      end
+      build_multi_duration_secs = Time.now.to_f - start_time
 
       # Loop over all array of multi-event requests, sending each multi-event to Scalyr
       sleep_interval = @retry_initial_interval
@@ -439,6 +464,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       if records_count > 0
         @stats_lock.synchronize do
           @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+          @plugin_metrics[:build_multi_duration_secs].observe(build_multi_duration_secs)
           @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
           @plugin_metrics[:multi_receive_event_count].observe(records_count)
           @plugin_metrics[:batches_per_multi_receive].observe(total_batches)
@@ -675,67 +701,75 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         scalyr_event[:log] = logs_ids[log_identifier]
       end
 
-      # get json string of event to keep track of how many bytes we are sending
-      begin
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
-        end
-      rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
-        @logger.warn "#{e.class}: #{e.message}"
-
-        # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
-        # TODO
-        # atime = Fluent::EventTime.new( sec, nsec )
-        # router.emit_error_event(serverHost, time, record, e)
-
-        scalyr_event[:attrs].each do |key, value|
-          @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
-          scalyr_event[:attrs][key] = value.encode(
-              "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
-          ).force_encoding('UTF-8')
-        end
-        event_json = scalyr_event.to_json
-      rescue Java::JavaLang::ClassCastException => e
-        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
-        # Because of the version of jruby logstash works with we don't have the option to just update this away,
-        # so if we run into it we convert bignums into strings so we can get the data in at least.
-        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
-        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
-        @stats_lock.synchronize do
-          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      if @estimate_each_event_size
+        # get json string of event to keep track of how many bytes we are sending
+        begin
+          event_json = self.json_encode(scalyr_event)
+          log_json = nil
+          if add_log
+            log_json = self.json_encode(logs[log_identifier])
+          end
+        rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
+          @logger.warn "#{e.class}: #{e.message}"
+
+          # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
+          # TODO
+          # atime = Fluent::EventTime.new( sec, nsec )
+          # router.emit_error_event(serverHost, time, record, e)
+
+          scalyr_event[:attrs].each do |key, value|
+            @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
+            scalyr_event[:attrs][key] = value.encode(
+                "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
+            ).force_encoding('UTF-8')
+          end
+          event_json = self.json_encode(scalyr_event)
+        rescue Java::JavaLang::ClassCastException => e
+          # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+          # Because of the version of jruby logstash works with we don't have the option to just update this away,
+          # so if we run into it we convert bignums into strings so we can get the data in at least.
+          # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+          @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_java_class_cast_errors] += 1
+          end
+          Scalyr::Common::Util.convert_bignums(scalyr_event)
+          event_json = self.json_encode(scalyr_event)
+          log_json = nil
+          if add_log
+            log_json = self.json_encode(logs[log_identifier])
+          end
         end
-        Scalyr::Common::Util.convert_bignums(scalyr_event)
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
+
+        # generate new request if json size of events in the array exceed maximum request buffer size
+        append_event = true
+        add_bytes = event_json.bytesize
+        if log_json
+          add_bytes = add_bytes + log_json.bytesize
         end
-      end
 
-      # generate new request if json size of events in the array exceed maximum request buffer size
-      append_event = true
-      add_bytes =  event_json.bytesize
-      if log_json
-        add_bytes = add_bytes + log_json.bytesize
-      end
-      if total_bytes + add_bytes > @max_request_buffer
-        # make sure we always have at least one event
-        if scalyr_events.size == 0
-          scalyr_events << scalyr_event
-          l_events << l_event
-          append_event = false
+        if total_bytes + add_bytes > @max_request_buffer
+          # make sure we always have at least one event
+          if scalyr_events.size == 0
+            scalyr_events << scalyr_event
+            l_events << l_event
+            append_event = false
+          end
+
+          multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
+          multi_event_request_array << multi_event_request
+
+          total_bytes = 0
+          current_threads = Hash.new
+          logs = Hash.new
+          logs_ids = Hash.new
+          scalyr_events = Array.new
+          l_events = Array.new
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
-        multi_event_request_array << multi_event_request
-
-        total_bytes = 0
-        current_threads = Hash.new
-        logs = Hash.new
-        logs_ids = Hash.new
-        scalyr_events = Array.new
-        l_events = Array.new
+      else
+        # If size estimation is disabled we simply append the event and handle splitting later on (if needed)
+        append_event = true
+        add_bytes = 0
       end
 
       # if we haven't consumed the current event already
@@ -748,9 +782,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     }
 
-    # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
-    multi_event_request_array << multi_event_request
+    # create a final request with any left over events (and make sure there is at least one event)
+    if scalyr_events.size >= 1
+      multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
+      multi_event_request_array << multi_event_request
+    end
+
     multi_event_request_array
   end
 
@@ -801,17 +838,26 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # We time serialization to get some insight on how long it takes to serialize the request body
     start_time = Time.now.to_f
     begin
-      serialized_body = body.to_json
+      serialized_body = self.json_encode(body)
     rescue Java::JavaLang::ClassCastException => e
       @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
       @stats_lock.synchronize do
         @multi_receive_statistics[:total_java_class_cast_errors] += 1
       end
       Scalyr::Common::Util.convert_bignums(body)
-      serialized_body = body.to_json
+      serialized_body = self.json_encode(body)
     end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
+
+    serialized_request_size = serialized_body.bytesize
+
+    # We give it "buffer" since the splitting code allows for some slack and doesn't take into account top-level non-event attributes
+    if not @estimate_each_event_size and serialized_request_size >= @max_request_buffer + 5000
+      # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
+      @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
+    end
+
     {
       :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
       :logstash_events => logstash_events

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.22.beta"
+PLUGIN_VERSION = "v0.1.26.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.22.beta'
+  s.version         = '0.1.26.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -18,11 +18,12 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
 
   # Gem dependencies
-  #
   s.add_runtime_dependency 'net-http-persistent'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'quantile'
+  s.add_runtime_dependency 'jrjackson'
+  s.add_runtime_dependency 'manticore'
   s.add_runtime_dependency 'ffi', '>= 1.9.18'
   s.add_runtime_dependency 'rbzip2', '0.3.0'
   s.add_development_dependency 'logstash-devutils'

data/spec/benchmarks/flattening_and_serialization.rb

@@ -2,6 +2,7 @@ require 'benchmark'
 require 'json'
 
 require_relative '../../lib/scalyr/common/util'
+require_relative './util'
 
 # NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
 # sense since it will just use JITed version of the code which will be very fast. If we
@@ -9,31 +10,6 @@ require_relative '../../lib/scalyr/common/util'
 # input data for each iteration.
 ITERATIONS = 500
 
-def rand_str(len)
-  return (0...len).map { (65 + rand(26)).chr }.join
-end
-
-def generate_hash(widths)
-  result = {}
-  if widths.empty?
-    return rand_str(20)
-  else
-    widths[0].times do
-      result[rand_str(9)] = generate_hash(widths[1..widths.length])
-    end
-    return result
-  end
-end
-
-def generate_data_array_for_spec(spec)
-  data = []
-  ITERATIONS.times do
-    data << generate_hash(spec)
-  end
-
-  data
-end
-
 def run_benchmark_and_print_results(data, run_benchmark_func)
   puts ""
   puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
@@ -70,7 +46,6 @@ DATASETS = {
   :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
 }
 
-
 puts "Using %s iterations" % [ITERATIONS]
 puts ""
 

data/spec/benchmarks/json_serialization.rb

@@ -0,0 +1,85 @@
+require 'benchmark'
+require 'json'
+require 'jrjackson'
+
+require_relative '../../lib/scalyr/common/util'
+require_relative './util'
+
+ITERATIONS = 500
+
+def json_serialize_data_native(data)
+  data.to_json
+end
+
+def json_serialize_data_jrjackson(data)
+  JrJackson::Json.dump(data)
+end
+
+DATASETS = {
+  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
+  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
+  :keys_200_flat => generate_data_array_for_spec([200]),
+  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
+  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
+  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
+}
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+puts "native"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:json_serialize_data_native))
+
+puts "jrjackson"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))