logstash-output-scalyr 0.1.22.beta → 0.1.23.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ea703e55bc28d2bb00445c2a3e2d822bfe978bbe889292caafa6265dff1de2b
-  data.tar.gz: 3dc35cf36a356e5651d69e1a7d4623f296e5818a4b1c0fc8965b1a55b94a20ab
+  metadata.gz: 8e2757da74eb03ea09a88597ed713db01bc050e17c31c771558bdb16a80ca40e
+  data.tar.gz: 1bc8519d1b30aea135d66b8b9ecfb632f688fd62abd58cd3b91cc92c4ad8957f
 SHA512:
-  metadata.gz: 9306c619ce51c8a9cf83cb545229d4c73ac496a133e07e7e55b434301e1efc2f668187c3aa1584997ab1252ce3c7510ac9c4d719ba363933160bf2f874d0a137
-  data.tar.gz: 621b8f9b4095f16e51b88b8aec2844d795de75cba980ba27e0510a04f57ee9e4b34eb42513455d7509acaa8af8e085febb92cd248401e642129e09d70987a476
+  metadata.gz: 327a3e2ba020f3166b0c435700bdfedf4f96598a352f1f66f45fe9e671a053ee684aa812d686a4d58b5dd9f319e8b0f87e2ea8290d876e504b8416a8010795ed
+  data.tar.gz: 1ba44520daddda5c17da810221063ce815af082645ae3fd12e76b031247ff8a13cb3e7dab4fbf69e84e5a876605290702785cc17639de69435f0b780a74691f9

data/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Beta
 
+## 0.1.23.beta
+- Add testing support for disabling estimation of serialized event size for each event in the batch.
+
 ## 0.1.22.beta
 - Add new plugin metric for tracking the duration of ``build_multi_event_request_array`` method.
 - Update internal dependencies (``manticore``) to latest stable version.

data/lib/logstash/outputs/scalyr.rb

@@ -140,11 +140,17 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
   # these stas this might be wanted or not.
   config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
-  
+
   # Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
   # after no work being done.
   config :noop_mode, :validate => :boolean, :default => false
 
+  # Set to true to disable estimiating the size of each serialized event to make sure we don't go over the max request
+  # size (5.5) and split batch into multiple Scalyr requests, if needed. Since this estimation is not "free", especially
+  # for large batches, it may make sense to disable this option when logstash batch size is configured in a way that
+  # Scalyr single request limit won't be reached.
+  config :estimate_each_event_size, :validate => :boolean, :default => true
+
   # Manticore related options
   config :http_connect_timeout, :validate => :number, :default => 10
   config :http_socket_timeout, :validate => :number, :default => 10
@@ -316,16 +322,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     begin
       records_count = events.to_a.length
 
-      # We also time the duration of the build_multi_event_request_array method
+      # We also time the duration of the build_multi_event_request_array method. To avoid locking twice,
+      # we store the duration value here and record metric at the end.
       start_time = Time.now.to_f
 
       multi_event_request_array = build_multi_event_request_array(events)
-
-      if records_count > 0
-        @stats_lock.synchronize do
-          @plugin_metrics[:build_multi_duration_secs].observe(Time.now.to_f - start_time)
-        end
-      end
+      build_multi_duration_secs = Time.now.to_f - start_time
 
       # Loop over all array of multi-event requests, sending each multi-event to Scalyr
       sleep_interval = @retry_initial_interval
@@ -439,6 +441,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       if records_count > 0
         @stats_lock.synchronize do
           @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+          @plugin_metrics[:build_multi_duration_secs].observe(build_multi_duration_secs)
           @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
           @plugin_metrics[:multi_receive_event_count].observe(records_count)
           @plugin_metrics[:batches_per_multi_receive].observe(total_batches)
@@ -675,67 +678,74 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         scalyr_event[:log] = logs_ids[log_identifier]
       end
 
-      # get json string of event to keep track of how many bytes we are sending
-      begin
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
-        end
-      rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
-        @logger.warn "#{e.class}: #{e.message}"
-
-        # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
-        # TODO
-        # atime = Fluent::EventTime.new( sec, nsec )
-        # router.emit_error_event(serverHost, time, record, e)
-
-        scalyr_event[:attrs].each do |key, value|
-          @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
-          scalyr_event[:attrs][key] = value.encode(
-              "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
-          ).force_encoding('UTF-8')
-        end
-        event_json = scalyr_event.to_json
-      rescue Java::JavaLang::ClassCastException => e
-        # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
-        # Because of the version of jruby logstash works with we don't have the option to just update this away,
-        # so if we run into it we convert bignums into strings so we can get the data in at least.
-        # This is fixed in JRuby 9.2.7, which includes json 2.2.0
-        @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
-        @stats_lock.synchronize do
-          @multi_receive_statistics[:total_java_class_cast_errors] += 1
+      if @estimate_each_event_size
+        # get json string of event to keep track of how many bytes we are sending
+        begin
+          event_json = scalyr_event.to_json
+          log_json = nil
+          if add_log
+            log_json = logs[log_identifier].to_json
+          end
+        rescue JSON::GeneratorError, Encoding::UndefinedConversionError => e
+          @logger.warn "#{e.class}: #{e.message}"
+
+          # Send the faulty event to a label @ERROR block and allow to handle it there (output to exceptions file for ex)
+          # TODO
+          # atime = Fluent::EventTime.new( sec, nsec )
+          # router.emit_error_event(serverHost, time, record, e)
+
+          scalyr_event[:attrs].each do |key, value|
+            @logger.debug "\t#{key} (#{value.encoding.name}): '#{value}'"
+            scalyr_event[:attrs][key] = value.encode(
+                "UTF-8", :invalid => :replace, :undef => :replace, :replace => "<?>"
+            ).force_encoding('UTF-8')
+          end
+          event_json = scalyr_event.to_json
+        rescue Java::JavaLang::ClassCastException => e
+          # Most likely we ran into the issue described here: https://github.com/flori/json/issues/336
+          # Because of the version of jruby logstash works with we don't have the option to just update this away,
+          # so if we run into it we convert bignums into strings so we can get the data in at least.
+          # This is fixed in JRuby 9.2.7, which includes json 2.2.0
+          @logger.warn("Error serializing events to JSON, likely due to the presence of Bignum values. Converting Bignum values to strings.")
+          @stats_lock.synchronize do
+            @multi_receive_statistics[:total_java_class_cast_errors] += 1
+          end
+          Scalyr::Common::Util.convert_bignums(scalyr_event)
+          event_json = scalyr_event.to_json
+          log_json = nil
+          if add_log
+            log_json = logs[log_identifier].to_json
+          end
         end
-        Scalyr::Common::Util.convert_bignums(scalyr_event)
-        event_json = scalyr_event.to_json
-        log_json = nil
-        if add_log
-          log_json = logs[log_identifier].to_json
+
+        # generate new request if json size of events in the array exceed maximum request buffer size
+        append_event = true
+        add_bytes = event_json.bytesize
+        if log_json
+          add_bytes = add_bytes + log_json.bytesize
         end
-      end
 
-      # generate new request if json size of events in the array exceed maximum request buffer size
-      append_event = true
-      add_bytes =  event_json.bytesize
-      if log_json
-        add_bytes = add_bytes + log_json.bytesize
-      end
-      if total_bytes + add_bytes > @max_request_buffer
-        # make sure we always have at least one event
-        if scalyr_events.size == 0
-          scalyr_events << scalyr_event
-          l_events << l_event
-          append_event = false
+        if total_bytes + add_bytes > @max_request_buffer
+          # make sure we always have at least one event
+          if scalyr_events.size == 0
+            scalyr_events << scalyr_event
+            l_events << l_event
+            append_event = false
+          end
+
+          multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
+          multi_event_request_array << multi_event_request
+
+          total_bytes = 0
+          current_threads = Hash.new
+          logs = Hash.new
+          logs_ids = Hash.new
+          scalyr_events = Array.new
+          l_events = Array.new
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, l_events, current_threads, logs)
-        multi_event_request_array << multi_event_request
-
-        total_bytes = 0
-        current_threads = Hash.new
-        logs = Hash.new
-        logs_ids = Hash.new
-        scalyr_events = Array.new
-        l_events = Array.new
+      else
+        # If size estimation is disabled we simply append the event and handle splitting later on (if needed)
+        append_event = true
       end
 
       # if we haven't consumed the current event already
@@ -812,6 +822,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
     end_time = Time.now.to_f
     serialization_duration = end_time - start_time
+
+    serialized_request_size = serialized_body.bytesize
+
+    if serialized_request_size >= @max_request_buffer
+      # TODO: If we end up here is estimate config opsion is false, split the request here into multiple ones
+      @logger.warn("Serialized request size (#{serialized_request_size}) is larger than max_request_buffer (#{max_request_buffer})!")
+    end
+
     {
       :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
       :logstash_events => logstash_events

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.22.beta"
+PLUGIN_VERSION = "v0.1.23.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.22.beta'
+  s.version         = '0.1.23.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.22.beta
+  version: 0.1.23.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-08-25 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement