logstash-output-scalyr 0.1.14.beta → 0.1.15.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1af6e37035c37c270036487e9b939f40e8454d39ddea9983a737f409957e0747
-  data.tar.gz: 95438909311aa595d2d4e771424765f3705cdc649225ca81c053e3b9c129b183
+  metadata.gz: b2e4aabe9bf971e1e0638899d713f5f16d7123e9be6f1dd98b82ab7ccfadb63f
+  data.tar.gz: 59ed216e6811e485f48ebed2a8fb3875f31c64cdf0f3a3b6e0a6112b2d8e11b6
 SHA512:
-  metadata.gz: d2a9f542cfaaf8bb9e84ebf976694c548e153e1e170417309644b9206bf4df2ba8844447cbb2b0afd4ba8d03922688f6348a6fb77a6c8489a3f2a49bb873fed9
-  data.tar.gz: 514fab3aa32f39e1ebe6e4bf164a9b9f648b82e7aa861d9cbc2edd6d44d9c6b4d9d3ea9b7c281ae26bf719e4373bde6059f220033dc660821a369b6186180798
+  metadata.gz: 45f6f63a99a7d232ea7cedaaa15ba5a64893d26788701ccfd6140367035771748cf4829a7a856642b704a20524a3d41710c9d9f23cc1daf3fea5b0b50394a626
+  data.tar.gz: 9a24eb6f935f9accd942b6091c33d9c3aca3601e7b833f07516c17a07e8e47660715f3e024748605ae57c40c38f3fc2f507151fa5708f6ab99280d3021c93e07

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Beta
 
+## 0.1.15.beta
+- Only call ``send_status`` method at the end of ``multi_receive()`` if there is at least one
+  record in the batch when ``report_status_for_empty_batches`` config option is set to ``false``.
+- Update ``register()`` method to use a separate short-lived client session for sending initial
+  client status.
+
 ## 0.1.14.beta
 - Add configurable max retries for requests when running into errors.
 - Add ability to send messages to the dead letter queue if we exhaust all retries and if it is configured.

data/lib/logstash/outputs/scalyr.rb

@@ -109,6 +109,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # True to also call send_status when multi_receive() is called with no events.
+  # In some situations (e.g. when logstash is configured with multiple scalyr
+  # plugins conditionally where most are idle) you may want to set this to false
+  config :report_status_for_empty_batches, :validate => :boolean, :default => true
+
   # Set to true to also log status messages with various metrics to stdout in addition to sending
   # this data to Scalyr
   config :log_status_messages_to_stdout, :validate => :boolean, :default => false
@@ -251,7 +256,18 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
-    send_status
+    # We use a special separate short lived client session for sending the initial client status.
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # plugins configured with conditionals and majority of them are inactive (aka receive no data).
+    # This way we don't need to keep idle long running connection open.
+    initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
+        @logger, @add_events_uri,
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
+    )
+    send_status(initial_send_status_client_session)
+    initial_send_status_client_session.close
 
   end # def register
 
@@ -394,7 +410,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      send_status
+      if @report_status_for_empty_batches or records_count > 0
+        send_status
+      end
+
       return result
 
     rescue => e
@@ -778,7 +797,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Finally, note that there could be multiple instances of this plugin (one per worker), in which case each worker
   # thread sends their own status updates.  This is intentional so that we know how much data each worker thread is
   # uploading to Scalyr over time.
-  def send_status
+  def send_status(client_session = nil)
+    client_session = @client_session if client_session.nil?
 
     status_event = {
       :ts => (Time.now.to_f * (10**9)).round,
@@ -797,7 +817,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # echee TODO: get instance stats from session and create a status log line
         msg = 'plugin_status: '
         cnt = 0
-        @client_session.get_stats.each do |k, v|
+        client_session.get_stats.each do |k, v|
           val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
           val = val.nil? ? 0 : val
           msg << ' ' if cnt > 0
@@ -817,7 +837,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
       multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
       begin
-        @client_session.post_add_events(multi_event_request[:body], true, 0)
+        client_session.post_add_events(multi_event_request[:body], true, 0)
       rescue => e
         if e.body
           @logger.warn(

data/lib/scalyr/common/client.rb

@@ -267,6 +267,7 @@ class ClientSession
 
 
   def close
+    @client.close if @client
   end  # def close
 
 

data/lib/scalyr/constants.rb

@@ -1,2 +1,2 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.1.14.beta"
+PLUGIN_VERSION = "v0.1.15.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.14.beta'
+  s.version         = '0.1.15.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Outputs::Scalyr do
       end
 
       it "it doesnt include flatten metrics if flattening is disabled" do
-          plugin1 = LogStash::Outputs::Scalyr.new({
+        plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
@@ -122,7 +122,7 @@ describe LogStash::Outputs::Scalyr do
         expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
-      it "send_stats is called when events list is empty, but otherwise noop" do
+      it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
@@ -137,6 +137,30 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive([])
       end
 
+      it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
+        plugin2 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                     'report_status_for_empty_batches' => false,
+                                                 })
+
+        mock_client_session = MockClientSession.new
+        quantile_estimator = Quantile::Estimator.new
+        plugin2.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin2.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin2).not_to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin2.multi_receive([])
+      end
+
       # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
       it "send_status only sends posts with is_status = true" do
         # 1. Initial send

data/vendor/bundle/jruby/2.5.0/bin/htmldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

data/vendor/bundle/jruby/2.5.0/bin/ldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.14.beta
+  version: 0.1.15.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-22 00:00:00.000000000 Z
+date: 2021-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement