logstash-output-scalyr 0.1.9 → 0.1.10.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 542910023101687e2d9bce1d355f3353a9f9eab2dbc8190ae47c948b17b1db58
-  data.tar.gz: 478acd0ae33f8b102da138ecfebe3a1ecab68f7d4b2d3779695812184e7a5618
+  metadata.gz: 290310d36108a3311f55a47efbb3a73e01f19ccdf025a15590f86493459963da
+  data.tar.gz: a5bf9f872c496fa65670cab36e0955f1417541db5f151e11f0d914abe5fc9920
 SHA512:
-  metadata.gz: 60777fcaababf8cc08559a582380fba24fa5a506a4751a0bb63e2a46957723dd4cbaf8c232ccb0ccfa91694c9723c24c054e708e27f59476c4734b58e1cd8269
-  data.tar.gz: f0f19ab3392f2d7a8f463b6c74b6e204024701502a86640bbfa3a207554ff69a25f53e36075aa14f65561f3187e2c24b0ae1db2f6ec7ac0630e3c83232a12ce4
+  metadata.gz: 35d60d700b6bb6ccc43aea5151d07c306b29c5524ac8d80145ae792263ffe5e7264d22a59de3a21de9aa549c5963b4cd3c9b230b845f67c6ff9618faca863cad
+  data.tar.gz: 41cf50b968a9a11efff957cda768cf9d05e1ccf3c36e4f4d8f12c3d1aca9fb9da049647432bf047d51c2b6fe33f7b711484632d108dcd390ca852bc45aa7a00d

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Beta
 
+## 0.1.10.beta
+
+- Switch to shared concurrency to allow the use of multiple worker threads for increased
+  throughput.
+- Switch HTTP client library to `manticore` to work better with new shared concurrency.
+
 ## 0.1.9
 
 - Add support for logging status messages with metrics to stdout in addition to sending this

data/Gemfile

@@ -13,3 +13,4 @@ end
 gem 'pry'
 gem 'pry-nav'
 gem 'quantile'
+gem 'manticore', platform: :jruby

data/lib/logstash/outputs/scalyr.rb

@@ -8,9 +8,7 @@ require "thread" # for safe queueing
 require "uri" # for escaping user input
 require 'json' # for converting event object to JSON for upload
 
-require 'net/http'
-require 'net/http/persistent'
-require 'net/https'
+require 'manticore'
 require 'rbzip2'
 require 'zlib'
 require 'stringio'
@@ -27,9 +25,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   config_name "scalyr"
 
-  # For correctness reasons we need to limit this plugin to a single worker, a single worker will be single concurrency
-  # anyway but we should be explicit.
-  concurrency :single
+  concurrency :shared
 
   # The Scalyr API write token, these are available at https://www.scalyr.com/keys.  This is the only compulsory configuration field required for proper upload
   config :api_write_token, :validate => :string, :required => true
@@ -37,12 +33,6 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
   config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
 
-  # Path to SSL bundle file.
-  config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-bundle.crt"
-
-  # If we should append our built-in Scalyr cert to the one we find at `ssl_ca_bundle_path`.
-  config :append_builtin_cert, :validate => :boolean, :default => true
-
   # server_attributes is a dictionary of key value pairs that represents/identifies the logstash aggregator server
   # (where this plugin is running).  Keys are arbitrary except for the 'serverHost' key which holds special meaning to
   # Scalyr and is given special treatment in the Scalyr UI.  All of these attributes are optional (not required for logs
@@ -91,9 +81,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Set max interval in seconds between bulk retries.
   config :retry_max_interval, :validate => :number, :default => 64
 
-  # The following two settings pertain to preventing Man-in-the-middle (MITM) attacks  # echee TODO: eliminate?
+  # Whether or not to verify the connection to Scalyr, only set to false for debugging.
   config :ssl_verify_peer, :validate => :boolean, :default => true
-  config :ssl_verify_depth, :validate => :number, :default => 5
+
+  # Path to SSL bundle file.
+  config :ssl_ca_bundle_path, :validate => :string, :default => "/etc/ssl/certs/ca-bundle.crt"
+
+  # If we should append our built-in Scalyr cert to the one we find at `ssl_ca_bundle_path`.
+  config :append_builtin_cert, :validate => :boolean, :default => true
 
   config :max_request_buffer, :validate => :number, :default => 5500000  # echee TODO: eliminate?
   config :force_message_encoding, :validate => :string, :default => nil
@@ -132,6 +127,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
   # these stas this might be wanted or not.
   config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
+  
+  # Causes this plugin to act as if it successfully uploaded the logs, while actually returning as quickly as possible
+  # after no work being done.
+  config :noop_mode, :validate => :boolean, :default => false
+
+  # Manticore related options
+  config :http_connect_timeout, :validate => :number, :default => 10
+  config :http_socket_timeout, :validate => :number, :default => 10
+  config :http_request_timeout, :validate => :number, :default => 60
+  config :http_pool_max, :validate => :number, :default => 50
+  config :http_pool_max_per_route, :validate => :number, :default => 25
+
+  def initialize(*params)
+    super
+    # Request statistics are accumulated across multiple threads and must be accessed through a mutex
+    @stats_lock = Mutex.new
+    @send_stats = Mutex.new
+  end
 
   def close
     @running = false
@@ -140,6 +153,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   public
   def register
+    # This prng is used exclusively to determine when to sample statistics and no security related purpose, for this
+    # reason we do not ensure thread safety for it.
     @prng = Random.new
 
     if @event_metrics_sample_rate < 0 or @event_metrics_sample_rate > 1
@@ -223,9 +238,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @running = true
     @client_session = Scalyr::Common::Client::ClientSession.new(
         @logger, @add_events_uri,
-        @compression_type, @compression_level,
-        @ssl_verify_peer, @ssl_ca_bundle_path, @ssl_verify_depth,
-        @append_builtin_cert, @record_stats_for_status, @flush_quantile_estimates_on_status_send
+        @compression_type, @compression_level, @ssl_verify_peer, @ssl_ca_bundle_path, @append_builtin_cert,
+        @record_stats_for_status, @flush_quantile_estimates_on_status_send,
+        @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
 
     @logger.info("Started Scalyr output plugin", :class => self.class.name)
@@ -257,6 +272,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   #
   public
   def multi_receive(events)
+    # Just return and pretend we did something if running in noop mode
+    return events if @noop_mode
+
     start_time = Time.now.to_f
 
     multi_event_request_array = build_multi_event_request_array(events)
@@ -276,6 +294,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         # ignore these.
         if !multi_event_request.nil?
           @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
+
           sleep_interval = 0
           result.push(multi_event_request)
         end
@@ -306,7 +325,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
         if e.is_commonly_retried?
           # well-known retriable errors should be debug
-          @logger.debug(message, exc_data)
+          @logger.error(message, exc_data)
         else
           # all other failed uploads should be errors
           @logger.error(message, exc_data)
@@ -328,9 +347,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
 
     if records_count > 0
-      @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
-      @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
-      @plugin_metrics[:multi_receive_event_count].observe(records_count)
+      @stats_lock.synchronize do
+        @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+        @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
+        @plugin_metrics[:multi_receive_event_count].observe(records_count)
+      end
     end
 
     send_status
@@ -501,10 +522,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
 
       if should_sample_event_metrics
-        @plugin_metrics[:event_attributes_count].observe(record.count)
+        @stats_lock.synchronize do
+          @plugin_metrics[:event_attributes_count].observe(record.count)
 
-        if @flatten_nested_values
-          @plugin_metrics[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
+          if @flatten_nested_values
+            @plugin_metrics[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
+          end
         end
       end
 
@@ -598,7 +621,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   def create_multi_event_request(scalyr_events, current_threads, current_logs)
 
     body = {
-      :session => @session_id,
+      :session => @session_id + Thread.current.object_id.to_s,
       :token => @api_write_token,
       :events => scalyr_events,
     }
@@ -638,33 +661,35 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   # Retrieve batch and other event level metric values
   def get_stats
-    current_stats = @multi_receive_statistics.clone
+    @stats_lock.synchronize do
+      current_stats = @multi_receive_statistics.clone
 
-    current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
-    current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
-    current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)
+      current_stats[:multi_receive_duration_p50] = @plugin_metrics[:multi_receive_duration_secs].query(0.5)
+      current_stats[:multi_receive_duration_p90] = @plugin_metrics[:multi_receive_duration_secs].query(0.9)
+      current_stats[:multi_receive_duration_p99] = @plugin_metrics[:multi_receive_duration_secs].query(0.99)
 
-    current_stats[:multi_receive_event_count_p50] = @plugin_metrics[:multi_receive_event_count].query(0.5)
-    current_stats[:multi_receive_event_count_p90] = @plugin_metrics[:multi_receive_event_count].query(0.9)
-    current_stats[:multi_receive_event_count_p99] = @plugin_metrics[:multi_receive_event_count].query(0.99)
+      current_stats[:multi_receive_event_count_p50] = @plugin_metrics[:multi_receive_event_count].query(0.5)
+      current_stats[:multi_receive_event_count_p90] = @plugin_metrics[:multi_receive_event_count].query(0.9)
+      current_stats[:multi_receive_event_count_p99] = @plugin_metrics[:multi_receive_event_count].query(0.99)
 
-    current_stats[:event_attributes_count_p50] = @plugin_metrics[:event_attributes_count].query(0.5)
-    current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
-    current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
+      current_stats[:event_attributes_count_p50] = @plugin_metrics[:event_attributes_count].query(0.5)
+      current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
+      current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
 
-    if @flatten_nested_values
-      # We only return those metrics in case flattening is enabled
-      current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
-      current_stats[:flatten_values_duration_secs_p90] = @plugin_metrics[:flatten_values_duration_secs].query(0.9)
-      current_stats[:flatten_values_duration_secs_p99] = @plugin_metrics[:flatten_values_duration_secs].query(0.99)
-    end
+      if @flatten_nested_values
+        # We only return those metrics in case flattening is enabled
+        current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
+        current_stats[:flatten_values_duration_secs_p90] = @plugin_metrics[:flatten_values_duration_secs].query(0.9)
+        current_stats[:flatten_values_duration_secs_p99] = @plugin_metrics[:flatten_values_duration_secs].query(0.99)
+      end
 
-    if @flush_quantile_estimates_on_status_send
-      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
-      @plugin_metrics = get_new_metrics
-    end
+      if @flush_quantile_estimates_on_status_send
+        @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+        @plugin_metrics = get_new_metrics
+      end
 
-    current_stats
+      current_stats
+    end
   end
 
 
@@ -683,33 +708,34 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         'plugin_id' => self.id,
       }
     }
-
-    if !@last_status_transmit_time
-      status_event[:attrs]['message'] = "Started Scalyr LogStash output plugin."
-      status_event[:attrs]['serverHost'] = @node_hostname
-    else
-      cur_time = Time.now()
-      return if (cur_time.to_i - @last_status_transmit_time.to_i) < @status_report_interval
-      # echee TODO: get instance stats from session and create a status log line
-      msg = 'plugin_status: '
-      cnt = 0
-      @client_session.get_stats.each do |k, v|
-        val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
-        val = val.nil? ? 0 : val
-        msg << ' ' if cnt > 0
-        msg << "#{k.to_s}=#{val}"
-        cnt += 1
-      end
-      get_stats.each do |k, v|
-        val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
-        val = val.nil? ? 0 : val
-        msg << ' ' if cnt > 0
-        msg << "#{k.to_s}=#{val}"
-        cnt += 1
+    @send_stats.synchronize do
+      if !@last_status_transmit_time
+        status_event[:attrs]['message'] = "Started Scalyr LogStash output plugin."
+        status_event[:attrs]['serverHost'] = @node_hostname
+      else
+        cur_time = Time.now()
+        return if (cur_time.to_i - @last_status_transmit_time.to_i) < @status_report_interval
+        # echee TODO: get instance stats from session and create a status log line
+        msg = 'plugin_status: '
+        cnt = 0
+        @client_session.get_stats.each do |k, v|
+          val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
+          val = val.nil? ? 0 : val
+          msg << ' ' if cnt > 0
+          msg << "#{k.to_s}=#{val}"
+          cnt += 1
+        end
+        get_stats.each do |k, v|
+          val = v.instance_of?(Float) ? sprintf("%.4f", v) : v
+          val = val.nil? ? 0 : val
+          msg << ' ' if cnt > 0
+          msg << "#{k.to_s}=#{val}"
+          cnt += 1
+        end
+        status_event[:attrs]['message'] = msg
+        status_event[:attrs]['serverHost'] = @node_hostname
+        status_event[:attrs]['parser'] = @status_parser
       end
-      status_event[:attrs]['message'] = msg
-      status_event[:attrs]['serverHost'] = @node_hostname
-      status_event[:attrs]['parser'] = @status_parser
     end
     multi_event_request = create_multi_event_request([status_event], nil, nil)
     @client_session.post_add_events(multi_event_request[:body], true, 0)

data/lib/scalyr/common/client.rb

@@ -52,8 +52,9 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert,
-                 record_stats_for_status, flush_quantile_estimates_on_status_send)
+                 ssl_verify_peer, ssl_ca_bundle_path, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send,
+                 connect_timeout, socket_timeout, request_timeout, pool_max, pool_max_per_route)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
@@ -61,9 +62,13 @@ class ClientSession
     @ssl_verify_peer = ssl_verify_peer
     @ssl_ca_bundle_path = ssl_ca_bundle_path
     @append_builtin_cert = append_builtin_cert
-    @ssl_verify_depth = ssl_verify_depth
     @record_stats_for_status = record_stats_for_status
     @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
+    @connect_timeout = connect_timeout
+    @socket_timeout = socket_timeout
+    @request_timeout = request_timeout
+    @pool_max = pool_max
+    @pool_max_per_route = pool_max_per_route
 
     # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
     @cert_string = "" \
@@ -126,11 +131,29 @@ class ClientSession
         :compression_type => @compression_type,
         :compression_level => @compression_level,
     }
+  end  # def initialize
 
-    @http = Net::HTTP::Persistent.new
+  def client_config
+    # TODO: Eventually expose some more of these as config options, though nothing here really needs tuning normally
+    # besides SSL
+    c = {
+      connect_timeout: @connect_timeout,
+      socket_timeout: @socket_timeout,
+      request_timeout: @request_timeout,
+      follow_redirects: true,
+      automatic_retries: 1,
+      retry_non_idempotent: false,
+      check_connection_timeout: 200,
+      pool_max: @pool_max,
+      pool_max_per_route: @pool_max_per_route,
+      cookies: true,
+      keepalive: true,
+      ssl: {}
+    }
 
     # verify peers to prevent potential MITM attacks
     if @ssl_verify_peer
+      c[:ssl][:verify] = :strict
       @ca_cert = Tempfile.new("ca_cert")
       if File.file?(@ssl_ca_bundle_path)
         @ca_cert.write(File.read(@ssl_ca_bundle_path))
@@ -142,13 +165,17 @@ class ClientSession
         end
       end
       @ca_cert.flush
-      @http.ca_file = @ca_cert.path
-      @http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      @http.verify_depth = @ssl_verify_depth
+      c[:ssl][:ca_file] = @ca_cert.path
     else
-      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      c[:ssl][:verify] = :disable
     end
-  end  # def initialize
+
+    c
+  end
+
+  def client
+    @client ||= Manticore::Client.new(client_config)
+  end
 
   # Convenience method to create a fresh quantile estimator
   def get_new_latency_stats
@@ -163,60 +190,56 @@ class ClientSession
 
   # Get a clone of current statistics hash and calculate percentiles
   def get_stats
-    current_stats = @stats.clone
-
-    current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
-    current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
-    current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
-    current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
-    current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
-    current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
-    current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
-    current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
-    current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
-    current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
-    current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
-    current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
-
-    if @flush_quantile_estimates_on_status_send
-      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
-      @latency_stats = get_new_latency_stats
+    @stats_lock.synchronize do
+      current_stats = @stats.clone
+
+      current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+      current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+      current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+      current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+      current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+      current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+      current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+      current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+      current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+      current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+      current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+      current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+      if @flush_quantile_estimates_on_status_send
+        @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+        @latency_stats = get_new_latency_stats
+      end
+      current_stats
     end
-    current_stats
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
   def post_add_events(body, is_status, body_serialization_duration = 0)
-    post, compression_duration = prepare_post_object @add_events_uri.path, body
+    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
     compressed_bytes_sent = 0
     bytes_received = 0
     begin
-      response = @http.request(@add_events_uri, post)
+      response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
       handle_response(response)
 
       fail_count -= 1  # success means we negate the putative failure
       uncompressed_bytes_sent = (body.bytesize + @add_events_uri.path.bytesize)
-      compressed_bytes_sent = (post.body.bytesize + @add_events_uri.path.bytesize)
+      compressed_bytes_sent = (post_body.bytesize + @add_events_uri.path.bytesize)
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
+    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
+    # way to detect that it is from SSL.
     rescue OpenSSL::SSL::SSLError => e
-      if @ssl_verify_peer and @ssl_ca_bundle_path.nil? and !File.file?(@ca_cert.path)
-        @ca_cert = Tempfile.new("ca_cert")
-        @ca_cert.write(@cert_string)
-        @ca_cert.flush
-        @http.ca_file = @ca_cert.path
-        raise ClientError.new("Packaged certificate appears to have been deleted, writing a new one.", @add_events_uri)
-      else
-        raise e
-      end
+      raise e
 
-    rescue Net::HTTP::Persistent::Error => e
+    rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
       raise ClientError.new(e.message, @add_events_uri)
@@ -245,7 +268,6 @@ class ClientSession
 
 
   def close
-    @http.shutdown
   end  # def close
 
 
@@ -273,18 +295,20 @@ class ClientSession
       compression_duration = end_time - start_time
     end
 
-    post = Net::HTTP::Post.new uri_path
-    post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.9'
-    post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
+    version = 'output-logstash-scalyr 0.1.10.beta'
+    post_headers = {
+      'Content-Type': 'application/json',
+      'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM
+    }
 
+    post_body = nil
     if not encoding.nil?
-      post.add_field('Content-Encoding', encoding)
-      post.body = compressed_body
+      post_headers['Content-Encoding'] = encoding
+      post_body = compressed_body
     else
-      post.body = body
+      post_body = body
     end
-    return post, compression_duration
+    return post_body, post_headers, compression_duration
   end  # def prepare_post_object
 
 

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.9'
+  s.version         = '0.1.10.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -24,7 +24,7 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
         end
       end
 
@@ -38,7 +38,7 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -65,7 +65,7 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "hostname \"invalid.mitm.should.fail.test.agent.scalyr.com\" does not match the server certificate")
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)")
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`

data/spec/logstash/outputs/scalyr_spec.rb

@@ -78,6 +78,7 @@ describe LogStash::Outputs::Scalyr do
         mock_client_session = MockClientSession.new
         plugin1.instance_variable_set(:@last_status_transmit_time, 100)
         plugin1.instance_variable_set(:@client_session, mock_client_session)
+        plugin1.instance_variable_set(:@session_id, "some_session_id")
         plugin1.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
@@ -95,6 +96,7 @@ describe LogStash::Outputs::Scalyr do
         # 1. Initial send
         plugin.instance_variable_set(:@last_status_transmit_time, nil)
         plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@session_id, "some_session_id")
         status_event = plugin.send_status
         expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
 
@@ -138,6 +140,7 @@ describe LogStash::Outputs::Scalyr do
         # 1. Initial send
         plugin.instance_variable_set(:@last_status_transmit_time, nil)
         plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@session_id, "some_session_id")
         expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything)
         plugin.send_status
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-07 00:00:00.000000000 Z
+date: 2021-06-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -3745,9 +3745,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10