logstash-output-scalyr 0.1.8 → 0.1.13

data/lib/scalyr/common/client.rb

@@ -1,3 +1,5 @@
+require "scalyr/constants"
+
 module Scalyr; module Common; module Client
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -52,8 +54,9 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert,
-                 record_stats_for_status, flush_quantile_estimates_on_status_send)
+                 ssl_verify_peer, ssl_ca_bundle_path, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send,
+                 connect_timeout, socket_timeout, request_timeout, pool_max, pool_max_per_route)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
@@ -61,9 +64,13 @@ class ClientSession
     @ssl_verify_peer = ssl_verify_peer
     @ssl_ca_bundle_path = ssl_ca_bundle_path
     @append_builtin_cert = append_builtin_cert
-    @ssl_verify_depth = ssl_verify_depth
     @record_stats_for_status = record_stats_for_status
     @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
+    @connect_timeout = connect_timeout
+    @socket_timeout = socket_timeout
+    @request_timeout = request_timeout
+    @pool_max = pool_max
+    @pool_max_per_route = pool_max_per_route
 
     # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
     @cert_string = "" \
@@ -126,11 +133,29 @@ class ClientSession
         :compression_type => @compression_type,
         :compression_level => @compression_level,
     }
+  end  # def initialize
 
-    @http = Net::HTTP::Persistent.new
+  def client_config
+    # TODO: Eventually expose some more of these as config options, though nothing here really needs tuning normally
+    # besides SSL
+    c = {
+      connect_timeout: @connect_timeout,
+      socket_timeout: @socket_timeout,
+      request_timeout: @request_timeout,
+      follow_redirects: true,
+      automatic_retries: 1,
+      retry_non_idempotent: false,
+      check_connection_timeout: 200,
+      pool_max: @pool_max,
+      pool_max_per_route: @pool_max_per_route,
+      cookies: true,
+      keepalive: true,
+      ssl: {}
+    }
 
     # verify peers to prevent potential MITM attacks
     if @ssl_verify_peer
+      c[:ssl][:verify] = :strict
       @ca_cert = Tempfile.new("ca_cert")
       if File.file?(@ssl_ca_bundle_path)
         @ca_cert.write(File.read(@ssl_ca_bundle_path))
@@ -142,13 +167,17 @@ class ClientSession
         end
       end
       @ca_cert.flush
-      @http.ca_file = @ca_cert.path
-      @http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      @http.verify_depth = @ssl_verify_depth
+      c[:ssl][:ca_file] = @ca_cert.path
     else
-      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      c[:ssl][:verify] = :disable
     end
-  end  # def initialize
+
+    c
+  end
+
+  def client
+    @client ||= Manticore::Client.new(client_config)
+  end
 
   # Convenience method to create a fresh quantile estimator
   def get_new_latency_stats
@@ -163,60 +192,56 @@ class ClientSession
 
   # Get a clone of current statistics hash and calculate percentiles
   def get_stats
-    current_stats = @stats.clone
-
-    current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
-    current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
-    current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
-    current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
-    current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
-    current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
-    current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
-    current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
-    current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
-    current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
-    current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
-    current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
-
-    if @flush_quantile_estimates_on_status_send
-      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
-      @latency_stats = get_new_latency_stats
+    @stats_lock.synchronize do
+      current_stats = @stats.clone
+
+      current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+      current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+      current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+      current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+      current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+      current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+      current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+      current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+      current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+      current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+      current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+      current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+      if @flush_quantile_estimates_on_status_send
+        @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+        @latency_stats = get_new_latency_stats
+      end
+      current_stats
     end
-    current_stats
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
   def post_add_events(body, is_status, body_serialization_duration = 0)
-    post, compression_duration = prepare_post_object @add_events_uri.path, body
+    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
     compressed_bytes_sent = 0
     bytes_received = 0
     begin
-      response = @http.request(@add_events_uri, post)
+      response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
       handle_response(response)
 
       fail_count -= 1  # success means we negate the putative failure
       uncompressed_bytes_sent = (body.bytesize + @add_events_uri.path.bytesize)
-      compressed_bytes_sent = (post.body.bytesize + @add_events_uri.path.bytesize)
+      compressed_bytes_sent = (post_body.bytesize + @add_events_uri.path.bytesize)
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
+    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
+    # way to detect that it is from SSL.
     rescue OpenSSL::SSL::SSLError => e
-      if @ssl_verify_peer and @ssl_ca_bundle_path.nil? and !File.file?(@ca_cert.path)
-        @ca_cert = Tempfile.new("ca_cert")
-        @ca_cert.write(@cert_string)
-        @ca_cert.flush
-        @http.ca_file = @ca_cert.path
-        raise ClientError.new("Packaged certificate appears to have been deleted, writing a new one.", @add_events_uri)
-      else
-        raise e
-      end
+      raise e
 
-    rescue Net::HTTP::Persistent::Error => e
+    rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
       raise ClientError.new(e.message, @add_events_uri)
@@ -245,7 +270,6 @@ class ClientSession
 
 
   def close
-    @http.shutdown
   end  # def close
 
 
@@ -273,18 +297,20 @@ class ClientSession
       compression_duration = end_time - start_time
     end
 
-    post = Net::HTTP::Post.new uri_path
-    post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.8'
-    post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
+    version = sprintf('output-logstash-scalyr %s' % [PLUGIN_VERSION])
+    post_headers = {
+      'Content-Type': 'application/json',
+      'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM
+    }
 
+    post_body = nil
     if not encoding.nil?
-      post.add_field('Content-Encoding', encoding)
-      post.body = compressed_body
+      post_headers['Content-Encoding'] = encoding
+      post_body = compressed_body
     else
-      post.body = body
+      post_body = body
     end
-    return post, compression_duration
+    return post_body, post_headers, compression_duration
   end  # def prepare_post_object
 
 

data/lib/scalyr/common/util.rb

@@ -45,5 +45,12 @@ def self.flatten(obj, delimiter='_')
   return result
 end
 
+def self.truncate(content, max)
+  if content.length > max
+    return "#{content[0...(max-3)]}..."
+  end
+  return content
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+PLUGIN_VERSION = "v0.1.13"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.8'
+  s.version         = '0.1.13'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -4,27 +4,63 @@ require "logstash/outputs/scalyr"
 require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
+require 'webmock/rspec'
+WebMock.allow_net_connect!
 
 describe LogStash::Outputs::Scalyr do
+  let(:sample_events) {
+    events = []
+    for i in 1..3 do
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+    events
+  }
 
   describe "#ssl_tests" do
       context "with default SSL configuration" do
         it "throws a ServerError due to fake api key" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :code=>401,
+                  :message=>"error/client/badParam",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
         end
       end
 
       context "when pointing at a location without any valid certs and not using builtin" do
         it "throws an SSLError" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
         end
       end
 
@@ -34,11 +70,21 @@ describe LogStash::Outputs::Scalyr do
           `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -61,11 +107,21 @@ describe LogStash::Outputs::Scalyr do
           `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(OpenSSL::SSL::SSLError, "hostname \"invalid.mitm.should.fail.test.agent.scalyr.com\" does not match the server certificate")
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -74,4 +130,88 @@ describe LogStash::Outputs::Scalyr do
         end
       end
   end
+
+  describe "response_handling_tests" do
+    context "when receiving a 503 response" do
+      it "don't throw an error but do log one to debug" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 503, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:debug)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :batch_num=>1,
+            :code=>503,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
+          }
+        )
+      end
+    end
+
+    context "when receiving a 500 response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
+          }
+        )
+      end
+    end
+
+    context "when receiving a long non-json response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "0123456789" * 52, headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>("0123456789" * 50) + "012345678..."
+          }
+        )
+      end
+    end
+  end
+
 end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -78,25 +78,28 @@ describe LogStash::Outputs::Scalyr do
         mock_client_session = MockClientSession.new
         plugin1.instance_variable_set(:@last_status_transmit_time, 100)
         plugin1.instance_variable_set(:@client_session, mock_client_session)
+        plugin1.instance_variable_set(:@session_id, "some_session_id")
         plugin1.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
         plugin1.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(1)
         plugin1.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
 
         status_event = plugin1.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.500 total_compression_duration_secs=10.200 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0")
       end
 
       it "returns and sends correct status event on send_stats on initial and subsequent send" do
         # 1. Initial send
         plugin.instance_variable_set(:@last_status_transmit_time, nil)
         plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@session_id, "some_session_id")
         status_event = plugin.send_status
-        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
+        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin (%s)." % [PLUGIN_VERSION])
 
         # 2. Second send
         plugin.instance_variable_set(:@last_status_transmit_time, 100)
@@ -106,7 +109,8 @@ describe LogStash::Outputs::Scalyr do
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
 
         (1..20).each do |n|
@@ -115,7 +119,7 @@ describe LogStash::Outputs::Scalyr do
 
         plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
         status_event = plugin.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.500 total_compression_duration_secs=10.200 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
       it "send_stats is called when events list is empty, but otherwise noop" do
@@ -138,6 +142,7 @@ describe LogStash::Outputs::Scalyr do
         # 1. Initial send
         plugin.instance_variable_set(:@last_status_transmit_time, nil)
         plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@session_id, "some_session_id")
         expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything)
         plugin.send_status
 
@@ -148,7 +153,8 @@ describe LogStash::Outputs::Scalyr do
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
         (1..20).each do |n|
           plugin.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(n)