logstash-output-scalyr 0.1.2 → 0.1.7

data/lib/scalyr/common/client.rb

@@ -52,17 +52,64 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth)
+                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
     @compression_level = compression_level
     @ssl_verify_peer = ssl_verify_peer
     @ssl_ca_bundle_path = ssl_ca_bundle_path
+    @append_builtin_cert = append_builtin_cert
     @ssl_verify_depth = ssl_verify_depth
+    @record_stats_for_status = record_stats_for_status
+    @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
+
+    # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
+    @cert_string = "" \
+        "-----BEGIN CERTIFICATE-----\n" \
+        "MIIG6zCCBNOgAwIBAgIJAM5aknNWtN6oMA0GCSqGSIb3DQEBCwUAMIGpMQswCQYD\n" \
+        "VQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEXMBUGA1UEBxMOUG9ydG9sYSBW\n" \
+        "YWxsZXkxEzARBgNVBAoTClNjYWx5ciBJbmMxFTATBgNVBAsTDFNjYWx5ciBBZ2Vu\n" \
+        "dDEdMBsGA1UEAxMUU2NhbHlyIEFnZW50IENBIFJvb3QxITAfBgkqhkiG9w0BCQEW\n" \
+        "EmNvbnRhY3RAc2NhbHlyLmNvbTAeFw0xNDA5MDkyMTUyMDVaFw0yNDA5MDYyMTUy\n" \
+        "MDVaMIGpMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEXMBUGA1UE\n" \
+        "BxMOUG9ydG9sYSBWYWxsZXkxEzARBgNVBAoTClNjYWx5ciBJbmMxFTATBgNVBAsT\n" \
+        "DFNjYWx5ciBBZ2VudDEdMBsGA1UEAxMUU2NhbHlyIEFnZW50IENBIFJvb3QxITAf\n" \
+        "BgkqhkiG9w0BCQEWEmNvbnRhY3RAc2NhbHlyLmNvbTCCAiIwDQYJKoZIhvcNAQEB\n" \
+        "BQADggIPADCCAgoCggIBALdNamcMNVxkIB6qVWmNCi1jeyeqOX00rYAWDlyBHff7\n" \
+        "vU833Evuixgrf0HxrOQNiPsOK66ehG6LfJd2UIBDEHBCXRo+aeFQLrCLIVXiqJ2W\n" \
+        "Tvl7dUU9d7zfw/XXif3lMQTiyQAWYTyjfugDczEScEUk93EWFfW47j9PTGh96yKm\n" \
+        "nVbfOxD4XbN0ykdo85cs7M/NOHQj4q34l77XGXrit+nb1cL3wS9ZzJG8s40J2+Dp\n" \
+        "LUA8KBQuvim6hfqrjaDX0bXVvc52a7TSh/zb58gkLbiqvBuPo5P8PBLHCx8bJtZu\n" \
+        "fjWRdjaftgw7CcsdIuMhbm3823WI/A+/p4s1B5KOPqOYRkgG8FBqFIRTecKAV5wC\n" \
+        "Z2ruTytoOUBWItrheyJhm+99X1I2y/6mdecBdk7j3+8U+nCsGHkH5Jwjl2BH9tfT\n" \
+        "RUhVTCQs25XLNm41kZo7xK464xZsJKHXj9jr5gLIdF6CgzU2uYsQHKcw1pAVITLe\n" \
+        "bfGEob8AcL0E7+1hurRjyYxtxZpsZeGMwI0/BStT+fLEAOJ1byGUgSUbhi9lJ8Hc\n" \
+        "+NZDfaCaCZKRxjePCqeWjZUUdVoH3fNSi2GuNLqtOFzxlkP5tBErnXufE6XZAtEQ\n" \
+        "lv/9qxa4ZLsvhbt+6qQryIAHL4aReh/VReER438ARdwG2QDK+vRfhNpke69em5Kb\n" \
+        "AgMBAAGjggESMIIBDjAdBgNVHQ4EFgQUENX6MjnzqTJdTQMAEakSdXV/I80wgd4G\n" \
+        "A1UdIwSB1jCB04AUENX6MjnzqTJdTQMAEakSdXV/I82hga+kgawwgakxCzAJBgNV\n" \
+        "BAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRcwFQYDVQQHEw5Qb3J0b2xhIFZh\n" \
+        "bGxleTETMBEGA1UEChMKU2NhbHlyIEluYzEVMBMGA1UECxMMU2NhbHlyIEFnZW50\n" \
+        "MR0wGwYDVQQDExRTY2FseXIgQWdlbnQgQ0EgUm9vdDEhMB8GCSqGSIb3DQEJARYS\n" \
+        "Y29udGFjdEBzY2FseXIuY29tggkAzlqSc1a03qgwDAYDVR0TBAUwAwEB/zANBgkq\n" \
+        "hkiG9w0BAQsFAAOCAgEAmmgm1AeO7wfuR36HHgpZCKZxboRFwc2FzKaHNSg2FQ0G\n" \
+        "MuOP6HZUQWsaXLe0Kc8emJKrIwrn6x2jMm19Bjbps2bPW6ao6UE/6fb5Z7CX82IX\n" \
+        "pKlDDH6OfYjDplBzoqf5PkPgxZNyiZ7nyNUWz+P2vesLFVynmej2MvLIZVnEJ2Wp\n" \
+        "xzyHMKQo92DP8yNEudoK8QQpoLcuNcAli9blt8+NIV9RSDrI9CvArLNpZJMlS1Vx\n" \
+        "gdzEU3wEQYWc36j3XCsp7ZDvgTm6FpyHS5ccMpXR1E62tVINGX9r+97ZHyxjqurb\n" \
+        "606y1FzV/5Mf/aihPYSSreq63UVqdsaQfyS77Q4tpJofq875w8nd2Vs3guDs2T0h\n" \
+        "1bOlV3e2HfglWsHKwNguQZo2nfMUp11IYfV/HOKWNQkbrPhuayXMi3i2wCZe9JNt\n" \
+        "P9uZ2OjzsVu2QFcSlvZF6y02/bjbNATRfj/J/SHNFyCDu6bXhtAu0yZzFLiOZxjD\n" \
+        "LwzunBMoWcJj+P2Vx3OhbE9FMyMeKdOWdTgiI1GLEkfJi6s7d/tk1ayLmbBTRD/e\n" \
+        "XkjSeLBss6mA1INuE1+gKVA4MABsUiLqGZ8xCPN16CyPcTqL2TJFo1IOqivMxKDh\n" \
+        "H4Z/mHoGi5SRnye+Wo+jyiQiWjJQ5LrlQPbHmuO0tLs9lM1t9nhzLifzga5F4+o=\n" \
+        "-----END CERTIFICATE-----"
 
     # Request statistics are accumulated across multiple threads and must be accessed through a mutex
     @stats_lock = Mutex.new
+    @latency_stats = get_new_latency_stats
     @stats = {
         :total_requests_sent => 0, # The total number of RPC requests sent.
         :total_requests_failed => 0, # The total number of RPC requests that failed.
@@ -73,37 +120,90 @@ class ClientSession
         # can be calculated by dividing this number by @total_requests_sent).
         # This includes connection establishment time.
         :total_connections_created => 0, # The total number of HTTP connections successfully created.
+        :total_serialization_duration_secs => 0, # The total duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_compression_duration_secs => 0, # The total duration (in seconds) it took to compress all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_flatten_values_duration_secs => 0, # The total duration (in seconds) it took to flatten nested record values.
+        # In case flattening is disabled, this value will always be 0. Can infer average per-request value by dividing this
+        # value by total_requests_sent
+        :compression_type => @compression_type,
+        :compression_level => @compression_level,
     }
 
     @http = Net::HTTP::Persistent.new
 
     # verify peers to prevent potential MITM attacks
     if @ssl_verify_peer
-      @http.ca_file = @ssl_ca_bundle_path
+      @ca_cert = Tempfile.new("ca_cert")
+      if File.file?(@ssl_ca_bundle_path)
+        @ca_cert.write(File.read(@ssl_ca_bundle_path))
+        @ca_cert.flush
+      end
+      if @append_builtin_cert
+        open(@ca_cert.path, 'a') do |f|
+          f.puts @cert_string
+        end
+      end
+      @ca_cert.flush
+      @http.ca_file = @ca_cert.path
       @http.verify_mode = OpenSSL::SSL::VERIFY_PEER
       @http.verify_depth = @ssl_verify_depth
+    else
+      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
     end
   end  # def initialize
 
-
+  # Convenience method to create a fresh quantile estimator
+  def get_new_latency_stats
+    return {
+      # The total number of HTTP connections successfully created.
+      :serialization_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+      :compression_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to compress all the request bodies.
+      :flatten_values_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to flatten nested record values.
+      # In case flattening is disabled, this value will always be 0.
+      :request_latency_secs => Quantile::Estimator.new, #  Secs spent waiting for a responses. This includes connection establishment time.
+      :bytes_sent => Quantile::Estimator.new  # The number of bytes sent over the network. Batch size with a bit more overhead.
+    }
+  end
 
   # Get a clone of current statistics hash
   def get_stats
-    @stats.clone
+    current_stats = @stats.clone
+
+    current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+    current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+    current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+    current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+    current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+    current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+    current_stats[:flatten_values_duration_secs_p50] = @latency_stats[:flatten_values_duration_secs].query(0.5)
+    current_stats[:flatten_values_duration_secs_p90] = @latency_stats[:flatten_values_duration_secs].query(0.9)
+    current_stats[:flatten_values_duration_secs_p99] = @latency_stats[:flatten_values_duration_secs].query(0.99)
+    current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+    current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+    current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+    current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+    current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+    current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+    if @flush_quantile_estimates_on_status_send
+      @latency_stats = get_new_latency_stats
+    end
+    current_stats
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
-  def post_add_events(body)
-    post = prepare_post_object @add_events_uri.path, body
+  def post_add_events(body, is_status, body_serialization_duration = 0, flatten_nested_values_duration = 0)
+    post, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
     compressed_bytes_sent = 0
     bytes_received = 0
     begin
-
       response = @http.request(@add_events_uri, post)
       handle_response(response)
 
@@ -113,22 +213,42 @@ class ClientSession
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
+    rescue OpenSSL::SSL::SSLError => e
+      if @ssl_verify_peer and @ssl_ca_bundle_path.nil? and !File.file?(@ca_cert.path)
+        @ca_cert = Tempfile.new("ca_cert")
+        @ca_cert.write(@cert_string)
+        @ca_cert.flush
+        @http.ca_file = @ca_cert.path
+        raise ClientError.new("Packaged certificate appears to have been deleted, writing a new one.", @add_events_uri)
+      else
+        raise e
+      end
+
     rescue Net::HTTP::Persistent::Error => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
       raise ClientError.new(e.message, @add_events_uri)
 
     ensure
-      @stats_lock.synchronize do
-        @stats[:total_requests_sent] += 1
-        @stats[:total_requests_failed] += fail_count
-        @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
-        @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
-        @stats[:total_response_bytes_received] += bytes_received
-        end_time = Time.now
-        @stats[:total_request_latency_secs] += (end_time - start_time)
+      if @record_stats_for_status or !is_status
+        @stats_lock.synchronize do
+          @stats[:total_requests_sent] += 1
+          @stats[:total_requests_failed] += fail_count
+          @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
+          @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
+          @stats[:total_response_bytes_received] += bytes_received
+          @stats[:total_serialization_duration_secs] += body_serialization_duration
+          @stats[:total_flatten_values_duration_secs] += flatten_nested_values_duration
+          @stats[:total_compression_duration_secs] += compression_duration
+          end_time = Time.now
+          @stats[:total_request_latency_secs] += (end_time - start_time)
+          @latency_stats[:request_latency_secs].observe(end_time - start_time)
+          @latency_stats[:serialization_duration_secs].observe(body_serialization_duration)
+          @latency_stats[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
+          @latency_stats[:compression_duration_secs].observe(compression_duration)
+          @latency_stats[:bytes_sent].observe(uncompressed_bytes_sent)
+        end
       end
-
     end
   end  # def post_request
 
@@ -145,7 +265,9 @@ class ClientSession
   def prepare_post_object(uri_path, body)
     # use compression if enabled
     encoding = nil
+    compression_duration = 0
     if @compression_type
+      start_time = Time.now.to_f
       if @compression_type == 'deflate'
         encoding = 'deflate'
         compressed_body = Zlib::Deflate.deflate(body, @compression_level)
@@ -157,11 +279,13 @@ class ClientSession
         bz2.close
         compressed_body = io.string
       end
+      end_time = Time.now.to_f
+      compression_duration = end_time - start_time
     end
 
     post = Net::HTTP::Post.new uri_path
     post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.2'
+    version = 'output-logstash-scalyr 0.1.7'
     post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
 
     if not encoding.nil?
@@ -170,7 +294,7 @@ class ClientSession
     else
       post.body = body
     end
-    post
+    return post, compression_duration
   end  # def prepare_post_object
 
 

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.2'
+  s.version         = '0.1.7'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -22,7 +22,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'net-http-persistent'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
-  s.add_runtime_dependency 'ffi', '>= 1.9.25'
+  s.add_runtime_dependency 'quantile'
+  s.add_runtime_dependency 'ffi', '>= 1.9.18'
   s.add_runtime_dependency 'rbzip2', '0.3.0'
   s.add_development_dependency 'logstash-devutils'
 end

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/scalyr"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe LogStash::Outputs::Scalyr do
+
+  describe "#ssl_tests" do
+      context "with default SSL configuration" do
+        it "throws a ServerError due to fake api key" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
+        end
+      end
+
+      context "when pointing at a location without any valid certs and not using builtin" do
+        it "throws an SSLError" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+        end
+      end
+
+      context "when system certs are missing and not using builtin" do
+        it "throws an SSLError" do
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_FILE} /tmp/system_cert.pem`
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+          end
+          ensure
+            `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
+            `sudo mv /tmp/system_cert.pem #{OpenSSL::X509::DEFAULT_CERT_FILE}`
+        end
+      end
+
+      context "when server hostname doesn't match the cert" do
+        it "throws an SSLError" do
+          agent_scalyr_com_ip = `dig +short agent.scalyr.com 2> /dev/null | tail -n 1 | tr -d "\n"`
+          if agent_scalyr_com_ip.empty?
+            agent_scalyr_com_ip = `getent hosts agent.scalyr.com \
+            | awk '{ print $1 }' | tail -n 1 | tr -d "\n"`
+          end
+          mock_host = "invalid.mitm.should.fail.test.agent.scalyr.com"
+          etc_hosts_entry = "#{agent_scalyr_com_ip} #{mock_host}"
+          hosts_bkp = `sudo cat /etc/hosts`
+          hosts_bkp = hosts_bkp.chomp
+          # Add mock /etc/hosts entry and config scalyr_server entry
+          `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "hostname \"invalid.mitm.should.fail.test.agent.scalyr.com\" does not match the server certificate")
+          ensure
+            # Clean up the hosts file
+            `sudo truncate -s 0 /etc/hosts`
+            `echo "#{hosts_bkp}" | sudo tee -a /etc/hosts`
+          end
+        end
+      end
+  end
+end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -4,8 +4,39 @@ require "logstash/outputs/scalyr"
 require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
+require "quantile"
 
 
+class MockClientSession
+  DEFAULT_STATS = {
+    :total_requests_sent => 20,
+    :total_requests_failed => 10,
+    :total_request_bytes_sent => 100,
+    :total_compressed_request_bytes_sent => 50,
+    :total_response_bytes_received => 100,
+    :total_request_latency_secs => 100,
+    :total_connections_created => 10,
+    :total_serialization_duration_secs => 100.5,
+    :total_compression_duration_secs => 10.20,
+    :total_flatten_values_duration_secs => 33.3,
+    :compression_type => "deflate",
+    :compression_level => 9,
+  }
+
+  def initialize(stats = DEFAULT_STATS)
+    @stats = stats
+    @sent_events = []
+  end
+
+  def get_stats
+    @stats.clone
+  end
+
+  def post_add_events(body, is_status, body_serialization_duration = 0, flatten_nested_values_duration = 0)
+    @sent_events << { :body => body, :body_serialization_duration => body_serialization_duration, :flatten_nested_values_duration => flatten_nested_values_duration }
+  end
+end
+
 
 describe LogStash::Outputs::Scalyr do
   let(:sample_events) {
@@ -24,6 +55,72 @@ describe LogStash::Outputs::Scalyr do
 
   describe "#build_multi_event_request_array" do
 
+    context "test get_stats and send_status" do
+      plugin = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                 })
+
+      mock_client_session = MockClientSession.new
+
+      it "returns correct stats on get_stats" do
+        stats = mock_client_session.get_stats
+        expect(stats[:total_requests_sent]).to eq(20)
+      end
+
+      it "returns and sends correct status event on send_stats on initial and subsequent send" do
+        # 1. Initial send
+        plugin.instance_variable_set(:@last_status_transmit_time, nil)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        status_event = plugin.send_status
+        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
+
+        # 2. Second send
+        plugin.instance_variable_set(:@last_status_transmit_time, 100)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        # Setup one quantile calculation to make sure at least one of them calculates as expected
+        plugin.instance_variable_set(:@multi_receive_metrics, {:multi_receive_duration_secs => Quantile::Estimator.new, :multi_receive_event_count => Quantile::Estimator.new})
+        (1..20).each do |n|
+          plugin.instance_variable_get(:@multi_receive_metrics)[:multi_receive_duration_secs].observe(n)
+        end
+        plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
+        status_event = plugin.send_status
+        puts status_event[:attrs]["message"]
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20, total_requests_failed=10, total_request_bytes_sent=100, total_compressed_request_bytes_sent=50, total_response_bytes_received=100, total_request_latency_secs=100, total_connections_created=10, total_serialization_duration_secs=100.500, total_compression_duration_secs=10.200, total_flatten_values_duration_secs=33.300, compression_type=deflate, compression_level=9, total_multi_receive_secs=0, multi_receive_duration_p50=10, multi_receive_duration_p90=18, multi_receive_duration_p99=19, multi_receive_event_count_p50=0, multi_receive_event_count_p90=0, multi_receive_event_count_p99=0")
+      end
+
+      it "send_stats is called when events list is empty, but otherwise noop" do
+        quantile_estimator = Quantile::Estimator.new
+        plugin.instance_variable_set(:@multi_receive_metrics, {:multi_receive_duration_secs => quantile_estimator, :multi_receive_event_count => quantile_estimator})
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin).to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin.multi_receive([])
+      end
+
+      # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
+      it "send_status only sends posts with is_status = true" do
+        # 1. Initial send
+        plugin.instance_variable_set(:@last_status_transmit_time, nil)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything, anything)
+        plugin.send_status
+
+        # 2. Second send
+        plugin.instance_variable_set(:@last_status_transmit_time, 100)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@multi_receive_metrics, {:multi_receive_duration_secs => Quantile::Estimator.new, :multi_receive_event_count => Quantile::Estimator.new})
+        (1..20).each do |n|
+          plugin.instance_variable_get(:@multi_receive_metrics)[:multi_receive_duration_secs].observe(n)
+        end
+        plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
+        expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything, anything)
+        plugin.send_status
+      end
+    end
+
     context "when a field is configured as a log attribute" do
       it "creates logfile from serverHost" do
         plugin = LogStash::Outputs::Scalyr.new({
@@ -36,7 +133,6 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('/logstash/my host 3')
@@ -67,7 +163,6 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('/logstash/my host 3')
@@ -86,13 +181,45 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('my file 3')
       end
     end
 
+    context "when configured to flatten values with custom delimiter" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b_0" => 3,
+                                                     "nested.b_1" => 4,
+                                                     "nested.b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
     context "when configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
@@ -116,9 +243,11 @@ describe LogStash::Outputs::Scalyr do
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
                                                      "tag_prefix_t1" => "true",
                                                      "tag_prefix_t2" => "true",
                                                      "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
                                                  })
       end
     end
@@ -139,7 +268,9 @@ describe LogStash::Outputs::Scalyr do
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
                                                      "tags" => ["t1", "t2", "t3"],
+                                                     "parser" => "logstashParser",
                                                  })
       end
     end