logstash-output-scalyr 0.1.4 → 0.1.9

data/lib/scalyr/common/client.rb

@@ -52,7 +52,8 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert)
+                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
@@ -61,6 +62,8 @@ class ClientSession
     @ssl_ca_bundle_path = ssl_ca_bundle_path
     @append_builtin_cert = append_builtin_cert
     @ssl_verify_depth = ssl_verify_depth
+    @record_stats_for_status = record_stats_for_status
+    @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
 
     # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
     @cert_string = "" \
@@ -106,6 +109,7 @@ class ClientSession
 
     # Request statistics are accumulated across multiple threads and must be accessed through a mutex
     @stats_lock = Mutex.new
+    @latency_stats = get_new_latency_stats
     @stats = {
         :total_requests_sent => 0, # The total number of RPC requests sent.
         :total_requests_failed => 0, # The total number of RPC requests that failed.
@@ -115,7 +119,12 @@ class ClientSession
         :total_request_latency_secs => 0, #  The total number of secs spent waiting for a responses (so average latency
         # can be calculated by dividing this number by @total_requests_sent).
         # This includes connection establishment time.
-        :total_connections_created => 0, # The total number of HTTP connections successfully created.
+        :total_serialization_duration_secs => 0, # The total duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_compression_duration_secs => 0, # The total duration (in seconds) it took to compress all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :compression_type => @compression_type,
+        :compression_level => @compression_level,
     }
 
     @http = Net::HTTP::Persistent.new
@@ -141,18 +150,46 @@ class ClientSession
     end
   end  # def initialize
 
+  # Convenience method to create a fresh quantile estimator
+  def get_new_latency_stats
+    return {
+      # The total number of HTTP connections successfully created.
+      :serialization_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+      :compression_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to compress all the request bodies.
+      :request_latency_secs => Quantile::Estimator.new, #  Secs spent waiting for a responses. This includes connection establishment time.
+      :bytes_sent => Quantile::Estimator.new  # The number of bytes sent over the network. Batch size with a bit more overhead.
+    }
+  end
 
-
-  # Get a clone of current statistics hash
+  # Get a clone of current statistics hash and calculate percentiles
   def get_stats
-    @stats.clone
+    current_stats = @stats.clone
+
+    current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+    current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+    current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+    current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+    current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+    current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+    current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+    current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+    current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+    current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+    current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+    current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+    if @flush_quantile_estimates_on_status_send
+      @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+      @latency_stats = get_new_latency_stats
+    end
+    current_stats
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
-  def post_add_events(body)
-    post = prepare_post_object @add_events_uri.path, body
+  def post_add_events(body, is_status, body_serialization_duration = 0)
+    post, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
@@ -185,16 +222,23 @@ class ClientSession
       raise ClientError.new(e.message, @add_events_uri)
 
     ensure
-      @stats_lock.synchronize do
-        @stats[:total_requests_sent] += 1
-        @stats[:total_requests_failed] += fail_count
-        @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
-        @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
-        @stats[:total_response_bytes_received] += bytes_received
-        end_time = Time.now
-        @stats[:total_request_latency_secs] += (end_time - start_time)
+      if @record_stats_for_status or !is_status
+        @stats_lock.synchronize do
+          @stats[:total_requests_sent] += 1
+          @stats[:total_requests_failed] += fail_count
+          @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
+          @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
+          @stats[:total_response_bytes_received] += bytes_received
+          @stats[:total_serialization_duration_secs] += body_serialization_duration
+          @stats[:total_compression_duration_secs] += compression_duration
+          end_time = Time.now
+          @stats[:total_request_latency_secs] += (end_time - start_time)
+          @latency_stats[:request_latency_secs].observe(end_time - start_time)
+          @latency_stats[:serialization_duration_secs].observe(body_serialization_duration)
+          @latency_stats[:compression_duration_secs].observe(compression_duration)
+          @latency_stats[:bytes_sent].observe(uncompressed_bytes_sent)
+        end
       end
-
     end
   end  # def post_request
 
@@ -211,7 +255,9 @@ class ClientSession
   def prepare_post_object(uri_path, body)
     # use compression if enabled
     encoding = nil
+    compression_duration = 0
     if @compression_type
+      start_time = Time.now.to_f
       if @compression_type == 'deflate'
         encoding = 'deflate'
         compressed_body = Zlib::Deflate.deflate(body, @compression_level)
@@ -223,11 +269,13 @@ class ClientSession
         bz2.close
         compressed_body = io.string
       end
+      end_time = Time.now.to_f
+      compression_duration = end_time - start_time
     end
 
     post = Net::HTTP::Post.new uri_path
     post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.4'
+    version = 'output-logstash-scalyr 0.1.9'
     post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
 
     if not encoding.nil?
@@ -236,7 +284,7 @@ class ClientSession
     else
       post.body = body
     end
-    post
+    return post, compression_duration
   end  # def prepare_post_object
 
 

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.4'
+  s.version         = '0.1.9'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'net-http-persistent'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'quantile'
   s.add_runtime_dependency 'ffi', '>= 1.9.18'
   s.add_runtime_dependency 'rbzip2', '0.3.0'
   s.add_development_dependency 'logstash-devutils'

data/spec/benchmarks/flattening_and_serialization.rb

@@ -0,0 +1,125 @@
+require 'benchmark'
+require 'json'
+
+require_relative '../../lib/scalyr/common/util'
+
+# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
+# sense since it will just use JITed version of the code which will be very fast. If we
+# wanted to accurately measure using multiple iterations we would need te different
+# input data for each iteration.
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_str(20)
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+def flatten_data_func(data)
+  Scalyr::Common::Util.flatten(data)
+end
+
+def json_serialize_data(data)
+  data.to_json
+end
+
+DATASETS = {
+  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
+  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
+  :keys_200_flat => generate_data_array_for_spec([200]),
+  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
+  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
+  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
+}
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+puts "Scalyr::Common::Util.flatten()"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~512 keys in a hash
+data = DATASETS[:keys_512]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~960 keys in a hash
+data = DATASETS[:keys_960]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+puts "JSON.dumps (hash.to_dict)"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))

data/spec/benchmarks/metrics_overhead.rb

@@ -0,0 +1,48 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
+# where making sampling (e.g. on event level metrics) is desired
+
+ITERATIONS = 10000
+
+def run_benchmark_and_print_results(run_benchmark_func)
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.() }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def quantile_observe()
+  @value.observe(5)
+end
+
+def random_sample()
+  return @prng.rand(0.0..1.0) < 0.5
+end
+
+puts "Quartile.observe()"
+puts "==============================="
+
+run_benchmark_and_print_results(method(:quantile_observe))
+
+puts "random sample"
+puts "==============================="
+run_benchmark_and_print_results(method(:random_sample))

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/scalyr"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe LogStash::Outputs::Scalyr do
+
+  describe "#ssl_tests" do
+      context "with default SSL configuration" do
+        it "throws a ServerError due to fake api key" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
+        end
+      end
+
+      context "when pointing at a location without any valid certs and not using builtin" do
+        it "throws an SSLError" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+        end
+      end
+
+      context "when system certs are missing and not using builtin" do
+        it "throws an SSLError" do
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_FILE} /tmp/system_cert.pem`
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "certificate verify failed")
+          end
+          ensure
+            `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
+            `sudo mv /tmp/system_cert.pem #{OpenSSL::X509::DEFAULT_CERT_FILE}`
+        end
+      end
+
+      context "when server hostname doesn't match the cert" do
+        it "throws an SSLError" do
+          agent_scalyr_com_ip = `dig +short agent.scalyr.com 2> /dev/null | tail -n 1 | tr -d "\n"`
+          if agent_scalyr_com_ip.empty?
+            agent_scalyr_com_ip = `getent hosts agent.scalyr.com \
+            | awk '{ print $1 }' | tail -n 1 | tr -d "\n"`
+          end
+          mock_host = "invalid.mitm.should.fail.test.agent.scalyr.com"
+          etc_hosts_entry = "#{agent_scalyr_com_ip} #{mock_host}"
+          hosts_bkp = `sudo cat /etc/hosts`
+          hosts_bkp = hosts_bkp.chomp
+          # Add mock /etc/hosts entry and config scalyr_server entry
+          `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(OpenSSL::SSL::SSLError, "hostname \"invalid.mitm.should.fail.test.agent.scalyr.com\" does not match the server certificate")
+          ensure
+            # Clean up the hosts file
+            `sudo truncate -s 0 /etc/hosts`
+            `echo "#{hosts_bkp}" | sudo tee -a /etc/hosts`
+          end
+        end
+      end
+  end
+end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -4,8 +4,37 @@ require "logstash/outputs/scalyr"
 require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
+require "quantile"
 
 
+class MockClientSession
+  DEFAULT_STATS = {
+    :total_requests_sent => 20,
+    :total_requests_failed => 10,
+    :total_request_bytes_sent => 100,
+    :total_compressed_request_bytes_sent => 50,
+    :total_response_bytes_received => 100,
+    :total_request_latency_secs => 100,
+    :total_serialization_duration_secs => 100.5,
+    :total_compression_duration_secs => 10.20,
+    :compression_type => "deflate",
+    :compression_level => 9,
+  }
+
+  def initialize(stats = DEFAULT_STATS)
+    @stats = stats
+    @sent_events = []
+  end
+
+  def get_stats
+    @stats.clone
+  end
+
+  def post_add_events(body, is_status, body_serialization_duration = 0)
+    @sent_events << { :body => body, :body_serialization_duration => body_serialization_duration }
+  end
+end
+
 
 describe LogStash::Outputs::Scalyr do
   let(:sample_events) {
@@ -24,6 +53,112 @@ describe LogStash::Outputs::Scalyr do
 
   describe "#build_multi_event_request_array" do
 
+    context "test get_stats and send_status" do
+      plugin = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => true,
+                                                 })
+
+      mock_client_session = MockClientSession.new
+
+      it "returns correct stats on get_stats" do
+        stats = mock_client_session.get_stats
+        expect(stats[:total_requests_sent]).to eq(20)
+      end
+
+      it "it doesnt include flatten metrics if flattening is disabled" do
+          plugin1 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                 })
+        mock_client_session = MockClientSession.new
+        plugin1.instance_variable_set(:@last_status_transmit_time, 100)
+        plugin1.instance_variable_set(:@client_session, mock_client_session)
+        plugin1.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count =>  Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin1.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(1)
+        plugin1.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
+
+        status_event = plugin1.send_status
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0")
+      end
+
+      it "returns and sends correct status event on send_stats on initial and subsequent send" do
+        # 1. Initial send
+        plugin.instance_variable_set(:@last_status_transmit_time, nil)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        status_event = plugin.send_status
+        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
+
+        # 2. Second send
+        plugin.instance_variable_set(:@last_status_transmit_time, 100)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        # Setup one quantile calculation to make sure at least one of them calculates as expected
+        plugin.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count =>  Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+
+        (1..20).each do |n|
+          plugin.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(n)
+        end
+
+        plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
+        status_event = plugin.send_status
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
+      end
+
+      it "send_stats is called when events list is empty, but otherwise noop" do
+        quantile_estimator = Quantile::Estimator.new
+        plugin.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin).to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin.multi_receive([])
+      end
+
+      # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
+      it "send_status only sends posts with is_status = true" do
+        # 1. Initial send
+        plugin.instance_variable_set(:@last_status_transmit_time, nil)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything)
+        plugin.send_status
+
+        # 2. Second send
+        plugin.instance_variable_set(:@last_status_transmit_time, 100)
+        plugin.instance_variable_set(:@client_session, mock_client_session)
+        plugin.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count =>  Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        (1..20).each do |n|
+          plugin.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(n)
+        end
+        plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
+        expect(mock_client_session).to receive(:post_add_events).with(anything, true, anything)
+        plugin.send_status
+      end
+    end
+
     context "when a field is configured as a log attribute" do
       it "creates logfile from serverHost" do
         plugin = LogStash::Outputs::Scalyr.new({
@@ -36,7 +171,6 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('/logstash/my host 3')
@@ -67,7 +201,6 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('/logstash/my host 3')
@@ -86,13 +219,45 @@ describe LogStash::Outputs::Scalyr do
         result = plugin.build_multi_event_request_array(sample_events)
         body = JSON.parse(result[0][:body])
         expect(body['events'].size).to eq(3)
-        attrs2 = body['events'][2]['attrs']
         logattrs2 = body['logs'][2]['attrs']
         expect(logattrs2.fetch('serverHost', nil)).to eq('my host 3')
         expect(logattrs2.fetch('logfile', nil)).to eq('my file 3')
       end
     end
 
+    context "when configured to flatten values with custom delimiter" do
+      config = {
+          'api_write_token' => '1234',
+          'flatten_tags' => true,
+          'flat_tag_value' => 'true',
+          'flat_tag_prefix' => 'tag_prefix_',
+          'flatten_nested_values' => true,  # this converts into string 'true'
+          'flatten_nested_values_delimiter' => ".",
+      }
+      plugin = LogStash::Outputs::Scalyr.new(config)
+      it "flattens nested values with a period" do
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+        expect(body['events'][2]['attrs']).to eq({
+                                                     "nested.a" => 1,
+                                                     "nested.b_0" => 3,
+                                                     "nested.b_1" => 4,
+                                                     "nested.b_2" => 5,
+                                                     'seq' => 3,
+                                                     'source_file' => 'my file 3',
+                                                     'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
+                                                     "tag_prefix_t1" => "true",
+                                                     "tag_prefix_t2" => "true",
+                                                     "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
+                                                 })
+      end
+    end
+
     context "when configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
@@ -116,9 +281,11 @@ describe LogStash::Outputs::Scalyr do
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
                                                      "tag_prefix_t1" => "true",
                                                      "tag_prefix_t2" => "true",
                                                      "tag_prefix_t3" => "true",
+                                                     "parser" => "logstashParser",
                                                  })
       end
     end
@@ -139,7 +306,9 @@ describe LogStash::Outputs::Scalyr do
                                                      'seq' => 3,
                                                      'source_file' => 'my file 3',
                                                      'source_host' => 'my host 3',
+                                                     'serverHost' => 'Logstash',
                                                      "tags" => ["t1", "t2", "t3"],
+                                                     "parser" => "logstashParser",
                                                  })
       end
     end