logstash-output-scalyr 0.1.6 → 0.1.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fab6a3a6d6d6b2f6f2a3fbe2dc2eb3f58cb6e0f3fba4661c483288f1f6f9f9a8
-  data.tar.gz: 954a437080e6fd03afc46fa08787ee35a02bad271e199e4347ba0728dc4699f1
+  metadata.gz: 9a0be59b3c2ecaeb444f9fb76e6c97129ab9e1eeaa4a907976e4f6e62a44a582
+  data.tar.gz: 4ddecf13bd1b7800d6d5bad0a271014257159aedfb7eb332ebc8f1c1dd0dd3c5
 SHA512:
-  metadata.gz: c2a23046dd52329648c0581f80065c530eac90a84a19ce3916d52910a9786bead1f15f2fb341f79dd6561fb101e742d1c6af0bc705aaa9698bdb7d4cf7b1a7b8
-  data.tar.gz: 2de8f9bb2f7b3b98021a163169317fa17219f9e698ff153429dd1540dc57bfe1e6a1bf525fb29a4cca2f29caf8d00a429e74c753b0c8ec70142893078dfb67ba
+  metadata.gz: 8ed321b57270cf4bc6fbbc3375f516d08df0592d0476fa71d15af40a7afc1ddb4a97634678b65735101f3c4a17422c0d73e3f40524db06444f8c99cfd3f3a375
+  data.tar.gz: 818943ede0fcad23642366f465c63134500cd70fea625d52313dcc628b3bde3f19aecc6739766690f7d5985c9188af78b5c32e63bcfc4e9a1ca60ee0ceac8743

data/CHANGELOG.md

@@ -1,4 +1,9 @@
 # Beta
+## 0.1.7
+ - Tracking of new statistics such as `multi_receive` method duration and batch sizes.
+ - Addition of percentiles for both existing and new stats.
+ - Add ability to define a parser name for status messages using the `status_parser` configuration option.
+
 ## 0.1.6
  - Allow for a customer delimiter when flattening values using the `flatten_nested_values_delimiter` configuration option
 

data/Gemfile

@@ -12,3 +12,4 @@ end
 
 gem 'pry'
 gem 'pry-nav'
+gem 'quantile'

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.6.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.7.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -48,7 +48,7 @@ In the above example, the Logstash pipeline defines a file input that reads from
 
 ---
 
-- If your Scalyr backend is located in other geographies (such as Europe which would use `https://agent.eu.scalyr.com/`), you may need to modify this
+- If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
 
 `config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"`
 
@@ -158,7 +158,6 @@ In the above example, the Logstash pipeline defines a file input that reads from
 
 # Conceptual Overview
 
-
 ## Persistence
 
 Logstash itself supports [Persistent Queues](https://www.elastic.co/guide/en/logstash/current/persistent-queues.html) with at-least-once delivery semantics.  It expects output plugins to retry uploads until success or else to write failures into a Dead-Letter Queue (DLQ). Since Logstash offers Persistent Queues, the Scalyr plugin does not perform its own buffering or persistence.  More specifically, invocation of `multi_receive` is synchronously retried until success or written to the DLQ upon failure.  Note: the `multi_receive` interface does not provide a feedback mechanism (outcome codes etc).
@@ -312,6 +311,16 @@ sudo bundle exec rspec
 
 in the root of the repo.
 
+By default this will run all the tests (including integration ones which require sudo access and
+may not pass everywhere).
+
+If you want to run just the unit tests, you can run the command displayed below.
+
+
+```bash
+bundle exec rspec spec/logstash/outputs/scalyr_spec.rb spec/scalyr/common/util_spec.rb
+```
+
 # Releasing
 
 ## Updating version

data/lib/logstash/outputs/scalyr.rb

@@ -14,6 +14,7 @@ require 'net/https'
 require 'rbzip2'
 require 'zlib'
 require 'stringio'
+require 'quantile'
 
 require 'scalyr/common/client'
 require "scalyr/common/util"
@@ -33,7 +34,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # The Scalyr API write token, these are available at https://www.scalyr.com/keys.  This is the only compulsory configuration field required for proper upload
   config :api_write_token, :validate => :string, :required => true
 
-  # If your Scalyr backend is located in other geographies (such as Europe which would use `https://agent.eu.scalyr.com/`), you may need to modify this
+  # If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
   config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
 
   # Path to SSL bundle file.
@@ -104,6 +105,20 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # An int containing the compression level of compression to use, from 1-9. Defaults to 6
   config :compression_level, :validate => :number, :default => 6
 
+  # How often to log and report status metrics to Scalyr. Defaults to every 5
+  # minutes.
+  config :status_report_interval, :validate => :number, :default => 300
+
+  # Whether or not to count status event uploads in the statistics such as request latency etc.
+  config :record_stats_for_status, :validate => :boolean, :default => false
+
+  # Parser to attach to status events
+  config :status_parser, :validate => :string, :default => "logstash_plugin_metrics"
+
+  # Whether or not to create fresh quantile estimators after a status send. Depending on what you want to gather from
+  # these stas this might be wanted or not.
+  config :flush_quantile_estimates_on_status_send, :validate => :boolean, :default => false
+
   def close
     @running = false
     @client_session.close if @client_session
@@ -176,6 +191,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     @last_status_transmit_time_lock = Mutex.new
     @last_status_transmit_time = nil
     @last_status_ = false
+    
+    # Per-multi-receive statistics
+    @multi_receive_statistics = {
+      :total_multi_receive_secs => 0
+    }
+    @multi_receive_metrics = get_new_multi_receive_metrics
 
     # create a client session for uploading to Scalyr
     @running = true
@@ -183,7 +204,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @logger, @add_events_uri,
         @compression_type, @compression_level,
         @ssl_verify_peer, @ssl_ca_bundle_path, @ssl_verify_depth,
-        @append_builtin_cert
+        @append_builtin_cert, @record_stats_for_status, @flush_quantile_estimates_on_status_send
     )
 
     @logger.info("Started Scalyr output plugin", :class => self.class.name)
@@ -193,6 +214,13 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
   end # def register
 
+  # Convenience method to create a fresh quantile estimator
+  def get_new_multi_receive_metrics
+    return {
+      :multi_receive_duration_secs => Quantile::Estimator.new,
+      :multi_receive_event_count => Quantile::Estimator.new
+    }
+  end
 
   # Receive an array of events and immediately upload them (without buffering).
   # The Logstash framework will call this plugin method whenever there is a list of events to upload to Scalyr.
@@ -206,6 +234,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   #
   public
   def multi_receive(events)
+    start_time = Time.now.to_f
 
     multi_event_request_array = build_multi_event_request_array(events)
     # Loop over all array of multi-event requests, sending each multi-event to Scalyr
@@ -215,22 +244,24 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
     result = []
+    records_count = events.to_a.length
+
     while !multi_event_request_array.to_a.empty?
       begin
         multi_event_request = multi_event_request_array.pop
         # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
         # ignore these.
         if !multi_event_request.nil?
-          @client_session.post_add_events(multi_event_request[:body])
+          @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration], multi_event_request[:flatten_nested_values_duration])
           sleep_interval = 0
           result.push(multi_event_request)
         end
 
       rescue OpenSSL::SSL::SSLError => e
         # cannot rely on exception message, so we always log the following warning
-        @logger.error "SSL certificate verification failed.  "
-        "Please make sure your certificate bundle is configured correctly and points to a valid file.  "
-        "You can configure this with the ssl_ca_bundle_path configuration option.  "
+        @logger.error "SSL certificate verification failed.  " +
+        "Please make sure your certificate bundle is configured correctly and points to a valid file.  " +
+        "You can configure this with the ssl_ca_bundle_path configuration option.  " +
         "The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
         @logger.error e.message
         @logger.error "Discarding buffer chunk without retrying."
@@ -273,6 +304,12 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
     end
 
+    if records_count > 0
+      @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+      @multi_receive_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
+      @multi_receive_metrics[:multi_receive_event_count].observe(records_count)
+    end
+
     send_status
     return result
   end  # def multi_receive
@@ -296,6 +333,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Tags are either propagated as a comma-separated string, or optionally transposed into key-values where the keys
   # are tag names and the values are 1 (may be configured.)
   def build_multi_event_request_array(logstash_events)
+    if logstash_events.nil? or logstash_events.empty?
+     return []
+    end
 
     multi_event_request_array = Array.new
     total_bytes = 0
@@ -312,6 +352,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     logs_ids = Hash.new
     next_log_id = 1
 
+    flatten_nested_values_duration = 0
+
     logstash_events.each {|l_event|
 
       record = l_event.to_hash
@@ -338,8 +380,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         if standard_field != renamed_field
           if record.key? renamed_field
             if record.key? standard_field
-              @logger.warn "Overwriting log record field '#{standard_field}'.  You are seeing this warning because in "
-              "your LogStash config file you have configured the '#{renamed_field}' field to be converted to the "
+              @logger.warn "Overwriting log record field '#{standard_field}'.  You are seeing this warning because in " +
+              "your LogStash config file you have configured the '#{renamed_field}' field to be converted to the " +
               "'#{standard_field}' field, but the event already contains a field called '#{standard_field}' and "
               "this is now being overwritten."
             end
@@ -426,7 +468,14 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
 
       # flatten record
-      record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter) if @flatten_nested_values
+      flatten_nested_values_duration = 0
+
+      if @flatten_nested_values
+        start_time = Time.now.to_f
+        record = Scalyr::Common::Util.flatten(record, delimiter=@flatten_nested_values_delimiter) if @flatten_nested_values
+        end_time = Time.now.to_f
+        flatten_nested_values_duration = end_time - start_time
+      end
 
       # Use LogStash event.timestamp as the 'ts' Scalyr timestamp.  Note that this may be overwritten by input
       # filters so may not necessarily reflect the actual originating timestamp.
@@ -477,7 +526,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           scalyr_events << scalyr_event
           append_event = false
         end
-        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+        multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs, flatten_nested_values_duration)
         multi_event_request_array << multi_event_request
 
         total_bytes = 0
@@ -497,7 +546,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     }
 
     # create a final request with any left over events
-    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs)
+    multi_event_request = self.create_multi_event_request(scalyr_events, current_threads, logs, flatten_nested_values_duration)
     multi_event_request_array << multi_event_request
     multi_event_request_array
   end
@@ -517,7 +566,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # A request comprises multiple Scalyr Events.  This function creates a request hash for
   # final upload to Scalyr (from an array of events, and an optional hash of current threads)
   # Note: The request body field will be json-encoded.
-  def create_multi_event_request(scalyr_events, current_threads, current_logs)
+  def create_multi_event_request(scalyr_events, current_threads, current_logs, flatten_nested_values_duration = 0)
 
     body = {
       :session => @session_id,
@@ -548,12 +597,35 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # add serverAttributes
     body[:sessionInfo] = @server_attributes if @server_attributes
 
-    { :body => body.to_json, :record_count => scalyr_events.size }
+    # We time serialization to get some insight on how long it takes to serialize the request body
+    start_time = Time.now.to_f
+    serialized_body = body.to_json
+    end_time = Time.now.to_f
+    serialization_duration = end_time - start_time
+    { :body => serialized_body, :record_count => scalyr_events.size, :serialization_duration => serialization_duration,
+      :flatten_nested_values_duration => flatten_nested_values_duration }
 
   end  # def create_multi_event_request
 
 
 
+  def get_batch_stats
+    current_stats = @multi_receive_statistics.clone
+
+    current_stats[:multi_receive_duration_p50] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.5)
+    current_stats[:multi_receive_duration_p90] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.9)
+    current_stats[:multi_receive_duration_p99] = @multi_receive_metrics[:multi_receive_duration_secs].query(0.99)
+    current_stats[:multi_receive_event_count_p50] = @multi_receive_metrics[:multi_receive_event_count].query(0.5)
+    current_stats[:multi_receive_event_count_p90] = @multi_receive_metrics[:multi_receive_event_count].query(0.9)
+    current_stats[:multi_receive_event_count_p99] = @multi_receive_metrics[:multi_receive_event_count].query(0.99)
+
+    if @flush_quantile_estimates_on_status_send
+      @multi_receive_metrics = get_new_multi_receive_metrics
+    end
+    current_stats
+  end
+
+
 
   # Sends a status update to Scalyr by posting a log entry under the special logfile of 'logstash_plugin.log'
   # Instead of creating a separate thread, let this method be invoked once at startup and then every 5 minutes
@@ -576,22 +648,32 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       status_event[:attrs]['serverHost'] = @node_hostname
     else
       cur_time = Time.now()
-      return if (cur_time.to_i - @last_status_transmit_time.to_i) < 300
+      return if (cur_time.to_i - @last_status_transmit_time.to_i) < @status_report_interval
       # echee TODO: get instance stats from session and create a status log line
       msg = 'plugin_status: '
       cnt = 0
       @client_session.get_stats.each do |k, v|
-        val = v.instance_of?(Float) ? sprintf("%.1f", v) : v
+        val = v.instance_of?(Float) ? sprintf("%.3f", v) : v
+        val = val.nil? ? 0 : val
+        msg << ', ' if cnt > 0
+        msg << "#{k.to_s}=#{val}"
+        cnt += 1
+      end
+      get_batch_stats.each do |k, v|
+        val = v.instance_of?(Float) ? sprintf("%.3f", v) : v
+        val = val.nil? ? 0 : val
         msg << ', ' if cnt > 0
         msg << "#{k.to_s}=#{val}"
         cnt += 1
       end
       status_event[:attrs]['message'] = msg
       status_event[:attrs]['serverHost'] = @node_hostname
+      status_event[:attrs]['parser'] = @status_parser
     end
-    multi_event_request = create_multi_event_request([status_event], nil, nil)
-    @client_session.post_add_events(multi_event_request[:body])
+    multi_event_request = create_multi_event_request([status_event], nil, nil, nil)
+    @client_session.post_add_events(multi_event_request[:body], true, 0, 0)
     @last_status_transmit_time = Time.now()
+    status_event
   end
 
 
@@ -600,7 +682,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   def should_transmit_status?
     @last_status_transmit_time_lock.synchronize do
       saved_last_time = @last_status_transmit_time
-      if Time.now.to_i - saved_last_time.to_i > 300
+      if Time.now.to_i - saved_last_time.to_i > @status_report_interval
         @last_status_transmit_time = Float::INFINITY
         return saved_last_time
       end

data/lib/scalyr/common/client.rb

@@ -52,7 +52,8 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert)
+                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
@@ -61,6 +62,8 @@ class ClientSession
     @ssl_ca_bundle_path = ssl_ca_bundle_path
     @append_builtin_cert = append_builtin_cert
     @ssl_verify_depth = ssl_verify_depth
+    @record_stats_for_status = record_stats_for_status
+    @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
 
     # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
     @cert_string = "" \
@@ -106,6 +109,7 @@ class ClientSession
 
     # Request statistics are accumulated across multiple threads and must be accessed through a mutex
     @stats_lock = Mutex.new
+    @latency_stats = get_new_latency_stats
     @stats = {
         :total_requests_sent => 0, # The total number of RPC requests sent.
         :total_requests_failed => 0, # The total number of RPC requests that failed.
@@ -116,6 +120,15 @@ class ClientSession
         # can be calculated by dividing this number by @total_requests_sent).
         # This includes connection establishment time.
         :total_connections_created => 0, # The total number of HTTP connections successfully created.
+        :total_serialization_duration_secs => 0, # The total duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_compression_duration_secs => 0, # The total duration (in seconds) it took to compress all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_flatten_values_duration_secs => 0, # The total duration (in seconds) it took to flatten nested record values.
+        # In case flattening is disabled, this value will always be 0. Can infer average per-request value by dividing this
+        # value by total_requests_sent
+        :compression_type => @compression_type,
+        :compression_level => @compression_level,
     }
 
     @http = Net::HTTP::Persistent.new
@@ -141,18 +154,50 @@ class ClientSession
     end
   end  # def initialize
 
-
+  # Convenience method to create a fresh quantile estimator
+  def get_new_latency_stats
+    return {
+      # The total number of HTTP connections successfully created.
+      :serialization_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+      :compression_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to compress all the request bodies.
+      :flatten_values_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to flatten nested record values.
+      # In case flattening is disabled, this value will always be 0.
+      :request_latency_secs => Quantile::Estimator.new, #  Secs spent waiting for a responses. This includes connection establishment time.
+      :bytes_sent => Quantile::Estimator.new  # The number of bytes sent over the network. Batch size with a bit more overhead.
+    }
+  end
 
   # Get a clone of current statistics hash
   def get_stats
-    @stats.clone
+    current_stats = @stats.clone
+
+    current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+    current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+    current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+    current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+    current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+    current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+    current_stats[:flatten_values_duration_secs_p50] = @latency_stats[:flatten_values_duration_secs].query(0.5)
+    current_stats[:flatten_values_duration_secs_p90] = @latency_stats[:flatten_values_duration_secs].query(0.9)
+    current_stats[:flatten_values_duration_secs_p99] = @latency_stats[:flatten_values_duration_secs].query(0.99)
+    current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+    current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+    current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+    current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+    current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+    current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+    if @flush_quantile_estimates_on_status_send
+      @latency_stats = get_new_latency_stats
+    end
+    current_stats
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
-  def post_add_events(body)
-    post = prepare_post_object @add_events_uri.path, body
+  def post_add_events(body, is_status, body_serialization_duration = 0, flatten_nested_values_duration = 0)
+    post, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
@@ -185,16 +230,25 @@ class ClientSession
       raise ClientError.new(e.message, @add_events_uri)
 
     ensure
-      @stats_lock.synchronize do
-        @stats[:total_requests_sent] += 1
-        @stats[:total_requests_failed] += fail_count
-        @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
-        @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
-        @stats[:total_response_bytes_received] += bytes_received
-        end_time = Time.now
-        @stats[:total_request_latency_secs] += (end_time - start_time)
+      if @record_stats_for_status or !is_status
+        @stats_lock.synchronize do
+          @stats[:total_requests_sent] += 1
+          @stats[:total_requests_failed] += fail_count
+          @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
+          @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
+          @stats[:total_response_bytes_received] += bytes_received
+          @stats[:total_serialization_duration_secs] += body_serialization_duration
+          @stats[:total_flatten_values_duration_secs] += flatten_nested_values_duration
+          @stats[:total_compression_duration_secs] += compression_duration
+          end_time = Time.now
+          @stats[:total_request_latency_secs] += (end_time - start_time)
+          @latency_stats[:request_latency_secs].observe(end_time - start_time)
+          @latency_stats[:serialization_duration_secs].observe(body_serialization_duration)
+          @latency_stats[:flatten_values_duration_secs].observe(flatten_nested_values_duration)
+          @latency_stats[:compression_duration_secs].observe(compression_duration)
+          @latency_stats[:bytes_sent].observe(uncompressed_bytes_sent)
+        end
       end
-
     end
   end  # def post_request
 
@@ -211,7 +265,9 @@ class ClientSession
   def prepare_post_object(uri_path, body)
     # use compression if enabled
     encoding = nil
+    compression_duration = 0
     if @compression_type
+      start_time = Time.now.to_f
       if @compression_type == 'deflate'
         encoding = 'deflate'
         compressed_body = Zlib::Deflate.deflate(body, @compression_level)
@@ -223,11 +279,13 @@ class ClientSession
         bz2.close
         compressed_body = io.string
       end
+      end_time = Time.now.to_f
+      compression_duration = end_time - start_time
     end
 
     post = Net::HTTP::Post.new uri_path
     post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.6'
+    version = 'output-logstash-scalyr 0.1.7'
     post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
 
     if not encoding.nil?
@@ -236,7 +294,7 @@ class ClientSession
     else
       post.body = body
     end
-    post
+    return post, compression_duration
   end  # def prepare_post_object