logstash-output-scalyr 0.1.5 → 0.1.10.beta

data/lib/scalyr/common/client.rb

@@ -52,7 +52,9 @@ end
 class ClientSession
 
   def initialize(logger, add_events_uri, compression_type, compression_level,
-                 ssl_verify_peer, ssl_ca_bundle_path, ssl_verify_depth, append_builtin_cert)
+                 ssl_verify_peer, ssl_ca_bundle_path, append_builtin_cert,
+                 record_stats_for_status, flush_quantile_estimates_on_status_send,
+                 connect_timeout, socket_timeout, request_timeout, pool_max, pool_max_per_route)
     @logger = logger
     @add_events_uri = add_events_uri  # typically /addEvents
     @compression_type = compression_type
@@ -60,7 +62,13 @@ class ClientSession
     @ssl_verify_peer = ssl_verify_peer
     @ssl_ca_bundle_path = ssl_ca_bundle_path
     @append_builtin_cert = append_builtin_cert
-    @ssl_verify_depth = ssl_verify_depth
+    @record_stats_for_status = record_stats_for_status
+    @flush_quantile_estimates_on_status_send = flush_quantile_estimates_on_status_send
+    @connect_timeout = connect_timeout
+    @socket_timeout = socket_timeout
+    @request_timeout = request_timeout
+    @pool_max = pool_max
+    @pool_max_per_route = pool_max_per_route
 
     # A cert to use by default to avoid issues caused by the OpenSSL library not validating certs according to standard
     @cert_string = "" \
@@ -106,6 +114,7 @@ class ClientSession
 
     # Request statistics are accumulated across multiple threads and must be accessed through a mutex
     @stats_lock = Mutex.new
+    @latency_stats = get_new_latency_stats
     @stats = {
         :total_requests_sent => 0, # The total number of RPC requests sent.
         :total_requests_failed => 0, # The total number of RPC requests that failed.
@@ -115,13 +124,36 @@ class ClientSession
         :total_request_latency_secs => 0, #  The total number of secs spent waiting for a responses (so average latency
         # can be calculated by dividing this number by @total_requests_sent).
         # This includes connection establishment time.
-        :total_connections_created => 0, # The total number of HTTP connections successfully created.
+        :total_serialization_duration_secs => 0, # The total duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :total_compression_duration_secs => 0, # The total duration (in seconds) it took to compress all the request bodies.
+        # You can calculate avg compression duration by diving this value with total_requests_sent
+        :compression_type => @compression_type,
+        :compression_level => @compression_level,
     }
+  end  # def initialize
 
-    @http = Net::HTTP::Persistent.new
+  def client_config
+    # TODO: Eventually expose some more of these as config options, though nothing here really needs tuning normally
+    # besides SSL
+    c = {
+      connect_timeout: @connect_timeout,
+      socket_timeout: @socket_timeout,
+      request_timeout: @request_timeout,
+      follow_redirects: true,
+      automatic_retries: 1,
+      retry_non_idempotent: false,
+      check_connection_timeout: 200,
+      pool_max: @pool_max,
+      pool_max_per_route: @pool_max_per_route,
+      cookies: true,
+      keepalive: true,
+      ssl: {}
+    }
 
     # verify peers to prevent potential MITM attacks
     if @ssl_verify_peer
+      c[:ssl][:verify] = :strict
       @ca_cert = Tempfile.new("ca_cert")
       if File.file?(@ssl_ca_bundle_path)
         @ca_cert.write(File.read(@ssl_ca_bundle_path))
@@ -133,75 +165,109 @@ class ClientSession
         end
       end
       @ca_cert.flush
-      @http.ca_file = @ca_cert.path
-      @http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-      @http.verify_depth = @ssl_verify_depth
+      c[:ssl][:ca_file] = @ca_cert.path
     else
-      @http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      c[:ssl][:verify] = :disable
     end
-  end  # def initialize
 
+    c
+  end
+
+  def client
+    @client ||= Manticore::Client.new(client_config)
+  end
 
+  # Convenience method to create a fresh quantile estimator
+  def get_new_latency_stats
+    return {
+      # The total number of HTTP connections successfully created.
+      :serialization_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to serialize (JSON dumos) all the request bodies.
+      :compression_duration_secs => Quantile::Estimator.new, # The duration (in seconds) it took to compress all the request bodies.
+      :request_latency_secs => Quantile::Estimator.new, #  Secs spent waiting for a responses. This includes connection establishment time.
+      :bytes_sent => Quantile::Estimator.new  # The number of bytes sent over the network. Batch size with a bit more overhead.
+    }
+  end
 
-  # Get a clone of current statistics hash
+  # Get a clone of current statistics hash and calculate percentiles
   def get_stats
-    @stats.clone
+    @stats_lock.synchronize do
+      current_stats = @stats.clone
+
+      current_stats[:request_latency_p50] = @latency_stats[:request_latency_secs].query(0.5)
+      current_stats[:request_latency_p90] = @latency_stats[:request_latency_secs].query(0.9)
+      current_stats[:request_latency_p99] = @latency_stats[:request_latency_secs].query(0.99)
+      current_stats[:serialization_duration_secs_p50] = @latency_stats[:serialization_duration_secs].query(0.5)
+      current_stats[:serialization_duration_secs_p90] = @latency_stats[:serialization_duration_secs].query(0.9)
+      current_stats[:serialization_duration_secs_p99] = @latency_stats[:serialization_duration_secs].query(0.99)
+      current_stats[:compression_duration_secs_p50] = @latency_stats[:compression_duration_secs].query(0.5)
+      current_stats[:compression_duration_secs_p90] = @latency_stats[:compression_duration_secs].query(0.9)
+      current_stats[:compression_duration_secs_p99] = @latency_stats[:compression_duration_secs].query(0.99)
+      current_stats[:bytes_sent_p50] = @latency_stats[:bytes_sent].query(0.5)
+      current_stats[:bytes_sent_p90] = @latency_stats[:bytes_sent].query(0.9)
+      current_stats[:bytes_sent_p99] = @latency_stats[:bytes_sent].query(0.99)
+
+      if @flush_quantile_estimates_on_status_send
+        @logger.debug "Recreating / reseting quantile estimator classes for plugin metrics"
+        @latency_stats = get_new_latency_stats
+      end
+      current_stats
+    end
   end
 
 
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
-  def post_add_events(body)
-    post = prepare_post_object @add_events_uri.path, body
+  def post_add_events(body, is_status, body_serialization_duration = 0)
+    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
     fail_count = 1  # putative assume failure
     start_time = Time.now
     uncompressed_bytes_sent = 0
     compressed_bytes_sent = 0
     bytes_received = 0
     begin
-      response = @http.request(@add_events_uri, post)
+      response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
       handle_response(response)
 
       fail_count -= 1  # success means we negate the putative failure
       uncompressed_bytes_sent = (body.bytesize + @add_events_uri.path.bytesize)
-      compressed_bytes_sent = (post.body.bytesize + @add_events_uri.path.bytesize)
+      compressed_bytes_sent = (post_body.bytesize + @add_events_uri.path.bytesize)
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
+    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
+    # way to detect that it is from SSL.
     rescue OpenSSL::SSL::SSLError => e
-      if @ssl_verify_peer and @ssl_ca_bundle_path.nil? and !File.file?(@ca_cert.path)
-        @ca_cert = Tempfile.new("ca_cert")
-        @ca_cert.write(@cert_string)
-        @ca_cert.flush
-        @http.ca_file = @ca_cert.path
-        raise ClientError.new("Packaged certificate appears to have been deleted, writing a new one.", @add_events_uri)
-      else
-        raise e
-      end
+      raise e
 
-    rescue Net::HTTP::Persistent::Error => e
+    rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
       raise ClientError.new(e.message, @add_events_uri)
 
     ensure
-      @stats_lock.synchronize do
-        @stats[:total_requests_sent] += 1
-        @stats[:total_requests_failed] += fail_count
-        @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
-        @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
-        @stats[:total_response_bytes_received] += bytes_received
-        end_time = Time.now
-        @stats[:total_request_latency_secs] += (end_time - start_time)
+      if @record_stats_for_status or !is_status
+        @stats_lock.synchronize do
+          @stats[:total_requests_sent] += 1
+          @stats[:total_requests_failed] += fail_count
+          @stats[:total_request_bytes_sent] += uncompressed_bytes_sent
+          @stats[:total_compressed_request_bytes_sent] += compressed_bytes_sent
+          @stats[:total_response_bytes_received] += bytes_received
+          @stats[:total_serialization_duration_secs] += body_serialization_duration
+          @stats[:total_compression_duration_secs] += compression_duration
+          end_time = Time.now
+          @stats[:total_request_latency_secs] += (end_time - start_time)
+          @latency_stats[:request_latency_secs].observe(end_time - start_time)
+          @latency_stats[:serialization_duration_secs].observe(body_serialization_duration)
+          @latency_stats[:compression_duration_secs].observe(compression_duration)
+          @latency_stats[:bytes_sent].observe(uncompressed_bytes_sent)
+        end
       end
-
     end
   end  # def post_request
 
 
 
   def close
-    @http.shutdown
   end  # def close
 
 
@@ -211,7 +277,9 @@ class ClientSession
   def prepare_post_object(uri_path, body)
     # use compression if enabled
     encoding = nil
+    compression_duration = 0
     if @compression_type
+      start_time = Time.now.to_f
       if @compression_type == 'deflate'
         encoding = 'deflate'
         compressed_body = Zlib::Deflate.deflate(body, @compression_level)
@@ -223,20 +291,24 @@ class ClientSession
         bz2.close
         compressed_body = io.string
       end
+      end_time = Time.now.to_f
+      compression_duration = end_time - start_time
     end
 
-    post = Net::HTTP::Post.new uri_path
-    post.add_field('Content-Type', 'application/json')
-    version = 'output-logstash-scalyr 0.1.5'
-    post.add_field('User-Agent', version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM)
+    version = 'output-logstash-scalyr 0.1.10.beta'
+    post_headers = {
+      'Content-Type': 'application/json',
+      'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM
+    }
 
+    post_body = nil
     if not encoding.nil?
-      post.add_field('Content-Encoding', encoding)
-      post.body = compressed_body
+      post_headers['Content-Encoding'] = encoding
+      post_body = compressed_body
     else
-      post.body = body
+      post_body = body
     end
-    post
+    return post_body, post_headers, compression_duration
   end  # def prepare_post_object
 
 

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.5'
+  s.version         = '0.1.10.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -22,6 +22,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'net-http-persistent'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
   s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_runtime_dependency 'quantile'
   s.add_runtime_dependency 'ffi', '>= 1.9.18'
   s.add_runtime_dependency 'rbzip2', '0.3.0'
   s.add_development_dependency 'logstash-devutils'

data/spec/benchmarks/flattening_and_serialization.rb

@@ -0,0 +1,125 @@
+require 'benchmark'
+require 'json'
+
+require_relative '../../lib/scalyr/common/util'
+
+# NOTE: When using jRuby using multiple iterations with the same dataset doesn't make
+# sense since it will just use JITed version of the code which will be very fast. If we
+# wanted to accurately measure using multiple iterations we would need te different
+# input data for each iteration.
+ITERATIONS = 500
+
+def rand_str(len)
+  return (0...len).map { (65 + rand(26)).chr }.join
+end
+
+def generate_hash(widths)
+  result = {}
+  if widths.empty?
+    return rand_str(20)
+  else
+    widths[0].times do
+      result[rand_str(9)] = generate_hash(widths[1..widths.length])
+    end
+    return result
+  end
+end
+
+def generate_data_array_for_spec(spec)
+  data = []
+  ITERATIONS.times do
+    data << generate_hash(spec)
+  end
+
+  data
+end
+
+def run_benchmark_and_print_results(data, run_benchmark_func)
+  puts ""
+  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
+  puts ""
+
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.(data[0]) }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+def flatten_data_func(data)
+  Scalyr::Common::Util.flatten(data)
+end
+
+def json_serialize_data(data)
+  data.to_json
+end
+
+DATASETS = {
+  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
+  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
+  :keys_200_flat => generate_data_array_for_spec([200]),
+  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
+  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
+  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
+}
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+puts "Scalyr::Common::Util.flatten()"
+puts "==============================="
+
+# Around ~50 keys in a hash
+data = DATASETS[:keys_50]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash
+data = DATASETS[:keys_200]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~512 keys in a hash
+data = DATASETS[:keys_512]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~960 keys in a hash
+data = DATASETS[:keys_960]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+# Around ~2700 keys in a hash
+data = DATASETS[:keys_2700]
+run_benchmark_and_print_results(data, method(:flatten_data_func))
+
+puts "JSON.dumps (hash.to_dict)"
+puts "==============================="
+
+# Around ~200 keys in a hash
+data = generate_data_array_for_spec([4, 4, 3, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~200 keys in a hash (single level)
+data = DATASETS[:keys_200_flat]
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~512 keys in a hash
+data = generate_data_array_for_spec([8, 4, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~960 keys in a hash
+data = generate_data_array_for_spec([12, 5, 4, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))
+
+# Around ~2700 keys in a hash
+data = generate_data_array_for_spec([14, 8, 6, 4])
+run_benchmark_and_print_results(data, method(:json_serialize_data))

data/spec/benchmarks/metrics_overhead.rb

@@ -0,0 +1,48 @@
+require 'benchmark'
+require 'quantile'
+
+require_relative '../../lib/scalyr/common/util'
+
+# Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
+# where making sampling (e.g. on event level metrics) is desired
+
+ITERATIONS = 10000
+
+def run_benchmark_and_print_results(run_benchmark_func)
+  result = []
+  ITERATIONS.times do |i|
+    result << Benchmark.measure { run_benchmark_func.() }
+  end
+
+  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
+  avg = sum / result.size
+
+  Benchmark.bm(7, "sum:", "avg:") do |b|
+    [sum, avg]
+  end
+  puts ""
+end
+
+
+puts "Using %s iterations" % [ITERATIONS]
+puts ""
+
+@value = Quantile::Estimator.new
+@prng = Random.new
+
+def quantile_observe()
+  @value.observe(5)
+end
+
+def random_sample()
+  return @prng.rand(0.0..1.0) < 0.5
+end
+
+puts "Quartile.observe()"
+puts "==============================="
+
+run_benchmark_and_print_results(method(:quantile_observe))
+
+puts "random sample"
+puts "==============================="
+run_benchmark_and_print_results(method(:random_sample))

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -0,0 +1,77 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/scalyr"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe LogStash::Outputs::Scalyr do
+
+  describe "#ssl_tests" do
+      context "with default SSL configuration" do
+        it "throws a ServerError due to fake api key" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
+        end
+      end
+
+      context "when pointing at a location without any valid certs and not using builtin" do
+        it "throws an SSLError" do
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
+        end
+      end
+
+      context "when system certs are missing and not using builtin" do
+        it "throws an SSLError" do
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_FILE} /tmp/system_cert.pem`
+          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
+          end
+          ensure
+            `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
+            `sudo mv /tmp/system_cert.pem #{OpenSSL::X509::DEFAULT_CERT_FILE}`
+        end
+      end
+
+      context "when server hostname doesn't match the cert" do
+        it "throws an SSLError" do
+          agent_scalyr_com_ip = `dig +short agent.scalyr.com 2> /dev/null | tail -n 1 | tr -d "\n"`
+          if agent_scalyr_com_ip.empty?
+            agent_scalyr_com_ip = `getent hosts agent.scalyr.com \
+            | awk '{ print $1 }' | tail -n 1 | tr -d "\n"`
+          end
+          mock_host = "invalid.mitm.should.fail.test.agent.scalyr.com"
+          etc_hosts_entry = "#{agent_scalyr_com_ip} #{mock_host}"
+          hosts_bkp = `sudo cat /etc/hosts`
+          hosts_bkp = hosts_bkp.chomp
+          # Add mock /etc/hosts entry and config scalyr_server entry
+          `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
+
+          begin
+            expect {
+              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
+              plugin.register
+              plugin.multi_receive(sample_events)
+            }.to raise_error(Scalyr::Common::Client::ClientError, "Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)")
+          ensure
+            # Clean up the hosts file
+            `sudo truncate -s 0 /etc/hosts`
+            `echo "#{hosts_bkp}" | sudo tee -a /etc/hosts`
+          end
+        end
+      end
+  end
+end