logstash-output-scalyr 0.2.6 → 0.2.8.beta

data/spec/benchmarks/json_serialization.rb

@@ -1,85 +0,0 @@
-require 'benchmark'
-require 'json'
-require 'jrjackson'
-
-require_relative '../../lib/scalyr/common/util'
-require_relative './util'
-
-ITERATIONS = 500
-
-def json_serialize_data_native(data)
-  data.to_json
-end
-
-def json_serialize_data_jrjackson(data)
-  JrJackson::Json.dump(data)
-end
-
-DATASETS = {
-  :keys_50 => generate_data_array_for_spec([3, 3, 3, 2]),
-  :keys_200 => generate_data_array_for_spec([4, 4, 3, 4]),
-  :keys_200_flat => generate_data_array_for_spec([200]),
-  :keys_512 => generate_data_array_for_spec([8, 4, 4, 4]),
-  :keys_960 => generate_data_array_for_spec([12, 5, 4, 4]),
-  :keys_2700 => generate_data_array_for_spec([14, 8, 6, 4])
-}
-
-def run_benchmark_and_print_results(data, run_benchmark_func)
-  puts ""
-  puts "Using %s total keys in a hash" % [Scalyr::Common::Util.flatten(data[0]).count]
-  puts ""
-
-  result = []
-  ITERATIONS.times do |i|
-    result << Benchmark.measure { run_benchmark_func.(data[i]) }
-  end
-
-  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
-  avg = sum / result.size
-
-  Benchmark.bm(7, "sum:", "avg:") do |b|
-    [sum, avg]
-  end
-  puts ""
-end
-
-puts "Using %s iterations" % [ITERATIONS]
-puts ""
-
-puts "native"
-puts "==============================="
-
-# Around ~50 keys in a hash
-data = DATASETS[:keys_50]
-run_benchmark_and_print_results(data, method(:json_serialize_data_native))
-
-# Around ~200 keys in a hash
-data = DATASETS[:keys_200]
-run_benchmark_and_print_results(data, method(:json_serialize_data_native))
-
-# Around ~200 keys in a hash (single level)
-data = DATASETS[:keys_200_flat]
-run_benchmark_and_print_results(data, method(:json_serialize_data_native))
-
-# Around ~2700 keys in a hash
-data = DATASETS[:keys_2700]
-run_benchmark_and_print_results(data, method(:json_serialize_data_native))
-
-puts "jrjackson"
-puts "==============================="
-
-# Around ~50 keys in a hash
-data = DATASETS[:keys_50]
-run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
-
-# Around ~200 keys in a hash
-data = DATASETS[:keys_200]
-run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
-
-# Around ~200 keys in a hash (single level)
-data = DATASETS[:keys_200_flat]
-run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))
-
-# Around ~2700 keys in a hash
-data = DATASETS[:keys_2700]
-run_benchmark_and_print_results(data, method(:json_serialize_data_jrjackson))

data/spec/benchmarks/metrics_overhead.rb

@@ -1,48 +0,0 @@
-require 'benchmark'
-require 'quantile'
-
-require_relative '../../lib/scalyr/common/util'
-
-# Micro benchmark which measures how much overhead Quantile.observe adds vs random sampling to see
-# where making sampling (e.g. on event level metrics) is desired
-
-ITERATIONS = 10000
-
-def run_benchmark_and_print_results(run_benchmark_func)
-  result = []
-  ITERATIONS.times do |i|
-    result << Benchmark.measure { run_benchmark_func.() }
-  end
-
-  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
-  avg = sum / result.size
-
-  Benchmark.bm(7, "sum:", "avg:") do |b|
-    [sum, avg]
-  end
-  puts ""
-end
-
-
-puts "Using %s iterations" % [ITERATIONS]
-puts ""
-
-@value = Quantile::Estimator.new
-@prng = Random.new
-
-def quantile_observe()
-  @value.observe(5)
-end
-
-def random_sample()
-  return @prng.rand(0.0..1.0) < 0.5
-end
-
-puts "Quartile.observe()"
-puts "==============================="
-
-run_benchmark_and_print_results(method(:quantile_observe))
-
-puts "random sample"
-puts "==============================="
-run_benchmark_and_print_results(method(:random_sample))

data/spec/benchmarks/set_session_level_serverhost_on_events.rb

@@ -1,107 +0,0 @@
-require 'benchmark'
-require 'quantile'
-
-require_relative '../../lib/scalyr/constants'
-require_relative '../../lib/scalyr/common/util'
-require_relative './util'
-
-# Micro benchmark which measures how long "set_session_level_serverhost_on_events" takes
-
-ITERATIONS = 100
-
-def run_benchmark_and_print_results(data, run_benchmark_func)
-  puts ""
-  puts "Using %s total events in a batch" % [data[0].size]
-  puts ""
-
-  result = []
-  ITERATIONS.times do |i|
-    result << Benchmark.measure { run_benchmark_func.(data[i]) }
-  end
-
-  sum = result.inject(nil) { |sum, t| sum.nil? ? sum = t : sum += t }
-  avg = sum / result.size
-
-  Benchmark.bm(7, "sum:", "avg:") do |b|
-    [sum, avg]
-  end
-  puts ""
-end
-
-# Generate random event with only single event having special server host attribute set which
-# represents a worst case scenario since we need to backfill rest of the events.
-def generate_events(count)
-  result = []
-
-  ITERATIONS.times do |iteration|
-    events = []
-
-    count.times do |index|
-      event = generate_hash([2])
-      event[:attrs] = Hash.new
-      event[:log] = 1
-
-      if index == count - 1
-        event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME] = format("test-host-%s", index)
-      end
-
-      events << event
-    end
-
-    raise "Assertion failed" unless events.size == count
-
-    result << events
-  end
-
-  raise "Assertion failed" unless result.size == ITERATIONS
-  result
-end
-
-def run_func(events)
-  # NOTE: This function manipulates events in place
-  events.each_with_index do |event, index|
-    if index < events.size - 1
-      # Last event will have _origServerHost set, but others won't
-      raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil?
-    end
-  end
-
-  Scalyr::Common::Util.set_session_level_serverhost_on_events("session-server-host-dummy", events, {}, true)
-
-  events.each do |event|
-    raise "Assertion failed" unless event[:attrs][EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME].nil? == false
-  end
-end
-
-
-puts "Using %s iterations" % [ITERATIONS]
-puts ""
-
-@value = Quantile::Estimator.new
-
-puts "Util.set_session_level_serverhost_on_events()"
-puts "==============================="
-
-# 100 events in a batch
-data = generate_events(100)
-run_benchmark_and_print_results(data, method(:run_func))
-
-# 500 events in a batch
-data = generate_events(500)
-run_benchmark_and_print_results(data, method(:run_func))
-
-# 1000 events in a batch
-data = generate_events(1000)
-run_benchmark_and_print_results(data, method(:run_func))
-
-# 2000 events in a batch
-data = generate_events(2000)
-run_benchmark_and_print_results(data, method(:run_func))
-
-# 3000 events in a batch
-data = generate_events(3000)
-run_benchmark_and_print_results(data, method(:run_func))
-
-# 5000 events in a batch
-data = generate_events(5000)
-run_benchmark_and_print_results(data, method(:run_func))

data/spec/benchmarks/util.rb

@@ -1,24 +0,0 @@
-def rand_str(len)
-  return (0...len).map { (65 + rand(26)).chr }.join
-end
-
-def generate_hash(widths)
-  result = {}
-  if widths.empty?
-    return rand_str(20)
-  else
-    widths[0].times do
-      result[rand_str(9)] = generate_hash(widths[1..widths.length])
-    end
-    return result
-  end
-end
-
-def generate_data_array_for_spec(spec)
-  data = []
-  ITERATIONS.times do
-    data << generate_hash(spec)
-  end
-
-  data
-end

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -1,337 +0,0 @@
-# encoding: utf-8
-require "logstash/devutils/rspec/spec_helper"
-require "logstash/outputs/scalyr"
-require "logstash/codecs/plain"
-require "logstash/event"
-require "json"
-require 'webmock/rspec'
-
-# Require the specific version of `json` used in logstash
-gem 'json', '1.8.6'
-require 'json'
-
-WebMock.allow_net_connect!
-
-RSpec.configure do |rspec|
-  rspec.expect_with :rspec do |c|
-    c.max_formatted_output_length = nil
-  end
-end
-
-describe LogStash::Outputs::Scalyr do
-  let(:sample_events) {
-    events = []
-    for i in 1..3 do
-      e = LogStash::Event.new
-      e.set('source_host', "my host #{i}")
-      e.set('source_file', "my file #{i}")
-      e.set('seq', i)
-      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
-      e.set('tags', ['t1', 't2', 't3'])
-      events.push(e)
-    end
-    events
-  }
-
-  describe "#ssl_tests" do
-      context "with default SSL configuration" do
-        it "throws a ServerError due to fake api key" do
-              plugin = LogStash::Outputs::Scalyr.new({
-                'api_write_token' => '1234',
-                'perform_connectivity_check' => false,
-                'max_retries' => 2,
-                'retry_max_interval' => 2,
-                'retry_initial_interval' => 0.2,
-              })
-              plugin.register
-              plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-              plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-                {
-                  :error_class=>"Scalyr::Common::Client::ServerError",
-                  :batch_num=>1,
-                  :code=>401,
-                  :message=>"error/client/badParam",
-                  :payload_size=>737,
-                  :record_count=>3,
-                  :total_batches=>1,
-                  :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>0.4,
-                  :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
-                }
-              )
-        end
-      end
-
-      context "when pointing at a location without any valid certs and not using builtin" do
-        it "throws an SSLError" do
-              plugin = LogStash::Outputs::Scalyr.new({
-                'api_write_token' => '1234',
-                'perform_connectivity_check' => false,
-                'ssl_ca_bundle_path' => '/fakepath/nocerts',
-                'append_builtin_cert' => false,
-                'max_retries' => 2,
-                'retry_max_interval' => 2,
-                'retry_initial_interval' => 0.2,
-              })
-              plugin.register
-              plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-              plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-                {
-                  :error_class=>"Manticore::UnknownException",
-                  :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
-                  #:message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
-                  :payload_size=>737,
-                  :record_count=>3,
-                  :total_batches=>1,
-                  :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>0.4
-                }
-              )
-        end
-      end
-
-      context "when system certs are missing and not using builtin" do
-        it "throws an SSLError" do
-          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_FILE} /tmp/system_cert.pem`
-          `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
-
-          begin
-              plugin = LogStash::Outputs::Scalyr.new({
-                'api_write_token' => '1234',
-                'perform_connectivity_check' => false,
-                'append_builtin_cert' => false,
-                'max_retries' => 2,
-                'retry_max_interval' => 2,
-                'retry_initial_interval' => 0.2,
-              })
-              plugin.register
-              plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-              plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-                {
-                  :error_class=>"Manticore::UnknownException",
-                  :batch_num=>1,
-                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
-                  #:message=>"java.lang.RuntimeException: Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
-                  :payload_size=>737,
-                  :record_count=>3,
-                  :total_batches=>1,
-                  :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>0.4
-                }
-              )
-          end
-          ensure
-            `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
-            `sudo mv /tmp/system_cert.pem #{OpenSSL::X509::DEFAULT_CERT_FILE}`
-        end
-      end
-
-      context "when server hostname doesn't match the cert" do
-        it "throws an SSLError" do
-          agent_scalyr_com_ip = `dig +short agent.scalyr.com 2> /dev/null | tail -n 1 | tr -d "\n"`
-          if agent_scalyr_com_ip.empty?
-            agent_scalyr_com_ip = `getent hosts agent.scalyr.com \
-            | awk '{ print $1 }' | tail -n 1 | tr -d "\n"`
-          end
-          mock_host = "invalid.mitm.should.fail.test.agent.scalyr.com"
-          etc_hosts_entry = "#{agent_scalyr_com_ip} #{mock_host}"
-          hosts_bkp = `sudo cat /etc/hosts`
-          hosts_bkp = hosts_bkp.chomp
-          # Add mock /etc/hosts entry and config scalyr_server entry
-          `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
-
-          begin
-              plugin = LogStash::Outputs::Scalyr.new({
-                'api_write_token' => '1234',
-                'perform_connectivity_check' => false,
-                'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
-                'max_retries' => 2,
-                'retry_max_interval' => 2,
-                'retry_initial_interval' => 0.2,
-              })
-              plugin.register
-              plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-              plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-                {
-                  :error_class=>"Manticore::UnknownException",
-                  :batch_num=>1,
-                  :message=>"Certificate for <invalid.mitm.should.fail.test.agent.scalyr.com> doesn't match any of the subject alternative names: [*.scalyr.com, scalyr.com]",
-                  :payload_size=>737,
-                  :record_count=>3,
-                  :total_batches=>1,
-                  :url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>0.4
-                }
-              )
-          ensure
-            # Clean up the hosts file
-            `sudo truncate -s 0 /etc/hosts`
-            `echo "#{hosts_bkp}" | sudo tee -a /etc/hosts`
-          end
-        end
-      end
-
-      context "when an error occurs with retries at 15" do
-        it "exits after 5 retries and emits a log" do
-              plugin = LogStash::Outputs::Scalyr.new({
-                'api_write_token' => '1234',
-                'perform_connectivity_check' => false,
-                'ssl_ca_bundle_path' => '/fakepath/nocerts',
-                'append_builtin_cert' => false,
-                'max_retries' => 15,
-                'retry_max_interval' => 0.5,
-                'retry_initial_interval' => 0.2,
-              })
-              plugin.register
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
-              plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 15 tries.", anything
-              )
-        end
-      end
-  end
-
-  describe "response_handling_tests" do
-    context "when receiving a 503 response" do
-      it "don't throw an error but do log one to debug" do
-        stub_request(:post, "https://agent.scalyr.com/addEvents").
-          to_return(status: 503, body: "stubbed response", headers: {})
-
-        plugin = LogStash::Outputs::Scalyr.new({
-          'api_write_token' => '1234',
-          'perform_connectivity_check' => false,
-          'ssl_ca_bundle_path' => '/fakepath/nocerts',
-          'append_builtin_cert' => false,
-          'max_retries' => 2,
-          'retry_max_interval' => 2,
-          'retry_initial_interval' => 0.2,
-        })
-        plugin.register
-        plugin.instance_variable_set(:@running, false)
-
-        allow(plugin.instance_variable_get(:@logger)).to receive(:debug)
-        plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
-          {
-            :error_class=>"Scalyr::Common::Client::ServerError",
-            :batch_num=>1,
-            :code=>503,
-            :message=>"Invalid JSON response from server",
-            :payload_size=>737,
-            :record_count=>3,
-            :total_batches=>1,
-            :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>0.4,
-            :body=>"stubbed response"
-          }
-        )
-      end
-    end
-
-    context "when receiving a 500 response" do
-      it "don't throw an error but do log one to error" do
-        stub_request(:post, "https://agent.scalyr.com/addEvents").
-          to_return(status: 500, body: "stubbed response", headers: {})
-
-        plugin = LogStash::Outputs::Scalyr.new({
-          'api_write_token' => '1234',
-          'perform_connectivity_check' => false,
-          'ssl_ca_bundle_path' => '/fakepath/nocerts',
-          'append_builtin_cert' => false,
-          'max_retries' => 2,
-          'retry_max_interval' => 2,
-          'retry_initial_interval' => 0.2,
-        })
-        plugin.register
-        plugin.instance_variable_set(:@running, false)
-
-        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-        plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-          {
-            :error_class=>"Scalyr::Common::Client::ServerError",
-            :batch_num=>1,
-            :code=>500,
-            :message=>"Invalid JSON response from server",
-            :payload_size=>737,
-            :record_count=>3,
-            :total_batches=>1,
-            :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>0.4,
-            :body=>"stubbed response"
-          }
-        )
-      end
-    end
-
-    context "when receiving a long non-json response" do
-      it "don't throw an error but do log one to error" do
-        stub_request(:post, "https://agent.scalyr.com/addEvents").
-          to_return(status: 500, body: "0123456789" * 52, headers: {})
-
-        plugin = LogStash::Outputs::Scalyr.new({
-            'api_write_token' => '1234',
-            'perform_connectivity_check' => false,
-            'ssl_ca_bundle_path' => '/fakepath/nocerts',
-            'append_builtin_cert' => false,
-            'max_retries' => 2,
-            'retry_max_interval' => 2,
-            'retry_initial_interval' => 0.2,
-        })
-        plugin.register
-        plugin.instance_variable_set(:@running, false)
-
-        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
-        plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
-          {
-            :error_class=>"Scalyr::Common::Client::ServerError",
-            :batch_num=>1,
-            :code=>500,
-            :message=>"Invalid JSON response from server",
-            :payload_size=>737,
-            :record_count=>3,
-            :total_batches=>1,
-            :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>0.4,
-            :body=>("0123456789" * 50) + "012345678..."
-          }
-        )
-      end
-    end
-
-    context 'when DLQ is enabled' do
-      let(:dlq_writer) { double('DLQ writer') }
-      it 'should send the event to the DLQ' do
-        stub_request(:post, "https://agent.scalyr.com/addEvents").
-          to_return(status: 500, body: "stubbed response", headers: {})
-
-        plugin = LogStash::Outputs::Scalyr.new({
-            'api_write_token' => '1234',
-            'perform_connectivity_check' => false,
-            'ssl_ca_bundle_path' => '/fakepath/nocerts',
-            'append_builtin_cert' => false,
-            'max_retries' => 2,
-            'retry_max_interval' => 2,
-            'retry_initial_interval' => 0.2,
-        })
-        plugin.register
-        plugin.instance_variable_set(:@running, false)
-        plugin.instance_variable_set('@dlq_writer', dlq_writer)
-
-        expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
-        plugin.multi_receive(sample_events)
-      end
-    end
-  end
-
-end