logstash-output-scalyr 0.1.10.beta → 0.1.15.beta

data/lib/scalyr/common/client.rb

@@ -1,3 +1,5 @@
+require "scalyr/constants"
+
 module Scalyr; module Common; module Client
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -5,13 +7,14 @@ module Scalyr; module Common; module Client
 #---------------------------------------------------------------------------------------------------------------------
 class ServerError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, code=nil, url=nil, body=nil)
+  def initialize(msg=nil, code=nil, url=nil, body=nil, e_class="Scalyr::Common::Client::ServerError")
     super(msg)
     @code = code.to_i
     @url = url
     @body = body
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -31,13 +34,14 @@ end
 #---------------------------------------------------------------------------------------------------------------------
 class ClientError < StandardError
 
-  attr_reader :code, :url, :body
+  attr_reader :code, :url, :body, :e_class
 
-  def initialize(msg=nil, url=nil)
+  def initialize(msg=nil, url=nil, e_class="Scalyr::Common::Client::ClientError")
     super(msg)
     @code = nil  # currently no way to get this from Net::HTTP::Persistent::Error
     @url = url
     @body = nil
+    @e_class = e_class
   end
 
   def is_commonly_retried?
@@ -234,15 +238,10 @@ class ClientSession
       bytes_received = response.body.bytesize  # echee: double check
         # echee TODO add more statistics
 
-    # TODO: Manticore doesn't raise SSL errors as this but as "UnknownExceptions", need to dig in and see if there is a
-    # way to detect that it is from SSL.
-    rescue OpenSSL::SSL::SSLError => e
-      raise e
-
     rescue Manticore::ManticoreException => e
       # The underlying persistent-connection library automatically retries when there are network-related errors.
       # Eventually, it will give up and raise this generic error, at which time, we convert it to a ClientError
-      raise ClientError.new(e.message, @add_events_uri)
+      raise ClientError.new(e.message, @add_events_uri, e.class.name)
 
     ensure
       if @record_stats_for_status or !is_status
@@ -268,6 +267,7 @@ class ClientSession
 
 
   def close
+    @client.close if @client
   end  # def close
 
 
@@ -295,7 +295,7 @@ class ClientSession
       compression_duration = end_time - start_time
     end
 
-    version = 'output-logstash-scalyr 0.1.10.beta'
+    version = sprintf('output-logstash-scalyr %s' % [PLUGIN_VERSION])
     post_headers = {
       'Content-Type': 'application/json',
       'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM

data/lib/scalyr/common/util.rb

@@ -45,5 +45,12 @@ def self.flatten(obj, delimiter='_')
   return result
 end
 
+def self.truncate(content, max)
+  if content.length > max
+    return "#{content[0...(max-3)]}..."
+  end
+  return content
+end
+
 end; end; end;
 

data/lib/scalyr/constants.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+PLUGIN_VERSION = "v0.1.15.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.10.beta'
+  s.version         = '0.1.15.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -4,27 +4,68 @@ require "logstash/outputs/scalyr"
 require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
+require 'webmock/rspec'
+WebMock.allow_net_connect!
 
 describe LogStash::Outputs::Scalyr do
+  let(:sample_events) {
+    events = []
+    for i in 1..3 do
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+    events
+  }
 
   describe "#ssl_tests" do
       context "with default SSL configuration" do
         it "throws a ServerError due to fake api key" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :error_class=>"Scalyr::Common::Client::ServerError",
+                  :batch_num=>1,
+                  :code=>401,
+                  :message=>"error/client/badParam",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2,
+                  :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
+                }
+              )
         end
       end
 
       context "when pointing at a location without any valid certs and not using builtin" do
         it "throws an SSLError" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :error_class=>"Manticore::UnknownException",
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
         end
       end
 
@@ -34,11 +75,23 @@ describe LogStash::Outputs::Scalyr do
           `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :error_class=>"Manticore::UnknownException",
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -61,11 +114,23 @@ describe LogStash::Outputs::Scalyr do
           `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)")
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :error_class=>"Manticore::UnknownException",
+                  :batch_num=>1,
+                  :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -73,5 +138,119 @@ describe LogStash::Outputs::Scalyr do
           end
         end
       end
+
+      context "when an error occurs with retries at 5" do
+        it "exits after 5 retries and emits a log" do
+              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin.register
+              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              plugin.multi_receive(sample_events)
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              )
+        end
+      end
   end
+
+  describe "response_handling_tests" do
+    context "when receiving a 503 response" do
+      it "don't throw an error but do log one to debug" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 503, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:debug)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :error_class=>"Scalyr::Common::Client::ServerError",
+            :batch_num=>1,
+            :code=>503,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
+          }
+        )
+      end
+    end
+
+    context "when receiving a 500 response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :error_class=>"Scalyr::Common::Client::ServerError",
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>"stubbed response"
+          }
+        )
+      end
+    end
+
+    context "when receiving a long non-json response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "0123456789" * 52, headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :error_class=>"Scalyr::Common::Client::ServerError",
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2,
+            :body=>("0123456789" * 50) + "012345678..."
+          }
+        )
+      end
+    end
+
+    context 'when DLQ is enabled' do
+      let(:dlq_writer) { double('DLQ writer') }
+      it 'should send the event to the DLQ' do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+        plugin.instance_variable_set('@dlq_writer', dlq_writer)
+
+        expect(dlq_writer).to receive(:write).exactly(3).times.with(anything, anything)
+        plugin.multi_receive(sample_events)
+      end
+    end
+  end
+
 end

data/spec/logstash/outputs/scalyr_spec.rb

@@ -69,7 +69,7 @@ describe LogStash::Outputs::Scalyr do
       end
 
       it "it doesnt include flatten metrics if flattening is disabled" do
-          plugin1 = LogStash::Outputs::Scalyr.new({
+        plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
@@ -83,13 +83,14 @@ describe LogStash::Outputs::Scalyr do
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
         plugin1.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(1)
         plugin1.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
 
         status_event = plugin1.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=1 multi_receive_duration_p90=1 multi_receive_duration_p99=1 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0")
       end
 
       it "returns and sends correct status event on send_stats on initial and subsequent send" do
@@ -98,7 +99,7 @@ describe LogStash::Outputs::Scalyr do
         plugin.instance_variable_set(:@client_session, mock_client_session)
         plugin.instance_variable_set(:@session_id, "some_session_id")
         status_event = plugin.send_status
-        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin.")
+        expect(status_event[:attrs]["message"]).to eq("Started Scalyr LogStash output plugin (%s)." % [PLUGIN_VERSION])
 
         # 2. Second send
         plugin.instance_variable_set(:@last_status_transmit_time, 100)
@@ -108,7 +109,8 @@ describe LogStash::Outputs::Scalyr do
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
 
         (1..20).each do |n|
@@ -117,10 +119,10 @@ describe LogStash::Outputs::Scalyr do
 
         plugin.instance_variable_set(:@multi_receive_statistics, {:total_multi_receive_secs => 0})
         status_event = plugin.send_status
-        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
+        expect(status_event[:attrs]["message"]).to eq("plugin_status: total_requests_sent=20 total_requests_failed=10 total_request_bytes_sent=100 total_compressed_request_bytes_sent=50 total_response_bytes_received=100 total_request_latency_secs=100 total_serialization_duration_secs=100.5000 total_compression_duration_secs=10.2000 compression_type=deflate compression_level=9 total_multi_receive_secs=0 multi_receive_duration_p50=10 multi_receive_duration_p90=18 multi_receive_duration_p99=19 multi_receive_event_count_p50=0 multi_receive_event_count_p90=0 multi_receive_event_count_p99=0 event_attributes_count_p50=0 event_attributes_count_p90=0 event_attributes_count_p99=0 batches_per_multi_receive_p50=0 batches_per_multi_receive_p90=0 batches_per_multi_receive_p99=0 flatten_values_duration_secs_p50=0 flatten_values_duration_secs_p90=0 flatten_values_duration_secs_p99=0")
       end
 
-      it "send_stats is called when events list is empty, but otherwise noop" do
+      it "send_stats is called when events list is empty, but otherwise is noop" do
         quantile_estimator = Quantile::Estimator.new
         plugin.instance_variable_set(:@plugin_metrics, {
           :multi_receive_duration_secs => Quantile::Estimator.new,
@@ -135,6 +137,30 @@ describe LogStash::Outputs::Scalyr do
         plugin.multi_receive([])
       end
 
+      it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
+        plugin2 = LogStash::Outputs::Scalyr.new({
+                                                     'api_write_token' => '1234',
+                                                     'serverhost_field' => 'source_host',
+                                                     'log_constants' => ['tags'],
+                                                     'flatten_nested_values' => false,
+                                                     'report_status_for_empty_batches' => false,
+                                                 })
+
+        mock_client_session = MockClientSession.new
+        quantile_estimator = Quantile::Estimator.new
+        plugin2.instance_variable_set(:@plugin_metrics, {
+          :multi_receive_duration_secs => Quantile::Estimator.new,
+          :multi_receive_event_count => Quantile::Estimator.new,
+          :event_attributes_count => Quantile::Estimator.new,
+          :flatten_values_duration_secs => Quantile::Estimator.new
+        })
+        plugin2.instance_variable_set(:@client_session, mock_client_session)
+        expect(plugin2).not_to receive(:send_status)
+        expect(quantile_estimator).not_to receive(:observe)
+        expect(mock_client_session).not_to receive(:post_add_events)
+        plugin2.multi_receive([])
+      end
+
       # Kind of a weak test but I don't see a decent way to write a stronger one without a live client session
       it "send_status only sends posts with is_status = true" do
         # 1. Initial send
@@ -151,7 +177,8 @@ describe LogStash::Outputs::Scalyr do
           :multi_receive_duration_secs => Quantile::Estimator.new,
           :multi_receive_event_count => Quantile::Estimator.new,
           :event_attributes_count =>  Quantile::Estimator.new,
-          :flatten_values_duration_secs => Quantile::Estimator.new
+          :flatten_values_duration_secs => Quantile::Estimator.new,
+          :batches_per_multi_receive => Quantile::Estimator.new
         })
         (1..20).each do |n|
           plugin.instance_variable_get(:@plugin_metrics)[:multi_receive_duration_secs].observe(n)

data/vendor/bundle/jruby/2.5.0/bin/htmldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #

data/vendor/bundle/jruby/2.5.0/bin/ldiff

@@ -1,6 +1,6 @@
 #!/bin/sh
 'exec' "jruby" '-x' "$0" "$@"
-#!/Users/yans/.rvm/rubies/jruby-9.2.9.0/bin/jruby
+#!/Users/tomaz/.rvm/rubies/jruby-9.2.9.0/bin/jruby
 #
 # This file was generated by RubyGems.
 #