logstash-output-scalyr 0.1.10.beta → 0.1.11.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 290310d36108a3311f55a47efbb3a73e01f19ccdf025a15590f86493459963da
-  data.tar.gz: a5bf9f872c496fa65670cab36e0955f1417541db5f151e11f0d914abe5fc9920
+  metadata.gz: d6df8debd88e7675c00715a8b4fa2888e14cea861436c0049c1cbb99b96f2d0c
+  data.tar.gz: f07b38d925f732664390d391da3e84b3930644b77197b871043cabbf28bfd218
 SHA512:
-  metadata.gz: 35d60d700b6bb6ccc43aea5151d07c306b29c5524ac8d80145ae792263ffe5e7264d22a59de3a21de9aa549c5963b4cd3c9b230b845f67c6ff9618faca863cad
-  data.tar.gz: 41cf50b968a9a11efff957cda768cf9d05e1ccf3c36e4f4d8f12c3d1aca9fb9da049647432bf047d51c2b6fe33f7b711484632d108dcd390ca852bc45aa7a00d
+  metadata.gz: 542cacbdb0ecb8d9fbabfd955b3c88dcd826af141ded1924eeb9863b8a075fcb8c323448bf0dbf4ac06254309309cdc8a60b1899e5aaa9a51f167243ff1fb818
+  data.tar.gz: f19aeb46e95f7207e11dccd1b89aa6927e9bec88276cc1a5393ac144139b559c40ac2102da0a22624ac532c27dc753fca7b83f860b6b4763b9f28f67e4662555

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Beta
 
+## 0.1.11.beta
+- Fixes to retry mechanisms.
+- More thorough catching of events, preferring to retry requests rather than crashing the plugin.
+
 ## 0.1.10.beta
 
 - Switch to shared concurrency to allow the use of multiple worker threads for increased

data/Gemfile

@@ -10,6 +10,10 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
 
+group :test do
+  gem "webmock"
+end
+
 gem 'pry'
 gem 'pry-nav'
 gem 'quantile'

data/README.md

@@ -10,7 +10,7 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.9.gem` or follow the latest official instructions on working with plugins from Logstash.
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.1.11.beta.gem` or follow the latest official instructions on working with plugins from Logstash.
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -369,7 +369,7 @@ If you want to change status reporting interval you can do that by changing the
 ## Updating version
 
 Currently references to the version need to be manually updated, files to look in for this are `logstash-putput-scalyr.gemspec`,
- `client.rb`, twice in the CircleCI `Dockerfile`, and under "Quick Start" in this `README.md`.
+ `lib/scalyr/constants.rb`, and under "Quick Start" in this `README.md`.
 
 The changelog should also be updated with the latest version and changes of note.
 

data/lib/logstash/outputs/scalyr.rb

@@ -16,6 +16,7 @@ require 'quantile'
 
 require 'scalyr/common/client'
 require "scalyr/common/util"
+require "scalyr/constants"
 
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -243,7 +244,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
 
-    @logger.info("Started Scalyr output plugin", :class => self.class.name)
+    @logger.info(sprintf("Started Scalyr output plugin (%s)." % [PLUGIN_VERSION]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
     send_status
@@ -256,7 +257,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       :multi_receive_duration_secs => Quantile::Estimator.new,
       :multi_receive_event_count => Quantile::Estimator.new,
       :event_attributes_count => Quantile::Estimator.new,
-      :flatten_values_duration_secs => Quantile::Estimator.new
+      :flatten_values_duration_secs => Quantile::Estimator.new,
+      :batches_per_multi_receive => Quantile::Estimator.new
     }
   end
 
@@ -275,87 +277,93 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     # Just return and pretend we did something if running in noop mode
     return events if @noop_mode
 
-    start_time = Time.now.to_f
+    begin
+      start_time = Time.now.to_f
 
-    multi_event_request_array = build_multi_event_request_array(events)
-    # Loop over all array of multi-event requests, sending each multi-event to Scalyr
+      multi_event_request_array = build_multi_event_request_array(events)
+      # Loop over all array of multi-event requests, sending each multi-event to Scalyr
 
-    sleep_interval = @retry_initial_interval
-    batch_num = 1
-    total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
+      sleep_interval = @retry_initial_interval
+      batch_num = 1
+      total_batches = multi_event_request_array.length unless multi_event_request_array.nil?
 
-    result = []
-    records_count = events.to_a.length
+      result = []
+      records_count = events.to_a.length
 
-    while !multi_event_request_array.to_a.empty?
-      begin
+      while !multi_event_request_array.to_a.empty?
         multi_event_request = multi_event_request_array.pop
-        # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
-        # ignore these.
-        if !multi_event_request.nil?
-          @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
+        begin
+          # For some reason a retry on the multi_receive may result in the request array containing `nil` elements, we
+          # ignore these.
+          if !multi_event_request.nil?
+            @client_session.post_add_events(multi_event_request[:body], false, multi_event_request[:serialization_duration])
+
+            sleep_interval = @retry_initial_interval
+            batch_num += 1
+            result.push(multi_event_request)
+          end
 
-          sleep_interval = 0
-          result.push(multi_event_request)
+        rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
+          sleep_interval = sleep_for(sleep_interval)
+          message = "Error uploading to Scalyr (will backoff-retry)"
+          exc_data = {
+              :url => e.url.to_s,
+              :message => e.message,
+              :batch_num => batch_num,
+              :total_batches => total_batches,
+              :record_count => multi_event_request[:record_count],
+              :payload_size => multi_event_request[:body].bytesize,
+              :will_retry_in_seconds => sleep_interval,
+          }
+          exc_data[:code] = e.code if e.code
+          exc_data[:body] = e.body if @logger.debug? and e.body
+          exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
+          if e.is_commonly_retried?
+            # well-known retriable errors should be debug
+            @logger.debug(message, exc_data)
+          else
+            # all other failed uploads should be errors
+            @logger.error(message, exc_data)
+          end
+          sleep_interval *= 2
+          retry if @running
+
+        rescue => e
+          # Any unexpected errors should be fully logged
+          @logger.error(
+              "Unexpected error occurred while uploading to Scalyr (will backoff-retry)",
+              :error_message => e.message,
+              :error_class => e.class.name,
+              :backtrace => e.backtrace
+          )
+          @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
+          sleep_interval = sleep_for(sleep_interval)
+          sleep_interval *= 2
+          retry if @running
         end
+      end
 
-      rescue OpenSSL::SSL::SSLError => e
-        # cannot rely on exception message, so we always log the following warning
-        @logger.error "SSL certificate verification failed.  " +
-        "Please make sure your certificate bundle is configured correctly and points to a valid file.  " +
-        "You can configure this with the ssl_ca_bundle_path configuration option.  " +
-        "The current value of ssl_ca_bundle_path is '#{@ssl_ca_bundle_path}'"
-        @logger.error e.message
-        @logger.error "Discarding buffer chunk without retrying."
-
-      rescue Scalyr::Common::Client::ServerError, Scalyr::Common::Client::ClientError => e
-        sleep_interval = sleep_for(sleep_interval)
-        message = "Error uploading to Scalyr (will backoff-retry)"
-        exc_data = {
-            :url => e.url.to_s,
-            :message => e.message,
-            :batch_num => batch_num,
-            :total_batches => total_batches,
-            :record_count => multi_event_request[:record_count],
-            :payload_size => multi_event_request[:body].bytesize,
-            :will_retry_in_seconds => sleep_interval,
-        }
-        exc_data[:code] = e.response_code if e.code
-        exc_data[:body] = e.response_body if @logger.debug? and e.body
-        exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
-        if e.is_commonly_retried?
-          # well-known retriable errors should be debug
-          @logger.error(message, exc_data)
-        else
-          # all other failed uploads should be errors
-          @logger.error(message, exc_data)
+      if records_count > 0
+        @stats_lock.synchronize do
+          @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
+          @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
+          @plugin_metrics[:multi_receive_event_count].observe(records_count)
+          @plugin_metrics[:batches_per_multi_receive].observe(total_batches)
         end
-        retry if @running
-
-      rescue => e
-        # Any unexpected errors should be fully logged
-        @logger.error(
-            "Unexpected error occurred while uploading to Scalyr (will backoff-retry)",
-            :error_message => e.message,
-            :error_class => e.class.name,
-            :backtrace => e.backtrace
-        )
-        @logger.debug("Failed multi_event_request", :multi_event_request => multi_event_request)
-        sleep_interval = sleep_for(sleep_interval)
-        retry if @running
       end
-    end
 
-    if records_count > 0
-      @stats_lock.synchronize do
-        @multi_receive_statistics[:total_multi_receive_secs] += (Time.now.to_f - start_time)
-        @plugin_metrics[:multi_receive_duration_secs].observe(Time.now.to_f - start_time)
-        @plugin_metrics[:multi_receive_event_count].observe(records_count)
-      end
+      send_status
+      return result
+
+    rescue => e
+      # Any unexpected errors should be fully logged
+      @logger.error(
+          "Unexpected error occurred while executing multi_receive.",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+      )
     end
-
-    send_status
-    return result
   end  # def multi_receive
 
 
@@ -676,6 +684,10 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       current_stats[:event_attributes_count_p90] = @plugin_metrics[:event_attributes_count].query(0.9)
       current_stats[:event_attributes_count_p99] = @plugin_metrics[:event_attributes_count].query(0.99)
 
+      current_stats[:batches_per_multi_receive_p50] = @plugin_metrics[:batches_per_multi_receive].query(0.5)
+      current_stats[:batches_per_multi_receive_p90] = @plugin_metrics[:batches_per_multi_receive].query(0.9)
+      current_stats[:batches_per_multi_receive_p99] = @plugin_metrics[:batches_per_multi_receive].query(0.99)
+
       if @flatten_nested_values
         # We only return those metrics in case flattening is enabled
         current_stats[:flatten_values_duration_secs_p50] = @plugin_metrics[:flatten_values_duration_secs].query(0.5)
@@ -710,7 +722,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     }
     @send_stats.synchronize do
       if !@last_status_transmit_time
-        status_event[:attrs]['message'] = "Started Scalyr LogStash output plugin."
+        status_event[:attrs]['message'] = sprintf("Started Scalyr LogStash output plugin (%s)." % [PLUGIN_VERSION])
         status_event[:attrs]['serverHost'] = @node_hostname
       else
         cur_time = Time.now()
@@ -738,7 +750,16 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       end
     end
     multi_event_request = create_multi_event_request([status_event], nil, nil)
-    @client_session.post_add_events(multi_event_request[:body], true, 0)
+    begin
+      @client_session.post_add_events(multi_event_request[:body], true, 0)
+    rescue => e
+      @logger.warn(
+        "Unexpected error occurred while uploading status to Scalyr",
+        :error_message => e.message,
+        :error_class => e.class.name
+      )
+      return
+    end
     @last_status_transmit_time = Time.now()
 
     if @log_status_messages_to_stdout

data/lib/scalyr/common/client.rb

@@ -1,3 +1,5 @@
+require "scalyr/constants"
+
 module Scalyr; module Common; module Client
 
 #---------------------------------------------------------------------------------------------------------------------
@@ -295,7 +297,7 @@ class ClientSession
       compression_duration = end_time - start_time
     end
 
-    version = 'output-logstash-scalyr 0.1.10.beta'
+    version = sprintf('output-logstash-scalyr %s' % [PLUGIN_VERSION])
     post_headers = {
       'Content-Type': 'application/json',
       'User-Agent': version + ';' + RUBY_VERSION + ';' + RUBY_PLATFORM

data/lib/scalyr/constants.rb

@@ -0,0 +1,2 @@
+# encoding: utf-8
+PLUGIN_VERSION = "v0.1.11.beta"

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.1.10.beta'
+  s.version         = '0.1.11.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -4,27 +4,63 @@ require "logstash/outputs/scalyr"
 require "logstash/codecs/plain"
 require "logstash/event"
 require "json"
+require 'webmock/rspec'
+WebMock.allow_net_connect!
 
 describe LogStash::Outputs::Scalyr do
+  let(:sample_events) {
+    events = []
+    for i in 1..3 do
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+    events
+  }
 
   describe "#ssl_tests" do
       context "with default SSL configuration" do
         it "throws a ServerError due to fake api key" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :code=>401,
+                  :message=>"error/client/badParam",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ServerError, "error/client/badParam")
         end
       end
 
       context "when pointing at a location without any valid certs and not using builtin" do
         it "throws an SSLError" do
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
         end
       end
 
@@ -34,11 +70,21 @@ describe LogStash::Outputs::Scalyr do
           `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Unexpected error: java.security.InvalidAlgorithmParameterException: the trustAnchors parameter must be non-empty")
           end
           ensure
             `sudo mv /tmp/system_certs #{OpenSSL::X509::DEFAULT_CERT_DIR}`
@@ -61,11 +107,21 @@ describe LogStash::Outputs::Scalyr do
           `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
 
           begin
-            expect {
               plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
               plugin.register
+              plugin.instance_variable_set(:@running, false)
+              expect(plugin.instance_variable_get(:@logger)).to receive(:error).with("Error uploading to Scalyr (will backoff-retry)",
+                {
+                  :batch_num=>1,
+                  :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
+                  :payload_size=>781,
+                  :record_count=>3,
+                  :total_batches=>1,
+                  :url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
+                  :will_retry_in_seconds=>2
+                }
+              )
               plugin.multi_receive(sample_events)
-            }.to raise_error(Scalyr::Common::Client::ClientError, "Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)")
           ensure
             # Clean up the hosts file
             `sudo truncate -s 0 /etc/hosts`
@@ -74,4 +130,59 @@ describe LogStash::Outputs::Scalyr do
         end
       end
   end
+
+  describe "response_handling_tests" do
+    context "when receiving a 503 response" do
+      it "don't throw an error but do log one to debug" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 503, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:debug)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:debug).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :batch_num=>1,
+            :code=>503,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2
+          }
+        )
+      end
+    end
+
+    context "when receiving a 500 response" do
+      it "don't throw an error but do log one to error" do
+        stub_request(:post, "https://agent.scalyr.com/addEvents").
+          to_return(status: 500, body: "stubbed response", headers: {})
+
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin.register
+        plugin.instance_variable_set(:@running, false)
+
+        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        plugin.multi_receive(sample_events)
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+          {
+            :batch_num=>1,
+            :code=>500,
+            :message=>"Invalid JSON response from server",
+            :payload_size=>781,
+            :record_count=>3,
+            :total_batches=>1,
+            :url=>"https://agent.scalyr.com/addEvents",
+            :will_retry_in_seconds=>2
+          }
+        )
+      end
+    end
+  end
+
 end