logstash-output-scalyr 0.2.0 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17884743262e123a7719335dd18559236505f4bfbf50ff8b2e2ca24e9416c35f
-  data.tar.gz: 34db0e1f15cc4f9a207fb5a681a5f4d6a1337f1187e01dc6a0f9467440ab53d0
+  metadata.gz: ba6f9b828f98401b5f19e040e84f64a27de65fb6c41d4405c44b8038aac52450
+  data.tar.gz: 2aeed49b1518ea70df9f41bf4b627b748e23cd85104b45d96ce140d428220939
 SHA512:
-  metadata.gz: c55678d99d117c2dd08fe48c4505b2da39a57553cf8cc36a84cab7aa38dbf539bb63b564772cf5a2e575c334b617bdb3b5b61d91af13c5cff0c4ea4c18738834
-  data.tar.gz: d8e819de557a452f4129bd94eb581ff40dbdd902a1eba07ef6e7ec838f72ee5dfdd266827100232f03f9269cd933329426846081dcfaf5b3e66084075b79989c
+  metadata.gz: d14976aa00f8d3a8021b9b955bf76102e003def61dff4b89069dff3ad6ca3c5861ed93ce3c500c7991aa809bcdbc3d3b4f601ae151bcf20e87d56ee3ce59d848
+  data.tar.gz: 9728d3b1925621e80e5623c208fe4125d18ba856d4358b109f6879ea1533da97a587b2f659c15ec90f04140dbf92640d122f5c2ed8e1c8f4cc5ec7bb5a12aa77

data/CHANGELOG.md

@@ -1,5 +1,31 @@
 # Beta
 
+## 0.2.3
+
+- Increase default number of maximum retry attempts on failure from `5` to `15`.
+- Change "Unexpected error occurred while uploading to Scalyr (will backoff-retry)" message to
+  be logged under WARNING and not ERROR log level. This error is not fatal and simply indicates
+  client will retry a failed request. We use WARNING and not INFO so we still have visibility into
+  those messages (since most deployments have log level set to WARNING or above).
+
+## 0.2.2
+
+- No longer vendor dependencies in the gem. This gem used to vendor a vulnerable log4j version
+  but because logstash uses the system log4j this should not make earlier versions of this gem
+  vulnerable. This brings the additional benefit of reducing the file size of this gem.
+
+## 0.2.1.beta
+
+- Update plugin to fail fast on register and throw more user-friendly error on invalid URL for
+  ``scalyr_server`` configuration option value and other fatal server errors (e.g. invalid
+  hostname).
+- On plugin init / register we now perform connectivity check and verify that we can talk to
+  Scalyr API and validate that the API key is valid. This ensures that the plugin doesn't start and
+  start consuming events until we can successfully perform a connectivity check. This means
+  we can't end up in situation when we could potentially drop some events to the ground in case of
+  an invalid API key or similar when reaching retry limit and DLQ disabled. If you want to disable
+  this check on register, you can set ``perform_connectivity_check`` config option to ``false``.
+
 ## 0.2.0.beta, 0.2.0
 
 - Fix a bug and correctly handle ``serverHost`` event level attribute. Now if an event contains

data/README.md

@@ -9,9 +9,9 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.2.0.gem` 
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.2.3.gem` 
    or follow the latest official instructions on working with plugins from Logstash. As an alternative, you can directly install latest
-   stable version from RubyGems - ``/usr/share/logstash/bin/logstash-plugin --version 0.2.0 logstash-output-scalyr``
+   stable version from RubyGems - ``/usr/share/logstash/bin/logstash-plugin --version 0.2.3 logstash-output-scalyr``
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -439,6 +439,21 @@ bundle exec rspec
 bundle exec rake publish_gem
 ```
 
+Or as an alternative if ``rake publish_gem`` task doesn't appear to work for whatever reason
+(``publish_gem`` logstash Rake task silently swallows all the errors):
+
+```
+rm -rf vendor/
+bundle check --path vendor/bundle || bundle install --deployment
+curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
+chmod 0600 ~/.gem/credentials
+bundle exec rake vendor
+bundle exec rspec
+rvm use jruby
+bundle exec gem build logstash-output-scalyr.gemspec
+bundle exec gem push logstash-output-scalyr-<version>.gem
+```
+
 `RUBY_USER` and `RUBY_PASSWORD` should be replaced with the username and password to the RubyGems.org account you wish to release to,
  these credentials should be found in Keeper.
 
@@ -462,7 +477,7 @@ git clone https://github.com/Kami/logstash-config-tester ~/
 gem build logstash-output-scalyr.gemspec
 
 # 2. Copy it to the config test repo
-cp logstash-output-scalyr-0.2.0.gem ~/logstash-config-test/logstash-output-scalyr.gem
+cp logstash-output-scalyr-0.2.3.gem ~/logstash-config-test/logstash-output-scalyr.gem
 
 # 3. Build docker image with the latest dev version of the plugin (may take a while)
 docker-compose build

data/lib/logstash/outputs/scalyr.rb

@@ -35,6 +35,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
   config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
 
+  # True to perform connectivity check with Scalyr on plugin start up / register phase. This
+  # ensures an exception is thrown if we can't communicate with Scalyr and we don't start
+  # consuming events until plugin is correctly configured.
+  config :perform_connectivity_check, :validate => :boolean, :default => true
+
   # server_attributes is a dictionary of key value pairs that represents/identifies the logstash aggregator server
   # (where this plugin is running).  Keys are arbitrary except for the 'serverHost' key which holds special meaning to
   # Scalyr and is given special treatment in the Scalyr UI.  All of these attributes are optional (not required for logs
@@ -100,7 +105,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
   # How many times to retry sending an event before giving up on it
-  config :max_retries, :validate => :number, :default => 5
+  # This will result in a total of around 12 minutes of retrying / sleeping with a default value
+  # for retry_max_interval
+  config :max_retries, :validate => :number, :default => 15
   # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
   config :send_to_dlq, :validate => :boolean, :default => true
 
@@ -130,6 +137,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # Status will be reported regardless if the plugin receives empty batch if the plugin is
+  # "active" (received and processed some events during the plugin life time) and more than this
+  # amount of time has passed since the last reporting.
+  config :idle_status_report_deadline, :validate => :number, :default => 300
+
   # True to also call send_status when multi_receive() is called with no events.
   # In some situations (e.g. when logstash is configured with multiple scalyr
   # plugins conditionally where most are idle) you may want to set this to false
@@ -287,6 +299,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     @scalyr_server << '/' unless @scalyr_server.end_with?('/')
 
+    # Validate the URL
+    uri = URI.parse(@scalyr_server)
+
+    if not uri.kind_of?(URI::HTTP) and not uri.kind_of?(URI::HTTPS)
+      raise LogStash::ConfigurationError, "scalyr_server configuration option value is not a valid URL. " \
+                                          "This value needs contain a full URL with the protocol. e.g. " \
+                                          "https://agent.scalyr.com for US and https://eu.scalyr.com for EU"
+    end
+
     @add_events_uri = URI(@scalyr_server) + "addEvents"
 
     @logger.info "Scalyr LogStash Plugin ID - #{self.id}"
@@ -317,12 +338,23 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
 
+    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
+    # Here priming just means setting up the client parameters without opening any connections.
+    # Since client writes certs to a temporary file there could be a race in case we don't do that
+    # here since multi_receive() is multi threaded. An alternative would be to put a look around
+    # client init method (aka client_config())
+    @client_session.client
+
+    # Send a ping to verify that the configuration API key is correct and that we can establish
+    # connection with Scalyr API
+    connectivity_check
+
     @logger.info(sprintf("Started Scalyr LogStash output plugin %s (compression_type=%s,compression_level=%s,json_library=%s)." %
                          [PLUGIN_VERSION, @compression_type, @compression_type, @json_library]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
     # We use a special separate short lived client session for sending the initial client status.
-    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output
     # plugins configured with conditionals and majority of them are inactive (aka receive no data).
     # This way we don't need to keep idle long running connection open.
     initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
@@ -331,18 +363,52 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @record_stats_for_status, @flush_quantile_estimates_on_status_send,
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
+
+    # We propagate errors on intial request to better handle errors related to invalid hostname
+    # or similar
     send_status(initial_send_status_client_session)
     initial_send_status_client_session.close
 
-    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
-    # Here priming just means setting up the client parameters without opening any connections.
-    # Since client writes certs to a temporary file there could be a race in case we don't do that
-    # here since multi_receive() is multi threaded. An alternative would be to put a look around
-    # client init method (aka client_config())
-    @client_session.client
-
   end # def register
 
+  # Method which performs connectivity check with Scalyr API, verifies that wt can talk to the API
+  # and that the API token is valid.
+  def connectivity_check()
+    if not @perform_connectivity_check
+      return
+    end
+
+    @logger.debug("Performing connectivity check against the Scalyr API")
+
+    body = create_multi_event_request([], nil, nil, nil)[:body]
+
+    begin
+      @client_session.send_ping(body)
+    rescue Scalyr::Common::Client::ClientError, Manticore::ResolutionFailure => e
+      if not e.message.nil? and (e.message.include?("nodename nor servname provided") or
+          e.message.downcase.include?("name or service not know"))
+        raise LogStash::ConfigurationError,
+                    format("Received error when trying to communicate with Scalyr API. This likely means that " \
+                           "the configured value for 'scalyr_server' config option is invalid. Original error: %s",
+                           e.message)
+      end
+
+      # For now, we consider rest of the errors non fatal and just log them and don't propagate
+      # them and fail register
+      @logger.warn("Received error when trying to send connectivity check request to Scalyr",
+                    :error => e.message)
+    rescue Scalyr::Common::Client::ServerError => e
+      if e.code == 401
+        raise LogStash::ConfigurationError,
+                    format("Received 401 from Scalyr API during connectivity check which indicates " \
+                           "an invalid API key. Server Response: %s", e.body)
+      end
+
+    rescue => e
+      @logger.warn("Received non-fatal error during connectivity check", :error => e.message)
+    end
+  end
+
   # Convenience method to create a fresh quantile estimator
   def get_new_metrics
     return {
@@ -440,8 +506,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @logger.debug(message, exc_data)
             exc_commonly_retried = true
           else
-            # all other failed uploads should be errors
-            @logger.error(message, exc_data)
+            # all other failed uploads should be warning
+            @logger.warn(message, exc_data)
             exc_commonly_retried = false
           end
           retry if @running and exc_retries < @max_retries
@@ -499,7 +565,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      if @report_status_for_empty_batches or records_count > 0
+      if @report_status_for_empty_batches or records_count > 0 or should_call_send_status_for_empty_batch()
         send_status
       end
 
@@ -516,6 +582,28 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
   end  # def multi_receive
 
+  # Return true if "send_status()" should be called even for an empty batch.
+  # NOTE: We always report status even if record count is 0 if more than X minutes have passed
+  # from the last status reporting for active plugins. Logstash sends empty batches for flush
+  # purposes quite often, but we only want to report status if we haven't reported it for a
+  # longer period (aka there hasn't been any data flowing in for a while). If we reported it
+  # for every flush event (which happens very often), this would skew metrics for regular
+  # events and output plugins which receive no data at all.
+  # We perform check against total events processed so we don't send status for output plugins
+  # which receive no data at all.
+  def should_call_send_status_for_empty_batch()
+    if @multi_receive_statistics[:total_events_processed] < 100
+      # Ignore idle plugins with no data
+      return false
+    end
+
+    if @last_status_transmit_time.nil?
+      # If this value is not set it means plugin is idle and hasn't received any data at all
+      return false
+    end
+
+    return (Time.now.to_i - @last_status_transmit_time.to_i > @idle_status_report_deadline)
+  end
 
   def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
     @stats_lock.synchronize do
@@ -1030,6 +1118,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             :error_class => e.class.name
           )
         end
+
         return
       end
       @last_status_transmit_time = Time.now()

data/lib/scalyr/common/client.rb

@@ -219,7 +219,15 @@ class ClientSession
     end
   end
 
+  # Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
+  # and verify that the API key is valid.
+  def send_ping(body)
+    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
+    response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
+    handle_response(response)
 
+    response
+  end
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
   def post_add_events(body, is_status, body_serialization_duration = 0)

data/lib/scalyr/constants.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.2.0"
+PLUGIN_VERSION = "v0.2.3"
 
 # Special event level attribute name which can be used for setting event level serverHost attribute
 EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.2.0'
+  s.version         = '0.2.3'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"
@@ -10,7 +10,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -36,12 +36,15 @@ describe LogStash::Outputs::Scalyr do
   describe "#ssl_tests" do
       context "with default SSL configuration" do
         it "throws a ServerError due to fake api key" do
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Scalyr::Common::Client::ServerError",
                   :batch_num=>1,
@@ -60,12 +63,17 @@ describe LogStash::Outputs::Scalyr do
 
       context "when pointing at a location without any valid certs and not using builtin" do
         it "throws an SSLError" do
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'ssl_ca_bundle_path' => '/fakepath/nocerts',
+                'append_builtin_cert' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
@@ -86,12 +94,16 @@ describe LogStash::Outputs::Scalyr do
           `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
 
           begin
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'append_builtin_cert' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
@@ -125,12 +137,16 @@ describe LogStash::Outputs::Scalyr do
           `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
 
           begin
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
@@ -150,13 +166,19 @@ describe LogStash::Outputs::Scalyr do
         end
       end
 
-      context "when an error occurs with retries at 5" do
+      context "when an error occurs with retries at 15" do
         it "exits after 5 retries and emits a log" do
-              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'retry_initial_interval' => 0.1,
+                'ssl_ca_bundle_path' => '/fakepath/nocerts',
+                'append_builtin_cert' => false
+              })
               plugin.register
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 15 tries.", anything
               )
         end
       end
@@ -168,7 +190,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 503, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+          'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
+          'ssl_ca_bundle_path' => '/fakepath/nocerts',
+          'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
@@ -196,13 +223,18 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+          'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
+          'ssl_ca_bundle_path' => '/fakepath/nocerts',
+          'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
-        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
         plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
           {
             :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
@@ -224,13 +256,18 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "0123456789" * 52, headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'ssl_ca_bundle_path' => '/fakepath/nocerts',
+            'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
-        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
         plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
           {
             :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
@@ -253,7 +290,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'ssl_ca_bundle_path' => '/fakepath/nocerts',
+            'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
         plugin.instance_variable_set('@dlq_writer', dlq_writer)