logstash-output-scalyr 0.2.0 → 0.2.1.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17884743262e123a7719335dd18559236505f4bfbf50ff8b2e2ca24e9416c35f
-  data.tar.gz: 34db0e1f15cc4f9a207fb5a681a5f4d6a1337f1187e01dc6a0f9467440ab53d0
+  metadata.gz: a42ad86b12842e9e684f6e155a5a18f72e976a116af35b3bd7a15b38f4ab4fe8
+  data.tar.gz: 7b89fcda801a9d8b2a0df34c95288d4d1599c4c08795f600aba78206fe45407c
 SHA512:
-  metadata.gz: c55678d99d117c2dd08fe48c4505b2da39a57553cf8cc36a84cab7aa38dbf539bb63b564772cf5a2e575c334b617bdb3b5b61d91af13c5cff0c4ea4c18738834
-  data.tar.gz: d8e819de557a452f4129bd94eb581ff40dbdd902a1eba07ef6e7ec838f72ee5dfdd266827100232f03f9269cd933329426846081dcfaf5b3e66084075b79989c
+  metadata.gz: 2da035c07ee249d16c5f1041f6dcc72fbe3ae39edbc161ddc602ea2af3cac3823584563d5ccb03bc0e099dc256bc0396898d26562558dcf03777a5eccc690920
+  data.tar.gz: d9491cd2bb02bc2f5d5e612b69ca9ba3923b773d21c37e8b2bc22d270c467552fe1569a85e0755961857f360d8a59841134156e390b4a5c536ede5299f196b89

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 # Beta
 
+## 0.2.1.beta
+
+- Update plugin to fail fast on register and throw more user-friendly error on invalid URL for
+  ``scalyr_server`` configuration option value and other fatal server errors (e.g. invalid
+  hostname).
+- On plugin init / register we now perform connectivity check and verify that we can talk to
+  Scalyr API and validate that the API key is valid. This ensures that the plugin doesn't start and
+  start consuming events until we can successfully perform a connectivity check. This means
+  we can't end up in situation when we could potentially drop some events to the ground in case of
+  an invalid API key or similar when reaching retry limit and DLQ disabled. If you want to disable
+  this check on register, you can set ``perform_connectivity_check`` config option to ``false``.
+
 ## 0.2.0.beta, 0.2.0
 
 - Fix a bug and correctly handle ``serverHost`` event level attribute. Now if an event contains

data/README.md

@@ -439,6 +439,21 @@ bundle exec rspec
 bundle exec rake publish_gem
 ```
 
+Or as an alternative if ``rake publish_gem`` task doesn't appear to work for whatever reason
+(``publish_gem`` logstash Rake task silently swallows all the errors):
+
+```
+rm -rf vendor/
+bundle check --path vendor/bundle || bundle install --deployment
+curl -u RUBY_USER:RUBY_PASSWORD https://rubygems.org/api/v1/api_key.yaml > ~/.gem/credentials
+chmod 0600 ~/.gem/credentials
+bundle exec rake vendor
+bundle exec rspec
+rvm use jruby
+bundle exec gem build logstash-output-scalyr.gemspec
+bundle exec gem push logstash-output-scalyr-<version>.gem
+```
+
 `RUBY_USER` and `RUBY_PASSWORD` should be replaced with the username and password to the RubyGems.org account you wish to release to,
  these credentials should be found in Keeper.
 

data/lib/logstash/outputs/scalyr.rb

@@ -35,6 +35,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # If you have an EU-based Scalyr account, please use https://eu.scalyr.com/
   config :scalyr_server, :validate => :string, :default => "https://agent.scalyr.com/"
 
+  # True to perform connectivity check with Scalyr on plugin start up / register phase. This
+  # ensures an exception is thrown if we can't communicate with Scalyr and we don't start
+  # consuming events until plugin is correctly configured.
+  config :perform_connectivity_check, :validate => :boolean, :default => true
+
   # server_attributes is a dictionary of key value pairs that represents/identifies the logstash aggregator server
   # (where this plugin is running).  Keys are arbitrary except for the 'serverHost' key which holds special meaning to
   # Scalyr and is given special treatment in the Scalyr UI.  All of these attributes are optional (not required for logs
@@ -130,6 +135,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # minutes.
   config :status_report_interval, :validate => :number, :default => 300
 
+  # Status will be reported regardless if the plugin receives empty batch if the plugin is
+  # "active" (received and processed some events during the plugin life time) and more than this
+  # amount of time has passed since the last reporting.
+  config :idle_status_report_deadline, :validate => :number, :default => 300
+
   # True to also call send_status when multi_receive() is called with no events.
   # In some situations (e.g. when logstash is configured with multiple scalyr
   # plugins conditionally where most are idle) you may want to set this to false
@@ -287,6 +297,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
 
     @scalyr_server << '/' unless @scalyr_server.end_with?('/')
 
+    # Validate the URL
+    uri = URI.parse(@scalyr_server)
+
+    if not uri.kind_of?(URI::HTTP) and not uri.kind_of?(URI::HTTPS)
+      raise LogStash::ConfigurationError, "scalyr_server configuration option value is not a valid URL. " \
+                                          "This value needs contain a full URL with the protocol. e.g. " \
+                                          "https://agent.scalyr.com for US and https://eu.scalyr.com for EU"
+    end
+
     @add_events_uri = URI(@scalyr_server) + "addEvents"
 
     @logger.info "Scalyr LogStash Plugin ID - #{self.id}"
@@ -317,12 +336,23 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
 
+    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
+    # Here priming just means setting up the client parameters without opening any connections.
+    # Since client writes certs to a temporary file there could be a race in case we don't do that
+    # here since multi_receive() is multi threaded. An alternative would be to put a look around
+    # client init method (aka client_config())
+    @client_session.client
+
+    # Send a ping to verify that the configuration API key is correct and that we can establish
+    # connection with Scalyr API
+    connectivity_check
+
     @logger.info(sprintf("Started Scalyr LogStash output plugin %s (compression_type=%s,compression_level=%s,json_library=%s)." %
                          [PLUGIN_VERSION, @compression_type, @compression_type, @json_library]), :class => self.class.name)
 
     # Finally, send a status line to Scalyr
     # We use a special separate short lived client session for sending the initial client status.
-    # This is done to avoid the overhead in case single logstash instance has many scalyr output 
+    # This is done to avoid the overhead in case single logstash instance has many scalyr output
     # plugins configured with conditionals and majority of them are inactive (aka receive no data).
     # This way we don't need to keep idle long running connection open.
     initial_send_status_client_session = Scalyr::Common::Client::ClientSession.new(
@@ -331,18 +361,52 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         @record_stats_for_status, @flush_quantile_estimates_on_status_send,
         @http_connect_timeout, @http_socket_timeout, @http_request_timeout, @http_pool_max, @http_pool_max_per_route
     )
+
+    # We propagate errors on intial request to better handle errors related to invalid hostname
+    # or similar
     send_status(initial_send_status_client_session)
     initial_send_status_client_session.close
 
-    # We also "prime" the main HTTP client here, one which is used for sending subsequent requests.
-    # Here priming just means setting up the client parameters without opening any connections.
-    # Since client writes certs to a temporary file there could be a race in case we don't do that
-    # here since multi_receive() is multi threaded. An alternative would be to put a look around
-    # client init method (aka client_config())
-    @client_session.client
-
   end # def register
 
+  # Method which performs connectivity check with Scalyr API, verifies that wt can talk to the API
+  # and that the API token is valid.
+  def connectivity_check()
+    if not @perform_connectivity_check
+      return
+    end
+
+    @logger.debug("Performing connectivity check against the Scalyr API")
+
+    body = create_multi_event_request([], nil, nil, nil)[:body]
+
+    begin
+      @client_session.send_ping(body)
+    rescue Scalyr::Common::Client::ClientError, Manticore::ResolutionFailure => e
+      if not e.message.nil? and (e.message.include?("nodename nor servname provided") or
+          e.message.downcase.include?("name or service not know"))
+        raise LogStash::ConfigurationError,
+                    format("Received error when trying to communicate with Scalyr API. This likely means that " \
+                           "the configured value for 'scalyr_server' config option is invalid. Original error: %s",
+                           e.message)
+      end
+
+      # For now, we consider rest of the errors non fatal and just log them and don't propagate
+      # them and fail register
+      @logger.warn("Received error when trying to send connectivity check request to Scalyr",
+                    :error => e.message)
+    rescue Scalyr::Common::Client::ServerError => e
+      if e.code == 401
+        raise LogStash::ConfigurationError,
+                    format("Received 401 from Scalyr API during connectivity check which indicates " \
+                           "an invalid API key. Server Response: %s", e.body)
+      end
+
+    rescue => e
+      @logger.warn("Received non-fatal error during connectivity check", :error => e.message)
+    end
+  end
+
   # Convenience method to create a fresh quantile estimator
   def get_new_metrics
     return {
@@ -499,7 +563,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
-      if @report_status_for_empty_batches or records_count > 0
+      if @report_status_for_empty_batches or records_count > 0 or should_call_send_status_for_empty_batch()
         send_status
       end
 
@@ -516,6 +580,28 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
     end
   end  # def multi_receive
 
+  # Return true if "send_status()" should be called even for an empty batch.
+  # NOTE: We always report status even if record count is 0 if more than X minutes have passed
+  # from the last status reporting for active plugins. Logstash sends empty batches for flush
+  # purposes quite often, but we only want to report status if we haven't reported it for a
+  # longer period (aka there hasn't been any data flowing in for a while). If we reported it
+  # for every flush event (which happens very often), this would skew metrics for regular
+  # events and output plugins which receive no data at all.
+  # We perform check against total events processed so we don't send status for output plugins
+  # which receive no data at all.
+  def should_call_send_status_for_empty_batch()
+    if @multi_receive_statistics[:total_events_processed] < 100
+      # Ignore idle plugins with no data
+      return false
+    end
+
+    if @last_status_transmit_time.nil?
+      # If this value is not set it means plugin is idle and hasn't received any data at all
+      return false
+    end
+
+    return (Time.now.to_i - @last_status_transmit_time.to_i > @idle_status_report_deadline)
+  end
 
   def log_retry_failure(multi_event_request, exc_data, exc_retries, exc_sleep)
     @stats_lock.synchronize do
@@ -1030,6 +1116,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             :error_class => e.class.name
           )
         end
+
         return
       end
       @last_status_transmit_time = Time.now()

data/lib/scalyr/common/client.rb

@@ -219,7 +219,15 @@ class ClientSession
     end
   end
 
+  # Send "ping" request to the API. This is mostly used to test the connecting with Scalyr API
+  # and verify that the API key is valid.
+  def send_ping(body)
+    post_body, post_headers, compression_duration = prepare_post_object @add_events_uri.path, body
+    response = client.send(:post, @add_events_uri, body: post_body, headers: post_headers)
+    handle_response(response)
 
+    response
+  end
 
   # Upload data to Scalyr. Assumes that the body size complies with Scalyr limits
   def post_add_events(body, is_status, body_serialization_duration = 0)

data/lib/scalyr/constants.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.2.0"
+PLUGIN_VERSION = "v0.2.1.beta"
 
 # Special event level attribute name which can be used for setting event level serverHost attribute
 EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.2.0'
+  s.version         = '0.2.1.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -36,7 +36,10 @@ describe LogStash::Outputs::Scalyr do
   describe "#ssl_tests" do
       context "with default SSL configuration" do
         it "throws a ServerError due to fake api key" do
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
@@ -60,7 +63,12 @@ describe LogStash::Outputs::Scalyr do
 
       context "when pointing at a location without any valid certs and not using builtin" do
         it "throws an SSLError" do
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'ssl_ca_bundle_path' => '/fakepath/nocerts',
+                'append_builtin_cert' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
@@ -86,7 +94,11 @@ describe LogStash::Outputs::Scalyr do
           `sudo mv #{OpenSSL::X509::DEFAULT_CERT_DIR} /tmp/system_certs`
 
           begin
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'append_builtin_cert' => false,
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
@@ -125,7 +137,11 @@ describe LogStash::Outputs::Scalyr do
           `echo "#{etc_hosts_entry}" | sudo tee -a /etc/hosts`
 
           begin
-              plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443'})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
+              })
               plugin.register
               plugin.instance_variable_set(:@running, false)
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
@@ -152,7 +168,13 @@ describe LogStash::Outputs::Scalyr do
 
       context "when an error occurs with retries at 5" do
         it "exits after 5 retries and emits a log" do
-              plugin = LogStash::Outputs::Scalyr.new({'retry_initial_interval' => 0.1, 'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+              plugin = LogStash::Outputs::Scalyr.new({
+                'api_write_token' => '1234',
+                'perform_connectivity_check' => false,
+                'retry_initial_interval' => 0.1,
+                'ssl_ca_bundle_path' => '/fakepath/nocerts',
+                'append_builtin_cert' => false
+              })
               plugin.register
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
@@ -168,7 +190,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 503, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+          'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
+          'ssl_ca_bundle_path' => '/fakepath/nocerts',
+          'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
@@ -196,7 +223,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+          'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
+          'ssl_ca_bundle_path' => '/fakepath/nocerts',
+          'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
@@ -224,7 +256,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "0123456789" * 52, headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'ssl_ca_bundle_path' => '/fakepath/nocerts',
+            'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
@@ -253,7 +290,12 @@ describe LogStash::Outputs::Scalyr do
         stub_request(:post, "https://agent.scalyr.com/addEvents").
           to_return(status: 500, body: "stubbed response", headers: {})
 
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'ssl_ca_bundle_path' => '/fakepath/nocerts', 'append_builtin_cert' => false})
+        plugin = LogStash::Outputs::Scalyr.new({
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'ssl_ca_bundle_path' => '/fakepath/nocerts',
+            'append_builtin_cert' => false
+        })
         plugin.register
         plugin.instance_variable_set(:@running, false)
         plugin.instance_variable_set('@dlq_writer', dlq_writer)

data/spec/logstash/outputs/scalyr_spec.rb

@@ -61,6 +61,7 @@ describe LogStash::Outputs::Scalyr do
     context "test get_stats and send_status" do
       plugin = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
+                                                     'perform_connectivity_check' => false,
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
                                                      'flatten_nested_values' => true,
@@ -77,6 +78,7 @@ describe LogStash::Outputs::Scalyr do
       it "it doesnt include flatten metrics if flattening is disabled" do
         plugin1 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
+                                                     'perform_connectivity_check' => false,
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
                                                      'flatten_nested_values' => false,
@@ -151,6 +153,7 @@ describe LogStash::Outputs::Scalyr do
       it "send_stats is not called when events list is empty and report_status_for_empty_batches is false" do
         plugin2 = LogStash::Outputs::Scalyr.new({
                                                      'api_write_token' => '1234',
+                                                     'perform_connectivity_check' => false,
                                                      'serverhost_field' => 'source_host',
                                                      'log_constants' => ['tags'],
                                                      'flatten_nested_values' => false,
@@ -206,6 +209,7 @@ describe LogStash::Outputs::Scalyr do
       it "creates logfile from serverHost" do
         plugin = LogStash::Outputs::Scalyr.new({
                                                    'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
                                                    'serverhost_field' => 'source_host',
                                                    'log_constants' => ['tags'],
                                                })
@@ -223,7 +227,7 @@ describe LogStash::Outputs::Scalyr do
 
     context "when serverhost_field is missing" do
       it "does not contain log file" do
-        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234'})
+        plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'perform_connectivity_check' => false})
         allow(plugin).to receive(:send_status).and_return(nil)
         plugin.register
         result = plugin.build_multi_event_request_array(sample_events)
@@ -237,6 +241,7 @@ describe LogStash::Outputs::Scalyr do
       it "creates logfile from serverHost" do
         plugin = LogStash::Outputs::Scalyr.new({
                                                    'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
                                                    'serverhost_field' => 'source_host',
                                                })
         allow(plugin).to receive(:send_status).and_return(nil)
@@ -254,6 +259,7 @@ describe LogStash::Outputs::Scalyr do
       it "does not contain log file" do
         plugin = LogStash::Outputs::Scalyr.new({
                                                    'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
                                                    'serverhost_field' => 'source_host',
                                                    'logfile_field' => 'source_file',
                                                })
@@ -271,6 +277,7 @@ describe LogStash::Outputs::Scalyr do
     context "when configured to flatten values with custom delimiter" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
           'flatten_tags' => true,
           'flat_tag_value' => 'true',
           'flat_tag_prefix' => 'tag_prefix_',
@@ -303,6 +310,7 @@ describe LogStash::Outputs::Scalyr do
     context "when configured to flatten values with custom delimiter and deep delimiter fix" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
           'flatten_tags' => true,
           'flat_tag_value' => 'true',
           'flat_tag_prefix' => 'tag_prefix_',
@@ -336,6 +344,7 @@ describe LogStash::Outputs::Scalyr do
     context "when configured to flatten values with custom delimiter, no array flattening" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
           'flatten_tags' => true,
           'flat_tag_value' => 'true',
           'flat_tag_prefix' => 'tag_prefix_',
@@ -367,6 +376,7 @@ describe LogStash::Outputs::Scalyr do
     context "when configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
           'flatten_tags' => true,
           'flat_tag_value' => 'true',
           'flat_tag_prefix' => 'tag_prefix_',
@@ -399,6 +409,7 @@ describe LogStash::Outputs::Scalyr do
       it "estimate_each_event_size is true explicit (default) batch split into 3 scalyr requests" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'flatten_tags' => true,
             'flat_tag_value' => 'true',
             'flat_tag_prefix' => 'tag_prefix_',
@@ -439,6 +450,7 @@ describe LogStash::Outputs::Scalyr do
       it "estimate_each_event_size is true implicit (default) batch split into 3 scalyr requests" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'flatten_tags' => true,
             'flat_tag_value' => 'true',
             'flat_tag_prefix' => 'tag_prefix_',
@@ -478,6 +490,7 @@ describe LogStash::Outputs::Scalyr do
       it "estimate_each_event_size is false batch not split into multiple scalyr requests" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'flatten_tags' => true,
             'flat_tag_value' => 'true',
             'flat_tag_prefix' => 'tag_prefix_',
@@ -513,6 +526,7 @@ describe LogStash::Outputs::Scalyr do
     context "when not configured to flatten values and tags" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
       }
       plugin = LogStash::Outputs::Scalyr.new(config)
       it "does not flatten" do
@@ -535,6 +549,7 @@ describe LogStash::Outputs::Scalyr do
     context "when configured to flatten with max keys configured to 3" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
           'flatten_nested_values' => true,  # this converts into string 'true'
           'flattening_max_key_count' => 3,
       }
@@ -567,6 +582,7 @@ describe LogStash::Outputs::Scalyr do
       it "no serverHost defined in server_attributes, no serverHost defined on event level - should use node hostname as the default session level value" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
 
@@ -585,6 +601,7 @@ describe LogStash::Outputs::Scalyr do
       it "serverHost defined in server_attributes, nothing defined on event level - server_attributes value should be used" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'serverHost' => 'fooHost'}
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -604,6 +621,7 @@ describe LogStash::Outputs::Scalyr do
       it "serverHost defined in server_attributes (explicitly defined), event level serverHost defined - event level value should be used" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'serverHost' => 'fooHost', 'attr1' => 'val1'}
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -660,6 +678,7 @@ describe LogStash::Outputs::Scalyr do
       it "serverHost defined in server_attributes (defined via node hostname), event level serverHost defined - event level value should be used" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'attr1' => 'val1'}
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -719,6 +738,7 @@ describe LogStash::Outputs::Scalyr do
       it "serverHost defined in server_attributes (explicitly defined), event level serverHost defined - event level value should be used and server level one for events without server host" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'serverHost' => 'top-level-session-host', 'attr1' => 'val1'}
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -771,6 +791,7 @@ describe LogStash::Outputs::Scalyr do
       it "no serverHost defined, event level serverHost defined via non-default serverhost_field - event level value should be used" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'attr1' => 'val1'},
             'use_hostname_for_serverhost' => false,
             'serverhost_field' => 'custom_server_host',
@@ -827,6 +848,7 @@ describe LogStash::Outputs::Scalyr do
       it "no serverHost defined, event level serverHost defined - event level value should be used" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'server_attributes' => {'attr1' => 'val1'},
             'use_hostname_for_serverhost' => false
         }
@@ -882,6 +904,7 @@ describe LogStash::Outputs::Scalyr do
     context "when receiving an event with Bignums" do
       config = {
           'api_write_token' => '1234',
+          'perform_connectivity_check' => false,
       }
       plugin = LogStash::Outputs::Scalyr.new(config)
       it "doesn't throw an error" do
@@ -902,6 +925,7 @@ describe LogStash::Outputs::Scalyr do
       it "host attribute removed by default" do
        config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
 
@@ -927,6 +951,7 @@ describe LogStash::Outputs::Scalyr do
       it "host attribute not removed if config option set" do
        config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'remove_host_attribute_from_events' => false,
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -955,6 +980,7 @@ describe LogStash::Outputs::Scalyr do
       it "stdlib (implicit)" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
 
@@ -971,6 +997,7 @@ describe LogStash::Outputs::Scalyr do
       it "stdlib (explicit)" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'json_library' => 'stdlib'
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -988,6 +1015,7 @@ describe LogStash::Outputs::Scalyr do
       it "jrjackson" do
         config = {
             'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
             'json_library' => 'jrjackson'
         }
         plugin = LogStash::Outputs::Scalyr.new(config)
@@ -1002,5 +1030,65 @@ describe LogStash::Outputs::Scalyr do
         expect(body['events'].size).to eq(1)
       end
     end
+
+    context "scalyr_server config option handling and connectivity check" do
+      it "doesn't throw an error on valid url" do
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'https://agent.scalyr.com'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        plugin.register
+
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'https://eu.scalyr.com'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        plugin.register
+      end
+
+      it "throws on invalid URL" do
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'agent.scalyr.com'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /is not a valid URL/)
+
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'eu.scalyr.com'
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /is not a valid URL/)
+      end
+
+      it "throws on invalid hostname" do
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'https://agent.invalid.foo.scalyr.com',
+            'perform_connectivity_check' => true
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Received error when trying to communicate/)
+      end
+
+      it "throws on invalid api key" do
+        config = {
+            'api_write_token' => '1234',
+            'perform_connectivity_check' => false,
+            'scalyr_server' => 'https://agent.scalyr.com',
+            'perform_connectivity_check' => true
+        }
+        plugin = LogStash::Outputs::Scalyr.new(config)
+        expect { plugin.register }.to raise_error(LogStash::ConfigurationError, /Received 401 from Scalyr API during/)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-14 00:00:00.000000000 Z
+date: 2021-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -4149,9 +4149,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10