logstash-output-scalyr 0.2.2 → 0.2.5.beta

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b6b9327052bb3d05f9d043f6c9d2c02456f3b300ceb0355abae8b7d050e99fb9
-  data.tar.gz: 1ff559076075d0b49728fce8bf103452d21d3c48ce9eff1a58d5d3c786b582f6
+  metadata.gz: a1f4ab67bba48557efab9493bdd6e85ef80ba6aab10f1b21e029acc78e69fdc7
+  data.tar.gz: c081bfd418f30502ca4fde78f8d8e15316ea18bd8f2834adff004d40913f2c39
 SHA512:
-  metadata.gz: a1ac7b6792503f9cff444962efbc33f349cad7c9d525beaed05bcdc57ebadc5d688381c235fa21d2ff5004d6ca3277ac62336f027ba3880be1d9b01d40006f75
-  data.tar.gz: f07bcc497143581032ded69de2a6c833e4ea422e88fbbf12080adb07388e1f57cedc42319b1aca84f668e47c1d567ac25b4d3787bae74c84674416227ca9ba93
+  metadata.gz: c1e6019904b05043552ea46d303d861f3a539c94fa644686d5a12ecd5bf40a69ec1a9afde09589a676ccecc0b03ee737bc24456ca5b19026b580421c18ae6605
+  data.tar.gz: ce5b38bbff35964ce84062fd647645a222958e72ea3a7f973900b182edb8e527f045dedac15668817898f6119e32057927f102b07bdffdfceb3099ea4be47beb

data/CHANGELOG.md

@@ -1,5 +1,34 @@
 # Beta
 
+## 0.2.5.beta
+
+* Allow user to specify value for the DataSet event severity  (``sev``) field. "sev" field is a
+  special top level event field which denotes the event severity (log level).
+
+  To enable this functionality, user needs to configure ``severity_field`` plugin config option and
+  set it to the logstash event field which carries the severity field value. This field value
+  needs to be an integer and contain a value from 0 to 6 (inclusive).
+* Upgrade dependencies (manticore -> 0.9.1, jrjackson -> 0.4.15).
+* Fix experimental ``zstandard`` support.
+
+  NOTE: For zstandard compression to be used zstd / libstd system package needs to be installed
+  (https://github.com/msievers/zstandard-ruby/#examples-for-installing-libzstd) and ``zstandard``
+  gem needs to be installed inside the Logstash jRuby environment
+  (e.g. ``/usr/share/logstash/bin/ruby -S /usr/share/logstash/vendor/jruby/bin/gem install 
+  zstandard ; echo 'gem "zstandard"' >> /opt/logstash/Gemfile``).
+
+## 0.2.4.beta
+
+* Experimental zstandard support - in development, not to be used in production.
+
+## 0.2.3
+
+- Increase default number of maximum retry attempts on failure from `5` to `15`.
+- Change "Unexpected error occurred while uploading to Scalyr (will backoff-retry)" message to
+  be logged under WARNING and not ERROR log level. This error is not fatal and simply indicates
+  client will retry a failed request. We use WARNING and not INFO so we still have visibility into
+  those messages (since most deployments have log level set to WARNING or above).
+
 ## 0.2.2
 
 - No longer vendor dependencies in the gem. This gem used to vendor a vulnerable log4j version

data/Gemfile

@@ -20,5 +20,5 @@ end
 gem 'pry'
 gem 'pry-nav'
 gem 'quantile', '~> 0.2.1'
-gem 'manticore', '~> 0.7.1', platform: :jruby
-gem 'jrjackson', '~> 0.4.14', platform: :jruby
+gem 'manticore', '~> 0.9.1', platform: :jruby
+gem 'jrjackson', '~> 0.4.15', platform: :jruby

data/README.md

@@ -1,4 +1,4 @@
-[![CircleCI](https://circleci.com/gh/scalyr/logstash-output-scalyr.svg?style=svg)](https://circleci.com/gh/scalyr/logstash-output-scalyr)
+[![CircleCI](https://circleci.com/gh/scalyr/logstash-output-scalyr.svg?style=svg)](https://circleci.com/gh/scalyr/logstash-output-scalyr) [![Gem Version](https://badge.fury.io/rb/logstash-output-scalyr.svg)](https://badge.fury.io/rb/logstash-output-scalyr)
 
 # [Scalyr output plugin for Logstash]
 
@@ -9,9 +9,9 @@ You can view documentation for this plugin [on the Scalyr website](https://app.s
 # Quick start
 
 1. Build the gem, run `gem build logstash-output-scalyr.gemspec` 
-2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.2.2.gem` 
+2. Install the gem into a Logstash installation, run `/usr/share/logstash/bin/logstash-plugin install logstash-output-scalyr-0.2.3.gem` 
    or follow the latest official instructions on working with plugins from Logstash. As an alternative, you can directly install latest
-   stable version from RubyGems - ``/usr/share/logstash/bin/logstash-plugin --version 0.2.2 logstash-output-scalyr``
+   stable version from RubyGems - ``/usr/share/logstash/bin/logstash-plugin --version 0.2.3 logstash-output-scalyr``
 3. Configure the output plugin (e.g. add it to a pipeline .conf)
 4. Restart Logstash 
 
@@ -86,6 +86,40 @@ output {
   }
 ```
 
+## Notes on severity (sev) attribute handling
+
+``sev`` is a special top level DataSet event field which denotes the event severity / log level.
+
+To enable this functionality, user needs to define ``severity_field`` plugin config option. This
+config option tells the plugin which Logstash event field carries the value for the severity field.
+
+The actual value needs to be an integer between 0 and 6 inclusive. Those values are mapped to
+different severity / log levels on DataSet server side as shown below:
+
+- 0 -> finest
+- 1 -> trace
+- 2 -> debut
+- 3 -> info
+- 4 -> warning
+- 5 -> error
+- 6 -> fatal / emergency / critical
+
+```
+output {
+ scalyr {
+   api_write_token => 'SCALYR_API_KEY'
+   ...
+   severity_field => 'severity',
+ }
+}
+```
+
+In the example above, value for the DataSet severity field should be included in the ``severity``
+Logstash event field.
+
+In case the field value doesn't contain a valid severity number (0 - 6), ``sev`` field won't be
+set on the event object to prevent API from rejecting an invalid request.
+
 ## Options
 
 - The Scalyr API write token, these are available at https://www.scalyr.com/keys.  This is the only compulsory configuration field required for proper upload
@@ -477,7 +511,7 @@ git clone https://github.com/Kami/logstash-config-tester ~/
 gem build logstash-output-scalyr.gemspec
 
 # 2. Copy it to the config test repo
-cp logstash-output-scalyr-0.2.2.gem ~/logstash-config-test/logstash-output-scalyr.gem
+cp logstash-output-scalyr-0.2.3.gem ~/logstash-config-test/logstash-output-scalyr.gem
 
 # 3. Build docker image with the latest dev version of the plugin (may take a while)
 docker-compose build

data/lib/logstash/outputs/scalyr.rb

@@ -60,6 +60,22 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # (Warning: events with an existing 'logfile' field, it will be overwritten)
   config :logfile_field, :validate => :string, :default => 'logfile'
 
+  # Record field which includes the value for the "severity" field. severity is a special field which tells
+  # Scalyr severity / log level for a particulat event. This field is a top level event field and not
+  # event attribute field. Actual field value must be an integer and is mapped to different severity /
+  # log level on DataSet server side as shown below:
+  #
+  # - 0 -> finest
+  # - 1 -> trace
+  # - 2 -> debut
+  # - 3 -> info
+  # - 4 -> warning
+  # - 5 -> error
+  # - 6 -> fatal / emergency / critical
+  #
+  # By default, if Event contains no severity field, default value of 3 (info) will be used.
+  config :severity_field, :validate => :string, :default => nil
+
   # The Scalyr Output Plugin expects the main log message to be contained in the Event['message'].  If your main log
   # content is contained in a different field, specify it here.  It will be renamed to 'message' before upload.
   # (Warning: events with an existing 'message' field, it will be overwritten)
@@ -105,7 +121,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Initial interval in seconds between bulk retries. Doubled on each retry up to `retry_max_interval`
   config :retry_initial_interval, :validate => :number, :default => 1
   # How many times to retry sending an event before giving up on it
-  config :max_retries, :validate => :number, :default => 5
+  # This will result in a total of around 12 minutes of retrying / sleeping with a default value
+  # for retry_max_interval
+  config :max_retries, :validate => :number, :default => 15
   # Whether or not to send messages that failed to send a max_retries amount of times to the DLQ or just drop them
   config :send_to_dlq, :validate => :boolean, :default => true
 
@@ -128,7 +146,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
   # Valid options are bz2, deflate, or none.
   config :compression_type, :validate => :string, :default => 'deflate'
 
-  # An int containing the compression level of compression to use, from 1-9. Defaults to 6
+  # An int containing the compression level of compression to use, from 1-9. Defaults to 6. Only
+  # applicable when compression type is "deflate" or "bz2".
   config :compression_level, :validate => :number, :default => 6
 
   # How often to log and report status metrics to Scalyr. Defaults to every 5
@@ -493,9 +512,9 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
               :will_retry_in_seconds => sleep_interval,
           }
           exc_data[:code] = e.code if e.code
-          if @logger.debug? and e.body
+          if @logger.debug? and defined?(e.body) and e.body
             exc_data[:body] = e.body
-          elsif e.body
+          elsif defined?(e.body) and e.body
             exc_data[:body] = Scalyr::Common::Util.truncate(e.body, 512)
           end
           exc_data[:payload] = "\tSample payload: #{request[:body][0,1024]}..." if @logger.debug?
@@ -504,8 +523,8 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
             @logger.debug(message, exc_data)
             exc_commonly_retried = true
           else
-            # all other failed uploads should be errors
-            @logger.error(message, exc_data)
+            # all other failed uploads should be warning
+            @logger.warn(message, exc_data)
             exc_commonly_retried = false
           end
           retry if @running and exc_retries < @max_retries
@@ -720,6 +739,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       # Rename user-specified logfile field -> 'logfile'
       rename.call(@logfile_field, 'logfile')
 
+      # Rename user-specified severity field -> 'severity' (if configured)
+      if not @severity_field.nil?
+        rename.call(@severity_field, 'severity')
+      end
+
       # Remove "host" attribute
       if @remove_host_attribute_from_events and record.key? "host"
         record.delete("host")
@@ -764,6 +788,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
           logs[log_identifier]['attrs']['logfile'] = record['logfile']
           record.delete('logfile')
         end
+
         if @log_constants
           @log_constants.each {|log_constant|
             if record.key? log_constant
@@ -825,6 +850,21 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         end
       end
 
+      severity = record['severity']
+      severity_int = nil
+
+      # Server won't accept the payload in case severity value is not valid. To avoid events from
+      # being dropped, we only set Event.sev field in case this field contains a valid value.
+      if not @severity_field.nil? and severity and severity.is_integer?
+        severity_int = severity.to_i
+
+        if severity_int >= 0 and severity_int <= 6
+          record.delete('severity')
+        else
+          severity_int = nil
+        end
+      end
+
       # Use LogStash event.timestamp as the 'ts' Scalyr timestamp.  Note that this may be overwritten by input
       # filters so may not necessarily reflect the actual originating timestamp.
       scalyr_event = {
@@ -838,6 +878,11 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         scalyr_event[:log] = logs_ids[log_identifier]
       end
 
+      # optionally set severity (if available and valid)
+      if @severity_field and not severity_int.nil?
+        scalyr_event[:sev] = severity_int
+      end
+
       if @estimate_each_event_size
         # get json string of event to keep track of how many bytes we are sending
         begin
@@ -1102,7 +1147,7 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
       begin
         client_session.post_add_events(multi_event_request[:body], true, 0)
       rescue => e
-        if e.body
+        if defined?(e.body) and e.body
           @logger.warn(
             "Unexpected error occurred while uploading status to Scalyr",
             :error_message => e.message,
@@ -1168,3 +1213,15 @@ class LogStash::Outputs::Scalyr < LogStash::Outputs::Base
         !execution_context.dlq_writer.inner_writer.is_a?(::LogStash::Util::DummyDeadLetterQueueWriter)
   end
 end
+
+class String
+  def is_integer?
+    self.to_i.to_s == self
+  end
+end
+
+class Integer
+  def is_integer?
+    true
+  end
+end

data/lib/scalyr/common/client.rb

@@ -299,6 +299,19 @@ class ClientSession
         bz2.write body
         bz2.close
         compressed_body = io.string
+      elsif @compression_type == "zstandard"
+        # NOTE: zstandard requires libzstd to be installed on the system and
+        # zstandard gem. Since libzstd may not be installed out of the box, we
+        # don't directly depend on this gem and it's up to the user to install
+        # both dependencies manually in case they want to use zstandard.
+        begin
+          require 'zstandard'
+        rescue LoadError
+          raise SystemExit, "zstandard gem is missing. If you want to use zstandard compression you need to make sure zstandard and and libzstd dependency is installed. See TODO for more information."
+        end
+
+        encoding = 'zstandard'
+        compressed_body = Zstandard.deflate(body)
       end
       end_time = Time.now.to_f
       compression_duration = end_time - start_time

data/lib/scalyr/constants.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
-PLUGIN_VERSION = "v0.2.2"
+PLUGIN_VERSION = "v0.2.5.beta"
 
 # Special event level attribute name which can be used for setting event level serverHost attribute
 EVENT_LEVEL_SERVER_HOST_ATTRIBUTE_NAME = '__origServerHost'

data/logstash-output-scalyr.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-scalyr'
-  s.version         = '0.2.2'
+  s.version         = '0.2.5.beta'
   s.licenses = ['Apache-2.0']
   s.summary = "Scalyr output plugin for Logstash"
   s.description     = "Sends log data collected by Logstash to Scalyr (https://www.scalyr.com)"

data/spec/logstash/outputs/scalyr_integration_spec.rb

@@ -39,12 +39,15 @@ describe LogStash::Outputs::Scalyr do
               plugin = LogStash::Outputs::Scalyr.new({
                 'api_write_token' => '1234',
                 'perform_connectivity_check' => false,
+                'max_retries' => 2,
+                'retry_max_interval' => 2,
+                'retry_initial_interval' => 0.2,
               })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Scalyr::Common::Client::ServerError",
                   :batch_num=>1,
@@ -54,7 +57,7 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2,
+                  :will_retry_in_seconds=>0.4,
                   :body=>"{\n  \"message\": \"Couldn't decode API token ...234.\",\n  \"status\": \"error/client/badParam\"\n}"
                 }
               )
@@ -68,12 +71,15 @@ describe LogStash::Outputs::Scalyr do
                 'perform_connectivity_check' => false,
                 'ssl_ca_bundle_path' => '/fakepath/nocerts',
                 'append_builtin_cert' => false,
+                'max_retries' => 2,
+                'retry_max_interval' => 2,
+                'retry_initial_interval' => 0.2,
               })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
@@ -82,7 +88,7 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>0.4
                 }
               )
         end
@@ -98,12 +104,15 @@ describe LogStash::Outputs::Scalyr do
                 'api_write_token' => '1234',
                 'perform_connectivity_check' => false,
                 'append_builtin_cert' => false,
+                'max_retries' => 2,
+                'retry_max_interval' => 2,
+                'retry_initial_interval' => 0.2,
               })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
@@ -112,7 +121,7 @@ describe LogStash::Outputs::Scalyr do
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>0.4
                 }
               )
           end
@@ -141,21 +150,24 @@ describe LogStash::Outputs::Scalyr do
                 'api_write_token' => '1234',
                 'perform_connectivity_check' => false,
                 'scalyr_server' => 'https://invalid.mitm.should.fail.test.agent.scalyr.com:443',
+                'max_retries' => 2,
+                'retry_max_interval' => 2,
+                'retry_initial_interval' => 0.2,
               })
               plugin.register
               plugin.instance_variable_set(:@running, false)
-              allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+              allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
                 {
                   :error_class=>"Manticore::UnknownException",
                   :batch_num=>1,
-                  :message=>"Host name 'invalid.mitm.should.fail.test.agent.scalyr.com' does not match the certificate subject provided by the peer (CN=*.scalyr.com)",
+                  :message=>"Certificate for <invalid.mitm.should.fail.test.agent.scalyr.com> doesn't match any of the subject alternative names: [*.scalyr.com, scalyr.com]",
                   :payload_size=>737,
                   :record_count=>3,
                   :total_batches=>1,
                   :url=>"https://invalid.mitm.should.fail.test.agent.scalyr.com/addEvents",
-                  :will_retry_in_seconds=>2
+                  :will_retry_in_seconds=>0.4
                 }
               )
           ensure
@@ -166,19 +178,21 @@ describe LogStash::Outputs::Scalyr do
         end
       end
 
-      context "when an error occurs with retries at 5" do
+      context "when an error occurs with retries at 15" do
         it "exits after 5 retries and emits a log" do
               plugin = LogStash::Outputs::Scalyr.new({
                 'api_write_token' => '1234',
                 'perform_connectivity_check' => false,
-                'retry_initial_interval' => 0.1,
                 'ssl_ca_bundle_path' => '/fakepath/nocerts',
-                'append_builtin_cert' => false
+                'append_builtin_cert' => false,
+                'max_retries' => 15,
+                'retry_max_interval' => 0.5,
+                'retry_initial_interval' => 0.2,
               })
               plugin.register
               allow(plugin.instance_variable_get(:@logger)).to receive(:error)
               plugin.multi_receive(sample_events)
-              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 5 tries.", anything
+              expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Failed to send 3 events after 15 tries.", anything
               )
         end
       end
@@ -194,7 +208,10 @@ describe LogStash::Outputs::Scalyr do
           'api_write_token' => '1234',
           'perform_connectivity_check' => false,
           'ssl_ca_bundle_path' => '/fakepath/nocerts',
-          'append_builtin_cert' => false
+          'append_builtin_cert' => false,
+          'max_retries' => 2,
+          'retry_max_interval' => 2,
+          'retry_initial_interval' => 0.2,
         })
         plugin.register
         plugin.instance_variable_set(:@running, false)
@@ -211,7 +228,7 @@ describe LogStash::Outputs::Scalyr do
             :record_count=>3,
             :total_batches=>1,
             :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>2,
+            :will_retry_in_seconds=>0.4,
             :body=>"stubbed response"
           }
         )
@@ -227,14 +244,17 @@ describe LogStash::Outputs::Scalyr do
           'api_write_token' => '1234',
           'perform_connectivity_check' => false,
           'ssl_ca_bundle_path' => '/fakepath/nocerts',
-          'append_builtin_cert' => false
+          'append_builtin_cert' => false,
+          'max_retries' => 2,
+          'retry_max_interval' => 2,
+          'retry_initial_interval' => 0.2,
         })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
-        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
         plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
           {
             :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
@@ -244,7 +264,7 @@ describe LogStash::Outputs::Scalyr do
             :record_count=>3,
             :total_batches=>1,
             :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>2,
+            :will_retry_in_seconds=>0.4,
             :body=>"stubbed response"
           }
         )
@@ -260,14 +280,17 @@ describe LogStash::Outputs::Scalyr do
             'api_write_token' => '1234',
             'perform_connectivity_check' => false,
             'ssl_ca_bundle_path' => '/fakepath/nocerts',
-            'append_builtin_cert' => false
+            'append_builtin_cert' => false,
+            'max_retries' => 2,
+            'retry_max_interval' => 2,
+            'retry_initial_interval' => 0.2,
         })
         plugin.register
         plugin.instance_variable_set(:@running, false)
 
-        allow(plugin.instance_variable_get(:@logger)).to receive(:error)
+        allow(plugin.instance_variable_get(:@logger)).to receive(:warn)
         plugin.multi_receive(sample_events)
-        expect(plugin.instance_variable_get(:@logger)).to have_received(:error).with("Error uploading to Scalyr (will backoff-retry)",
+        expect(plugin.instance_variable_get(:@logger)).to have_received(:warn).with("Error uploading to Scalyr (will backoff-retry)",
           {
             :error_class=>"Scalyr::Common::Client::ServerError",
             :batch_num=>1,
@@ -277,7 +300,7 @@ describe LogStash::Outputs::Scalyr do
             :record_count=>3,
             :total_batches=>1,
             :url=>"https://agent.scalyr.com/addEvents",
-            :will_retry_in_seconds=>2,
+            :will_retry_in_seconds=>0.4,
             :body=>("0123456789" * 50) + "012345678..."
           }
         )
@@ -294,7 +317,10 @@ describe LogStash::Outputs::Scalyr do
             'api_write_token' => '1234',
             'perform_connectivity_check' => false,
             'ssl_ca_bundle_path' => '/fakepath/nocerts',
-            'append_builtin_cert' => false
+            'append_builtin_cert' => false,
+            'max_retries' => 2,
+            'retry_max_interval' => 2,
+            'retry_initial_interval' => 0.2,
         })
         plugin.register
         plugin.instance_variable_set(:@running, false)

data/spec/logstash/outputs/scalyr_spec.rb

@@ -56,6 +56,65 @@ describe LogStash::Outputs::Scalyr do
     events
   }
 
+  let(:sample_events_with_severity) {
+    events = []
+    for i in 0..6 do
+      # valid severity - integer
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('severity', i)
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+    for i in 0..6 do
+      # valid severity - string
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('severity', i.to_s)
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+
+    # invalid severity values
+    e = LogStash::Event.new
+    e.set('source_host', "my host a")
+    e.set('severity', -1)
+    events.push(e)
+
+    e = LogStash::Event.new
+    e.set('source_host', "my host a")
+    e.set('severity', 7)
+    events.push(e)
+
+    e = LogStash::Event.new
+    e.set('source_host', "my host a")
+    e.set('severity', "invalid")
+    events.push(e)
+
+    events
+  }
+
+  let(:sample_events_with_level) {
+    events = []
+    for i in 0..6 do
+      e = LogStash::Event.new
+      e.set('source_host', "my host #{i}")
+      e.set('source_file', "my file #{i}")
+      e.set('level', i)
+      e.set('seq', i)
+      e.set('nested', {'a'=>1, 'b'=>[3,4,5]})
+      e.set('tags', ['t1', 't2', 't3'])
+      events.push(e)
+    end
+    events
+  }
+
   describe "#build_multi_event_request_array" do
 
     context "test get_stats and send_status" do
@@ -225,6 +284,117 @@ describe LogStash::Outputs::Scalyr do
       end
     end
 
+    context "when severity field is configured" do
+      it "works correctly when severity event attribute is specified" do
+        plugin = LogStash::Outputs::Scalyr.new({
+                                                   'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
+                                                   'severity_field' => 'severity',
+                                               })
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events_with_severity)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(7 + 7 + 3)
+
+        (0..6).each do |index|
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq(nil)
+          expect(body['events'][index]['attrs'].fetch('sev', nil)).to eq(nil)
+          expect(body['events'][index]['sev']).to eq(index)
+        end
+
+        (7..13).each do |index|
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq(nil)
+          expect(body['events'][index]['attrs'].fetch('sev', nil)).to eq(nil)
+          expect(body['events'][index]['sev']).to eq(index - 7)
+        end
+
+        expect(body['events'][14]['attrs'].fetch('severity', nil)).to eq(-1)
+        expect(body['events'][14].key?("sev")).to eq(false)
+        expect(body['events'][14]['sev']).to eq(nil)
+        expect(body['events'][15]['attrs'].fetch('severity', nil)).to eq(7)
+        expect(body['events'][15].key?("sev")).to eq(false)
+        expect(body['events'][15]['sev']).to eq(nil)
+        expect(body['events'][16]['attrs'].fetch('severity', nil)).to eq("invalid")
+        expect(body['events'][16].key?("sev")).to eq(false)
+        expect(body['events'][16]['sev']).to eq(nil)
+      end
+
+      it "works correctly when level event attribute is specified" do
+        plugin = LogStash::Outputs::Scalyr.new({
+                                                   'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
+                                                   'severity_field' => 'level',
+                                               })
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events_with_level)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(7)
+
+        (0..6).each do |index|
+          expect(body['events'][index]['attrs'].fetch('level', nil)).to eq(nil)
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq(nil)
+          expect(body['events'][index]['attrs'].fetch('sev', nil)).to eq(nil)
+          expect(body['events'][index]['sev']).to eq(index)
+        end
+      end
+
+      it "works correctly when severity event attribute is not specified" do
+        plugin = LogStash::Outputs::Scalyr.new({
+                                                   'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
+                                                   'severity_field' => 'severity',
+                                               })
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(3)
+
+        (0..2).each do |index|
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq(nil)
+          expect(body['events'][index]['attrs'].fetch('sev', nil)).to eq(nil)
+          expect(body['events'][index]['sev']).to eq(nil)
+        end
+      end
+
+      it "works correctly when severity event attribute is not specified but severity field is not set" do
+        # Since severity_field config option is not set, severity field should be treated as a
+        # regular event attribute and not as s a special top level Event.sev field
+        plugin = LogStash::Outputs::Scalyr.new({
+                                                   'api_write_token' => '1234',
+                                                   'perform_connectivity_check' => false,
+                                                   'severity_field' => nil,
+                                               })
+        allow(plugin).to receive(:send_status).and_return(nil)
+        plugin.register
+        result = plugin.build_multi_event_request_array(sample_events_with_severity)
+        body = JSON.parse(result[0][:body])
+        expect(body['events'].size).to eq(7 + 7 + 3)
+
+        (0..6).each do |index|
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq(index)
+          expect(body['events'][index]['sev']).to eq(nil)
+        end
+
+        (7..13).each do |index|
+          expect(body['events'][index]['attrs'].fetch('severity', nil)).to eq((index - 7).to_s)
+          expect(body['events'][index]['sev']).to eq(nil)
+        end
+
+        expect(body['events'][14]['attrs'].fetch('severity', nil)).to eq(-1)
+        expect(body['events'][14].key?("sev")).to eq(false)
+        expect(body['events'][14]['sev']).to eq(nil)
+        expect(body['events'][15]['attrs'].fetch('severity', nil)).to eq(7)
+        expect(body['events'][15].key?("sev")).to eq(false)
+        expect(body['events'][15]['sev']).to eq(nil)
+        expect(body['events'][16]['attrs'].fetch('severity', nil)).to eq("invalid")
+        expect(body['events'][16].key?("sev")).to eq(false)
+        expect(body['events'][16]['sev']).to eq(nil)
+      end
+    end
+
     context "when serverhost_field is missing" do
       it "does not contain log file" do
         plugin = LogStash::Outputs::Scalyr.new({'api_write_token' => '1234', 'perform_connectivity_check' => false})
@@ -1071,7 +1241,6 @@ describe LogStash::Outputs::Scalyr do
       it "throws on invalid hostname" do
         config = {
             'api_write_token' => '1234',
-            'perform_connectivity_check' => false,
             'scalyr_server' => 'https://agent.invalid.foo.scalyr.com',
             'perform_connectivity_check' => true
         }
@@ -1082,7 +1251,6 @@ describe LogStash::Outputs::Scalyr do
       it "throws on invalid api key" do
         config = {
             'api_write_token' => '1234',
-            'perform_connectivity_check' => false,
             'scalyr_server' => 'https://agent.scalyr.com',
             'perform_connectivity_check' => true
         }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-scalyr
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.5.beta
 platform: ruby
 authors:
 - Edward Chee
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-12-15 00:00:00.000000000 Z
+date: 2022-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -186,9 +186,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project:
 rubygems_version: 2.7.10