logstash-output-newrelic 1.1.2 → 1.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e0d7a8723786c8081fd2e39cf95cab7e3dae1d79a4bf8dece2a08c61597b385
-  data.tar.gz: 753a456d7596ff42faada9d95be59527304a6a71ff7010d342807c2ce9ce5f80
+  metadata.gz: '068a9ac838ba356ffc7cc6746933ad3970d5c72df54c6633e1c7453283d700c0'
+  data.tar.gz: 8f6557b7b8ab66e4d2de7d772cbf470403be8f125d2bb90d371c0af4581ec84e
 SHA512:
-  metadata.gz: f4c351f708c77f81f496f07e381f5bd6d9fbb75cb067f6d00c5bad69010ef4df37cd037a98dcf32d1d9db59dddd2ecf5ec6be9fb784e5d69b69ad716104600a1
-  data.tar.gz: 207d9d589b285b058f16953ac428e076448f017c3887f7872b0fc861d9fbeb2870e66404adde0df7428b9de32f048bdf2b245c1f9dcad7182078bd2f4ece647a
+  metadata.gz: c200662a9b6e2157193f9d21c7dadc9562db4ca0245525775742b5518afa73b7db801dc05b506c8a5c4df74e127c7c8e520a684b5a9cf709f29350c09c0b9ebc
+  data.tar.gz: b6f15e63a3a625fc86c4dc6423b37973a01ae8406f31aa944aac62c3343d7c7dcfb3a7642f54d0464c170b824f59126d5b25b87a459cea1c79f2d14e9fd087ab

data/DEVELOPER.md

@@ -0,0 +1,26 @@
+# Developing the plugin
+
+# Getting started
+
+* Install JRuby: `rbenv install jruby-9.2.5.0`
+* Use that JRuby: `rbenv local jruby-9.2.5.0`
+* Install Bundler gem: `jruby -S gem install bundler`
+
+# Developing
+
+* Install dependencies: `jruby -S bundle install`
+* Write tests and production code!
+* Bump version: edit version file `version.rb`
+* Run tests: `jruby -S bundle exec rspec`
+* Build the gem: `jruby -S gem build logstash-output-newrelic.gemspec`
+
+# Testing it with a local Logstash install
+
+Note: you may need to run the following commands outside of your checkout, since these should not
+be run with the JRuby version that you've configured your checkout to use (by using rbenv). 
+
+* Remove previous version: `logstash-plugin remove logstash-output-newrelic`
+* Add new version: `logstash-plugin install logstash-output-newrelic-<version>.gem`
+* Restart logstash: For Homebrew: `brew services restart logstash`
+* Cause a change that you've configured Logstash to pick up (for instance, append to a file you're having it monitor)
+* Look in `https://one.newrelic.com/launcher/logger.log-launcher` for your log message

data/README.md

@@ -57,7 +57,7 @@ Exactly one of the following:
 |---|---|---|
 | concurrent_requests | The number of threads to make requests from | 1 |
 | base_uri | New Relic ingestion endpoint | https://log-api.newrelic.com/log/v1 |
-
+| max_retries | Maximum number attempts to retry to send a message. If set to 0, no re-attempts will be made. | 3 |
 
 ### EU plugin configuration
 

data/lib/logstash/outputs/config/bigdecimal_patch.rb

@@ -0,0 +1,24 @@
+require 'bigdecimal'
+
+class BigDecimal
+    # Floating-point numbers that go through the 'json' Logstash filter get automatically converted into BigDecimals.
+    # Example of such a filter:
+    #
+    # filter {
+    #   json {
+    #     source => "message"
+    #   }
+    # }
+    #
+    # The problem is that { "value" => BigDecimal('0.12345') } gets serialized into { "value": "0.12345e0"}. We do
+    # want to keep floating point numbers serialized as floating point numbers, even at the expense of loosing a little
+    # bit of precision during the conversion. So, in the above example, the correct serialization would be:
+    # { "value": 0.12345}
+    def to_json(options = nil) #:nodoc:
+      if finite?
+        self.to_f.to_s
+      else
+        'null'
+      end
+    end
+  end

data/lib/logstash/outputs/exception/error.rb

@@ -0,0 +1,14 @@
+class Error
+  class BadResponseCodeError < StandardError
+    attr_reader :url, :response_code
+
+    def initialize(response_code, url)
+      @response_code = response_code
+      @url = url
+    end
+
+    def message
+      "Got response code '#{@response_code}' contacting NewRelic at URL '#{@url}'"
+    end
+  end
+end

data/lib/logstash/outputs/newrelic.rb

@@ -6,17 +6,23 @@ require 'uri'
 require 'zlib'
 require 'json'
 require 'java'
+require 'set'
+require_relative './config/bigdecimal_patch'
+require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
   java_import java.util.concurrent.Semaphore;
 
+  NON_RETRYABLE_CODES = Set[401, 403]
+
   config_name "newrelic"
 
   config :api_key, :validate => :password, :required => false
   config :license_key, :validate => :password, :required => false
   config :concurrent_requests, :validate => :number, :default => 1
   config :base_uri, :validate => :string, :default => "https://log-api.newrelic.com/log/v1"
+  config :max_retries, :validate => :number, :default => 3
 
   public
 
@@ -26,12 +32,12 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       raise LogStash::ConfigurationError, "Must provide a license key or api key", caller
     end
     auth = {
-      @api_key.nil? ? 'X-License-Key': 'X-Insert-Key' => 
-      @api_key.nil? ? @license_key.value : @api_key.value
+      @api_key.nil? ? 'X-License-Key' : 'X-Insert-Key' =>
+        @api_key.nil? ? @license_key.value : @api_key.value
     }
     @header = {
-        'X-Event-Source' => 'logs',
-        'Content-Encoding' => 'gzip'
+      'X-Event-Source' => 'logs',
+      'Content-Encoding' => 'gzip'
     }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
     @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
@@ -58,11 +64,10 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     end
   end
 
-
   def encode(event_hash)
     log_message_hash = {
-        # non-intrinsic attributes get put into 'attributes'
-        :attributes => event_hash
+      # non-intrinsic attributes get put into 'attributes'
+      :attributes => event_hash
     }
 
     # intrinsic attributes go at the top level
@@ -88,15 +93,15 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       payload.push(encode(event.to_hash))
     end
     payload = {
-        :common => {
-            :attributes => {
-                :plugin => {
-                    :type => 'logstash',
-                    :version => LogStash::Outputs::NewRelicVersion::VERSION,
-                }
-            }
-        },
-        :logs => payload
+      :common => {
+        :attributes => {
+          :plugin => {
+            :type => 'logstash',
+            :version => LogStash::Outputs::NewRelicVersion::VERSION,
+          }
+        }
+      },
+      :logs => payload
     }
     @semaphor.acquire()
     execute = @executor.java_method :submit, [java.lang.Runnable]
@@ -115,17 +120,58 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def handle_response(response)
     if !(200 <= response.code.to_i && response.code.to_i < 300)
-      @logger.error("Request returned " + response.code + " " + response.body)
+      raise Error::BadResponseCodeError.new(response.code.to_i, @base_uri)
     end
   end
 
   def nr_send(payload)
-    http = Net::HTTP.new(@end_point.host, 443)
-    request = Net::HTTP::Post.new(@end_point.request_uri)
-    http.use_ssl = true
-    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-    @header.each {|k, v| request[k] = v}
-    request.body = payload
-    handle_response(http.request(request))
+    retries = 0
+    begin
+      http = Net::HTTP.new(@end_point.host, 443)
+      request = Net::HTTP::Post.new(@end_point.request_uri)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      @header.each { |k, v| request[k] = v }
+      request.body = payload
+      handle_response(http.request(request))
+    rescue Error::BadResponseCodeError => e
+      @logger.error(e.message)
+      if (should_retry(retries) && is_retryable_code(e))
+        retries += 1
+        sleep(1)
+        retry
+      end
+    rescue => e
+      # Stuff that should never happen
+      # For all other errors print out full issues
+      if (should_retry(retries))
+        retries += 1
+        @logger.warn(
+          "An unknown error occurred sending a bulk request to NewRelic. Retrying...",
+          :retries => "attempt #{retries} of #{@max_retries}",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+        )
+        sleep(1)
+        retry
+      else
+        @logger.error(
+          "An unknown error occurred sending a bulk request to NewRelic. Maximum of attempts reached, dropping logs.",
+          :error_message => e.message,
+          :error_class => e.class.name,
+          :backtrace => e.backtrace
+        )
+      end
+    end
+  end
+
+  def should_retry(retries)
+    retries < @max_retries
+  end
+
+  def is_retryable_code(response_error)
+    error_code = response_error.response_code
+    !NON_RETRYABLE_CODES.include?(error_code)
   end
 end # class LogStash::Outputs::NewRelic

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.1.2"
+      VERSION = "1.2.2"
     end
   end
 end

data/logstash-output-newrelic.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.name          = 'logstash-output-newrelic'
   s.version       = LogStash::Outputs::NewRelicVersion::VERSION
   s.licenses      = ['Apache-2.0']
-  s.summary       = "Sends Lostash events to New Relic"
+  s.summary       = "Sends Logstash events to New Relic"
   s.homepage      = 'https://github.com/newrelic/logstash-output-plugin'
   s.authors       = ['New Relic Logging Team']
   s.email         = 'logging-team@newrelic.com'

data/spec/outputs/newrelic_spec.rb

@@ -21,7 +21,6 @@ describe LogStash::Outputs::NewRelic do
     }
   }
 
-
   before(:each) do
     @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(simple_config)
     @newrelic_output.register
@@ -32,6 +31,7 @@ describe LogStash::Outputs::NewRelic do
       @newrelic_output.shutdown
     end
   end
+
   context "license key tests" do
     it "sets license key when given in the header" do
       stub_request(:any, base_uri).to_return(status: 200)
@@ -279,5 +279,104 @@ describe LogStash::Outputs::NewRelic do
         .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
         .to have_been_made
     end
+
+    it "retry when receive retryable http error code" do
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(2)
+    end
+
+    it "not retry when receive a non retryable http error code" do
+      stub_request(:any, base_uri)
+        .to_return(status: 401)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      # Due the async behavior we need to wait to be sure that the method was not called more than 1 time
+      sleep(2)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(1)
+    end
+
+    it "not retries when retry is disabled" do
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
+        { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '0' }
+      )
+      @newrelic_output.register
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      # Due the async behavior we need to wait to be sure that the method was not called more than 1 time
+      sleep(2)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(1)
+    end
+
+    it "retry when receive a not expected exception" do
+      stub_request(:any, base_uri)
+        .to_raise(StandardError.new("from test"))
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(2)
+    end
+  end
+
+  context "JSON serialization" do
+    it "serializes floating point numbers as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "floatingpoint" => 0.12345 })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['floatingpoint'] == 0.12345
+        }
+      ).to have_been_made
+    end
+
+    it "serializes BigDecimals as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "bigdecimal" => BigDecimal('0.12345') })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['bigdecimal'] == 0.12345
+        }
+      ).to have_been_made
+    end
+
+    it "serializes NaN as null" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "nan" => BigDecimal('NaN') })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['nan'] == nil
+        }
+      ).to have_been_made
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.1.2
+  version: 1.2.2
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-17 00:00:00.000000000 Z
+date: 2021-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,8 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -31,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -45,8 +45,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,8 +59,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: webmock
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -73,8 +73,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rspec
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -87,8 +87,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rspec-wait
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -101,8 +101,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rspec_junit_formatter
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -116,9 +116,12 @@ extra_rdoc_files: []
 files:
 - CHANGELOG.md
 - CONTRIBUTORS
+- DEVELOPER.md
 - Gemfile
 - LICENSE
 - README.md
+- lib/logstash/outputs/config/bigdecimal_patch.rb
+- lib/logstash/outputs/exception/error.rb
 - lib/logstash/outputs/newrelic.rb
 - lib/logstash/outputs/newrelic_version/version.rb
 - logstash-output-newrelic.gemspec
@@ -144,10 +147,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
-summary: Sends Lostash events to New Relic
+summary: Sends Logstash events to New Relic
 test_files:
 - spec/outputs/newrelic_spec.rb