logstash-output-newrelic 1.1.1 → 1.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 45a98fd90dd92f095b34d4593459e6b5587ab779c331597c291ea3b94ad4291e
-  data.tar.gz: f7f612da2d0040da95ca329c25e6dae097762246f5fa5061ad01082d22df0733
+  metadata.gz: a4569ef6cca71b35b48be2b7ad4f42c5648368a082ec71af161c4024d841df53
+  data.tar.gz: d8e2e7397ddc36ad08710f4dbe67b4cfb6f795ca6fa379502991bc0323673169
 SHA512:
-  metadata.gz: 816c2f5fa309328ffdf78f58894885cf791c3f8fb91867620e8494e1761e2ca4b8d9744b4c6a385c13482a7654ecfc677fb0792dc4174141e0b92beb6bc757ec
-  data.tar.gz: b2b41b31f4af73a28bdad58416c10f0a25e54a56bc295987d96c825268bce4bcfcdc01e80ca0027dfaa71af95edb9c88d8fe254ad5be6aa6b025b692742d50b1
+  metadata.gz: accb5c5d6ee856dc360e0522ac052cffee7e65ab8808e0bc19a588ad247ae7bb0c1132d96e105afc520560ebcdb85033af290a3af1b96cdb838623535f2510bc
+  data.tar.gz: cec14c1c4e073bba5533cc77a60b9abe4b85d5cde2a6cad8a563140273ac965a02d02e26e778bbab6285027ebd9123f58e7f94b7f5b5fca6d4e240d27432c48e

data/DEVELOPER.md

@@ -14,14 +14,6 @@
 * Run tests: `jruby -S bundle exec rspec`
 * Build the gem: `jruby -S gem build logstash-output-newrelic.gemspec`
 
-## Pushing changes to the public repo
-After updating the New Relic repo with changes, changes will need to be pushed to the public GitHub repo at: https://github.com/newrelic/logstash-output-plugin
-
-* `git remote add public git@github.com:newrelic/logstash-output-plugin.git`
-* `git push public master:name-of-branch-to-create`
-* Create a PR from that branch in https://github.com/newrelic/logstash-output-plugin
-* Get the PR reviewed, merged, and delete the branch!
-
 # Testing it with a local Logstash install
 
 Note: you may need to run the following commands outside of your checkout, since these should not
@@ -31,10 +23,4 @@ be run with the JRuby version that you've configured your checkout to use (by us
 * Add new version: `logstash-plugin install logstash-output-newrelic-<version>.gem`
 * Restart logstash: For Homebrew: `brew services restart logstash`
 * Cause a change that you've configured Logstash to pick up (for instance, append to a file you're having it monitor)
-* Look in `https://staging-one.newrelic.com/launcher/logger.log-launcher` for your log message
-
-# Push changes to RubyGems
-After updating the source code and gem version in `version.rb`, push the changes to RubyGems. There is an older version of the gem that we do not want to overwrite — `version 0.9.1`. Please be sure you are publishing changes to the correct gem i.e. `version 1.0.0` or higher. Note, you must be a gem owner to publish changes on [RubyGems.org](https://rubygems.org/profiles/NR-LOGGING). Once you've created the account, you will need to run `gem signin` to login to RubyGems via the command line.
-
-* Build the gem: `gem build logstash-output-newrelic.gemspec`
-* Publish the gem: `gem push logstash-output-newrelic-<VERSION>.gem` with the updated version (ex: `gem push logstash-output-newrelic-1.0.0.gem`)
+* Look in `https://one.newrelic.com/launcher/logger.log-launcher` for your log message

data/README.md

@@ -11,9 +11,11 @@ Versions of this plugin less than 1.0.0 are unsupported.
 
 ## Configuration
 
-Add the following block to your logstash.conf (with your specific API Insert key), then restart Logstash.
+Add one of the following blocks to your logstash.conf (with your specific key), then restart Logstash.
 There are other optional configuration properties, see below.
 
+### Using API Insert Key
+
 Get your API Insert Key:
 `https://insights.newrelic.com/accounts/<ACCOUNT_ID>/manage/api_keys`
 
@@ -26,12 +28,28 @@ output {
 }
 ```
 
+### Using License Key
+
+Get your License Key:
+`https://rpm.newrelic.com/accounts/<ACCOUNT_ID>`
+
+Example:
+```rb
+output {
+  newrelic {
+    license_key => "<LICENSE_KEY>"
+  }
+}
+```
 
 ### Required plugin configuration
 
+Exactly one of the following:
+
 | Property | Description |
 |---|---|
 | api_key | your New Relic API Insert key |
+| license_key | your New Relic License key |
 
 ### Optional plugin configuration
 
@@ -39,11 +57,11 @@ output {
 |---|---|---|
 | concurrent_requests | The number of threads to make requests from | 1 |
 | base_uri | New Relic ingestion endpoint | https://log-api.newrelic.com/log/v1 |
-
+| max_retries | Maximum number attempts to retry to send a message. If set to 0, no re-attempts will be made. | 3 |
 
 ### EU plugin configuration
 
-When using this plugin in the EU override the base_uri with `http://log-api.eu.newrelic.com/log/v1`
+When using this plugin in the EU override the base_uri with `https://log-api.eu.newrelic.com/log/v1`
 
 ## Testing
 
@@ -59,30 +77,3 @@ input {
 * Restart Logstash
 * Append a test log message to your log file: `echo "test message" >> /path/to/your/log/file`
 * Search New Relic Logs for `"test message"`
-
-## JSON message parsing
-
-This plugin will attempt to parse any 'message' attribute as JSON -- if it is JSON, it will be parsed and 
-the JSON attributes will be added to the event.
-
-For example, the event:
-```
-{
-  "timestamp": 1562767499238,
-  "message": "{\"service-name\": \"login-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}"
-}
-
-```
-
-Will be treated as:
-```
-{
-  "timestamp": 1562767499238,
-  "message": "{\"service-name\": \"my-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}",
-  "service-name": "login-service",
-  "user": {
-    "id": 123,
-    "name": "alice"
-  }
-}
-```

data/lib/logstash/outputs/config/bigdecimal_patch.rb

@@ -0,0 +1,24 @@
+require 'bigdecimal'
+
+class BigDecimal
+    # Floating-point numbers that go through the 'json' Logstash filter get automatically converted into BigDecimals.
+    # Example of such a filter:
+    #
+    # filter {
+    #   json {
+    #     source => "message"
+    #   }
+    # }
+    #
+    # The problem is that { "value" => BigDecimal('0.12345') } gets serialized into { "value": "0.12345e0"}. We do
+    # want to keep floating point numbers serialized as floating point numbers, even at the expense of loosing a little
+    # bit of precision during the conversion. So, in the above example, the correct serialization would be:
+    # { "value": 0.12345}
+    def to_json(options = nil) #:nodoc:
+      if finite?
+        self.to_f.to_s
+      else
+        'null'
+      end
+    end
+  end

data/lib/logstash/outputs/exception/error.rb

@@ -0,0 +1,14 @@
+class Error
+  class BadResponseCodeError < StandardError
+    attr_reader :url, :response_code
+
+    def initialize(response_code, url)
+      @response_code = response_code
+      @url = url
+    end
+
+    def message
+      "Got response code '#{@response_code}' contacting NewRelic at URL '#{@url}'"
+    end
+  end
+end

data/lib/logstash/outputs/newrelic.rb

@@ -6,17 +6,23 @@ require 'uri'
 require 'zlib'
 require 'json'
 require 'java'
+require 'set'
+require_relative './config/bigdecimal_patch'
+require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
   java_import java.util.concurrent.Semaphore;
 
+  NON_RETRYABLE_CODES = Set[401, 403]
+
   config_name "newrelic"
 
   config :api_key, :validate => :password, :required => false
   config :license_key, :validate => :password, :required => false
   config :concurrent_requests, :validate => :number, :default => 1
   config :base_uri, :validate => :string, :default => "https://log-api.newrelic.com/log/v1"
+  config :max_retries, :validate => :number, :default => 3
 
   public
 
@@ -26,12 +32,12 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       raise LogStash::ConfigurationError, "Must provide a license key or api key", caller
     end
     auth = {
-      @api_key.nil? ? 'X-License-Key': 'X-Insert-Key' => 
-      @api_key.nil? ? @license_key.value : @api_key.value
+      @api_key.nil? ? 'X-License-Key' : 'X-Insert-Key' =>
+        @api_key.nil? ? @license_key.value : @api_key.value
     }
     @header = {
-        'X-Event-Source' => 'logs',
-        'Content-Encoding' => 'gzip'
+      'X-Event-Source' => 'logs',
+      'Content-Encoding' => 'gzip'
     }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
     @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
@@ -58,11 +64,10 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     end
   end
 
-
   def encode(event_hash)
     log_message_hash = {
-        # non-intrinsic attributes get put into 'attributes'
-        :attributes => event_hash
+      # non-intrinsic attributes get put into 'attributes'
+      :attributes => event_hash
     }
 
     # intrinsic attributes go at the top level
@@ -88,15 +93,15 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       payload.push(encode(event.to_hash))
     end
     payload = {
-        :common => {
-            :attributes => {
-                :plugin => {
-                    :type => 'logstash',
-                    :version => LogStash::Outputs::NewRelicVersion::VERSION,
-                }
-            }
-        },
-        :logs => payload
+      :common => {
+        :attributes => {
+          :plugin => {
+            :type => 'logstash',
+            :version => LogStash::Outputs::NewRelicVersion::VERSION,
+          }
+        }
+      },
+      :logs => payload
     }
     @semaphor.acquire()
     execute = @executor.java_method :submit, [java.lang.Runnable]
@@ -115,17 +120,50 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def handle_response(response)
     if !(200 <= response.code.to_i && response.code.to_i < 300)
-      @logger.error("Request returned " + response.code + " " + response.body)
+      raise Error::BadResponseCodeError.new(response.code.to_i, @base_uri)
     end
   end
 
   def nr_send(payload)
-    http = Net::HTTP.new(@end_point.host, 443)
-    request = Net::HTTP::Post.new(@end_point.request_uri)
-    http.use_ssl = true
-    http.verify_mode = OpenSSL::SSL::VERIFY_PEER
-    @header.each {|k, v| request[k] = v}
-    request.body = payload
-    handle_response(http.request(request))
+    retries = 0
+    begin
+      http = Net::HTTP.new(@end_point.host, 443)
+      request = Net::HTTP::Post.new(@end_point.request_uri)
+      http.use_ssl = true
+      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      @header.each { |k, v| request[k] = v }
+      request.body = payload
+      handle_response(http.request(request))
+    rescue Error::BadResponseCodeError => e
+      @logger.error(e.message)
+      if (should_retry(retries) && is_retryable_code(e))
+        retries += 1
+        sleep(1)
+        retry
+      end
+    rescue => e
+      # Stuff that should never happen
+      # For all other errors print out full issues
+      @logger.error(
+        "An unknown error occurred sending a bulk request to NewRelic.",
+        :error_message => e.message,
+        :error_class => e.class.name,
+        :backtrace => e.backtrace
+      )
+      if (should_retry(retries))
+        retries += 1
+        sleep(1)
+        retry
+      end
+    end
+  end
+
+  def should_retry(retries)
+    retries < @max_retries
+  end
+
+  def is_retryable_code(response_error)
+    error_code = response_error.response_code
+    !NON_RETRYABLE_CODES.include?(error_code)
   end
 end # class LogStash::Outputs::NewRelic

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.1.1"
+      VERSION = "1.2.1"
     end
   end
 end

data/logstash-output-newrelic.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.name          = 'logstash-output-newrelic'
   s.version       = LogStash::Outputs::NewRelicVersion::VERSION
   s.licenses      = ['Apache-2.0']
-  s.summary       = "Sends Lostash events to New Relic"
+  s.summary       = "Sends Logstash events to New Relic"
   s.homepage      = 'https://github.com/newrelic/logstash-output-plugin'
   s.authors       = ['New Relic Logging Team']
   s.email         = 'logging-team@newrelic.com'
@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_development_dependency "logstash-devutils"
   s.add_development_dependency "webmock"
+  s.add_development_dependency "rspec"
   s.add_development_dependency "rspec-wait"
-
+  s.add_development_dependency "rspec_junit_formatter"
 end

data/spec/outputs/newrelic_spec.rb

@@ -21,7 +21,6 @@ describe LogStash::Outputs::NewRelic do
     }
   }
 
-
   before(:each) do
     @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(simple_config)
     @newrelic_output.register
@@ -32,6 +31,7 @@ describe LogStash::Outputs::NewRelic do
       @newrelic_output.shutdown
     end
   end
+
   context "license key tests" do
     it "sets license key when given in the header" do
       stub_request(:any, base_uri).to_return(status: 200)
@@ -279,5 +279,104 @@ describe LogStash::Outputs::NewRelic do
         .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
         .to have_been_made
     end
+
+    it "retry when receive retryable http error code" do
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(2)
+    end
+
+    it "not retry when receive a non retryable http error code" do
+      stub_request(:any, base_uri)
+        .to_return(status: 401)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      # Due the async behavior we need to wait to be sure that the method was not called more than 1 time
+      sleep(2)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(1)
+    end
+
+    it "not retries when retry is disabled" do
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
+        { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '0' }
+      )
+      @newrelic_output.register
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      # Due the async behavior we need to wait to be sure that the method was not called more than 1 time
+      sleep(2)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(1)
+    end
+
+    it "retry when receive a not expected exception" do
+      stub_request(:any, base_uri)
+        .to_raise(StandardError.new("from test"))
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      @newrelic_output.multi_receive([event1])
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
+        .to have_been_made.times(2)
+    end
+  end
+
+  context "JSON serialization" do
+    it "serializes floating point numbers as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "floatingpoint" => 0.12345 })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['floatingpoint'] == 0.12345
+        }
+      ).to have_been_made
+    end
+
+    it "serializes BigDecimals as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "bigdecimal" => BigDecimal('0.12345') })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['bigdecimal'] == 0.12345
+        }
+      ).to have_been_made
+    end
+
+    it "serializes NaN as null" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "nan" => BigDecimal('NaN') })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['nan'] == nil
+        }
+      ).to have_been_made
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.1
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-09-04 00:00:00.000000000 Z
+date: 2021-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -17,8 +17,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.0'
   name: logstash-core-plugin-api
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -31,8 +31,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
-  prerelease: false
   type: :runtime
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -45,8 +45,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
-  prerelease: false
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -59,8 +59,22 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: webmock
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -73,8 +87,22 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
   name: rspec-wait
+  type: :development
   prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec_junit_formatter
   type: :development
+  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -92,6 +120,8 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/logstash/outputs/config/bigdecimal_patch.rb
+- lib/logstash/outputs/exception/error.rb
 - lib/logstash/outputs/newrelic.rb
 - lib/logstash/outputs/newrelic_version/version.rb
 - logstash-output-newrelic.gemspec
@@ -117,10 +147,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 3.0.6
 signing_key:
 specification_version: 4
-summary: Sends Lostash events to New Relic
+summary: Sends Logstash events to New Relic
 test_files:
 - spec/outputs/newrelic_spec.rb