logstash-output-newrelic 1.0.8 → 1.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15a58f1472c4f00bc1decbd672cf520a247d192eb6b31b348bc6daa77f2ea27c
-  data.tar.gz: f71834a29f1507c3948d9044826b7c2cc23e249121b57dacd65103bc8d22f742
+  metadata.gz: f099ad9c408ec722141c72268c3bdc038adbe98c0760da028f5149f87077a0eb
+  data.tar.gz: 45817bcb9ada5fa8ca81a57f102878fc752111f7a0b7e3d3a7fc9c681a6df480
 SHA512:
-  metadata.gz: 9db4eb7268c9c401932ff4bfd2055ca8475c69c962d3a99f41dec89e51833f6b9c527eb887b65d520d20e322401798a231cb5ae53c7d8c78c3bc6581cef77d29
-  data.tar.gz: b95f9564bd9f6c51d38a3995c86733b78cdaa9cb474ffda178a05006d6f10f47e4fbdb31e1934e0e4e9f31821487dd26be65a49f617c503add87fa394d689982
+  metadata.gz: de3784b9f1ca8300c99a7adea011e95b5ac5140ffd01d7a18dfc6cf5c5c5bbd2767cc4ac519e763bea7e0e7d12a509b9dcde98cc890b52d41ae901d750086511
+  data.tar.gz: acfc744ccc74de945848c3b76ade20b6e858fce52d1951e1533b302a2c55b4ae2f6156f507a8b0f949967981a5a304f293973f632f8c3863ab04fc1f1463a82b

data/DEVELOPER.md

@@ -14,14 +14,6 @@
 * Run tests: `jruby -S bundle exec rspec`
 * Build the gem: `jruby -S gem build logstash-output-newrelic.gemspec`
 
-## Pushing changes to the public repo
-After updating the New Relic repo with changes, changes will need to be pushed to the public GitHub repo at: https://github.com/newrelic/logstash-output-plugin
-
-* `git remote add public git@github.com:newrelic/logstash-output-plugin.git`
-* `git push public master:name-of-branch-to-create`
-* Create a PR from that branch in https://github.com/newrelic/logstash-output-plugin
-* Get the PR reviewed, merged, and delete the branch!
-
 # Testing it with a local Logstash install
 
 Note: you may need to run the following commands outside of your checkout, since these should not
@@ -31,10 +23,4 @@ be run with the JRuby version that you've configured your checkout to use (by us
 * Add new version: `logstash-plugin install logstash-output-newrelic-<version>.gem`
 * Restart logstash: For Homebrew: `brew services restart logstash`
 * Cause a change that you've configured Logstash to pick up (for instance, append to a file you're having it monitor)
-* Look in `https://staging-one.newrelic.com/launcher/logger.log-launcher` for your log message
-
-# Push changes to RubyGems
-After updating the source code and gem version in `version.rb`, push the changes to RubyGems. There is an older version of the gem that we do not want to overwrite — `version 0.9.1`. Please be sure you are publishing changes to the correct gem i.e. `version 1.0.0` or higher. Note, you must be a gem owner to publish changes on [RubyGems.org](https://rubygems.org/profiles/NR-LOGGING). Once you've created the account, you will need to run `gem signin` to login to RubyGems via the command line.
-
-* Build the gem: `gem build logstash-output-newrelic.gemspec`
-* Publish the gem: `gem push logstash-output-newrelic-<VERSION>.gem` with the updated version (ex: `gem push logstash-output-newrelic-1.0.0.gem`)
+* Look in `https://one.newrelic.com/launcher/logger.log-launcher` for your log message

data/README.md

@@ -11,9 +11,11 @@ Versions of this plugin less than 1.0.0 are unsupported.
 
 ## Configuration
 
-Add the following block to your logstash.conf (with your specific API Insert key), then restart Logstash.
+Add one of the following blocks to your logstash.conf (with your specific key), then restart Logstash.
 There are other optional configuration properties, see below.
 
+### Using API Insert Key
+
 Get your API Insert Key:
 `https://insights.newrelic.com/accounts/<ACCOUNT_ID>/manage/api_keys`
 
@@ -26,23 +28,41 @@ output {
 }
 ```
 
+### Using License Key
+
+Get your License Key:
+`https://rpm.newrelic.com/accounts/<ACCOUNT_ID>`
+
+Example:
+```rb
+output {
+  newrelic {
+    license_key => "<LICENSE_KEY>"
+  }
+}
+```
 
 ### Required plugin configuration
 
+Exactly one of the following:
+
 | Property | Description |
 |---|---|
 | api_key | your New Relic API Insert key |
+| license_key | your New Relic License key |
 
 ### Optional plugin configuration
 
 | Property | Description | Default value |
 |---|---|---|
 | concurrent_requests | The number of threads to make requests from | 1 |
-| retries | The maximum number of times to retry a failed request, exponentially increasing delay between each retry | 5 |
-| retry_seconds | The inital delay between retries, in seconds | 5 |
-| max_delay | The maximum delay between retries, in seconds | 30 |
 | base_uri | New Relic ingestion endpoint | https://log-api.newrelic.com/log/v1 |
 
+
+### EU plugin configuration
+
+When using this plugin in the EU override the base_uri with `https://log-api.eu.newrelic.com/log/v1`
+
 ## Testing
 
 An easy way to test the plugin is to make sure Logstash is getting input from a log file you 
@@ -57,30 +77,3 @@ input {
 * Restart Logstash
 * Append a test log message to your log file: `echo "test message" >> /path/to/your/log/file`
 * Search New Relic Logs for `"test message"`
-
-## JSON message parsing
-
-This plugin will attempt to parse any 'message' attribute as JSON -- if it is JSON, it will be parsed and 
-the JSON attributes will be added to the event.
-
-For example, the event:
-```
-{
-  "timestamp": 1562767499238,
-  "message": "{\"service-name\": \"login-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}"
-}
-
-```
-
-Will be treated as:
-```
-{
-  "timestamp": 1562767499238,
-  "message": "{\"service-name\": \"my-service\", \"user\": {\"id\": 123, \"name\": \"alice\"}}",
-  "service-name": "login-service",
-  "user": {
-    "id": 123,
-    "name": "alice"
-  }
-}
-```

data/lib/logstash/outputs/config/bigdecimal_patch.rb

@@ -0,0 +1,24 @@
+require 'bigdecimal'
+
+class BigDecimal
+    # Floating-point numbers that go through the 'json' Logstash filter get automatically converted into BigDecimals.
+    # Example of such a filter:
+    #
+    # filter {
+    #   json {
+    #     source => "message"
+    #   }
+    # }
+    #
+    # The problem is that { "value" => BigDecimal('0.12345') } gets serialized into { "value": "0.12345e0"}. We do
+    # want to keep floating point numbers serialized as floating point numbers, even at the expense of loosing a little
+    # bit of precision during the conversion. So, in the above example, the correct serialization would be:
+    # { "value": 0.12345}
+    def to_json(options = nil) #:nodoc:
+      if finite?
+        self.to_f.to_s
+      else
+        'null'
+      end
+    end
+  end

data/lib/logstash/outputs/newrelic.rb

@@ -6,6 +6,7 @@ require 'uri'
 require 'zlib'
 require 'json'
 require 'java'
+require_relative './config/bigdecimal_patch'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
@@ -13,10 +14,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   config_name "newrelic"
 
-  config :api_key, :validate => :password, :required => true
-  config :retry_seconds, :validate => :number, :default => 5
-  config :max_delay, :validate => :number, :default => 30
-  config :retries, :validate => :number, :default => 5
+  config :api_key, :validate => :password, :required => false
+  config :license_key, :validate => :password, :required => false
   config :concurrent_requests, :validate => :number, :default => 1
   config :base_uri, :validate => :string, :default => "https://log-api.newrelic.com/log/v1"
 
@@ -24,23 +23,31 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def register
     @end_point = URI.parse(@base_uri)
+    if @api_key.nil? && @license_key.nil?
+      raise LogStash::ConfigurationError, "Must provide a license key or api key", caller
+    end
+    auth = {
+      @api_key.nil? ? 'X-License-Key': 'X-Insert-Key' => 
+      @api_key.nil? ? @license_key.value : @api_key.value
+    }
     @header = {
-        'X-Insert-Key' => @api_key.value,
         'X-Event-Source' => 'logs',
         'Content-Encoding' => 'gzip'
-    }.freeze
+    }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
     @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
   def shutdown
-    @executor&.shutdown
-    # We want this long enough to not have threading issues
-    terminationWaitInSeconds = 10
-    terminatedInTime = @executor&.awaitTermination(terminationWaitInSeconds, java.util.concurrent.TimeUnit::SECONDS)
-    if !terminatedInTime
-      raise "Did not shut down within #{terminationWaitInSeconds} seconds"
+    if @executor
+      @executor.shutdown
+      # We want this long enough to not have threading issues
+      terminationWaitInSeconds = 10
+      terminatedInTime = @executor.awaitTermination(terminationWaitInSeconds, java.util.concurrent.TimeUnit::SECONDS)
+      if !terminatedInTime
+        raise "Did not shut down within #{terminationWaitInSeconds} seconds"
+      end
     end
   end
 
@@ -100,39 +107,17 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
         gzip = Zlib::GzipWriter.new(io)
         gzip << [payload].to_json
         gzip.close
-        attempt_send(io.string, 0)
+        nr_send(io.string)
       ensure
         @semaphor.release()
       end
     end
   end
 
-  def should_retry?(attempt)
-    attempt < retries
-  end
-
-  def sleep_duration(attempt)
-    if attempt == 0
-      return 0
+  def handle_response(response)
+    if !(200 <= response.code.to_i && response.code.to_i < 300)
+      @logger.error("Request returned " + response.code + " " + response.body)
     end
-
-    [max_delay, (2 ** (attempt - 1)) * retry_seconds].min
-  end
-
-  def sleep_on_retry(attempt)
-    duration = sleep_duration(attempt)
-    if duration > 0
-      sleep duration
-    end
-  end
-
-  def attempt_send(payload, attempt)
-    sleep_on_retry(attempt)
-    attempt_send(payload, attempt + 1) unless was_successful?(nr_send(payload)) if should_retry?(attempt)
-  end
-
-  def was_successful?(response)
-    200 <= response.code.to_i && response.code.to_i < 300
   end
 
   def nr_send(payload)
@@ -142,6 +127,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     http.verify_mode = OpenSSL::SSL::VERIFY_PEER
     @header.each {|k, v| request[k] = v}
     request.body = payload
-    http.request(request)
+    handle_response(http.request(request))
   end
 end # class LogStash::Outputs::NewRelic

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.0.8"
+      VERSION = "1.1.4"
     end
   end
 end

data/logstash-output-newrelic.gemspec

@@ -26,6 +26,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_development_dependency "logstash-devutils"
   s.add_development_dependency "webmock"
+  s.add_development_dependency "rspec"
   s.add_development_dependency "rspec-wait"
-
+  s.add_development_dependency "rspec_junit_formatter"
 end

data/spec/outputs/newrelic_spec.rb

@@ -7,6 +7,48 @@ require "logstash/event"
 require "webmock/rspec"
 require "zlib"
 
+describe LogStash::Outputs::NewRelic do
+  let (:base_uri) { "https://testing-example-collector.com" }
+  let (:retry_seconds) { 0 }
+  # Don't sleep in tests, to keep tests fast. We have a test for the method that produces the sleep duration between retries.
+  let (:max_delay) { 0 }
+  let (:retries) { 3 }
+  let (:license_key) { 'cool-guy' }
+  let (:simple_config) {
+    {
+      "base_uri" => base_uri,
+      "license_key" => license_key
+    }
+  }
+
+  before(:each) do
+    @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(simple_config)
+    @newrelic_output.register
+  end
+
+  after(:each) do
+    if @newrelic_output
+      @newrelic_output.shutdown
+    end
+  end
+
+  context "license key tests" do
+    it "sets license key when given in the header" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({:message => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with(headers: {
+                "X-License-Key" => license_key,
+                "X-Event-Source" => "logs",
+                "Content-Encoding" => "gzip",
+              })).to have_been_made
+    end
+  end
+end
+
 describe LogStash::Outputs::NewRelic do
   let (:api_key) { "someAccountKey" }
   let (:base_uri) { "https://testing-example-collector.com" }
@@ -18,9 +60,6 @@ describe LogStash::Outputs::NewRelic do
     {
       "api_key" => api_key,
       "base_uri" => base_uri,
-      "retries" => retries,
-      "retry_seconds" => retry_seconds,
-      "max_delay" => max_delay,
     }
   }
 
@@ -62,15 +101,17 @@ describe LogStash::Outputs::NewRelic do
   end
 
   after(:each) do
-    @newrelic_output&.shutdown
+    if @newrelic_output
+      @newrelic_output.shutdown
+    end
   end
 
   context "validation of config" do
     it "requires api_key" do
       no_api_key_config = {
       }
-
-      expect { LogStash::Plugin.lookup("output", "newrelic").new(no_api_key_config) }.to raise_error LogStash::ConfigurationError
+      output =  LogStash::Plugin.lookup("output", "newrelic").new(no_api_key_config)
+      expect { output.register }.to raise_error LogStash::ConfigurationError
     end
   end
 
@@ -200,7 +241,7 @@ describe LogStash::Outputs::NewRelic do
 
       # Shut down the plugin so that it has the chance to send a request
       # (since we're verifying that nothing is sent)
-      @newrelic_output&.shutdown
+      @newrelic_output.shutdown
 
       expect(a_request(:post, base_uri))
           .not_to have_been_made
@@ -223,80 +264,64 @@ describe LogStash::Outputs::NewRelic do
     end
   end
 
-  context "retry" do
-    it "sleep periods double each time up to max time" do
-      specific_config = simple_config.clone
-      # Use non-trivial times -- they can be big, since this test doesn't do any sleeping, just
-      # tests the sleep duration
-      specific_config["max_delay"] = 60
-      specific_config["retry_seconds"] = 5
-
-      # Create a new plugin with this specific config that has longer retry sleep
-      # configuration than we normally want
-      @newrelic_output&.shutdown
-      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(specific_config)
-      @newrelic_output.register
-
-      expect(@newrelic_output.sleep_duration(0)).to equal(0)
-      expect(@newrelic_output.sleep_duration(1)).to equal(5)
-      expect(@newrelic_output.sleep_duration(2)).to equal(10)
-      expect(@newrelic_output.sleep_duration(3)).to equal(20)
-      expect(@newrelic_output.sleep_duration(4)).to equal(40)
-      expect(@newrelic_output.sleep_duration(5)).to equal(60)
-      expect(@newrelic_output.sleep_duration(6)).to equal(60) # Never gets bigger than this
-    end
-
-    it "first call fails, should retry" do
+  context "error handling" do
+    it "continues through errors, future calls should still succeed" do
       stub_request(:any, base_uri)
-        .to_return(status: 500)
+        .to_raise(StandardError.new("from test"))
         .to_return(status: 200)
 
-      event = LogStash::Event.new({ "message" => "Test message" })
-      @newrelic_output.multi_receive([event])
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      event2 = LogStash::Event.new({ "message" => "Test message 2" })
+      @newrelic_output.multi_receive([event1])
+      @newrelic_output.multi_receive([event2])
 
-      wait_for(a_request(:post, base_uri)).to have_been_made.times(2)
+      wait_for(a_request(:post, base_uri)
+        .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
+        .to have_been_made
     end
+  end
 
-    it "first two calls fail, should retry" do
-      stub_request(:any, base_uri)
-        .to_return(status: 500)
-        .to_return(status: 500)
-        .to_return(status: 200)
+  context "JSON serialization" do
+    it "serializes floating point numbers as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
 
-      event = LogStash::Event.new({ "message" => "Test message" })
+      event = LogStash::Event.new({ "floatingpoint" => 0.12345 })
       @newrelic_output.multi_receive([event])
 
-      wait_for(a_request(:post, base_uri)).to have_been_made.times(3)
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['floatingpoint'] == 0.12345
+        }
+      ).to have_been_made
     end
 
-    it "all calls fails, should stop retrying at some point" do
-      stub_request(:any, base_uri)
-        .to_return(status: 500)
+    it "serializes BigDecimals as floating point numbers" do
+      stub_request(:any, base_uri).to_return(status: 200)
 
-      event = LogStash::Event.new({ "message" => "Test message" })
+      event = LogStash::Event.new({ "bigdecimal" => BigDecimal('0.12345') })
       @newrelic_output.multi_receive([event])
 
-      # This may not fail if the wait_for is called exactly when there have been 'retries' calls.
-      # However, with zero sleep time (max_delay=0), on a laptop the POST was done 2000+ times by the
-      # time this was executed
-      wait_for(a_request(:post, base_uri)).to have_been_made.times(retries)
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['bigdecimal'] == 0.12345
+        }
+      ).to have_been_made
     end
-  end
 
-  context "error handling" do
-    it "continues through errors, future calls should still succeed" do
-      stub_request(:any, base_uri)
-        .to_raise(StandardError.new("from test"))
-        .to_return(status: 200)
+    it "serializes NaN as null" do
+      stub_request(:any, base_uri).to_return(status: 200)
 
-      event1 = LogStash::Event.new({ "message" => "Test message 1" })
-      event2 = LogStash::Event.new({ "message" => "Test message 2" })
-      @newrelic_output.multi_receive([event1])
-      @newrelic_output.multi_receive([event2])
+      event = LogStash::Event.new({ "nan" => BigDecimal('NaN') })
+      @newrelic_output.multi_receive([event])
 
       wait_for(a_request(:post, base_uri)
-        .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
-        .to have_been_made
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['attributes']['nan'] == nil
+        }
+      ).to have_been_made
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.0.8
+  version: 1.1.4
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-08-20 00:00:00.000000000 Z
+date: 2020-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -80,6 +94,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec_junit_formatter
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email: logging-team@newrelic.com
 executables: []
@@ -92,6 +120,7 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/logstash/outputs/config/bigdecimal_patch.rb
 - lib/logstash/outputs/newrelic.rb
 - lib/logstash/outputs/newrelic_version/version.rb
 - logstash-output-newrelic.gemspec