logstash-output-newrelic 0.9.1 → 1.0.0

data/spec/outputs/newrelic_internal_spec.rb

@@ -0,0 +1,284 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/newrelic_internal"
+require "logstash/outputs/newrelic_internal_version/version"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "webmock/rspec"
+require "zlib"
+
+describe LogStash::Outputs::NewRelicInternal do
+  let (:api_key) { "someAccountKey" }
+  let (:base_uri) { "https://testing-example-collector.com" }
+  let (:retry_seconds) { 0 }
+  # Don't sleep in tests, to keep tests fast. We have a test for the method that produces the sleep duration between retries.
+  let (:max_delay) { 0 }
+  let (:retries) { 3 }
+  let (:simple_config) {
+    {
+      "api_key" => api_key,
+      "base_uri" => base_uri,
+      "retries" => retries,
+      "retry_seconds" => retry_seconds,
+      "max_delay" => max_delay,
+    }
+  }
+
+  def gunzip(bytes)
+    gz = Zlib::GzipReader.new(StringIO.new(bytes))
+    gz.read
+  end
+
+  def single_gzipped_message(body)
+    message = JSON.parse(gunzip(body))[0]['logs']
+    expect(message.length).to equal(1)
+    message[0]
+  end
+
+  def multiple_gzipped_messages(body)
+    JSON.parse(gunzip(body))
+  end
+
+  before(:each) do
+    @newrelic_output = LogStash::Plugin.lookup("output", "newrelic_internal").new(simple_config)
+    @newrelic_output.register
+  end
+
+  after(:each) do
+    @newrelic_output&.shutdown
+  end
+
+  context "validation of config" do
+    it "requires api_key" do
+      no_api_key_config = {
+      }
+
+      expect { LogStash::Plugin.lookup("output", "newrelic_internal").new(no_api_key_config) }.to raise_error LogStash::ConfigurationError
+    end
+  end
+
+  context "request headers" do
+    it "all present" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with(headers: {
+                "X-Insert-Key" => api_key,
+                "X-Event-Source" => "logs",
+                "Content-Encoding" => "gzip",
+              })).to have_been_made
+    end
+  end
+  context "request body" do
+
+    it "message contains plugin information" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ :message => "Test message", :@timestamp => '123' })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+      .with { |request|
+        data = multiple_gzipped_messages(request.body)[0]
+        data['common']['attributes']['plugin']['type'] == 'logstash' &&
+        data['common']['attributes']['plugin']['version'] == LogStash::Outputs::NewRelicInternalVersion::VERSION })
+      .to have_been_made
+    end
+
+    # TODO: why is this field always removed?
+    it "'@timestamp' field is removed" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ :message => "Test message", :@timestamp => '123' })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request| single_gzipped_message(request.body)['@timestamp'] == nil })
+        .to have_been_made
+    end
+
+    it "all other fields passed through as is" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ :message => "Test message", :other => "Other value" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['message'] == 'Test message' &&
+          message['other'] == 'Other value' })
+        .to have_been_made
+    end
+
+    it "JSON object 'message' field is parsed, removed, and its data merged as attributes" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      message_json = '{ "in-json-1": "1", "in-json-2": "2", "sub-object": {"in-json-3": "3"} }'
+      event = LogStash::Event.new({ :message => message_json, :other => "Other value" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['in-json-1'] == '1' &&
+          message['in-json-2'] == '2' &&
+          message['sub-object'] == {"in-json-3" => "3"} &&
+          message['other'] == 'Other value' })
+        .to have_been_made
+    end
+
+    it "JSON array 'message' field is not parsed, left as is" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      message_json_array = '[{ "in-json-1": "1", "in-json-2": "2", "sub-object": {"in-json-3": "3"} }]'
+      event = LogStash::Event.new({ :message => message_json_array, :other => "Other value" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['message'] == message_json_array &&
+          message['other'] == 'Other value' })
+        .to have_been_made
+    end
+
+    it "JSON string 'message' field is not parsed, left as is" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      message_json_string = '"I can be parsed as JSON"'
+      event = LogStash::Event.new({ :message => message_json_string, :other => "Other value" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['message'] == message_json_string &&
+          message['other'] == 'Other value' })
+        .to have_been_made
+    end
+
+    it "other JSON fields are not parsed" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      other_json = '{ "key": "value" }'
+      event = LogStash::Event.new({ :message => "Test message", :other => other_json })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          message = single_gzipped_message(request.body)
+          message['message'] == 'Test message' &&
+          message['other'] == other_json })
+        .to have_been_made
+    end
+
+    it "handles messages without a 'message' field" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({ :other => 'Other value' })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+      .with { |request|
+        message = single_gzipped_message(request.body)
+        message['other'] == 'Other value' })
+      .to have_been_made
+    end
+
+    it "multiple events" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      event2 = LogStash::Event.new({ "message" => "Test message 2" })
+      @newrelic_output.multi_receive([event1, event2])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request|
+          messages = multiple_gzipped_messages(request.body)[0]['logs']
+          messages.length == 2 &&
+          messages[0]['message'] == 'Test message 1' &&
+          messages[1]['message'] == 'Test message 2' })
+        .to have_been_made
+    end
+  end
+
+  context "retry" do
+    it "sleep periods double each time up to max time" do
+      specific_config = simple_config.clone
+      # Use non-trivial times -- they can be big, since this test doesn't do any sleeping, just
+      # tests the sleep duration
+      specific_config["max_delay"] = 60
+      specific_config["retry_seconds"] = 5
+
+      # Create a new plugin with this specific config that has longer retry sleep
+      # configuration than we normally want
+      @newrelic_output&.shutdown
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic_internal").new(specific_config)
+      @newrelic_output.register
+
+      expect(@newrelic_output.sleep_duration(0)).to equal(5)
+      expect(@newrelic_output.sleep_duration(1)).to equal(10)
+      expect(@newrelic_output.sleep_duration(2)).to equal(20)
+      expect(@newrelic_output.sleep_duration(3)).to equal(40)
+      expect(@newrelic_output.sleep_duration(4)).to equal(60)
+      expect(@newrelic_output.sleep_duration(5)).to equal(60) # Never gets bigger than this
+    end
+
+    it "first call fails, should retry" do
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)).to have_been_made.times(2)
+    end
+
+    it "first two calls fail, should retry" do
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)).to have_been_made.times(3)
+    end
+
+    it "all calls fails, should stop retrying at some point" do
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+
+      event = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      # This may not fail if the wait_for is called exactly when there have been 'retries' calls.
+      # However, with zero sleep time (max_delay=0), on a laptop the POST was done 2000+ times by the
+      # time this was executed
+      wait_for(a_request(:post, base_uri)).to have_been_made.times(retries)
+    end
+  end
+
+  context "error handling" do
+    it "continues through errors, future calls should still succeed" do
+      stub_request(:any, base_uri)
+        .to_raise(StandardError.new("from test"))
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message 1" })
+      event2 = LogStash::Event.new({ "message" => "Test message 2" })
+      @newrelic_output.multi_receive([event1])
+      @newrelic_output.multi_receive([event2])
+
+      wait_for(a_request(:post, base_uri)
+        .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 2' })
+        .to have_been_made
+    end
+  end
+end

metadata

@@ -1,39 +1,33 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 1.0.0
 platform: ruby
 authors:
-- The Chocolate Factory
+- New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-22 00:00:00.000000000 Z
+date: 2019-06-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.60'
-    - - <=
-      - !ruby/object:Gem::Version
-        version: '2.99'
+        version: '2.0'
   name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '1.60'
-    - - <=
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.99'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-codec-plain
@@ -41,13 +35,13 @@ dependencies:
   type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   name: logstash-devutils
@@ -55,35 +49,57 @@ dependencies:
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webmock
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-wait
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description: Use logstash to ship log events to New Relic Insights. This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program.
-email: info@chocolatefirm.com
+description: Gzips and decorates logstash events to be properly formatted as custom
+  events
+email: logging-team@newrelic.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .github/CONTRIBUTING.md
-- .github/ISSUE_TEMPLATE.md
-- .github/PULL_REQUEST_TEMPLATE.md
-- .gitignore
-- .travis.yml
 - CHANGELOG.md
+- CONTRIBUTORS
+- DEVELOPER.md
 - Gemfile
 - LICENSE
-- NOTICE.TXT
 - README.md
-- Rakefile
-- bin/logstash-newrelic
-- bin/logstash-newrelic-debug
-- bin/logstash-newrelic.conf.template
-- lib/logstash/outputs/newrelic.rb
+- lib/logstash/outputs/newrelic_internal.rb
+- lib/logstash/outputs/newrelic_internal_version/version.rb
 - logstash-output-newrelic.gemspec
-- spec/outputs/newrelic_spec.rb
-homepage: http://www.chocolatefirm.com
+- spec/outputs/newrelic_internal_spec.rb
+homepage: https://source.datanerd.us/logging/logstash-output-newrelic
 licenses:
-- MIT
+- Apache-2.0
 metadata:
   logstash_plugin: 'true'
   logstash_group: output
@@ -93,19 +109,19 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.5
+rubygems_version: 2.7.6
 signing_key:
 specification_version: 4
-summary: New Relic Insights output plugin for Logstash
+summary: Forwards logs as custom events to insights
 test_files:
-- spec/outputs/newrelic_spec.rb
+- spec/outputs/newrelic_internal_spec.rb

data/.github/CONTRIBUTING.md

@@ -1,65 +0,0 @@
-# Contributing to Logstash
-
-All contributions are welcome: ideas, patches, documentation, bug reports,
-complaints, etc!
-
-Programming is not a required skill, and there are many ways to help out!
-It is more important to us that you are able to contribute.
-
-That said, some basic guidelines, which you are free to ignore :)
-
-## Want to learn?
-
-Want to lurk about and see what others are doing with Logstash? 
-
-* The irc channel (#logstash on irc.freenode.org) is a good place for this
-* The [forum](https://discuss.elastic.co/c/logstash) is also
-  great for learning from others.
-
-## Got Questions?
-
-Have a problem you want Logstash to solve for you? 
-
-* You can ask a question in the [forum](https://discuss.elastic.co/c/logstash)
-* Alternately, you are welcome to join the IRC channel #logstash on
-irc.freenode.org and ask for help there!
-
-## Have an Idea or Feature Request?
-
-* File a ticket on [GitHub](https://github.com/elastic/logstash/issues). Please remember that GitHub is used only for issues and feature requests. If you have a general question, the [forum](https://discuss.elastic.co/c/logstash) or IRC would be the best place to ask.
-
-## Something Not Working? Found a Bug?
-
-If you think you found a bug, it probably is a bug.
-
-* If it is a general Logstash or a pipeline issue, file it in [Logstash GitHub](https://github.com/elasticsearch/logstash/issues)
-* If it is specific to a plugin, please file it in the respective repository under [logstash-plugins](https://github.com/logstash-plugins)
-* or ask the [forum](https://discuss.elastic.co/c/logstash).
-
-# Contributing Documentation and Code Changes
-
-If you have a bugfix or new feature that you would like to contribute to
-logstash, and you think it will take more than a few minutes to produce the fix
-(ie; write code), it is worth discussing the change with the Logstash users and developers first! You can reach us via [GitHub](https://github.com/elastic/logstash/issues), the [forum](https://discuss.elastic.co/c/logstash), or via IRC (#logstash on freenode irc)
-Please note that Pull Requests without tests will not be merged. If you would like to contribute but do not have experience with writing tests, please ping us on IRC/forum or create a PR and ask our help.
-
-## Contributing to plugins
-
-Check our [documentation](https://www.elastic.co/guide/en/logstash/current/contributing-to-logstash.html) on how to contribute to plugins or write your own! It is super easy!
-
-## Contribution Steps
-
-1. Test your changes! [Run](https://github.com/elastic/logstash#testing) the test suite
-2. Please make sure you have signed our [Contributor License
-   Agreement](https://www.elastic.co/contributor-agreement/). We are not
-   asking you to assign copyright to us, but to give us the right to distribute
-   your code without restriction. We ask this of all contributors in order to
-   assure our users of the origin and continuing existence of the code. You
-   only need to sign the CLA once.
-3. Send a pull request! Push your changes to your fork of the repository and
-   [submit a pull
-   request](https://help.github.com/articles/using-pull-requests). In the pull
-   request, describe what your changes do and mention any bugs/issues related
-   to the pull request.
-
-

data/.github/ISSUE_TEMPLATE.md

@@ -1,9 +0,0 @@
-Please post all product and debugging questions on our [forum](https://discuss.elastic.co/c/logstash). Your questions will reach our wider community members there, and if we confirm that there is a bug, then we can open a new issue here.
-
-For all general issues, please provide the following details for fast resolution:
-
-- Version:
-- Operating System:
-- Config File (if you have sensitive info, please remove it):
-- Sample Data:
-- Steps to Reproduce:

data/.github/PULL_REQUEST_TEMPLATE.md

@@ -1 +0,0 @@
-Thanks for contributing to Logstash! If you haven't already signed our CLA, here's a handy link: https://www.elastic.co/contributor-agreement/

data/.gitignore

@@ -1,2 +0,0 @@
-
-.DS_Store

data/.travis.yml

@@ -1,11 +0,0 @@
----
-sudo: false
-language: ruby
-cache: bundler
-rvm:
-  - jruby-1.7.25
-script:
-  - bundle exec rspec spec
-jdk:
-  - oraclejdk8
-before_install: []

data/NOTICE.TXT

@@ -1,5 +0,0 @@
-Elasticsearch
-Copyright 2012-2015 Elasticsearch
-
-This product includes software developed by The Apache Software
-Foundation (http://www.apache.org/).

data/Rakefile

@@ -1,7 +0,0 @@
-@files=[]
-
-task :default do
-  system("rake -T")
-end
-
-require "logstash/devutils/rake"

data/bin/logstash-newrelic

@@ -1,2 +0,0 @@
-#!/bin/sh
-nohup ./logstash -f logstash-newrelic.conf 2>&1 &

data/bin/logstash-newrelic-debug

@@ -1,2 +0,0 @@
-#!/bin/sh
-nohup ./logstash -f logstash-newrelic.conf --debug 2>&1 &

data/bin/logstash-newrelic.conf.template

@@ -1,10 +0,0 @@
-output {
-  newrelic {
-    account_id => your_id_here
-    insert_key => your_key_here
-    event_type => your_event_type_here
-    batch => true
-    batch_events => 10
-    batch_timeout => 5
-  }
-}