logstash-output-newrelic 1.2.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '068a9ac838ba356ffc7cc6746933ad3970d5c72df54c6633e1c7453283d700c0'
-  data.tar.gz: 8f6557b7b8ab66e4d2de7d772cbf470403be8f125d2bb90d371c0af4581ec84e
+  metadata.gz: 9e6c4d247631f976e7a7fc6153a02cc764f583491c749654a827c1fd6846c6c2
+  data.tar.gz: 42389a3c04b5f1ef4804f9bf7efa820d7037c25e2df73c127354d180c624ff02
 SHA512:
-  metadata.gz: c200662a9b6e2157193f9d21c7dadc9562db4ca0245525775742b5518afa73b7db801dc05b506c8a5c4df74e127c7c8e520a684b5a9cf709f29350c09c0b9ebc
-  data.tar.gz: b6f15e63a3a625fc86c4dc6423b37973a01ae8406f31aa944aac62c3343d7c7dcfb3a7642f54d0464c170b824f59126d5b25b87a459cea1c79f2d14e9fd087ab
+  metadata.gz: deebd2f6b0c7f5c07ee4fc347f577a9011a481be03f8e170a9e9c368594009cbc33b853e05c025a1938b038febc8cc6d6db8c906aa3eacbeacfabab3483d7755
+  data.tar.gz: becf3aef35efc785f84ea956cabf2004d8d1ddbde4dac11f11bec3a25b65ab9a3277a97dcb35add2839581fb0b19ddf59114a83c9e5afd7a54da14ddd9b181d1

data/DEVELOPER.md

@@ -2,12 +2,23 @@
 
 # Getting started
 
-* Install JRuby: `rbenv install jruby-9.2.5.0`
-* Use that JRuby: `rbenv local jruby-9.2.5.0`
+**NOTE for Mac M1 users: ** note that `jruby-9.3.3.0` is the first jruby compatible with Mac M1 processors. In order to
+develop the plugin locally, we recommend using this version. Nevertheless, when building the gem file (in the GH workflows),
+we keep using jruby 9.2.13.0 in order to be backwards-compatible with older Logstash versions.
+
+* Install RVM:
+  * `command curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -`
+  * `\curl -sSL https://get.rvm.io | bash -s stable`
+  * Reopen the terminal for `rvm` command to be available
+* Install JRuby: `rvm install jruby-9.2.13.0`.
+* Use that JRuby: `rvm use jruby-9.2.13.0`
+* Ensure your terminal is using Java 11.
 * Install Bundler gem: `jruby -S gem install bundler`
 
 # Developing
 
+* Ensure you have `logstash` installed locally (required for unit testing): `brew install logstash`
+* Ensure your `logstash` path matches the one in `Gemfile` 
 * Install dependencies: `jruby -S bundle install`
 * Write tests and production code!
 * Bump version: edit version file `version.rb`

data/Gemfile

@@ -1,3 +1,15 @@
 source 'https://rubygems.org'
 gemspec
 
+# The following is required to locally develop this plugin. Note that this Gemfile is NOT used when building the gem
+# file for this plugin (see merge-to-master.yml), only when unit testing. When unit-testing, we need to have logstash-core
+# in our local machine, given that the logstash-core GEM has not been published since 5.6.0.
+# See: https://github.com/elastic/logstash/pull/14229 https://github.com/elastic/logstash/issues/14203
+# And: https://github.com/elastic/logstash/pull/14229
+
+logstash_path = ENV['LOGSTASH_PATH'] || '/opt/homebrew/Cellar/logstash/8.9.0/libexec'
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/README.md

@@ -1,3 +1,5 @@
+[![Community Plus header](https://github.com/newrelic/opensource-website/raw/master/src/images/categories/Community_Plus.png)](https://opensource.newrelic.com/oss-category/#community-plus)
+
 # New Relic Logstash Output Plugin
 
 This is a plugin for [Logstash](https://github.com/elastic/logstash) that outputs logs to New Relic.
@@ -65,7 +67,7 @@ When using this plugin in the EU override the base_uri with `https://log-api.eu.
 
 ## Testing
 
-An easy way to test the plugin is to make sure Logstash is getting input from a log file you 
+An easy way to test the plugin is to make sure Logstash is getting input from a log file you
 can write to. Something like this in your logstash.conf:
 ```
 input {
@@ -77,3 +79,18 @@ input {
 * Restart Logstash
 * Append a test log message to your log file: `echo "test message" >> /path/to/your/log/file`
 * Search New Relic Logs for `"test message"`
+
+## Community
+
+New Relic hosts and moderates an online forum where customers can interact with New Relic employees as well as other customers to get help and share best practices. Like all official New Relic open source projects, there's a related Community topic in the New Relic Explorers Hub: [Log forwarding](https://discuss.newrelic.com/tag/log-forwarding)
+
+## A note about vulnerabilities
+
+As noted in our [security policy](../../security/policy), New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals.
+
+If you believe you have found a security vulnerability in this project or any of New Relic's products or websites, we welcome and greatly appreciate you reporting it to New Relic through [HackerOne](https://hackerone.com/newrelic).
+
+If you would like to contribute to this project, review [these guidelines](https://opensource.newrelic.com/code-of-conduct/).
+
+## License
+logstash-output-plugin is licensed under the [Apache 2.0](http://apache.org/licenses/LICENSE-2.0.txt) License.

data/lib/logstash/outputs/newrelic.rb

@@ -12,10 +12,11 @@ require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
-  java_import java.util.concurrent.Semaphore;
 
   NON_RETRYABLE_CODES = Set[401, 403]
 
+  MAX_PAYLOAD_SIZE_BYTES = 1_000_000
+
   config_name "newrelic"
 
   config :api_key, :validate => :password, :required => false
@@ -23,6 +24,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   config :concurrent_requests, :validate => :number, :default => 1
   config :base_uri, :validate => :string, :default => "https://log-api.newrelic.com/log/v1"
   config :max_retries, :validate => :number, :default => 3
+  # Only used for E2E testing
+  config :custom_ca_cert, :validate => :string, :required => false
 
   public
 
@@ -37,10 +40,10 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     }
     @header = {
       'X-Event-Source' => 'logs',
-      'Content-Encoding' => 'gzip'
+      'Content-Encoding' => 'gzip',
+      'Content-Type' => 'application/json'
     }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
-    @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -64,34 +67,40 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     end
   end
 
-  def encode(event_hash)
-    log_message_hash = {
-      # non-intrinsic attributes get put into 'attributes'
-      :attributes => event_hash
-    }
+  def to_nr_logs(logstash_events)
+    logstash_events.map do |logstash_event|
+      event_hash = logstash_event.to_hash
 
-    # intrinsic attributes go at the top level
-    if event_hash['message']
-      log_message_hash['message'] = event_hash['message']
-      log_message_hash[:attributes].delete('message')
-    end
-    if event_hash['timestamp']
-      log_message_hash['timestamp'] = event_hash['timestamp']
-      log_message_hash[:attributes].delete('timestamp')
-    end
+      nr_log_message_hash = {
+        # non-intrinsic attributes get put into 'attributes'
+        :attributes => event_hash
+      }
 
-    log_message_hash
+      # intrinsic attributes go at the top level
+      if event_hash['message']
+        nr_log_message_hash['message'] = event_hash['message']
+        nr_log_message_hash[:attributes].delete('message')
+      end
+      if event_hash['timestamp']
+        nr_log_message_hash['timestamp'] = event_hash['timestamp']
+        nr_log_message_hash[:attributes].delete('timestamp')
+      end
+
+      nr_log_message_hash
+    end
   end
 
-  def multi_receive(events)
-    if events.size == 0
+  def multi_receive(logstash_events)
+    if logstash_events.empty?
       return
     end
 
-    payload = []
-    events.each do |event|
-      payload.push(encode(event.to_hash))
-    end
+    nr_logs = to_nr_logs(logstash_events)
+
+    package_and_send_recursively(nr_logs)
+  end
+
+  def package_and_send_recursively(nr_logs)
     payload = {
       :common => {
         :attributes => {
@@ -101,19 +110,29 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
           }
         }
       },
-      :logs => payload
+      :logs => nr_logs
     }
-    @semaphor.acquire()
+
     execute = @executor.java_method :submit, [java.lang.Runnable]
     execute.call do
-      begin
-        io = StringIO.new
-        gzip = Zlib::GzipWriter.new(io)
-        gzip << [payload].to_json
-        gzip.close
-        nr_send(io.string)
-      ensure
-        @semaphor.release()
+      compressed_payload = StringIO.new
+      gzip = Zlib::GzipWriter.new(compressed_payload)
+      gzip << [payload].to_json
+      gzip.close
+
+      compressed_size = compressed_payload.string.bytesize
+      log_record_count = nr_logs.length
+
+      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+        split_index = log_record_count / 2
+        package_and_send_recursively(nr_logs[0...split_index])
+        package_and_send_recursively(nr_logs[split_index..-1])
+      else
+        @logger.debug("Payload compressed size: #{compressed_size}")
+        nr_send(compressed_payload.string)
       end
     end
   end
@@ -131,6 +150,12 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       request = Net::HTTP::Post.new(@end_point.request_uri)
       http.use_ssl = true
       http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      if !@custom_ca_cert.nil?
+        store = OpenSSL::X509::Store.new
+        ca_cert = OpenSSL::X509::Certificate.new(File.read(@custom_ca_cert))
+        store.add_cert(ca_cert)
+        http.cert_store = store
+      end
       @header.each { |k, v| request[k] = v }
       request.body = payload
       handle_response(http.request(request))

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.2.2"
+      VERSION = "1.4.0"
     end
   end
 end