logstash-output-newrelic 1.3.0 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f6e7768d78135be1d77a25e79d4db1eacbe23c5d1a31b4c738073672dfb3aa3
-  data.tar.gz: 8844275d5c3e3d5200d3a3461050e5f46053a7614582377db4480aca600b780c
+  metadata.gz: 25593f8efc45dce5d2366a19ce89a4d722f6e5a55fdb3d6d8cad607d551dcbd7
+  data.tar.gz: d729dc031d6276a9dafdd1ae5a7615bd378993437a36b6828c540fc901839c88
 SHA512:
-  metadata.gz: b71c733f87e3e1ec200b1ee101a9d293a84e20f744fcd8aa2b572b93e0523bda83af67d216d94b90da9c273152874ec832f1e1c7c02250fabee0cafc9b53a315
-  data.tar.gz: '0617081a05a7a31f9e5f0469836fd55016f8c7fbe971851ad531c646819898a31dc7303d1aa569be3fbec98f173d39bc60c7d090594c074745f0c72d70a42545'
+  metadata.gz: 82c061a523aab278574d91373f8691c1499d0560b04e96dd8a8f178b14438eaceb7236dc62cea8ca6ac15d56e9ffa7acdfc2534338d4ae00306b0ef3e47b9f58
+  data.tar.gz: a6b92e08674d6d67e04d9ca2cb5bc0007c717e43c07c888399f715e4c5de546a58cf8237ae9128b742d5cd96ba7b9d52142e11875d1e02b9649af7e515e5e3a0

data/DEVELOPER.md

@@ -2,12 +2,23 @@
 
 # Getting started
 
-* Install JRuby: `rbenv install jruby-9.2.5.0`
-* Use that JRuby: `rbenv local jruby-9.2.5.0`
+**NOTE for Mac M1 users: ** note that `jruby-9.3.3.0` is the first jruby compatible with Mac M1 processors. In order to
+develop the plugin locally, we recommend using this version. Nevertheless, when building the gem file (in the GH workflows),
+we keep using jruby 9.2.13.0 in order to be backwards-compatible with older Logstash versions.
+
+* Install RVM:
+  * `command curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -`
+  * `\curl -sSL https://get.rvm.io | bash -s stable`
+  * Reopen the terminal for `rvm` command to be available
+* Install JRuby: `rvm install jruby-9.2.13.0`.
+* Use that JRuby: `rvm use jruby-9.2.13.0`
+* Ensure your terminal is using Java 11.
 * Install Bundler gem: `jruby -S gem install bundler`
 
 # Developing
 
+* Ensure you have `logstash` installed locally (required for unit testing): `brew install logstash`
+* Ensure your `logstash` path matches the one in `Gemfile` 
 * Install dependencies: `jruby -S bundle install`
 * Write tests and production code!
 * Bump version: edit version file `version.rb`

data/Gemfile

@@ -1,3 +1,15 @@
 source 'https://rubygems.org'
 gemspec
 
+# The following is required to locally develop this plugin. Note that this Gemfile is NOT used when building the gem
+# file for this plugin (see merge-to-master.yml), only when unit testing. When unit-testing, we need to have logstash-core
+# in our local machine, given that the logstash-core GEM has not been published since 5.6.0.
+# See: https://github.com/elastic/logstash/pull/14229 https://github.com/elastic/logstash/issues/14203
+# And: https://github.com/elastic/logstash/pull/14229
+
+logstash_path = ENV['LOGSTASH_PATH'] || '/opt/homebrew/Cellar/logstash/8.9.0/libexec'
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/lib/logstash/outputs/newrelic.rb

@@ -12,9 +12,10 @@ require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
-  java_import java.util.concurrent.Semaphore;
 
-  NON_RETRYABLE_CODES = Set[401, 403]
+  RETRIABLE_CODES = Set[408, 429, 500, 502, 503, 504, 599]
+
+  MAX_PAYLOAD_SIZE_BYTES = 1_000_000
 
   config_name "newrelic"
 
@@ -43,7 +44,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       'Content-Type' => 'application/json'
     }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
-    @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -67,34 +67,40 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     end
   end
 
-  def encode(event_hash)
-    log_message_hash = {
-      # non-intrinsic attributes get put into 'attributes'
-      :attributes => event_hash
-    }
+  def to_nr_logs(logstash_events)
+    logstash_events.map do |logstash_event|
+      event_hash = logstash_event.to_hash
 
-    # intrinsic attributes go at the top level
-    if event_hash['message']
-      log_message_hash['message'] = event_hash['message']
-      log_message_hash[:attributes].delete('message')
-    end
-    if event_hash['timestamp']
-      log_message_hash['timestamp'] = event_hash['timestamp']
-      log_message_hash[:attributes].delete('timestamp')
-    end
+      nr_log_message_hash = {
+        # non-intrinsic attributes get put into 'attributes'
+        :attributes => event_hash
+      }
 
-    log_message_hash
+      # intrinsic attributes go at the top level
+      if event_hash['message']
+        nr_log_message_hash['message'] = event_hash['message']
+        nr_log_message_hash[:attributes].delete('message')
+      end
+      if event_hash['timestamp']
+        nr_log_message_hash['timestamp'] = event_hash['timestamp']
+        nr_log_message_hash[:attributes].delete('timestamp')
+      end
+
+      nr_log_message_hash
+    end
   end
 
-  def multi_receive(events)
-    if events.size == 0
+  def multi_receive(logstash_events)
+    if logstash_events.empty?
       return
     end
 
-    payload = []
-    events.each do |event|
-      payload.push(encode(event.to_hash))
-    end
+    nr_logs = to_nr_logs(logstash_events)
+
+    package_and_send_recursively(nr_logs)
+  end
+
+  def package_and_send_recursively(nr_logs)
     payload = {
       :common => {
         :attributes => {
@@ -104,19 +110,29 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
           }
         }
       },
-      :logs => payload
+      :logs => nr_logs
     }
-    @semaphor.acquire()
+
     execute = @executor.java_method :submit, [java.lang.Runnable]
     execute.call do
-      begin
-        io = StringIO.new
-        gzip = Zlib::GzipWriter.new(io)
-        gzip << [payload].to_json
-        gzip.close
-        nr_send(io.string)
-      ensure
-        @semaphor.release()
+      compressed_payload = StringIO.new
+      gzip = Zlib::GzipWriter.new(compressed_payload)
+      gzip << [payload].to_json
+      gzip.close
+
+      compressed_size = compressed_payload.string.bytesize
+      log_record_count = nr_logs.length
+
+      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+        split_index = log_record_count / 2
+        package_and_send_recursively(nr_logs[0...split_index])
+        package_and_send_recursively(nr_logs[split_index..-1])
+      else
+        @logger.debug("Payload compressed size: #{compressed_size}")
+        nr_send(compressed_payload.string)
       end
     end
   end
@@ -129,6 +145,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def nr_send(payload)
     retries = 0
+    retry_duration = 1
+
     begin
       http = Net::HTTP.new(@end_point.host, 443)
       request = Net::HTTP::Post.new(@end_point.request_uri)
@@ -147,7 +165,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       @logger.error(e.message)
       if (should_retry(retries) && is_retryable_code(e))
         retries += 1
-        sleep(1)
+        sleep(retry_duration)
+        retry_duration *= 2
         retry
       end
     rescue => e
@@ -162,7 +181,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
           :error_class => e.class.name,
           :backtrace => e.backtrace
         )
-        sleep(1)
+        sleep(retry_duration)
+        retry_duration *= 2
         retry
       else
         @logger.error(
@@ -181,6 +201,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def is_retryable_code(response_error)
     error_code = response_error.response_code
-    !NON_RETRYABLE_CODES.include?(error_code)
+    RETRIABLE_CODES.include?(error_code)
   end
 end # class LogStash::Outputs::NewRelic

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.3.0"
+      VERSION = "1.5.0"
     end
   end
 end