logstash-output-newrelic 1.5.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25593f8efc45dce5d2366a19ce89a4d722f6e5a55fdb3d6d8cad607d551dcbd7
-  data.tar.gz: d729dc031d6276a9dafdd1ae5a7615bd378993437a36b6828c540fc901839c88
+  metadata.gz: b9339803f78539fb9240937234cc69240b289f8050c9ddfd37210f423570d828
+  data.tar.gz: 14746a11f4c482bebe359504f9a66987bb6d500fb6e4b38f2d839a508aa4d62b
 SHA512:
-  metadata.gz: 82c061a523aab278574d91373f8691c1499d0560b04e96dd8a8f178b14438eaceb7236dc62cea8ca6ac15d56e9ffa7acdfc2534338d4ae00306b0ef3e47b9f58
-  data.tar.gz: a6b92e08674d6d67e04d9ca2cb5bc0007c717e43c07c888399f715e4c5de546a58cf8237ae9128b742d5cd96ba7b9d52142e11875d1e02b9649af7e515e5e3a0
+  metadata.gz: 7cf438c882e3017df2597038d8812110d2fd8037c57c3d5e0440e1f11396143fa4aa52b1ad639d617cc32fb0c7ed05ac148bf70088d1e9dfab8b4170bf136f43
+  data.tar.gz: b6c5a7c7acf89da4da50a1e675a9339ccc8f17cd5bce2b4e0662f7be1ee4211ecce1ad0b120036084938bdf388aee3224ab6fdf5b83625856cc0a8caa8dbc38a

data/lib/logstash/outputs/newrelic.rb

@@ -11,7 +11,6 @@ require_relative './config/bigdecimal_patch'
 require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
-  java_import java.util.concurrent.Executors;
 
   RETRIABLE_CODES = Set[408, 429, 500, 502, 503, 504, 599]
 
@@ -43,7 +42,14 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       'Content-Encoding' => 'gzip',
       'Content-Type' => 'application/json'
     }.merge(auth).freeze
+
+    # We use a semaphore to ensure that at most there are @concurrent_requests inflight Logstash requests being processed
+    # by our plugin at the same time. Without this semaphore, given that @executor.submit() is an asynchronous method, it
+    # would cause that an unbounded amount of inflight requests may be processed by our plugin. Logstash then believes
+    # that our plugin has processed the request, and keeps reading more inflight requests in memory. This causes a memory
+    # leak and results in an OutOfMemoryError.
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
+    @semaphore = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -97,7 +103,19 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
     nr_logs = to_nr_logs(logstash_events)
 
-    package_and_send_recursively(nr_logs)
+    submit_logs_to_be_sent(nr_logs)
+  end
+
+  def submit_logs_to_be_sent(nr_logs)
+    @semaphore.acquire()
+    execute = @executor.java_method :submit, [java.lang.Runnable]
+    execute.call do
+      begin
+        package_and_send_recursively(nr_logs)
+      ensure
+        @semaphore.release()
+      end
+    end
   end
 
   def package_and_send_recursively(nr_logs)
@@ -113,27 +131,24 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       :logs => nr_logs
     }
 
-    execute = @executor.java_method :submit, [java.lang.Runnable]
-    execute.call do
-      compressed_payload = StringIO.new
-      gzip = Zlib::GzipWriter.new(compressed_payload)
-      gzip << [payload].to_json
-      gzip.close
-
-      compressed_size = compressed_payload.string.bytesize
-      log_record_count = nr_logs.length
-
-      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
-        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
-      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
-        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
-        split_index = log_record_count / 2
-        package_and_send_recursively(nr_logs[0...split_index])
-        package_and_send_recursively(nr_logs[split_index..-1])
-      else
-        @logger.debug("Payload compressed size: #{compressed_size}")
-        nr_send(compressed_payload.string)
-      end
+    compressed_payload = StringIO.new
+    gzip = Zlib::GzipWriter.new(compressed_payload)
+    gzip << [payload].to_json
+    gzip.close
+
+    compressed_size = compressed_payload.string.bytesize
+    log_record_count = nr_logs.length
+
+    if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+      @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+    elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+      @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+      split_index = log_record_count / 2
+      package_and_send_recursively(nr_logs[0...split_index])
+      package_and_send_recursively(nr_logs[split_index..-1])
+    else
+      @logger.debug("Payload compressed size: #{compressed_size}")
+      nr_send(compressed_payload.string)
     end
   end
 
@@ -161,6 +176,9 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       @header.each { |k, v| request[k] = v }
       request.body = payload
       handle_response(http.request(request))
+      if (retries > 0)
+        @logger.warn("Successfully sent logs at retry #{retries}")
+      end
     rescue Error::BadResponseCodeError => e
       @logger.error(e.message)
       if (should_retry(retries) && is_retryable_code(e))

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.5.0"
+      VERSION = "1.5.1"
     end
   end
 end

data/spec/outputs/newrelic_spec.rb

@@ -343,6 +343,28 @@ describe LogStash::Outputs::NewRelic do
                  .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
         .to have_been_made.times(2)
     end
+
+    it "performs the configured amount of retries, no more, no less" do
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
+        { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '3' }
+      )
+      @newrelic_output.register
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event1])
+
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_least_times(3)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_most_times(3)
+    end
   end
 
   context "JSON serialization" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.1
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2023-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement