logstash-output-newrelic 1.4.0 → 1.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9e6c4d247631f976e7a7fc6153a02cc764f583491c749654a827c1fd6846c6c2
-  data.tar.gz: 42389a3c04b5f1ef4804f9bf7efa820d7037c25e2df73c127354d180c624ff02
+  metadata.gz: b9339803f78539fb9240937234cc69240b289f8050c9ddfd37210f423570d828
+  data.tar.gz: 14746a11f4c482bebe359504f9a66987bb6d500fb6e4b38f2d839a508aa4d62b
 SHA512:
-  metadata.gz: deebd2f6b0c7f5c07ee4fc347f577a9011a481be03f8e170a9e9c368594009cbc33b853e05c025a1938b038febc8cc6d6db8c906aa3eacbeacfabab3483d7755
-  data.tar.gz: becf3aef35efc785f84ea956cabf2004d8d1ddbde4dac11f11bec3a25b65ab9a3277a97dcb35add2839581fb0b19ddf59114a83c9e5afd7a54da14ddd9b181d1
+  metadata.gz: 7cf438c882e3017df2597038d8812110d2fd8037c57c3d5e0440e1f11396143fa4aa52b1ad639d617cc32fb0c7ed05ac148bf70088d1e9dfab8b4170bf136f43
+  data.tar.gz: b6c5a7c7acf89da4da50a1e675a9339ccc8f17cd5bce2b4e0662f7be1ee4211ecce1ad0b120036084938bdf388aee3224ab6fdf5b83625856cc0a8caa8dbc38a

data/lib/logstash/outputs/newrelic.rb

@@ -11,9 +11,8 @@ require_relative './config/bigdecimal_patch'
 require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
-  java_import java.util.concurrent.Executors;
 
-  NON_RETRYABLE_CODES = Set[401, 403]
+  RETRIABLE_CODES = Set[408, 429, 500, 502, 503, 504, 599]
 
   MAX_PAYLOAD_SIZE_BYTES = 1_000_000
 
@@ -43,7 +42,14 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       'Content-Encoding' => 'gzip',
       'Content-Type' => 'application/json'
     }.merge(auth).freeze
+
+    # We use a semaphore to ensure that at most there are @concurrent_requests inflight Logstash requests being processed
+    # by our plugin at the same time. Without this semaphore, given that @executor.submit() is an asynchronous method, it
+    # would cause that an unbounded amount of inflight requests may be processed by our plugin. Logstash then believes
+    # that our plugin has processed the request, and keeps reading more inflight requests in memory. This causes a memory
+    # leak and results in an OutOfMemoryError.
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
+    @semaphore = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -97,7 +103,19 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
     nr_logs = to_nr_logs(logstash_events)
 
-    package_and_send_recursively(nr_logs)
+    submit_logs_to_be_sent(nr_logs)
+  end
+
+  def submit_logs_to_be_sent(nr_logs)
+    @semaphore.acquire()
+    execute = @executor.java_method :submit, [java.lang.Runnable]
+    execute.call do
+      begin
+        package_and_send_recursively(nr_logs)
+      ensure
+        @semaphore.release()
+      end
+    end
   end
 
   def package_and_send_recursively(nr_logs)
@@ -113,27 +131,24 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       :logs => nr_logs
     }
 
-    execute = @executor.java_method :submit, [java.lang.Runnable]
-    execute.call do
-      compressed_payload = StringIO.new
-      gzip = Zlib::GzipWriter.new(compressed_payload)
-      gzip << [payload].to_json
-      gzip.close
-
-      compressed_size = compressed_payload.string.bytesize
-      log_record_count = nr_logs.length
-
-      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
-        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
-      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
-        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
-        split_index = log_record_count / 2
-        package_and_send_recursively(nr_logs[0...split_index])
-        package_and_send_recursively(nr_logs[split_index..-1])
-      else
-        @logger.debug("Payload compressed size: #{compressed_size}")
-        nr_send(compressed_payload.string)
-      end
+    compressed_payload = StringIO.new
+    gzip = Zlib::GzipWriter.new(compressed_payload)
+    gzip << [payload].to_json
+    gzip.close
+
+    compressed_size = compressed_payload.string.bytesize
+    log_record_count = nr_logs.length
+
+    if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+      @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+    elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+      @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+      split_index = log_record_count / 2
+      package_and_send_recursively(nr_logs[0...split_index])
+      package_and_send_recursively(nr_logs[split_index..-1])
+    else
+      @logger.debug("Payload compressed size: #{compressed_size}")
+      nr_send(compressed_payload.string)
     end
   end
 
@@ -145,6 +160,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def nr_send(payload)
     retries = 0
+    retry_duration = 1
+
     begin
       http = Net::HTTP.new(@end_point.host, 443)
       request = Net::HTTP::Post.new(@end_point.request_uri)
@@ -159,11 +176,15 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       @header.each { |k, v| request[k] = v }
       request.body = payload
       handle_response(http.request(request))
+      if (retries > 0)
+        @logger.warn("Successfully sent logs at retry #{retries}")
+      end
     rescue Error::BadResponseCodeError => e
       @logger.error(e.message)
       if (should_retry(retries) && is_retryable_code(e))
         retries += 1
-        sleep(1)
+        sleep(retry_duration)
+        retry_duration *= 2
         retry
       end
     rescue => e
@@ -178,7 +199,8 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
           :error_class => e.class.name,
           :backtrace => e.backtrace
         )
-        sleep(1)
+        sleep(retry_duration)
+        retry_duration *= 2
         retry
       else
         @logger.error(
@@ -197,6 +219,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
   def is_retryable_code(response_error)
     error_code = response_error.response_code
-    !NON_RETRYABLE_CODES.include?(error_code)
+    RETRIABLE_CODES.include?(error_code)
   end
 end # class LogStash::Outputs::NewRelic

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.4.0"
+      VERSION = "1.5.1"
     end
   end
 end

data/spec/outputs/newrelic_spec.rb

@@ -265,7 +265,7 @@ describe LogStash::Outputs::NewRelic do
     end
   end
 
-  context "error handling" do
+  context "error handling and retry logic" do
     it "continues through errors, future calls should still succeed" do
       stub_request(:any, base_uri)
         .to_raise(StandardError.new("from test"))
@@ -281,33 +281,41 @@ describe LogStash::Outputs::NewRelic do
         .to have_been_made
     end
 
-    it "retry when receive retryable http error code" do
-      stub_request(:any, base_uri)
-        .to_return(status: 500)
-        .to_return(status: 200)
-
-      event1 = LogStash::Event.new({ "message" => "Test message 1" })
-      @newrelic_output.multi_receive([event1])
-
-      wait_for(a_request(:post, base_uri)
-                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
-        .to have_been_made.times(2)
-    end
-
-    it "not retry when receive a non retryable http error code" do
-      stub_request(:any, base_uri)
-        .to_return(status: 401)
-
-      event1 = LogStash::Event.new({ "message" => "Test message 1" })
-      @newrelic_output.multi_receive([event1])
-      # Due the async behavior we need to wait to be sure that the method was not called more than 1 time
-      sleep(2)
-      wait_for(a_request(:post, base_uri)
-                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
-        .to have_been_made.times(1)
+    [
+      { "returned_status_code" => 200, "expected_to_retry" => false },
+      { "returned_status_code" => 202, "expected_to_retry" => false },
+      { "returned_status_code" => 400, "expected_to_retry" => false },
+      { "returned_status_code" => 404, "expected_to_retry" => false },
+      { "returned_status_code" => 408, "expected_to_retry" => true },
+      { "returned_status_code" => 429, "expected_to_retry" => true },
+      { "returned_status_code" => 500, "expected_to_retry" => true },
+      { "returned_status_code" => 502, "expected_to_retry" => true },
+      { "returned_status_code" => 503, "expected_to_retry" => true },
+      { "returned_status_code" => 504, "expected_to_retry" => true },
+      { "returned_status_code" => 599, "expected_to_retry" => true }
+    ].each do |test_case|
+      returned_status_code = test_case["returned_status_code"]
+      expected_to_retry = test_case["expected_to_retry"]
+
+      it "should #{expected_to_retry ? "" : "not"} retry on status code #{returned_status_code}" do
+        stub_request(:any, base_uri)
+          .to_return(status: returned_status_code)
+          .to_return(status: 200)
+
+        logstash_event = LogStash::Event.new({ "message" => "Test message" })
+        @newrelic_output.multi_receive([logstash_event])
+
+        expected_retries = expected_to_retry ? 2 : 1
+        wait_for(a_request(:post, base_uri)
+                   .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+          .to have_been_made.at_least_times(expected_retries)
+        wait_for(a_request(:post, base_uri)
+                   .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+          .to have_been_made.at_most_times(expected_retries)
+      end
     end
 
-    it "not retries when retry is disabled" do
+    it "does not retry when max_retries is set to 0" do
       @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
         { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '0' }
       )
@@ -324,7 +332,7 @@ describe LogStash::Outputs::NewRelic do
         .to have_been_made.times(1)
     end
 
-    it "retry when receive a not expected exception" do
+    it "retries when receive a not expected exception" do
       stub_request(:any, base_uri)
         .to_raise(StandardError.new("from test"))
         .to_return(status: 200)
@@ -335,6 +343,28 @@ describe LogStash::Outputs::NewRelic do
                  .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
         .to have_been_made.times(2)
     end
+
+    it "performs the configured amount of retries, no more, no less" do
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
+        { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '3' }
+      )
+      @newrelic_output.register
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event1])
+
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_least_times(3)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_most_times(3)
+    end
   end
 
   context "JSON serialization" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.5.1
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2023-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement