logstash-output-newrelic 1.5.0 → 1.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 25593f8efc45dce5d2366a19ce89a4d722f6e5a55fdb3d6d8cad607d551dcbd7
-  data.tar.gz: d729dc031d6276a9dafdd1ae5a7615bd378993437a36b6828c540fc901839c88
+  metadata.gz: d582474ed8258ecfade02da25a5a32b690d70efb9c12e5e34d51cb0bf229afd6
+  data.tar.gz: 8c2d60d8d9910f5f70286e3898e1e71f8cf7dfc428a0626943808a7b44fdba13
 SHA512:
-  metadata.gz: 82c061a523aab278574d91373f8691c1499d0560b04e96dd8a8f178b14438eaceb7236dc62cea8ca6ac15d56e9ffa7acdfc2534338d4ae00306b0ef3e47b9f58
-  data.tar.gz: a6b92e08674d6d67e04d9ca2cb5bc0007c717e43c07c888399f715e4c5de546a58cf8237ae9128b742d5cd96ba7b9d52142e11875d1e02b9649af7e515e5e3a0
+  metadata.gz: 7a4b6e04dfccb15df29ebd92429756d2c15a4c22ad470ec008f7e59d9506d0e9c560b86e7ef9ce9657e4952dd46dd8cb4d7f2167769e3eb61a16da002fe47cd6
+  data.tar.gz: e75fa1ff04550653a7e2fcfbda24af1550cb4df68c8ec1949ea0e9bfe56fbdcb6a765855100cf2d7d8842e2ee9711fabdd9363d9bf80e6e1d2546c8e485b8e29

data/lib/logstash/outputs/newrelic.rb

@@ -11,7 +11,6 @@ require_relative './config/bigdecimal_patch'
 require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
-  java_import java.util.concurrent.Executors;
 
   RETRIABLE_CODES = Set[408, 429, 500, 502, 503, 504, 599]
 
@@ -43,7 +42,14 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       'Content-Encoding' => 'gzip',
       'Content-Type' => 'application/json'
     }.merge(auth).freeze
+
+    # We use a semaphore to ensure that at most there are @concurrent_requests inflight Logstash requests being processed
+    # by our plugin at the same time. Without this semaphore, given that @executor.submit() is an asynchronous method, it
+    # would cause that an unbounded amount of inflight requests may be processed by our plugin. Logstash then believes
+    # that our plugin has processed the request, and keeps reading more inflight requests in memory. This causes a memory
+    # leak and results in an OutOfMemoryError.
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
+    @semaphore = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -97,7 +103,19 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
 
     nr_logs = to_nr_logs(logstash_events)
 
-    package_and_send_recursively(nr_logs)
+    submit_logs_to_be_sent(nr_logs)
+  end
+
+  def submit_logs_to_be_sent(nr_logs)
+    @semaphore.acquire()
+    execute = @executor.java_method :submit, [java.lang.Runnable]
+    execute.call do
+      begin
+        package_and_send_recursively(nr_logs)
+      ensure
+        @semaphore.release()
+      end
+    end
   end
 
   def package_and_send_recursively(nr_logs)
@@ -113,27 +131,24 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       :logs => nr_logs
     }
 
-    execute = @executor.java_method :submit, [java.lang.Runnable]
-    execute.call do
-      compressed_payload = StringIO.new
-      gzip = Zlib::GzipWriter.new(compressed_payload)
-      gzip << [payload].to_json
-      gzip.close
-
-      compressed_size = compressed_payload.string.bytesize
-      log_record_count = nr_logs.length
-
-      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
-        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
-      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
-        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
-        split_index = log_record_count / 2
-        package_and_send_recursively(nr_logs[0...split_index])
-        package_and_send_recursively(nr_logs[split_index..-1])
-      else
-        @logger.debug("Payload compressed size: #{compressed_size}")
-        nr_send(compressed_payload.string)
-      end
+    compressed_payload = StringIO.new
+    gzip = Zlib::GzipWriter.new(compressed_payload)
+    gzip << [payload].to_json
+    gzip.close
+
+    compressed_size = compressed_payload.string.bytesize
+    log_record_count = nr_logs.length
+
+    if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+      @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+    elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+      @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+      split_index = log_record_count / 2
+      package_and_send_recursively(nr_logs[0...split_index])
+      package_and_send_recursively(nr_logs[split_index..-1])
+    else
+      @logger.debug("Payload compressed size: #{compressed_size}")
+      nr_send(compressed_payload.string)
     end
   end
 
@@ -148,10 +163,10 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     retry_duration = 1
 
     begin
-      http = Net::HTTP.new(@end_point.host, 443)
+      http = Net::HTTP.new(@end_point.host, @end_point.port || 443)
       request = Net::HTTP::Post.new(@end_point.request_uri)
-      http.use_ssl = true
-      http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+      http.use_ssl = (@end_point.scheme == 'https')
+      http.verify_mode = @end_point.scheme == 'https' ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
       if !@custom_ca_cert.nil?
         store = OpenSSL::X509::Store.new
         ca_cert = OpenSSL::X509::Certificate.new(File.read(@custom_ca_cert))
@@ -161,6 +176,9 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       @header.each { |k, v| request[k] = v }
       request.body = payload
       handle_response(http.request(request))
+      if (retries > 0)
+        @logger.warn("Successfully sent logs at retry #{retries}")
+      end
     rescue Error::BadResponseCodeError => e
       @logger.error(e.message)
       if (should_retry(retries) && is_retryable_code(e))

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.5.0"
+      VERSION = "1.5.2"
     end
   end
 end

data/spec/outputs/newrelic_spec.rb

@@ -7,6 +7,7 @@ require "logstash/event"
 require "thread"
 require "webmock/rspec"
 require "zlib"
+require "rspec/wait"
 
 describe LogStash::Outputs::NewRelic do
   let (:base_uri) { "https://testing-example-collector.com" }
@@ -48,6 +49,50 @@ describe LogStash::Outputs::NewRelic do
               })).to have_been_made
     end
   end
+
+  context "check https connection scheme" do
+    it "uses https by default" do
+      stub_request(:any, base_uri).to_return(status: 200)
+
+      event = LogStash::Event.new({:message => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, base_uri)
+        .with(headers: {
+                "X-License-Key" => license_key,
+                "X-Event-Source" => "logs",
+                "Content-Encoding" => "gzip",
+              })).to have_been_made
+
+      # Check if the requests were made using HTTPS
+      expect(WebMock).to have_requested(:post, base_uri).with { |req| req.uri.scheme == 'https' }
+      expect(WebMock).to have_requested(:post, base_uri).with { |req| req.uri.port == 443 }
+    end
+  end
+
+  context "check http connection scheme" do
+    it "uses http when http config is set" do
+      stub_request(:any, "http://localhost:5000/").to_return(status: 200)
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new({
+        "base_uri" => "http://localhost:5000/",
+        "license_key" => license_key
+      })
+      @newrelic_output.register
+      event = LogStash::Event.new({:message => "Test message" })
+      @newrelic_output.multi_receive([event])
+
+      wait_for(a_request(:post, "http://localhost:5000/")
+        .with(headers: {
+                "X-License-Key" => license_key,
+                "X-Event-Source" => "logs",
+                "Content-Encoding" => "gzip",
+              })).to have_been_made
+
+      # Check if the requests were made using HTTP to this endpoint
+      expect(WebMock).to have_requested(:post, "http://localhost:5000/").with { |req| req.uri.scheme == 'http' }
+      expect(WebMock).to have_requested(:post, "http://localhost:5000/").with { |req| req.uri.port == 5000 }
+    end
+  end
 end
 
 describe LogStash::Outputs::NewRelic do
@@ -343,6 +388,28 @@ describe LogStash::Outputs::NewRelic do
                  .with { |request| single_gzipped_message(request.body)['message'] == 'Test message 1' })
         .to have_been_made.times(2)
     end
+
+    it "performs the configured amount of retries, no more, no less" do
+      @newrelic_output = LogStash::Plugin.lookup("output", "newrelic").new(
+        { "base_uri" => base_uri, "license_key" => api_key, "max_retries" => '3' }
+      )
+      @newrelic_output.register
+      stub_request(:any, base_uri)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 500)
+        .to_return(status: 200)
+
+      event1 = LogStash::Event.new({ "message" => "Test message" })
+      @newrelic_output.multi_receive([event1])
+
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_least_times(3)
+      wait_for(a_request(:post, base_uri)
+                 .with { |request| single_gzipped_message(request.body)['message'] == 'Test message' })
+        .to have_been_made.at_most_times(3)
+    end
   end
 
   context "JSON serialization" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-newrelic
 version: !ruby/object:Gem::Version
-  version: 1.5.0
+  version: 1.5.2
 platform: ruby
 authors:
 - New Relic Logging Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-14 00:00:00.000000000 Z
+date: 2023-10-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement