logstash-output-newrelic 1.3.0 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7f6e7768d78135be1d77a25e79d4db1eacbe23c5d1a31b4c738073672dfb3aa3
-  data.tar.gz: 8844275d5c3e3d5200d3a3461050e5f46053a7614582377db4480aca600b780c
+  metadata.gz: 9e6c4d247631f976e7a7fc6153a02cc764f583491c749654a827c1fd6846c6c2
+  data.tar.gz: 42389a3c04b5f1ef4804f9bf7efa820d7037c25e2df73c127354d180c624ff02
 SHA512:
-  metadata.gz: b71c733f87e3e1ec200b1ee101a9d293a84e20f744fcd8aa2b572b93e0523bda83af67d216d94b90da9c273152874ec832f1e1c7c02250fabee0cafc9b53a315
-  data.tar.gz: '0617081a05a7a31f9e5f0469836fd55016f8c7fbe971851ad531c646819898a31dc7303d1aa569be3fbec98f173d39bc60c7d090594c074745f0c72d70a42545'
+  metadata.gz: deebd2f6b0c7f5c07ee4fc347f577a9011a481be03f8e170a9e9c368594009cbc33b853e05c025a1938b038febc8cc6d6db8c906aa3eacbeacfabab3483d7755
+  data.tar.gz: becf3aef35efc785f84ea956cabf2004d8d1ddbde4dac11f11bec3a25b65ab9a3277a97dcb35add2839581fb0b19ddf59114a83c9e5afd7a54da14ddd9b181d1

data/DEVELOPER.md

@@ -2,12 +2,23 @@
 
 # Getting started
 
-* Install JRuby: `rbenv install jruby-9.2.5.0`
-* Use that JRuby: `rbenv local jruby-9.2.5.0`
+**NOTE for Mac M1 users: ** note that `jruby-9.3.3.0` is the first jruby compatible with Mac M1 processors. In order to
+develop the plugin locally, we recommend using this version. Nevertheless, when building the gem file (in the GH workflows),
+we keep using jruby 9.2.13.0 in order to be backwards-compatible with older Logstash versions.
+
+* Install RVM:
+  * `command curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -`
+  * `\curl -sSL https://get.rvm.io | bash -s stable`
+  * Reopen the terminal for `rvm` command to be available
+* Install JRuby: `rvm install jruby-9.2.13.0`.
+* Use that JRuby: `rvm use jruby-9.2.13.0`
+* Ensure your terminal is using Java 11.
 * Install Bundler gem: `jruby -S gem install bundler`
 
 # Developing
 
+* Ensure you have `logstash` installed locally (required for unit testing): `brew install logstash`
+* Ensure your `logstash` path matches the one in `Gemfile` 
 * Install dependencies: `jruby -S bundle install`
 * Write tests and production code!
 * Bump version: edit version file `version.rb`

data/Gemfile

@@ -1,3 +1,15 @@
 source 'https://rubygems.org'
 gemspec
 
+# The following is required to locally develop this plugin. Note that this Gemfile is NOT used when building the gem
+# file for this plugin (see merge-to-master.yml), only when unit testing. When unit-testing, we need to have logstash-core
+# in our local machine, given that the logstash-core GEM has not been published since 5.6.0.
+# See: https://github.com/elastic/logstash/pull/14229 https://github.com/elastic/logstash/issues/14203
+# And: https://github.com/elastic/logstash/pull/14229
+
+logstash_path = ENV['LOGSTASH_PATH'] || '/opt/homebrew/Cellar/logstash/8.9.0/libexec'
+
+if Dir.exist?(logstash_path)
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/lib/logstash/outputs/newrelic.rb

@@ -12,10 +12,11 @@ require_relative './exception/error'
 
 class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
   java_import java.util.concurrent.Executors;
-  java_import java.util.concurrent.Semaphore;
 
   NON_RETRYABLE_CODES = Set[401, 403]
 
+  MAX_PAYLOAD_SIZE_BYTES = 1_000_000
+
   config_name "newrelic"
 
   config :api_key, :validate => :password, :required => false
@@ -43,7 +44,6 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
       'Content-Type' => 'application/json'
     }.merge(auth).freeze
     @executor = java.util.concurrent.Executors.newFixedThreadPool(@concurrent_requests)
-    @semaphor = java.util.concurrent.Semaphore.new(@concurrent_requests)
   end
 
   # Used by tests so that the test run can complete (background threads prevent JVM exit)
@@ -67,34 +67,40 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
     end
   end
 
-  def encode(event_hash)
-    log_message_hash = {
-      # non-intrinsic attributes get put into 'attributes'
-      :attributes => event_hash
-    }
+  def to_nr_logs(logstash_events)
+    logstash_events.map do |logstash_event|
+      event_hash = logstash_event.to_hash
 
-    # intrinsic attributes go at the top level
-    if event_hash['message']
-      log_message_hash['message'] = event_hash['message']
-      log_message_hash[:attributes].delete('message')
-    end
-    if event_hash['timestamp']
-      log_message_hash['timestamp'] = event_hash['timestamp']
-      log_message_hash[:attributes].delete('timestamp')
-    end
+      nr_log_message_hash = {
+        # non-intrinsic attributes get put into 'attributes'
+        :attributes => event_hash
+      }
+
+      # intrinsic attributes go at the top level
+      if event_hash['message']
+        nr_log_message_hash['message'] = event_hash['message']
+        nr_log_message_hash[:attributes].delete('message')
+      end
+      if event_hash['timestamp']
+        nr_log_message_hash['timestamp'] = event_hash['timestamp']
+        nr_log_message_hash[:attributes].delete('timestamp')
+      end
 
-    log_message_hash
+      nr_log_message_hash
+    end
   end
 
-  def multi_receive(events)
-    if events.size == 0
+  def multi_receive(logstash_events)
+    if logstash_events.empty?
       return
     end
 
-    payload = []
-    events.each do |event|
-      payload.push(encode(event.to_hash))
-    end
+    nr_logs = to_nr_logs(logstash_events)
+
+    package_and_send_recursively(nr_logs)
+  end
+
+  def package_and_send_recursively(nr_logs)
     payload = {
       :common => {
         :attributes => {
@@ -104,19 +110,29 @@ class LogStash::Outputs::NewRelic < LogStash::Outputs::Base
           }
         }
       },
-      :logs => payload
+      :logs => nr_logs
     }
-    @semaphor.acquire()
+
     execute = @executor.java_method :submit, [java.lang.Runnable]
     execute.call do
-      begin
-        io = StringIO.new
-        gzip = Zlib::GzipWriter.new(io)
-        gzip << [payload].to_json
-        gzip.close
-        nr_send(io.string)
-      ensure
-        @semaphor.release()
+      compressed_payload = StringIO.new
+      gzip = Zlib::GzipWriter.new(compressed_payload)
+      gzip << [payload].to_json
+      gzip.close
+
+      compressed_size = compressed_payload.string.bytesize
+      log_record_count = nr_logs.length
+
+      if compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count == 1
+        @logger.error("Can't compress record below required maximum packet size and it will be discarded.")
+      elsif compressed_size >= MAX_PAYLOAD_SIZE_BYTES && log_record_count > 1
+        @logger.debug("Compressed payload size (#{compressed_size}) exceededs maximum packet size (1MB) and will be split in two.")
+        split_index = log_record_count / 2
+        package_and_send_recursively(nr_logs[0...split_index])
+        package_and_send_recursively(nr_logs[split_index..-1])
+      else
+        @logger.debug("Payload compressed size: #{compressed_size}")
+        nr_send(compressed_payload.string)
       end
     end
   end

data/lib/logstash/outputs/newrelic_version/version.rb

@@ -1,7 +1,7 @@
 module LogStash
   module Outputs
     module NewRelicVersion
-      VERSION = "1.3.0"
+      VERSION = "1.4.0"
     end
   end
 end