logstash-output-dynatrace 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f41e7086fbd73db7471823a9bff842ad41783ba20b1f2ba112755f251af28c0a
-  data.tar.gz: 9b645d8c4f9eb4d416dc6c873cceb8779bae7c7745e63f009f2eaee4a5b29e52
+  metadata.gz: 69fbd3982ad32849fe3d185e3edd44f9c3d071cb3f74c31d2f6d32a1e7841df5
+  data.tar.gz: 24bfb2c6599648c80295860e90eb8e6600d75ad4af8ccd5252f42b46b3beb0fc
 SHA512:
-  metadata.gz: 043e6e396b6aa6e47e7dc6f2245440e83e295deeabc866a9bd96786f5d8c0c6b3298d936a231ec05d93f438424e92090f2f1cbff2edfd23fb6e4ebbbdf108471
-  data.tar.gz: 4940d3ab0bea54b9125da346c46157c7e16bf11e4fd529ce7d80fcbc07f7695131035b38ede87c13ca489b1f65815a1004e2666593983ce3fb8800227ad2265e
+  metadata.gz: ec160bc4be7fcdeed4704cba0c084d74cfb812de161a02f72ce4178dd25232505951d9237b23f433f0b195213ad49d6d36e9ac757f7970c0014b8920728536c0
+  data.tar.gz: 3ae1b197743a9b72e2e6530a78187f2d0acf822e4baf3835625ad3d348d62593cd7e9e7100b15475a4ccfb535409d835a652a92d534553abc9685dc391055665

data/README.md

@@ -1,6 +1,6 @@
 # Logstash Dynatrace output plugin
 
-[![Travis Build Status](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace.svg)](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace)
+[![Travis Build Status](https://api.travis-ci.com/dynatrace-oss/logstash-output-dynatrace.svg?branch=main)](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace)
 
 > This project is developed and maintained by Dynatrace R&D.
 

data/lib/logstash/outputs/dynatrace.rb

@@ -74,6 +74,9 @@ module LogStash
       # Include body in debug logs when HTTP errors occur. Body may be large and include sensitive data.
       config :debug_include_body, validate: :boolean, default: false
 
+      # Maximum size payload to send to the Dynatrace API in Bytes. Batches of events which would be larger than max_payload_size when serialized will be split into smaller batches of events.
+      config :max_payload_size, validate: :number, default: 4_500_000
+
       def register
         # ssl_verification_mode config is from mixin but ssl_verify_none is our documented config
         @ssl_verification_mode = 'none' if @ssl_verify_none
@@ -103,6 +106,35 @@ module LogStash
         end
       end
 
+      class Batcher
+        def initialize(max_batch_size)
+          @max_batch_size = max_batch_size
+          @batch_events_size = 0
+          @serialized_events = []
+        end
+
+        def offer(serialized_event)
+          # 2 square brackets, the length of all previously serialized strings, commas, and the current event size
+          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
+          return false if batch_size_bytes > @max_batch_size
+
+          @serialized_events.push(serialized_event)
+          @batch_events_size += serialized_event.length
+          true
+        end
+
+        def drain_and_serialize
+          out = "[#{@serialized_events.join(',')}]\n"
+          @batch_events_size = 0
+          @serialized_events = []
+          out
+        end
+
+        def empty?
+          @serialized_events.empty?
+        end
+      end
+
       def make_headers
         {
           'User-Agent' => "logstash-output-dynatrace/#{DynatraceConstants::VERSION} logstash/#{LOGSTASH_VERSION}",
@@ -135,7 +167,24 @@ module LogStash
         failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
 
         pending = Queue.new
-        pending << [events, 0]
+        batcher = Batcher.new(@max_payload_size)
+
+        events.each do |event|
+          serialized_event = LogStash::Json.dump(event.to_hash)
+          if serialized_event.length > @max_payload_size
+            log_params = { size: serialized_event.length }
+            log_params[:body] = serialized_event if @debug_include_body
+            log_warning('Event larger than max_payload_size dropped', log_params)
+            next
+          end
+
+          next if batcher.offer(serialized_event)
+
+          pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
+          batcher.offer(serialized_event)
+        end
+
+        pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
 
         while popped = pending.pop
           break if popped == :done
@@ -199,11 +248,10 @@ module LogStash
       end
 
       def send_event(event, attempt)
-        body = event_body(event)
         headers = make_headers
 
         # Create an async request
-        response = client.post(ingest_endpoint_url, body: body, headers: headers)
+        response = client.post(ingest_endpoint_url, body: event, headers: headers)
 
         if response_success?(response)
           [:success, event, attempt]
@@ -231,7 +279,7 @@ module LogStash
           end
           if @debug_include_body
             # body can be big and may have sensitive data
-            log_params[:body] = body
+            log_params[:body] = event
           end
         end
         log_failure('Could not fetch URL', log_params)
@@ -276,9 +324,9 @@ module LogStash
         @logger.error(message, opts)
       end
 
-      # Format the HTTP body
-      def event_body(event)
-        "#{LogStash::Json.dump(event.map(&:to_hash)).chomp}\n"
+      # This is split into a separate method mostly to help testing
+      def log_warning(message, opts)
+        @logger.warn(message, opts)
       end
     end
   end

data/spec/outputs/dynatrace_spec.rb

@@ -123,6 +123,7 @@ describe LogStash::Outputs::Dynatrace do
       .with(ingest_endpoint_url, hash_including(:body, :headers))
       .and_call_original
     allow(subject).to receive(:log_failure).with(any_args)
+    allow(subject).to receive(:log_warning).with(any_args)
     allow(subject).to receive(:log_retryable_response).with(any_args)
   end
 
@@ -132,7 +133,7 @@ describe LogStash::Outputs::Dynatrace do
     end
   end
 
-  context 'performing a get' do
+  context 'performing a request' do
     describe 'invoking the request' do
       before do
         subject.multi_receive([event])
@@ -186,6 +187,63 @@ describe LogStash::Outputs::Dynatrace do
         expect(subject).to have_received(:send_event).exactly(3).times
       end
     end
+
+    context 'with more than 4.5MB of events' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 2_500_001 }) })
+      end
+
+      it 'should split the chunk into multiple requests' do
+        expect(subject).to have_received(:send_event).exactly(2).times
+      end
+    end
+
+    shared_examples('send small and drop large') do
+      it 'should only send the small event' do
+        expect(subject).to have_received(:send_event).exactly(1).times
+      end
+
+      it 'should log a warning' do
+        expect(subject).to have_received(:log_warning)
+          .with('Event larger than max_payload_size dropped', hash_including(:size))
+          .exactly(:once)
+      end
+    end
+
+    context 'with one small event and one too large event' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([LogStash::Event.new({ 'event' => 'small' }),
+                               LogStash::Event.new({ 'event' => 'n' * 4_500_001 })])
+      end
+
+      include_examples('send small and drop large')
+    end
+
+    context 'with one too large event and one small event' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([LogStash::Event.new({ 'event' => 'n' * 4_500_001 }),
+                               LogStash::Event.new({ 'event' => 'small' })])
+      end
+
+      include_examples('send small and drop large')
+    end
+  end
+
+  context 'max_payload_size 2MB' do
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
+    subject { LogStash::Outputs::Dynatrace.new(config) }
+
+    before do
+      allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
+    end
+
+    it 'should split the chunk into multiple requests' do
+      expect(subject).to have_received(:send_event).exactly(2).times
+    end
   end
 
   context 'on retryable unknown exception' do

data/version.yaml

@@ -1 +1 @@
-logstash-output-dynatrace: '0.5.0'
+logstash-output-dynatrace: '0.5.1'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-09 00:00:00.000000000 Z
+date: 2023-09-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json