logstash-output-dynatrace 0.7.0 → 0.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4db961537a03cb5b21240ab63ea56877ceebcf0a075c5db85feb736b787d05df
-  data.tar.gz: 185005382c4ac576fde60eeffa8d3f1a5e037445b3dd92b2126be22f0a3481dc
+  metadata.gz: 6dc320145268685ae7585b0ffc68beb54ae932b3e9c5261e1ca6106803c487df
+  data.tar.gz: 223b65b82ffc6ed512d270eb879126a9f9fbe046f9dfc9e66ae9a37be2e60fcf
 SHA512:
-  metadata.gz: 0b14e9eba2edf866eff03c2fa7333ec23733bb6622dd841fcc3da9ddf5cfe5e5b1104238a2857cadc7474bc49444773ee0b5ae7cabcbbaddbefd28e4da403bb8
-  data.tar.gz: 334f37197001b662b86c451c1dcb1faa059f94d1d4045ad43e66590f08663e7a90ea5979bbe247d3d2d7774a0f0e386d6f97c33663e6bef70c0dc6b909557ce3
+  metadata.gz: 55a4b11c59aae913134c5972dff31b13c3e3eaa8761db6703a88e081df6d369b39d6be5f924219e30312b819a54c30d7aced19d98b24f6bcf4e988561929ce50
+  data.tar.gz: 401c4c51154584cee22c94f557a5869b8782759b5287498571e2f1ae345150f48625417126140bb54676560a3d75f7659b30ca392b2db2470dae7815b49904d2

data/CHANGELOG.md

@@ -1,3 +1,11 @@
+## 0.7.2
+
+- Log response bodies when the response is `200` indicating the request was only partially accepted
+ 
+## 0.7.1
+
+- Fix an error where max payload size was calculated using character count instead of bytes
+
 ## 0.7.0
 
 - Add new development dependency `rackup` for logstash 8.x compatibility

data/lib/logstash/outputs/dynatrace.rb

@@ -122,11 +122,11 @@ module LogStash
 
         def offer(serialized_event)
           # 2 square brackets, the length of all previously serialized strings, commas, and the current event size
-          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
+          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.bytesize
           return false if batch_size_bytes > @max_batch_size
 
           @serialized_events.push(serialized_event)
-          @batch_events_size += serialized_event.length
+          @batch_events_size += serialized_event.bytesize
           true
         end
 
@@ -160,6 +160,10 @@ module LogStash
         end
       end
 
+      def log_partial_success_response(response)
+        @logger.warn("Encountered partial success response", code: response.code, body: response.body)
+      end
+
       def log_error_response(response, ingest_endpoint_url, event)
         log_failure(
           "Encountered non-2xx HTTP code #{response.code}",
@@ -178,8 +182,8 @@ module LogStash
 
         events.each do |event|
           serialized_event = LogStash::Json.dump(event.to_hash)
-          if serialized_event.length > @max_payload_size
-            log_params = { size: serialized_event.length }
+          if serialized_event.bytesize > @max_payload_size
+            log_params = { size: serialized_event.bytesize }
             log_params[:body] = serialized_event if @debug_include_body
             log_warning('Event larger than max_payload_size dropped', log_params)
             next
@@ -267,6 +271,8 @@ module LogStash
         response = client.post(ingest_endpoint_url, body: event, headers: headers)
 
         if response_success?(response)
+          # some events were not accepted but we don't know which ones or why
+          log_partial_success_response(response) if response_partial_success?(response)
           [:success, event, attempt]
         elsif retryable_response?(response)
           log_retryable_response(response)
@@ -315,6 +321,10 @@ module LogStash
         response.code >= 200 && response.code <= 299
       end
 
+      def response_partial_success?(response)
+        response.code == 200
+      end
+
       def retryable_response?(response)
         RETRYABLE_CODES.include?(response.code)
       end

data/spec/outputs/dynatrace_spec.rb

@@ -168,6 +168,19 @@ describe LogStash::Outputs::Dynatrace do
       end
     end
 
+    context 'with partial success responses' do
+      let(:ingest_endpoint_url) { "http://localhost:#{port}/partial" }
+
+      before do
+        allow(subject).to receive(:log_partial_success_response)
+      end
+
+      it 'should warn on partial success' do
+        subject.multi_receive([event])
+        expect(subject).to have_received(:log_partial_success_response)
+      end
+    end
+
     context 'with retryable failing requests' do
       let(:ingest_endpoint_url) { "http://localhost:#{port}/retry" }
       let(:api_key) { 'placeholder-key' }
@@ -232,17 +245,38 @@ describe LogStash::Outputs::Dynatrace do
     end
   end
 
-  context 'max_payload_size 2MB' do
-    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
+  context 'max_payload_size 500' do
+    let(:max_payload_size) { 500 }
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => max_payload_size } }
     subject { LogStash::Outputs::Dynatrace.new(config) }
 
     before do
       allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
-      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
     end
 
-    it 'should split the chunk into multiple requests' do
-      expect(subject).to have_received(:send_event).exactly(2).times
+    it 'should send 2 400B messages in multiple requests' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 400 }) })
+      expect(subject).to have_received(:send_event).exactly(2).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should send 2 100B messages in a single request' do
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 100 }) })
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
+    end
+
+    it 'should drop messages larger than max_payload_size' do
+      subject.multi_receive([
+        LogStash::Event.new({ 'event' => '🤣' * 400 }),
+        LogStash::Event.new({ 'event' => 'n' * 600 }),
+        LogStash::Event.new({ 'event' => 'n' * 400 }),
+      ])
+      expect(subject).to have_received(:send_event).exactly(1).times do |s|
+        expect(s.bytesize).to be <= max_payload_size
+      end
     end
   end
 

data/spec/spec_helper.rb

@@ -98,7 +98,12 @@ class TestApp < Sinatra::Base
 
   multiroute(%w[get post put patch delete], '/good') do
     self.class.last_request = request
-    [200, 'YUP']
+    [204, 'YUP']
+  end
+
+  multiroute(%w[get post put patch delete], '/partial') do
+    self.class.last_request = request
+    [200, 'partial success']
   end
 
   multiroute(%w[get post put patch delete], '/bad') do

data/version.yaml

@@ -1 +1 @@
-logstash-output-dynatrace: '0.7.0'
+logstash-output-dynatrace: '0.7.2'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.7.2
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-08 00:00:00.000000000 Z
+date: 2025-05-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json