logstash-output-dynatrace 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f41e7086fbd73db7471823a9bff842ad41783ba20b1f2ba112755f251af28c0a
-  data.tar.gz: 9b645d8c4f9eb4d416dc6c873cceb8779bae7c7745e63f009f2eaee4a5b29e52
+  metadata.gz: 9281b9e498b9986e45d25eaadc74358f95547551baf97132483533997edf134d
+  data.tar.gz: 27bc3095ec54575becfaf75eb2b7bca7d186f95b2ab239713e83f8b9f58b363b
 SHA512:
-  metadata.gz: 043e6e396b6aa6e47e7dc6f2245440e83e295deeabc866a9bd96786f5d8c0c6b3298d936a231ec05d93f438424e92090f2f1cbff2edfd23fb6e4ebbbdf108471
-  data.tar.gz: 4940d3ab0bea54b9125da346c46157c7e16bf11e4fd529ce7d80fcbc07f7695131035b38ede87c13ca489b1f65815a1004e2666593983ce3fb8800227ad2265e
+  metadata.gz: f4863a8fae10fe23afc85d4a6e31290ba3c0628d974c57c8cec6306adc2ba1da5c73905c7597c0954cb095ea1a24c6e182d028d8d7289f05951f77c214255b48
+  data.tar.gz: 53281ec7c6a7a8f6ac5dffae9cde5babe61d0a1b962d5c903e7e60975bb00d0528efc57c21c3b35d6f9bfc18b84c1549221c18dbe8ca3ae75e8cde764a94d7b7

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.6.0
+
+- Disable cookie processing by default
+
+## 0.5.1
+
+- Split large batches into smaller requests in order to meet Dynatrace API payload size limitations
+
+## 0.5.0
+
+- Rewrite plugin using http client mixin
+
 ## 0.4.0
 
 - Add` OpenSSL::SSL::SSLError` to the list of retried exception

data/README.md

@@ -1,6 +1,7 @@
+<!-- omit in toc -->
 # Logstash Dynatrace output plugin
 
-[![Travis Build Status](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace.svg)](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace)
+[![Travis Build Status](https://api.travis-ci.com/dynatrace-oss/logstash-output-dynatrace.svg?branch=main)](https://app.travis-ci.com/dynatrace-oss/logstash-output-dynatrace)
 
 > This project is developed and maintained by Dynatrace R&D.
 
@@ -14,13 +15,12 @@
   - [`ingest_endpoint_url`](#ingest_endpoint_url)
   - [`api_key`](#api_key)
   - [`ssl_verify_none`](#ssl_verify_none)
-  - [`codec`](#codec)
   - [`enable_metric`](#enable_metric)
   - [`id`](#id)
 - [Troubleshooting issues](#troubleshooting-issues)
   - [Enable Debug Logs](#enable-debug-logs)
 
-A [Logstash](https://github.com/elastic/logstash) output plugin for sending logs to the Dynatrace [Generic log ingest API v2](https://www.dynatrace.com/support/help/how-to-use-dynatrace/log-monitoring/log-monitoring-v2/post-log-ingest/).
+A [Logstash](https://github.com/elastic/logstash) output plugin for sending logs to the Dynatrace [Generic log ingest API v2](https://docs.dynatrace.com/docs/shortlink/api-log-monitoring-v2-post-ingest).
 Please review the documentation for this API before using the plugin.
 
 ## Installation Prerequisites
@@ -81,7 +81,7 @@ The following configuration options are supported by all output plugins:
 * Value type is [string](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)
 * Required
 
-This is the full URL of the [Generic log ingest API v2](https://www.dynatrace.com/support/help/how-to-use-dynatrace/log-monitoring/log-monitoring-v2/post-log-ingest/) endpoint on your ActiveGate.
+This is the full URL of the [Generic log ingest API v2](https://docs.dynatrace.com/docs/shortlink/api-log-monitoring-v2-post-ingest/) endpoint on your ActiveGate.
 Example: `"ingest_endpoint_url" => "https://abc123456.live.dynatrace.com/api/v2/logs/ingest"`
 
 ### `api_key`
@@ -89,7 +89,7 @@ Example: `"ingest_endpoint_url" => "https://abc123456.live.dynatrace.com/api/v2/
 * Value type is [string](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#string)
 * Required
 
-This is the [Dynatrace API token](https://www.dynatrace.com/support/help/dynatrace-api/basics/dynatrace-api-authentication/) which will be used to authenticate log ingest requests.
+This is the [Dynatrace API token](https://docs.dynatrace.com/docs/shortlink/api-authentication) which will be used to authenticate log ingest requests.
 It requires the `logs.ingest` (Ingest Logs) scope to be set and it is recommended to limit scope to only this one.
 Example: `"api_key" => "dt0c01.4XLO3..."`
 

data/lib/logstash/outputs/dynatrace.rb

@@ -74,6 +74,12 @@ module LogStash
       # Include body in debug logs when HTTP errors occur. Body may be large and include sensitive data.
       config :debug_include_body, validate: :boolean, default: false
 
+      # Maximum size payload to send to the Dynatrace API in Bytes. Batches of events which would be larger than max_payload_size when serialized will be split into smaller batches of events.
+      config :max_payload_size, validate: :number, default: 4_500_000
+
+      # Disable cookie support. Overridden default value from LogStash::PluginMixins::HttpClient
+      config :cookies, :validate => :boolean, :default => false
+
       def register
         # ssl_verification_mode config is from mixin but ssl_verify_none is our documented config
         @ssl_verification_mode = 'none' if @ssl_verify_none
@@ -103,6 +109,35 @@ module LogStash
         end
       end
 
+      class Batcher
+        def initialize(max_batch_size)
+          @max_batch_size = max_batch_size
+          @batch_events_size = 0
+          @serialized_events = []
+        end
+
+        def offer(serialized_event)
+          # 2 square brackets, the length of all previously serialized strings, commas, and the current event size
+          batch_size_bytes = 2 + @batch_events_size + @serialized_events.length + serialized_event.length
+          return false if batch_size_bytes > @max_batch_size
+
+          @serialized_events.push(serialized_event)
+          @batch_events_size += serialized_event.length
+          true
+        end
+
+        def drain_and_serialize
+          out = "[#{@serialized_events.join(',')}]\n"
+          @batch_events_size = 0
+          @serialized_events = []
+          out
+        end
+
+        def empty?
+          @serialized_events.empty?
+        end
+      end
+
       def make_headers
         {
           'User-Agent' => "logstash-output-dynatrace/#{DynatraceConstants::VERSION} logstash/#{LOGSTASH_VERSION}",
@@ -135,7 +170,24 @@ module LogStash
         failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
 
         pending = Queue.new
-        pending << [events, 0]
+        batcher = Batcher.new(@max_payload_size)
+
+        events.each do |event|
+          serialized_event = LogStash::Json.dump(event.to_hash)
+          if serialized_event.length > @max_payload_size
+            log_params = { size: serialized_event.length }
+            log_params[:body] = serialized_event if @debug_include_body
+            log_warning('Event larger than max_payload_size dropped', log_params)
+            next
+          end
+
+          next if batcher.offer(serialized_event)
+
+          pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
+          batcher.offer(serialized_event)
+        end
+
+        pending << [batcher.drain_and_serialize, 0] unless batcher.empty?
 
         while popped = pending.pop
           break if popped == :done
@@ -199,11 +251,10 @@ module LogStash
       end
 
       def send_event(event, attempt)
-        body = event_body(event)
         headers = make_headers
 
         # Create an async request
-        response = client.post(ingest_endpoint_url, body: body, headers: headers)
+        response = client.post(ingest_endpoint_url, body: event, headers: headers)
 
         if response_success?(response)
           [:success, event, attempt]
@@ -231,7 +282,7 @@ module LogStash
           end
           if @debug_include_body
             # body can be big and may have sensitive data
-            log_params[:body] = body
+            log_params[:body] = event
           end
         end
         log_failure('Could not fetch URL', log_params)
@@ -276,9 +327,9 @@ module LogStash
         @logger.error(message, opts)
       end
 
-      # Format the HTTP body
-      def event_body(event)
-        "#{LogStash::Json.dump(event.map(&:to_hash)).chomp}\n"
+      # This is split into a separate method mostly to help testing
+      def log_warning(message, opts)
+        @logger.warn(message, opts)
       end
     end
   end

data/spec/outputs/dynatrace_spec.rb

@@ -123,6 +123,7 @@ describe LogStash::Outputs::Dynatrace do
       .with(ingest_endpoint_url, hash_including(:body, :headers))
       .and_call_original
     allow(subject).to receive(:log_failure).with(any_args)
+    allow(subject).to receive(:log_warning).with(any_args)
     allow(subject).to receive(:log_retryable_response).with(any_args)
   end
 
@@ -132,7 +133,7 @@ describe LogStash::Outputs::Dynatrace do
     end
   end
 
-  context 'performing a get' do
+  context 'performing a request' do
     describe 'invoking the request' do
       before do
         subject.multi_receive([event])
@@ -186,6 +187,63 @@ describe LogStash::Outputs::Dynatrace do
         expect(subject).to have_received(:send_event).exactly(3).times
       end
     end
+
+    context 'with more than 4.5MB of events' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 2_500_001 }) })
+      end
+
+      it 'should split the chunk into multiple requests' do
+        expect(subject).to have_received(:send_event).exactly(2).times
+      end
+    end
+
+    shared_examples('send small and drop large') do
+      it 'should only send the small event' do
+        expect(subject).to have_received(:send_event).exactly(1).times
+      end
+
+      it 'should log a warning' do
+        expect(subject).to have_received(:log_warning)
+          .with('Event larger than max_payload_size dropped', hash_including(:size))
+          .exactly(:once)
+      end
+    end
+
+    context 'with one small event and one too large event' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([LogStash::Event.new({ 'event' => 'small' }),
+                               LogStash::Event.new({ 'event' => 'n' * 4_500_001 })])
+      end
+
+      include_examples('send small and drop large')
+    end
+
+    context 'with one too large event and one small event' do
+      before do
+        allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+        subject.multi_receive([LogStash::Event.new({ 'event' => 'n' * 4_500_001 }),
+                               LogStash::Event.new({ 'event' => 'small' })])
+      end
+
+      include_examples('send small and drop large')
+    end
+  end
+
+  context 'max_payload_size 2MB' do
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'max_payload_size' => 2_000_000 } }
+    subject { LogStash::Outputs::Dynatrace.new(config) }
+
+    before do
+      allow(subject).to receive(:send_event) { |e, att| [:success, e, att] }
+      subject.multi_receive([1, 2].map { |n| LogStash::Event.new({ 'n' => n.to_s * 1_250_000 }) })
+    end
+
+    it 'should split the chunk into multiple requests' do
+      expect(subject).to have_received(:send_event).exactly(2).times
+    end
   end
 
   context 'on retryable unknown exception' do

data/version.yaml

@@ -1 +1 @@
-logstash-output-dynatrace: '0.5.0'
+logstash-output-dynatrace: '0.6.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-08-09 00:00:00.000000000 Z
+date: 2024-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json