logstash-output-dynatrace 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 657b814816f85318e7968d096c9d5c5de194002f39a1bdb3971442006001af8e
-  data.tar.gz: 7a94f714d34078f792a648f3c468518099601e1c0e23614461aa0ed6a97dc50d
+  metadata.gz: f41e7086fbd73db7471823a9bff842ad41783ba20b1f2ba112755f251af28c0a
+  data.tar.gz: 9b645d8c4f9eb4d416dc6c873cceb8779bae7c7745e63f009f2eaee4a5b29e52
 SHA512:
-  metadata.gz: 67579448d2e0034ba41b01cc5cdf5d6365d02e34c67c5b9698dc1af3638c885301610373fc774afba4c5b2a204aa42920d402a340520198057036c7362e00515
-  data.tar.gz: a6ee38c6ba636b731e343e6e63c2d190a9d8b90d9e241f435af404e39f9b25e3b043ffa79ae35f87cb5f9be8013929afdaa819570e9b6257e04e23cbcc7c1281
+  metadata.gz: 043e6e396b6aa6e47e7dc6f2245440e83e295deeabc866a9bd96786f5d8c0c6b3298d936a231ec05d93f438424e92090f2f1cbff2edfd23fb6e4ebbbdf108471
+  data.tar.gz: 4940d3ab0bea54b9125da346c46157c7e16bf11e4fd529ce7d80fcbc07f7695131035b38ede87c13ca489b1f65815a1004e2666593983ce3fb8800227ad2265e

data/Gemfile

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2021 Dynatrace LLC
+# Copyright 2023 Dynatrace LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

data/README.md

@@ -103,12 +103,8 @@ It is recommended to leave this optional configuration set to `false` unless abs
 Setting `ssl_verify_none` to `true` causes the output plugin to skip certificate verification when sending log ingest requests to SSL and TLS protected HTTPS endpoints.
 This option may be required if you are using a self-signed certificate, an expired certificate, or a certificate which was generated for a different domain than the one in use.
 
-### `codec` 
-
-* Value type is codec
-* Default value is "plain"
-
-The codec used for output data. Output codecs are a convenient method for encoding your data before it leaves the output without needing a separate filter in your Logstash pipeline.
+> NOTE: Starting in plugin version `0.5.0`, this option has no effect in versions of Logstash older than `8.1.0`.
+> If this functionality is required, it is recommended to update Logstash or stay at plugin version `0.4.x` or older.
 
 ### `enable_metric`
 

data/{version.rb → lib/dynatrace/version.rb}

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2021 Dynatrace LLC
+# Copyright 2023 Dynatrace LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,7 @@
 # limitations under the License.
 
 module DynatraceConstants
-  # Also required to change the version in lib/logstash/outputs/dynatrace.rb
-  VERSION = '0.4.0'
+  require 'yaml'
+  VERSION = YAML.load_file(File.expand_path('../../version.yaml',
+                                            File.dirname(__FILE__))).fetch('logstash-output-dynatrace')
 end

data/lib/logstash/outputs/dynatrace.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2021 Dynatrace LLC
+# Copyright 2023 Dynatrace LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,24 +14,48 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require 'logstash/namespace'
 require 'logstash/outputs/base'
+require 'logstash/namespace'
 require 'logstash/json'
-require 'openssl'
+require 'logstash/version'
+require 'dynatrace/version'
+require 'uri'
+require 'logstash/plugin_mixins/http_client'
 
-MAX_RETRIES = 5
-PLUGIN_VERSION = '0.4.0'
+# These constants came from the http plugin config but we don't want them configurable
+# If encountered as response codes this plugin will retry these requests
+RETRYABLE_CODES = [429, 500, 502, 503, 504].freeze
+RETRY_FAILED = true
 
 module LogStash
   module Outputs
-    class RetryableError < StandardError;
-    end
-
-    # An output which sends logs to the Dynatrace log ingest v2 endpoint formatted as JSON
     class Dynatrace < LogStash::Outputs::Base
-      config_name 'dynatrace'
+      include LogStash::PluginMixins::HttpClient
+
+      concurrency :shared
+
+      RETRYABLE_MANTICORE_EXCEPTIONS = [
+        ::Manticore::Timeout,
+        ::Manticore::SocketException,
+        ::Manticore::ClientProtocolException,
+        ::Manticore::ResolutionFailure,
+        ::Manticore::SocketTimeout
+      ].freeze
+
+      RETRYABLE_UNKNOWN_EXCEPTION_STRINGS = [
+        /Connection reset by peer/i,
+        /Read Timed out/i
+      ].freeze
 
-      concurrency :single
+      class PluginInternalQueueLeftoverError < StandardError; end
+
+      # This output will execute up to 'pool_max' requests in parallel for performance.
+      # Consider this when tuning this plugin for performance.
+      #
+      # Additionally, note that when parallel execution is used strict ordering of events is not
+      # guaranteed!
+
+      config_name 'dynatrace'
 
       # The full URL of the Dynatrace log ingestion endpoint:
       # - on SaaS:    https://{your-environment-id}.live.dynatrace.com/api/v2/logs/ingest
@@ -44,80 +68,217 @@ module LogStash
       # Disable SSL validation by setting :verify_mode OpenSSL::SSL::VERIFY_NONE
       config :ssl_verify_none, validate: :boolean, default: false
 
-      default :codec, 'json'
+      # Include headers in debug logs when HTTP errors occur. Headers include sensitive data such as API tokens.
+      config :debug_include_headers, validate: :boolean, default: false
 
-      attr_accessor :uri, :plugin_version
+      # Include body in debug logs when HTTP errors occur. Body may be large and include sensitive data.
+      config :debug_include_body, validate: :boolean, default: false
 
       def register
-        @logger.debug("Registering plugin")
-        require 'net/https'
-        require 'uri'
-        @uri = URI.parse(@ingest_endpoint_url.uri.to_s)
-        @client = Net::HTTP.new(@uri.host, @uri.port)
-
-        if uri.scheme == 'https'
-          @client.use_ssl = true
-          @client.verify_mode = OpenSSL::SSL::VERIFY_NONE if @ssl_verify_none
+        # ssl_verification_mode config is from mixin but ssl_verify_none is our documented config
+        @ssl_verification_mode = 'none' if @ssl_verify_none
+
+        @ingest_endpoint_url = @ingest_endpoint_url.to_s
+
+        # Run named Timer as daemon thread
+        @timer = java.util.Timer.new("HTTP Output #{params['id']}", true)
+      end
+
+      def multi_receive(events)
+        return if events.empty?
+
+        send_events(events)
+      end
+
+      class RetryTimerTask < java.util.TimerTask
+        def initialize(pending, event, attempt)
+          @pending = pending
+          @event = event
+          @attempt = attempt
+          super()
+        end
+
+        def run
+          @pending << [@event, @attempt]
         end
-        @logger.info('Client', client: @client.inspect)
       end
 
-      def headers
+      def make_headers
         {
-          'User-Agent' => "logstash-output-dynatrace/#{PLUGIN_VERSION}",
+          'User-Agent' => "logstash-output-dynatrace/#{DynatraceConstants::VERSION} logstash/#{LOGSTASH_VERSION}",
           'Content-Type' => 'application/json; charset=utf-8',
           'Authorization' => "Api-Token #{@api_key.value}"
         }
       end
 
-      # Takes an array of events
-      def multi_receive(events)
-        @logger.debug("Received #{events.length} events")
-        return if events.length.zero?
-
-        retries = 0
-        begin
-          request = Net::HTTP::Post.new(uri, headers)
-          request.body = "#{LogStash::Json.dump(events.map(&:to_hash)).chomp}\n"
-          response = @client.request(request)
-
-          case response
-          when Net::HTTPSuccess
-            @logger.debug("successfully sent #{events.length} events#{" with #{retries} retries" if retries > 0}")
-          when Net::HTTPServerError
-            @logger.error("Encountered an HTTP server error", :message => response.message, :code => response.code, :body => response.body) if retries == 0
-          when Net::HTTPNotFound
-            @logger.error("Encountered a 404 Not Found error. Please check that log ingest is enabled and your API token has the `logs.ingest` (Ingest Logs) scope.", :message => response.message, :code => response.code)
-          when Net::HTTPClientError
-            @logger.error("Encountered an HTTP client error", :message => response.message, :code => response.code, :body => response.body)
-          else
-            @logger.error("Encountered an unexpected response code", :message => response.message, :code => response.code)
+      def log_retryable_response(response)
+        retry_msg = RETRY_FAILED ? 'will retry' : "won't retry"
+        if response.code == 429
+          @logger.debug? && @logger.debug("Encountered a 429 response, #{retry_msg}. This is not serious, just flow control via HTTP")
+        else
+          @logger.warn("Encountered a retryable HTTP request in HTTP output, #{retry_msg}", code: response.code,
+                                                                                            body: response.body)
+        end
+      end
+
+      def log_error_response(response, ingest_endpoint_url, event)
+        log_failure(
+          "Encountered non-2xx HTTP code #{response.code}",
+          response_code: response.code,
+          ingest_endpoint_url: ingest_endpoint_url,
+          event: event
+        )
+      end
+
+      def send_events(events)
+        successes = java.util.concurrent.atomic.AtomicInteger.new(0)
+        failures  = java.util.concurrent.atomic.AtomicInteger.new(0)
+
+        pending = Queue.new
+        pending << [events, 0]
+
+        while popped = pending.pop
+          break if popped == :done
+
+          event, attempt = popped
+
+          if attempt > 2 && pipeline_shutdown_requested?
+            raise PluginInternalQueueLeftoverError, 'Received pipeline shutdown request but http output has unfinished events. ' \
+                    'If persistent queue is enabled, events will be retried.'
+          end
+
+          action, event, attempt = send_event(event, attempt)
+          begin
+            action = :failure if action == :retry && !RETRY_FAILED
+
+            case action
+            when :success
+              successes.incrementAndGet
+            when :retry
+              next_attempt = attempt + 1
+              sleep_for = sleep_for_attempt(next_attempt)
+              @logger.info("Retrying http request, will sleep for #{sleep_for} seconds")
+              timer_task = RetryTimerTask.new(pending, event, next_attempt)
+              @timer.schedule(timer_task, sleep_for * 1000)
+            when :failure
+              failures.incrementAndGet
+            else
+              # this should never happen. It means send_event returned a symbol we didn't recognize
+              raise "Unknown action #{action}"
+            end
+
+            pending << :done if %i[success failure].include?(action) && (successes.get + failures.get == 1)
+          rescue StandardError => e
+            # This should never happen unless there's a flat out bug in the code
+            @logger.error('Error sending HTTP Request',
+                          class: e.class.name,
+                          message: e.message,
+                          backtrace: e.backtrace)
+            failures.incrementAndGet
+            raise e
           end
+        end
+      rescue StandardError => e
+        @logger.error('Error in http output loop',
+                      class: e.class.name,
+                      message: e.message,
+                      backtrace: e.backtrace)
+        raise e
+      end
+
+      def pipeline_shutdown_requested?
+        return super if defined?(super) # since LS 8.1.0
+
+        nil
+      end
 
-          raise RetryableError.new "code #{response.code}" if retryable(response)
-
-        rescue Net::OpenTimeout, Net::HTTPBadResponse, OpenSSL::SSL::SSLError, RetryableError => e
-          # Net::OpenTimeout indicates a connection could not be established within the timeout period
-          # Net::HTTPBadResponse indicates a protocol error
-          # OpenSSL::SSL::SSLErrorWaitReadable indicates an error establishing the ssl connection
-          if retries < MAX_RETRIES
-            sleep_seconds = 2 ** retries
-            @logger.warn("Failed to contact dynatrace: #{e.message}. Trying again after #{sleep_seconds} seconds.")
-            sleep sleep_seconds
-            retries += 1
-            retry
-          else
-            @logger.error("Failed to export logs to Dynatrace.")
-            return
+      def sleep_for_attempt(attempt)
+        sleep_for = attempt**2
+        sleep_for = sleep_for <= 60 ? sleep_for : 60
+        (sleep_for / 2) + (rand(0..sleep_for) / 2)
+      end
+
+      def send_event(event, attempt)
+        body = event_body(event)
+        headers = make_headers
+
+        # Create an async request
+        response = client.post(ingest_endpoint_url, body: body, headers: headers)
+
+        if response_success?(response)
+          [:success, event, attempt]
+        elsif retryable_response?(response)
+          log_retryable_response(response)
+          [:retry, event, attempt]
+        else
+          log_error_response(response, ingest_endpoint_url, event)
+          [:failure, event, attempt]
+        end
+      rescue StandardError => e
+        will_retry = retryable_exception?(e)
+        log_params = {
+          ingest_endpoint_url: ingest_endpoint_url,
+          message: e.message,
+          class: e.class,
+          will_retry: will_retry
+        }
+        if @logger.debug?
+          # backtraces are big
+          log_params[:backtrace] = e.backtrace
+          if @debug_include_headers
+            # headers can have sensitive data
+            log_params[:headers] = headers
           end
-        rescue StandardError => e
-          @logger.error("Unknown error raised", :error => e.inspect)
-          raise e
+          if @debug_include_body
+            # body can be big and may have sensitive data
+            log_params[:body] = body
+          end
+        end
+        log_failure('Could not fetch URL', log_params)
+
+        if will_retry
+          [:retry, event, attempt]
+        else
+          [:failure, event, attempt]
         end
       end
 
-      def retryable(response)
-        return response.is_a? Net::HTTPServerError
+      def close
+        @timer.cancel
+        client.close
+      end
+
+      private
+
+      def response_success?(response)
+        response.code >= 200 && response.code <= 299
+      end
+
+      def retryable_response?(response)
+        RETRYABLE_CODES.include?(response.code)
+      end
+
+      def retryable_exception?(exception)
+        retryable_manticore_exception?(exception) || retryable_unknown_exception?(exception)
+      end
+
+      def retryable_manticore_exception?(exception)
+        RETRYABLE_MANTICORE_EXCEPTIONS.any? { |me| exception.is_a?(me) }
+      end
+
+      def retryable_unknown_exception?(exception)
+        exception.is_a?(::Manticore::UnknownException) &&
+          RETRYABLE_UNKNOWN_EXCEPTION_STRINGS.any? { |snippet| exception.message =~ snippet }
+      end
+
+      # This is split into a separate method mostly to help testing
+      def log_failure(message, opts)
+        @logger.error(message, opts)
+      end
+
+      # Format the HTTP body
+      def event_body(event)
+        "#{LogStash::Json.dump(event.map(&:to_hash)).chomp}\n"
       end
     end
   end

data/logstash-output-dynatrace.gemspec

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2021 Dynatrace LLC
+# Copyright 2023 Dynatrace LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,11 +14,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require_relative './version'
+require 'yaml'
 
 Gem::Specification.new do |s|
   s.name = 'logstash-output-dynatrace'
-  s.version = DynatraceConstants::VERSION
+  s.version = YAML.load_file(File.expand_path('./version.yaml',
+                                              File.dirname(__FILE__))).fetch('logstash-output-dynatrace')
   s.summary = 'A logstash output plugin for sending logs to the Dynatrace Generic log ingest API v2'
   s.description = <<-EOF
     This gem is a Logstash plugin required to be installed on top of the Logstash
@@ -32,7 +33,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # Files
-  s.files = Dir['lib/**/*', 'spec/**/*', '*.gemspec', '*.md', 'Gemfile', 'LICENSE','version.rb']
+  s.files = Dir['lib/**/*', 'spec/**/*', '*.gemspec', '*.md', 'Gemfile', 'LICENSE', 'version.yaml']
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
@@ -42,9 +43,12 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 2.0.0', '< 3'
+  s.add_runtime_dependency 'logstash-mixin-http_client', '>= 6.0.0', '< 8.0.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-input-generator'
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'webrick'
 
   s.add_development_dependency 'rubocop', '1.9.1'
   s.add_development_dependency 'rubocop-rake', '0.5.1'

data/spec/outputs/dynatrace_spec.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-# Copyright 2021 Dynatrace LLC
+# Copyright 2023 Dynatrace LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,138 +14,481 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-require_relative '../spec_helper'
-require_relative '../../version'
-require 'logstash/codecs/plain'
-require 'logstash/event'
-require 'net/http'
-require 'json'
+require File.expand_path('../spec_helper.rb', File.dirname(__FILE__))
 
 describe LogStash::Outputs::Dynatrace do
-  let(:events) do
-    [
-      LogStash::Event.new({ 'message' => 'message 1', '@timestamp' => "2021-06-25T15:46:45.693Z" }),
-      LogStash::Event.new({ 'message' => 'message 2', '@timestamp' => "2021-06-25T15:46:46.693Z" }),
-    ]
+  # Wait for the async request to finish in this spinlock
+  # Requires pool_max to be 1
+
+  before(:all) do
+    @server = start_app_and_wait(TestApp)
+  end
+
+  after(:all) do
+    @server.shutdown # WEBrick::HTTPServer
+    begin
+      TestApp.stop!
+    rescue StandardError
+      nil
+    end
+  end
+
+  let(:port) { PORT }
+  let(:event) do
+    LogStash::Event.new({ 'message' => 'hi' })
   end
-  let(:url) { "http://localhost/good" }
-  let(:key) { 'api.key' }
+  let(:ingest_endpoint_url) { "http://localhost:#{port}/good" }
+  let(:api_key) { 'placeholder-key' }
+
+  shared_examples('failure log behaviour') do
+    it 'logs failure' do
+      expect(subject).to have_received(:log_failure).with(any_args)
+    end
+
+    it 'does not log headers' do
+      expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
+    end
 
-  let(:subject) { LogStash::Outputs::Dynatrace.new({ 'api_key' => key, 'ingest_endpoint_url' => url }) }
-  let(:client) { subject.instance_variable_get(:@client) }
+    it 'does not log the message body' do
+      expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
+    end
+
+    context 'with debug log level' do
+      before :all do
+        @current_log_level = LogStash::Logging::Logger.get_logging_context.get_root_logger.get_level.to_s.downcase
+        LogStash::Logging::Logger.configure_logging 'debug'
+      end
+      after :all do
+        LogStash::Logging::Logger.configure_logging @current_log_level
+      end
+
+      it 'logs a failure' do
+        expect(subject).to have_received(:log_failure).with(anything, hash_including(:backtrace))
+      end
 
-  let(:ok) { Net::HTTPOK.new "1.1", "200", "OK" }
-  let(:server_error) { Net::HTTPServerError.new "1.1", "500", "Internal Server Error" }
-  let(:client_error) { Net::HTTPClientError.new("1.1", '400', 'Client error') }
-  let(:not_found) { Net::HTTPNotFound.new "1.1", "404", "Not Found" }
+      context 'with debug_include_headers false (default)' do
+        it 'does not log headers' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
+        end
+      end
+
+      context 'with debug_include_body false (default)' do
+        it 'does not log body' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
+        end
+      end
+
+      context 'with debug_include_headers true' do
+        let(:config) { super().merge 'debug_include_headers' => true }
+
+        it 'logs headers' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_including(:headers))
+        end
+      end
+
+      context 'with debug_include_body true' do
+        let(:config) { super().merge 'debug_include_body' => true }
+
+        it 'logs body' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_including(:body))
+        end
+      end
+
+      context 'with debug_include_headers false' do
+        let(:config) { super().merge 'debug_include_headers' => false }
+
+        it 'logs headers' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:headers))
+        end
+      end
+
+      context 'with debug_include_body false' do
+        let(:config) { super().merge 'debug_include_body' => false }
+
+        it 'logs body' do
+          expect(subject).to have_received(:log_failure).with(anything, hash_not_including(:body))
+        end
+      end
+    end
+  end
 
-  let(:body) { "this is a failure" }
+  let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'pool_max' => 1 } }
+  subject { LogStash::Outputs::Dynatrace.new(config) }
+
+  let(:client) { subject.client }
 
   before do
     subject.register
+    allow(client).to receive(:post)
+      .with(ingest_endpoint_url, hash_including(:body, :headers))
+      .and_call_original
+    allow(subject).to receive(:log_failure).with(any_args)
+    allow(subject).to receive(:log_retryable_response).with(any_args)
   end
 
-  it 'does not send empty events' do
-    expect(client).to_not receive(:request)
-    subject.multi_receive([])
+  context 'sending no events' do
+    it 'should not block the pipeline' do
+      subject.multi_receive([])
+    end
   end
 
-  context 'server response success' do
-    it 'sends events' do
-      expect(client).to receive(:request) do |req|
-        body = JSON.parse(req.body)
-        expect(body.length).to eql(2)
-        expect(body[0]['message']).to eql('message 1')
-        expect(body[0]['@timestamp']).to eql('2021-06-25T15:46:45.693Z')
-        expect(body[1]['message']).to eql('message 2')
-        expect(body[1]['@timestamp']).to eql('2021-06-25T15:46:46.693Z')
-        ok
+  context 'performing a get' do
+    describe 'invoking the request' do
+      before do
+        subject.multi_receive([event])
+      end
+
+      it 'should execute the request' do
+        expect(client).to have_received(:post)
+          .with(ingest_endpoint_url, hash_including(:body, :headers))
       end
-      subject.multi_receive(events)
     end
 
-    it 'includes authorization header' do
-      expect(client).to receive(:request) do |req|
-        expect(req['Authorization']).to eql("Api-Token #{key}")
-        ok
+    context 'with passing requests' do
+      before do
+        subject.multi_receive([event])
+      end
+
+      it 'should not log a failure' do
+        expect(subject).not_to have_received(:log_failure).with(any_args)
       end
-      subject.multi_receive(events)
     end
 
-    it 'includes content type header' do
-      expect(client).to receive(:request) do |req|
-        expect(req['Content-Type']).to eql('application/json; charset=utf-8')
-        ok
+    context 'with failing requests' do
+      let(:ingest_endpoint_url) { "http://localhost:#{port}/bad" }
+      let(:api_key) { 'placeholder-key' }
+
+      before do
+        subject.multi_receive([event])
+      end
+
+      it 'should log a failure' do
+        expect(subject).to have_received(:log_failure).with(any_args)
       end
-      subject.multi_receive(events)
     end
 
-    it 'includes user agent' do
-      expect(client).to receive(:request) do |req|
-        expect(req['User-Agent']).to eql("logstash-output-dynatrace/#{::DynatraceConstants::VERSION}")
-        ok
+    context 'with retryable failing requests' do
+      let(:ingest_endpoint_url) { "http://localhost:#{port}/retry" }
+      let(:api_key) { 'placeholder-key' }
+
+      before do
+        TestApp.retry_fail_count = 2
+        allow(subject).to receive(:send_event).and_call_original
+        allow(subject).to receive(:sleep_for_attempt) { 0 }
+        subject.multi_receive([event])
+      end
+
+      it 'should log a retryable response 2 times' do
+        expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+      end
+
+      it 'should make three total requests' do
+        expect(subject).to have_received(:send_event).exactly(3).times
+      end
+    end
+  end
+
+  context 'on retryable unknown exception' do
+    before :each do
+      raised = false
+      original_method = subject.client.method(:post)
+      allow(subject).to receive(:send_event).and_call_original
+      expect(subject.client).to receive(:post) do |*args|
+        unless raised
+          raised = true
+          raise ::Manticore::UnknownException, 'Read timed out'
+        end
+        original_method.call(args)
       end
-      subject.multi_receive(events)
+      subject.multi_receive([event])
     end
 
-    it 'does not log on success' do
-      allow(subject.logger).to receive(:debug)
-      expect(subject.logger).to_not receive(:info)
-      expect(subject.logger).to_not receive(:error)
-      expect(subject.logger).to_not receive(:warn)
-      expect(client).to receive(:request) { ok }
-      subject.multi_receive(events)
+    include_examples('failure log behaviour')
+
+    it 'retries' do
+      expect(subject).to have_received(:send_event).exactly(2).times
     end
   end
 
-  context 'with server error' do
-    it 'retries 5 times with exponential backoff' do
-      # This prevents the elusive "undefined method `close' for nil:NilClass" error.
-      expect(server_error).to receive(:body) { body }.once
-      expect(subject.logger).to receive(:error).with("Encountered an HTTP server error", {:body=>body, :code=>"500", :message=> "Internal Server Error"}).once
-      expect(client).to receive(:request) { server_error }.exactly(6).times
+  context 'on non-retryable unknown exception' do
+    before :each do
+      raised = false
+      original_method = subject.client.method(:post)
+      allow(subject).to receive(:send_event).and_call_original
+      expect(subject.client).to receive(:post) do |*args|
+        unless raised
+          raised = true
+          raise ::Manticore::UnknownException, 'broken'
+        end
+        original_method.call(args)
+      end
+      subject.multi_receive([event])
+    end
 
+    include_examples('failure log behaviour')
 
-      expect(subject).to receive(:sleep).with(1).ordered
-      expect(subject).to receive(:sleep).with(2).ordered
-      expect(subject).to receive(:sleep).with(4).ordered
-      expect(subject).to receive(:sleep).with(8).ordered
-      expect(subject).to receive(:sleep).with(16).ordered
+    it 'does not retry' do
+      expect(subject).to have_received(:send_event).exactly(1).times
+    end
+  end
 
-      expect(subject.logger).to receive(:error).with("Failed to export logs to Dynatrace.")
-      subject.multi_receive(events)
+  context 'on non-retryable exception' do
+    before :each do
+      raised = false
+      original_method = subject.client.method(:post)
+      allow(subject).to receive(:send_event).and_call_original
+      expect(subject.client).to receive(:post) do |*args|
+        unless raised
+          raised = true
+          raise 'broken'
+        end
+        original_method.call(args)
+      end
+      subject.multi_receive([event])
+    end
+
+    include_examples('failure log behaviour')
+
+    it 'does not retry' do
+      expect(subject).to have_received(:send_event).exactly(1).times
     end
   end
 
-  context 'with client error' do
-    it 'does not retry on 404' do
-      allow(subject.logger).to receive(:error)
-      expect(client).to receive(:request) { not_found }.once
-      subject.multi_receive(events)
+  context 'on retryable exception' do
+    before :each do
+      raised = false
+      original_method = subject.client.method(:post)
+      allow(subject).to receive(:send_event).and_call_original
+      expect(subject.client).to receive(:post) do |*args|
+        unless raised
+          raised = true
+          raise ::Manticore::Timeout, 'broken'
+        end
+        original_method.call(args)
+      end
+      subject.multi_receive([event])
     end
 
-    it 'logs the response body' do
-      expect(client).to receive(:request) { client_error }
-      # This prevents the elusive "undefined method `close' for nil:NilClass" error.
-      expect(client_error).to receive(:body) { body }
+    it 'retries' do
+      expect(subject).to have_received(:send_event).exactly(2).times
+    end
 
-      expect(subject.logger).to receive(:error).with("Encountered an HTTP client error",
-        {:body=>body, :code=>"400", :message=> "Client error"})
+    include_examples('failure log behaviour')
+  end
 
-      subject.multi_receive(events)
+  shared_examples('a received event') do
+    before do
+      TestApp.last_request = nil
+    end
+
+    let(:events) { [event] }
+
+    describe 'with a good code' do
+      before do
+        subject.multi_receive(events)
+      end
+
+      let(:last_request) { TestApp.last_request }
+      let(:body) { last_request.body.read }
+      let(:content_type) { last_request.env['CONTENT_TYPE'] }
+
+      it 'should receive the request' do
+        expect(last_request).to be_truthy
+      end
+
+      it 'should receive the event as a hash' do
+        expect(body).to eql(expected_body)
+      end
+
+      it 'should have the correct content type' do
+        expect(content_type).to eql(expected_content_type)
+      end
+    end
+
+    describe 'a retryable code' do
+      let(:ingest_endpoint_url) { "http://localhost:#{port}/retry" }
+      let(:api_key) { 'placeholder-key' }
+
+      before do
+        TestApp.retry_fail_count = 2
+        allow(subject).to receive(:send_event).and_call_original
+        allow(subject).to receive(:log_retryable_response)
+        subject.multi_receive(events)
+      end
+
+      it 'should retry' do
+        expect(subject).to have_received(:log_retryable_response).with(any_args).twice
+      end
     end
   end
 
-  context 'when an unknown error occurs' do
-    it 'logs and re-raises the error' do
-      class BadEvents
-        def length
-          1
-        end
+  shared_examples 'integration tests' do
+    let(:base_config) { {} }
+    let(:ingest_endpoint_url) { "http://localhost:#{port}/good" }
+    let(:api_key) { 'placeholder-key' }
+    let(:event) do
+      LogStash::Event.new('foo' => 'bar', 'baz' => 'bot', 'user' => 'McBest')
+    end
+
+    subject { LogStash::Outputs::Dynatrace.new(config) }
+
+    before do
+      subject.register
+    end
+
+    describe 'sending with the default (JSON) config' do
+      let(:config) do
+        base_config.merge({ 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'pool_max' => 1 })
+      end
+      let(:expected_body) { "#{LogStash::Json.dump([event].map(&:to_hash)).chomp}\n" }
+      let(:expected_content_type) { 'application/json; charset=utf-8' }
+
+      include_examples('a received event')
+    end
+  end
+
+  describe 'integration test without gzip compression' do
+    include_examples('integration tests')
+  end
+
+  # describe "integration test with gzip compression" do
+  #   include_examples("integration tests") do
+  #     let(:base_config) { { "http_compression" => true } }
+  #   end
+  # end
+
+  describe 'retryable error in termination' do
+    let(:ingest_endpoint_url) { "http://localhost:#{port - 1}/invalid" }
+    let(:api_key) { 'placeholder-key' }
+    let(:events) { [event] }
+    let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key, 'pool_max' => 1 } }
+
+    subject { LogStash::Outputs::Dynatrace.new(config) }
+
+    before do
+      subject.register
+      allow(subject).to receive(:pipeline_shutdown_requested?).and_return(true)
+    end
+
+    it 'raise exception to exit indefinitely retry' do
+      expect do
+        subject.multi_receive(events)
+      end.to raise_error(LogStash::Outputs::Dynatrace::PluginInternalQueueLeftoverError)
+    end
+  end
+end
+
+RSpec.describe LogStash::Outputs::Dynatrace do # different block as we're starting web server with TLS
+  @@default_server_settings = TestApp.server_settings.dup
+
+  before do
+    TestApp.server_settings = @@default_server_settings.merge(webrick_config)
+
+    TestApp.last_request = nil
+
+    @server = start_app_and_wait(TestApp)
+  end
+
+  let(:webrick_config) do
+    cert, key = WEBrick::Utils.create_self_signed_cert 2048, [['CN', ssl_cert_host]], 'Logstash testing'
+    {
+      SSLEnable: true,
+      SSLVerifyClient: OpenSSL::SSL::VERIFY_NONE,
+      SSLCertificate: cert,
+      SSLPrivateKey: key
+    }
+  end
+
+  after do
+    @server.shutdown # WEBrick::HTTPServer
+
+    begin
+      TestApp.stop!
+    rescue StandardError
+      nil
+    end
+    TestApp.server_settings = @@default_server_settings
+  end
+
+  let(:ssl_cert_host) { 'localhost' }
+
+  let(:port) { PORT }
+  let(:ingest_endpoint_url) { "https://localhost:#{port}/good" }
+  let(:api_key) { 'placeholder-key' }
+  let(:method) { 'post' }
+
+  let(:config) { { 'ingest_endpoint_url' => ingest_endpoint_url, 'api_key' => api_key } }
+
+  subject { LogStash::Outputs::Dynatrace.new(config) }
+
+  before { subject.register }
+  after  { subject.close }
+
+  let(:last_request) { TestApp.last_request }
+  let(:last_request_body) { last_request.body.read }
+
+  let(:event) { LogStash::Event.new('message' => 'hello!') }
+
+  context 'with default (full) verification' do
+    let(:config) { super() } # 'ssl_verification_mode' => 'full'
+
+    it 'does NOT process the request (due client protocol exception)' do
+      # Manticore's default verification does not accept self-signed certificates!
+      Thread.start do
+        subject.multi_receive [event]
+      end
+      sleep 1.5
+
+      expect(last_request).to be nil
+    end
+  end
+
+  context 'with verification disabled' do
+    let(:config) { super().merge 'ssl_verification_mode' => 'none' }
+
+    it 'should process the request' do
+      subject.multi_receive [event]
+      expect(last_request_body).to include '"message":"hello!"'
+    end
+  end
+
+  unless tls_version_enabled_by_default?('TLSv1.1')
+    context 'with supported_protocols set to (disabled) 1.1' do
+      let(:config) { super().merge 'ssl_supported_protocols' => ['TLSv1.1'], 'ssl_verification_mode' => 'none' }
+
+      it 'keeps retrying due a protocol exception' do # TLSv1.1 not enabled by default
+        expect(subject).to receive(:log_failure)
+          .with('Could not fetch URL', hash_including(message: 'No appropriate protocol (protocol is disabled or cipher suites are inappropriate)'))
+          .at_least(:once)
+        Thread.start { subject.multi_receive [event] }
+        sleep 1.0
       end
+    end
+  end
 
-      expect(subject.logger).to receive(:error)
-      expect { subject.multi_receive(BadEvents.new) }.to raise_error(StandardError)
+  context 'with supported_protocols set to 1.2/1.3' do
+    let(:config) do
+      super().merge 'ssl_supported_protocols' => ['TLSv1.2', 'TLSv1.3'], 'ssl_verification_mode' => 'none'
+    end
+
+    let(:webrick_config) { super().merge SSLVersion: 'TLSv1.2' }
+
+    it 'should process the request' do
+      subject.multi_receive [event]
+      expect(last_request_body).to include '"message":"hello!"'
+    end
+  end
+
+  if tls_version_enabled_by_default?('TLSv1.3') && JOpenSSL::VERSION > '0.12'
+    context 'with supported_protocols set to 1.3' do
+      let(:config) { super().merge 'ssl_supported_protocols' => ['TLSv1.3'], 'ssl_verification_mode' => 'none' }
+
+      let(:webrick_config) { super().merge SSLVersion: 'TLSv1.3' }
+
+      it 'should process the request' do
+        subject.multi_receive [event]
+        expect(last_request_body).to include '"message":"hello!"'
+      end
     end
   end
 end

data/spec/spec_helper.rb

@@ -1,4 +1,145 @@
 # frozen_string_literal: true
 
+# Copyright 2023 Dynatrace LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 require 'logstash/devutils/rspec/spec_helper'
 require 'logstash/outputs/dynatrace'
+require 'logstash/codecs/plain'
+
+require 'sinatra'
+require 'webrick'
+require 'webrick/https'
+require 'openssl'
+
+PORT = rand(65_535 - 1024) + 1025
+
+module LogStash
+  module Outputs
+    class Dynatrace
+      attr_writer :agent
+      attr_reader :request_tokens
+    end
+  end
+end
+
+# NOTE: extend WEBrick with support for config[:SSLVersion]
+WEBrick::GenericServer.class_eval do
+  alias_method :__setup_ssl_context, :setup_ssl_context
+
+  def setup_ssl_context(config)
+    ctx = __setup_ssl_context(config)
+    ctx.ssl_version = config[:SSLVersion] if config[:SSLVersion]
+    ctx
+  end
+end
+
+# NOTE: that Sinatra startup and shutdown messages are directly logged to stderr so
+# it is not really possible to disable them without reopening stderr which is not advisable.
+#
+# == Sinatra (v1.4.6) has taken the stage on 51572 for development with backup from WEBrick
+# == Sinatra has ended his set (crowd applauds)
+#
+class TestApp < Sinatra::Base
+  set :environment, :production
+  set :sessions, false
+
+  @@server_settings = {
+    AccessLog: [], # disable WEBrick logging
+    Logger: WEBrick::BasicLog.new(nil, WEBrick::BasicLog::FATAL)
+  }
+
+  def self.server_settings
+    @@server_settings
+  end
+
+  def self.server_settings=(settings)
+    @@server_settings = settings
+  end
+
+  def self.multiroute(methods, path, &block)
+    methods.each do |method|
+      method.to_sym
+      send method, path, &block
+    end
+  end
+
+  class << self
+    attr_writer :last_request
+  end
+
+  class << self
+    attr_reader :last_request
+  end
+
+  class << self
+    attr_writer :retry_fail_count
+  end
+
+  def self.retry_fail_count
+    @retry_fail_count || 2
+  end
+
+  multiroute(%w[get post put patch delete], '/good') do
+    self.class.last_request = request
+    [200, 'YUP']
+  end
+
+  multiroute(%w[get post put patch delete], '/bad') do
+    self.class.last_request = request
+    [400, 'YUP']
+  end
+
+  multiroute(%w[get post put patch delete], '/retry') do
+    self.class.last_request = request
+
+    if self.class.retry_fail_count > 0
+      self.class.retry_fail_count -= 1
+      [429, "Will succeed in #{self.class.retry_fail_count}"]
+    else
+      [200, 'Done Retrying']
+    end
+  end
+end
+
+RSpec.configure do |config|
+  # http://stackoverflow.com/questions/6557079/start-and-call-ruby-http-server-in-the-same-script
+  def start_app_and_wait(app, opts = {})
+    queue = Queue.new
+
+    Thread.start do
+      begin
+        app.start!({ server: 'WEBrick', port: PORT }.merge(opts)) do |server|
+          yield(server) if block_given?
+          queue.push(server)
+        end
+      rescue StandardError => e
+        warn "Error starting app: #{e.inspect}" # ignore
+      end
+    end
+
+    queue.pop # blocks until the start! callback runs
+  end
+
+  config.extend(Module.new do
+    def tls_version_enabled_by_default?(tls_version)
+      context = javax.net.ssl.SSLContext.getInstance('TLS')
+      context.init nil, nil, nil
+      context.getDefaultSSLParameters.getProtocols.include? tls_version.to_s
+    rescue StandardError => e
+      warn "#{__method__} failed : #{e.inspect}"
+      nil
+    end
+  end)
+end

data/version.yaml

@@ -0,0 +1 @@
+logstash-output-dynatrace: '0.5.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-07-10 00:00:00.000000000 Z
+date: 2023-08-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json
@@ -44,6 +44,26 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
+- !ruby/object:Gem::Dependency
+  name: logstash-mixin-http_client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 6.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 8.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
@@ -72,6 +92,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -115,11 +163,12 @@ files:
 - Gemfile
 - LICENSE
 - README.md
+- lib/dynatrace/version.rb
 - lib/logstash/outputs/dynatrace.rb
 - logstash-output-dynatrace.gemspec
 - spec/outputs/dynatrace_spec.rb
 - spec/spec_helper.rb
-- version.rb
+- version.yaml
 homepage: https://github.com/dynatrace-oss/logstash-output-dynatrace
 licenses:
 - Apache-2.0