logstash-output-dynatrace 0.2.1 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a8f5b5423d993317f8f73d5416c7c17b2961c833ca95da455f2f4742bcbe81b
-  data.tar.gz: 249eaf43d553a14a30ee84328514d6771acba98c8ef4da3bee738169b73008ec
+  metadata.gz: 06c22415ca3c16b7214c6aca478edd757db1302dff08fd706e80402bd3627392
+  data.tar.gz: af603917cd20c72d12d98add8409e1d171e7ca2edc632d42378f8a54f5a018a5
 SHA512:
-  metadata.gz: bdb3df6d43d9e2361958b925830af0ec1f686fa02d2d9adbd6488e063e890885aae42589cf5d60e4a2b53c89b6d4a121e9dff1b3500fe5773c1a1cf8203d415a
-  data.tar.gz: 20525e9f0cea6acb310656de596404e086ac755508dfa6d603a0d7b6f1dfb030c36ec32db60cf89fbe7942732c14dd2c0fbea375df5874c981d5805c16ce67a3
+  metadata.gz: 0c5b60e14c175f9e7937c79131f872768034e191df4c12e9c58b88abcb190208148fb77d76c2d93e42aff8a2599e1b89621f165740910b0ee0561273f529fcc4
+  data.tar.gz: ad04bc45337935f619619bef6765dee1335285fc7fa705147bfdce7888f88158c829f62ad1a84f4a26c729710292143ffd8efa217954ce5c1bc9f4c458a64abe

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.3.1
+ - Log and re-raise unknown errors
+ - Use [password](https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html#password) type for `api_key` configuration
+
+## 0.3.0
+ - Log response bodies on client errors
+
 ## 0.2.0
  - Add retries with exponential backoff (#8)
 

data/lib/logstash/outputs/dynatrace.rb

@@ -19,7 +19,7 @@ require 'logstash/outputs/base'
 require 'logstash/json'
 
 MAX_RETRIES = 5
-PLUGIN_VERSION = '0.2.1'
+PLUGIN_VERSION = '0.3.1'
 
 module LogStash
   module Outputs
@@ -36,7 +36,7 @@ module LogStash
       config :ingest_endpoint_url, validate: :uri, required: true
 
       # The API token to use to authenticate requests to the log ingestion endpoint. Must have `logs.ingest` (Ingest Logs) scope.
-      config :api_key, validate: :string, required: true
+      config :api_key, validate: :password, required: true
 
       # Disable SSL validation by setting :verify_mode OpenSSL::SSL::VERIFY_NONE
       config :ssl_verify_none, validate: :boolean, default: false
@@ -61,9 +61,9 @@ module LogStash
 
       def headers
         {
-          'User-Agent' => "logstash-output-dynatrace v#{PLUGIN_VERSION}",
+          'User-Agent' => "logstash-output-dynatrace/#{PLUGIN_VERSION}",
           'Content-Type' => 'application/json; charset=utf-8',
-          'Authorization' => "Api-Token #{@api_key}"
+          'Authorization' => "Api-Token #{@api_key.value}"
         }
       end
 
@@ -76,29 +76,26 @@ module LogStash
         begin
           request = Net::HTTP::Post.new(uri, headers)
           request.body = "#{LogStash::Json.dump(events.map(&:to_hash)).chomp}\n"
-          response = send(request)
-          return if response.is_a? Net::HTTPSuccess
-
-          failure_message = "Dynatrace returned #{response.code} #{response.message}."
-
-          if response.is_a? Net::HTTPServerError
-            raise RetryableError.new failure_message
-          end
-
-          if response.is_a? Net::HTTPNotFound
-            @logger.error("#{failure_message} Please check that log ingest is enabled and your API token has the `logs.ingest` (Ingest Logs) scope.")
-            return
+          response = @client.request(request)
+
+          case response
+          when Net::HTTPSuccess
+            @logger.debug("successfully sent #{events.length} events#{" with #{retries} retries" if retries > 0}")
+          when Net::HTTPServerError
+            @logger.error("Encountered an HTTP server error", :message => response.message, :code => response.code, :body => response.body) if retries == 0
+          when Net::HTTPNotFound
+            @logger.error("Encountered a 404 Not Found error. Please check that log ingest is enabled and your API token has the `logs.ingest` (Ingest Logs) scope.", :message => response.message, :code => response.code)
+          when Net::HTTPClientError
+            @logger.error("Encountered an HTTP client error", :message => response.message, :code => response.code, :body => response.body)
+          else
+            @logger.error("Encountered an unexpected response code", :message => response.message, :code => response.code)
           end
 
-          if response.is_a? Net::HTTPClientError
-            @logger.error(failure_message)
-            return
-          end
+          raise RetryableError.new "code #{response.code}" if retryable(response)
 
-          @logger.debug("successfully sent #{events.length} events")
         rescue Net::HTTPBadResponse, RetryableError => e
           # indicates a protocol error
-          if retries < MAX_RETRIES  
+          if retries < MAX_RETRIES
             sleep_seconds = 2 ** retries
             @logger.warn("Failed to contact dynatrace: #{e.message}. Trying again after #{sleep_seconds} seconds.")
             sleep sleep_seconds
@@ -108,13 +105,14 @@ module LogStash
             @logger.error("Failed to export logs to Dynatrace.")
             return
           end
+        rescue StandardError => e
+          @logger.error("Unknown error raised", :error => e.inspect)
+          raise e
         end
-
-        @logger.debug("Successfully exported #{events.length} events with #{retries} retries")
       end
 
-      def send(request)
-        @client.request(request)
+      def retryable(response)
+        return response.is_a? Net::HTTPServerError
       end
     end
   end

data/logstash-output-dynatrace.gemspec

@@ -43,11 +43,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-json'
   s.add_runtime_dependency 'logstash-core-plugin-api', '>= 2.0.0', '< 3'
 
-  s.add_development_dependency 'insist'
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'logstash-input-generator'
-  s.add_development_dependency 'sinatra'
-  s.add_development_dependency 'webrick'
 
   s.add_development_dependency 'rubocop', '1.9.1'
   s.add_development_dependency 'rubocop-rake', '0.5.1'

data/spec/outputs/dynatrace_spec.rb

@@ -18,8 +18,6 @@ require_relative '../spec_helper'
 require_relative '../../version'
 require 'logstash/codecs/plain'
 require 'logstash/event'
-require 'sinatra'
-require 'insist'
 require 'net/http'
 require 'json'
 
@@ -32,96 +30,122 @@ describe LogStash::Outputs::Dynatrace do
   end
   let(:url) { "http://localhost/good" }
   let(:key) { 'api.key' }
+
   let(:subject) { LogStash::Outputs::Dynatrace.new({ 'api_key' => key, 'ingest_endpoint_url' => url }) }
+  let(:client) { subject.instance_variable_get(:@client) }
+
+  let(:ok) { Net::HTTPOK.new "1.1", "200", "OK" }
+  let(:server_error) { Net::HTTPServerError.new "1.1", "500", "Internal Server Error" }
+  let(:client_error) { Net::HTTPClientError.new("1.1", '400', 'Client error') }
+  let(:not_found) { Net::HTTPNotFound.new "1.1", "404", "Not Found" }
+
+  let(:body) { "this is a failure" }
 
   before do
     subject.register
   end
 
   it 'does not send empty events' do
-    allow(subject).to receive(:send)
+    expect(client).to_not receive(:request)
     subject.multi_receive([])
-    expect(subject).to_not have_received(:send)
   end
 
   context 'server response success' do
     it 'sends events' do
-      allow(subject).to receive(:send) do |req|
+      expect(client).to receive(:request) do |req|
         body = JSON.parse(req.body)
         expect(body.length).to eql(2)
         expect(body[0]['message']).to eql('message 1')
         expect(body[0]['@timestamp']).to eql('2021-06-25T15:46:45.693Z')
         expect(body[1]['message']).to eql('message 2')
         expect(body[1]['@timestamp']).to eql('2021-06-25T15:46:46.693Z')
-        Net::HTTPOK.new "1.1", "200", "OK"
+        ok
       end
       subject.multi_receive(events)
-      expect(subject).to have_received(:send)
     end
 
     it 'includes authorization header' do
-      allow(subject).to receive(:send) do |req|
+      expect(client).to receive(:request) do |req|
         expect(req['Authorization']).to eql("Api-Token #{key}")
-        Net::HTTPOK.new "1.1", "200", "OK"
+        ok
       end
       subject.multi_receive(events)
-      expect(subject).to have_received(:send)
     end
 
     it 'includes content type header' do
-      allow(subject).to receive(:send) do |req|
+      expect(client).to receive(:request) do |req|
         expect(req['Content-Type']).to eql('application/json; charset=utf-8')
-        Net::HTTPOK.new "1.1", "200", "OK"
+        ok
       end
       subject.multi_receive(events)
-      expect(subject).to have_received(:send)
     end
 
     it 'includes user agent' do
-      allow(subject).to receive(:send) do |req|
-        expect(req['User-Agent']).to eql("logstash-output-dynatrace v#{::DynatraceConstants::VERSION}")
-        Net::HTTPOK.new "1.1", "200", "OK"
+      expect(client).to receive(:request) do |req|
+        expect(req['User-Agent']).to eql("logstash-output-dynatrace/#{::DynatraceConstants::VERSION}")
+        ok
       end
       subject.multi_receive(events)
-      expect(subject).to have_received(:send)
     end
 
     it 'does not log on success' do
       allow(subject.logger).to receive(:debug)
-      allow(subject.logger).to receive(:info) { raise "should not log" }
-      allow(subject.logger).to receive(:error) { raise "should not log" }
-      allow(subject.logger).to receive(:warn) { raise "should not log" }
-      allow(subject).to receive(:send) do |req|
-        Net::HTTPOK.new "1.1", "200", "OK"
-      end
+      expect(subject.logger).to_not receive(:info)
+      expect(subject.logger).to_not receive(:error)
+      expect(subject.logger).to_not receive(:warn)
+      expect(client).to receive(:request) { ok }
       subject.multi_receive(events)
-      expect(subject).to have_received(:send)
     end
   end
 
-  context 'with bad client request' do
-    it 'does not retry on 404' do
-      allow(subject).to receive(:send) { Net::HTTPNotFound.new "1.1", "404", "Not Found" }
+  context 'with server error' do
+    it 'retries 5 times with exponential backoff' do
+      # This prevents the elusive "undefined method `close' for nil:NilClass" error.
+      expect(server_error).to receive(:body) { body }.once
+      expect(subject.logger).to receive(:error).with("Encountered an HTTP server error", {:body=>body, :code=>"500", :message=> "Internal Server Error"}).once
+      expect(client).to receive(:request) { server_error }.exactly(6).times
+
+
+      expect(subject).to receive(:sleep).with(1).ordered
+      expect(subject).to receive(:sleep).with(2).ordered
+      expect(subject).to receive(:sleep).with(4).ordered
+      expect(subject).to receive(:sleep).with(8).ordered
+      expect(subject).to receive(:sleep).with(16).ordered
+
+      expect(subject.logger).to receive(:error).with("Failed to export logs to Dynatrace.")
       subject.multi_receive(events)
-      expect(subject).to have_received(:send).once
     end
   end
 
-  context 'with server error' do
-    it 'retries 5 times with exponential backoff' do
-      allow(subject).to receive(:sleep)
-      allow(subject).to receive(:send) { Net::HTTPInternalServerError.new "1.1", "500", "Internal Server Error" }
+  context 'with client error' do
+    it 'does not retry on 404' do
+      allow(subject.logger).to receive(:error)
+      expect(client).to receive(:request) { not_found }.once
+      subject.multi_receive(events)
+    end
+
+    it 'logs the response body' do
+      expect(client).to receive(:request) { client_error }
+      # This prevents the elusive "undefined method `close' for nil:NilClass" error.
+      expect(client_error).to receive(:body) { body }
+
+      expect(subject.logger).to receive(:error).with("Encountered an HTTP client error",
+        {:body=>body, :code=>"400", :message=> "Client error"})
 
       subject.multi_receive(events)
+    end
+  end
 
-      expect(subject).to have_received(:sleep).with(1).ordered
-      expect(subject).to have_received(:sleep).with(2).ordered
-      expect(subject).to have_received(:sleep).with(4).ordered
-      expect(subject).to have_received(:sleep).with(8).ordered
-      expect(subject).to have_received(:sleep).with(16).ordered
+  context 'when an unknown error occurs' do
+    it 'logs and re-raises the error' do
+      class BadEvents
+        def length
+          1
+        end
+      end
 
-      expect(subject).to have_received(:sleep).exactly(5).times
-      expect(subject).to have_received(:send).exactly(6).times
+      expect(subject.logger).to receive(:error)
+      expect { subject.multi_receive(BadEvents.new) }.to raise_error(StandardError)
     end
   end
 end

data/version.rb

@@ -16,5 +16,5 @@
 
 module DynatraceConstants
   # Also required to change the version in lib/logstash/outputs/dynatrace.rb
-  VERSION = '0.2.1'
+  VERSION = '0.3.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-dynatrace
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.1
 platform: ruby
 authors:
 - Dynatrace Open Source Engineering
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-14 00:00:00.000000000 Z
+date: 2022-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-codec-json
@@ -44,20 +44,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '3'
-- !ruby/object:Gem::Dependency
-  name: insist
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
@@ -86,34 +72,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: sinatra
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: webrick
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement