logstash-output-datadog_logs 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6f81e4d1f9e76634f0ee83cc649ad18a9ee9defc51c54c0cd630e8c9bae69d53
-  data.tar.gz: e15dfeba8935d842bc9db8ed8c1a8f166165376c4b0bc02d6feafca166313432
+  metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
+  data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
 SHA512:
-  metadata.gz: 46a621add6073375b653ce158b67d9aec0c1e6544c7d69ce7740b1b7930cb95a28525765048ebcc66975f9fa99b755f10fee32127a14b842ae0be80f034ebf11
-  data.tar.gz: e9276dc886a503ed450a20ce986861835496b5f8f1d991389afd5ece5a0933b18ae2cad57d211a74d068b51024c560eabb7a859ba17ebd6b6f9ca44fe71fb2a2
+  metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
+  data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.4.1
+  - Fix HTTP bug when remote server is timing out 
+
 ## 0.4.0
   - Enable HTTP forwarding for logs
   - Provide an option to disable SSL hostname verification for HTTPS

data/lib/logstash/outputs/datadog_logs.rb

@@ -50,15 +50,19 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   def multi_receive(events)
     return if events.empty?
     encoded_events = @codec.multi_encode(events)
-    if @use_http
-      batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
-      batches.each do |batched_event|
-        process_encoded_payload(format_http_event_batch(batched_event))
-      end
-    else
-      encoded_events.each do |encoded_event|
-        process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
+    begin
+      if @use_http
+        batches = batch_http_events(encoded_events, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
+        batches.each do |batched_event|
+          process_encoded_payload(format_http_event_batch(batched_event))
+        end
+      else
+        encoded_events.each do |encoded_event|
+          process_encoded_payload(format_tcp_event(encoded_event.last, @api_key, DD_MAX_BATCH_SIZE))
+        end
       end
+    rescue => e
+      @logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
     end
   end
 
@@ -158,12 +162,14 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
         send(payload)
       rescue RetryableError => e
         if retries < max_retries || max_retries < 0
-          @logger.warn("Retrying ", :exception => e, :backtrace => e.backtrace)
+          @logger.warn("Retrying send due to: #{e.message}")
           sleep backoff
           backoff = 2 * backoff unless backoff > max_backoff
           retries += 1
           retry
         end
+      rescue => ex
+        @logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
       end
     end
 
@@ -179,6 +185,13 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   class DatadogHTTPClient < DatadogClient
     require "manticore"
 
+    RETRYABLE_EXCEPTIONS = [
+        ::Manticore::Timeout,
+        ::Manticore::SocketException,
+        ::Manticore::ClientProtocolException,
+        ::Manticore::ResolutionFailure
+    ]
+
     def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
       @logger = logger
       protocol = use_ssl ? "https" : "http"
@@ -194,13 +207,27 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
     end
 
     def send(payload)
-      response = @client.post(@url, :body => payload, :headers => @headers).call
-      if response.code >= 500
-        raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
-      end
-      if response.code >= 400
-        @logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
+      begin
+        response = @client.post(@url, :body => payload, :headers => @headers).call
+        if response.code >= 500
+          raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
+        end
+        if response.code >= 400
+          @logger.error("Unable to send payload due to client error: #{response.code} #{response.body}")
+        end
+      rescue => client_exception
+        should_retry = retryable_exception?(client_exception)
+        if should_retry
+          raise RetryableError.new "Unable to send payload #{client_exception.message}"
+        else
+          raise client_exception
+        end
       end
+
+    end
+
+    def retryable_exception?(exception)
+      RETRYABLE_EXCEPTIONS.any? { |e| exception.is_a?(e) }
     end
 
     def close

data/logstash-output-datadog_logs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-datadog_logs'
-  s.version       = '0.4.0'
+  s.version       = '0.4.1'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
   s.homepage      = 'https://www.datadoghq.com/'
@@ -22,4 +22,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'logstash-codec-json'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'webmock'
 end

data/spec/outputs/datadog_logs_spec.rb

@@ -5,6 +5,7 @@
 
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/outputs/datadog_logs"
+require 'webmock/rspec'
 
 describe LogStash::Outputs::DatadogLogs do
   context "should register" do
@@ -79,7 +80,7 @@ describe LogStash::Outputs::DatadogLogs do
     end
 
     it "should truncate events whose length is bigger than the max request size" do
-      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}),"foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
+      input_events = [[LogStash::Event.new({"message" => "dd1"}), "dd1"], [LogStash::Event.new({"message" => "foobarfoobarfoobar"}), "foobarfoobarfoobar"], [LogStash::Event.new({"message" => "dd2"}), "dd2"]]
       actual_events = subject.batch_http_events(input_events, 10, 3)
       expect(actual_events.length).to eq(3)
       expect(actual_events[0][0]).to eq("dd1")
@@ -88,6 +89,80 @@ describe LogStash::Outputs::DatadogLogs do
     end
   end
 
+  context "when facing HTTP connection issues" do
+    it "should retry when server is returning 5XX" do
+      api_key = 'XXX'
+      stub_dd_request_with_return_code(api_key, 500)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should not retry when server is returning 4XX" do
+      api_key = 'XXX'
+      stub_dd_request_with_return_code(api_key, 400)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to_not raise_error
+    end
+
+    it "should retry when facing a timeout exception from manticore" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, Manticore::Timeout)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should retry when facing a socket exception from manticore" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, Manticore::SocketException)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should retry when facing a client protocol exception from manticore" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should retry when facing a dns failure from manticore" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should retry when facing a socket timeout from manticore" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+    end
+
+    it "should not retry when facing any other general error" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, StandardError)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send(payload) }.to raise_error(StandardError)
+    end
+
+    it "should not stop the forwarder when facing any client uncaught exception" do
+      api_key = 'XXX'
+      stub_dd_request_with_error(api_key, StandardError)
+      payload = '{}'
+      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
+      expect { client.send_retries(payload, 2, 2) }.to_not raise_error
+    end
+  end
+
   context "when using TCP" do
     it "should re-encode events" do
       input_event = "{message=dd}"
@@ -101,4 +176,25 @@ describe LogStash::Outputs::DatadogLogs do
       expect(encoded_event).to eq("xxx {...TRUNCATED...")
     end
   end
+
+  def stub_dd_request_with_return_code(api_key, return_code)
+    stub_dd_request(api_key).
+        to_return(status: return_code, body: "", headers: {})
+  end
+
+  def stub_dd_request_with_error(api_key, error)
+    stub_dd_request(api_key).
+        to_raise(error)
+  end
+
+  def stub_dd_request(api_key)
+    stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
+        with(
+            body: "{}",
+            headers: {
+                'Connection' => 'Keep-Alive',
+                'Content-Type' => 'application/json'
+            })
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-datadog_logs
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Datadog
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-25 00:00:00.000000000 Z
+date: 2020-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -73,6 +73,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: webmock
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email: support@datadoghq.com
 executables: []