logstash-output-datadog_logs 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8148c91e062cc8e1999b780d95f529d9cd1516664f7872a96da51e1656c2b71d
-  data.tar.gz: 6fd0ddbf65b8240e1609c0c74a57f7ffcf50e7945cf2601484007725ff5465b2
+  metadata.gz: 635ef3d038ba35b57528f006e81455e20b7cd27217a8fe6188f65d5229e13fe3
+  data.tar.gz: 6b4325a4cf44791c40aa18360ba7fd9c92a451fb94b182c5a0803de711e3d462
 SHA512:
-  metadata.gz: ce5a190b7c1550eb4edcec848aee451abd9814894da3e9c4302611c83c1e9414373086b69e60372bfde6cd6ad3da611146ac82413563b95c49721bdb0a024c68
-  data.tar.gz: 33c1d8afda9da935ccb5f9ee43388f424ab7c8fefdb61b16ef9ac474f5d1c5834c053f911bfadce596291ea62148ff0f0a1beaf7bf56819ad433ac7fea92ae71
+  metadata.gz: e555d642d57cd65971b9e05c2cd6d76c8919ec7413d1dab24eb9fe6eb8d9be3ce51dcfd0dc4751d77f052d87f49ea032fb6ce488c6b99350ae31b41f93908d2e
+  data.tar.gz: 2b89082813230155ebc747da6e5073aaa7f74d0ad2b1c922e8c577f670af6a2306447a0384229a0382f2c499de1741b20d11b03bc89f32f314939fee269e17c3

data/CHANGELOG.md

@@ -1,5 +1,8 @@
+## 0.5.0
+  - Support Datadog v2 endpoints #28
+
 ## 0.4.1
-  - Fix HTTP bug when remote server is timing out 
+  - Fix HTTP bug when remote server is timing out
 
 ## 0.4.0
   - Enable HTTP forwarding for logs

data/lib/logstash/outputs/datadog_logs.rb

@@ -8,6 +8,7 @@ require "logstash/outputs/base"
 require "logstash/namespace"
 require "zlib"
 
+require_relative "version"
 
 # DatadogLogs lets you send logs to Datadog
 # based on LogStash events.
@@ -33,11 +34,12 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   config :use_compression, :validate => :boolean, :required => false, :default => true
   config :compression_level, :validate => :number, :required => false, :default => 6
   config :no_ssl_validation, :validate => :boolean, :required => false, :default => false
+  config :force_v1_routes, :validate => :boolean, :required => false, :default => false # force using deprecated v1 routes
 
   # Register the plugin to logstash
   public
   def register
-    @client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression)
+    @client = new_client(@logger, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @port, @use_compression, @force_v1_routes)
   end
 
   # Logstash shutdown hook
@@ -143,9 +145,9 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   end
 
   # Build a new transport client
-  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression)
+  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression, force_v1_routes)
     if use_http
-      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key
+      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes
     else
       DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, port
     end
@@ -192,15 +194,26 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
         ::Manticore::ResolutionFailure
     ]
 
-    def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key)
+    def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes)
       @logger = logger
       protocol = use_ssl ? "https" : "http"
-      @url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
+
       @headers = {"Content-Type" => "application/json"}
       if use_compression
         @headers["Content-Encoding"] = "gzip"
       end
-      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled"))
+
+      if force_v1_routes
+        @url = "#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}"
+      else
+        @url = "#{protocol}://#{host}:#{port.to_s}/api/v2/logs"
+        @headers["DD-API-KEY"] = api_key
+        @headers["DD-EVP-ORIGIN"] = "logstash"
+        @headers["DD-EVP-ORIGIN-VERSION"] = DatadogLogStashPlugin::VERSION
+      end
+
+      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
+
       config = {}
       config[:ssl][:verify] = :disable if no_ssl_validation
       @client = Manticore::Client.new(config)
@@ -209,7 +222,8 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
     def send(payload)
       begin
         response = @client.post(@url, :body => payload, :headers => @headers).call
-        if response.code >= 500
+        # in case of error or 429, we will retry sending this payload
+        if response.code >= 500 || response.code == 429
           raise RetryableError.new "Unable to send payload: #{response.code} #{response.body}"
         end
         if response.code >= 400

data/lib/logstash/outputs/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DatadogLogStashPlugin
+  VERSION = '0.5.0'
+end

data/logstash-output-datadog_logs.gemspec

@@ -1,6 +1,12 @@
+# Load version.rb containing the DatadogLogStashPlugin::VERSION
+# for current Gem version.
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "logstash/outputs/version.rb"
+
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-datadog_logs'
-  s.version       = '0.4.1'
+  s.version       = DatadogLogStashPlugin::VERSION
   s.licenses      = ['Apache-2.0']
   s.summary       = 'DatadogLogs lets you send logs to Datadog based on LogStash events.'
   s.homepage      = 'https://www.datadoghq.com/'
@@ -21,6 +27,6 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
   s.add_runtime_dependency 'logstash-codec-json'
 
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', "= 1.3.6"
   s.add_development_dependency 'webmock'
 end

data/spec/outputs/datadog_logs_spec.rb

@@ -90,76 +90,86 @@ describe LogStash::Outputs::DatadogLogs do
   end
 
   context "when facing HTTP connection issues" do
-    it "should retry when server is returning 5XX" do
-      api_key = 'XXX'
-      stub_dd_request_with_return_code(api_key, 500)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should not retry when server is returning 4XX" do
-      api_key = 'XXX'
-      stub_dd_request_with_return_code(api_key, 400)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to_not raise_error
-    end
-
-    it "should retry when facing a timeout exception from manticore" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, Manticore::Timeout)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should retry when facing a socket exception from manticore" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, Manticore::SocketException)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should retry when facing a client protocol exception from manticore" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, Manticore::ClientProtocolException)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should retry when facing a dns failure from manticore" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, Manticore::ResolutionFailure)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should retry when facing a socket timeout from manticore" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, Manticore::SocketTimeout)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
-    end
-
-    it "should not retry when facing any other general error" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, StandardError)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send(payload) }.to raise_error(StandardError)
-    end
-
-    it "should not stop the forwarder when facing any client uncaught exception" do
-      api_key = 'XXX'
-      stub_dd_request_with_error(api_key, StandardError)
-      payload = '{}'
-      client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key
-      expect { client.send_retries(payload, 2, 2) }.to_not raise_error
+    [true, false].each do |force_v1_routes|
+        it "should retry when server is returning 5XX " + (force_v1_routes ? "using v1 routes" : "using v2 routes") do
+          api_key = 'XXX'
+          stub_dd_request_with_return_code(api_key, 500, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should not retry when server is returning 4XX" do
+          api_key = 'XXX'
+          stub_dd_request_with_return_code(api_key, 400, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to_not raise_error
+        end
+
+        it "should retry when server is returning 429" do
+          api_key = 'XXX'
+          stub_dd_request_with_return_code(api_key, 429, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should retry when facing a timeout exception from manticore" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, Manticore::Timeout, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should retry when facing a socket exception from manticore" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, Manticore::SocketException, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should retry when facing a client protocol exception from manticore" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, Manticore::ClientProtocolException, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should retry when facing a dns failure from manticore" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, Manticore::ResolutionFailure, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should retry when facing a socket timeout from manticore" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, Manticore::SocketTimeout, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(LogStash::Outputs::DatadogLogs::RetryableError)
+        end
+
+        it "should not retry when facing any other general error" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send(payload) }.to raise_error(StandardError)
+        end
+
+        it "should not stop the forwarder when facing any client uncaught exception" do
+          api_key = 'XXX'
+          stub_dd_request_with_error(api_key, StandardError, force_v1_routes)
+          payload = '{}'
+          client = LogStash::Outputs::DatadogLogs::DatadogHTTPClient.new Logger.new(STDOUT), false, false, "datadog.com", 80, false, api_key, force_v1_routes
+          expect { client.send_retries(payload, 2, 2) }.to_not raise_error
+        end
     end
   end
 
@@ -177,24 +187,36 @@ describe LogStash::Outputs::DatadogLogs do
     end
   end
 
-  def stub_dd_request_with_return_code(api_key, return_code)
-    stub_dd_request(api_key).
+  def stub_dd_request_with_return_code(api_key, return_code, force_v1_routes)
+    stub_dd_request(api_key, force_v1_routes).
         to_return(status: return_code, body: "", headers: {})
   end
 
-  def stub_dd_request_with_error(api_key, error)
-    stub_dd_request(api_key).
+  def stub_dd_request_with_error(api_key, error, force_v1_routes)
+    stub_dd_request(api_key, force_v1_routes).
         to_raise(error)
   end
 
-  def stub_dd_request(api_key)
-    stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
+  def stub_dd_request(api_key, force_v1_routes)
+    if force_v1_routes
+      stub_request(:post, "http://datadog.com/v1/input/#{api_key}").
+        with(
+          body: "{}",
+          headers: {
+            'Connection' => 'Keep-Alive',
+            'Content-Type' => 'application/json'
+        })
+    else
+      stub_request(:post, "http://datadog.com/api/v2/logs").
         with(
-            body: "{}",
-            headers: {
-                'Connection' => 'Keep-Alive',
-                'Content-Type' => 'application/json'
-            })
+          body: "{}",
+          headers: {
+            'Connection' => 'Keep-Alive',
+            'Content-Type' => 'application/json',
+            'DD-API-KEY' => "#{api_key}",
+            'DD-EVP-ORIGIN' => 'logstash',
+            'DD-EVP-ORIGIN-VERSION' => DatadogLogStashPlugin::VERSION
+       })
+    end
   end
-
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-datadog_logs
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Datadog
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-13 00:00:00.000000000 Z
+date: 2022-04-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -62,17 +62,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.6
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.6
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -100,6 +100,7 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/outputs/datadog_logs.rb
+- lib/logstash/outputs/version.rb
 - logstash-output-datadog_logs.gemspec
 - spec/outputs/datadog_logs_spec.rb
 homepage: https://www.datadoghq.com/
@@ -124,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.7.10
+rubygems_version: 2.7.6
 signing_key:
 specification_version: 4
 summary: DatadogLogs lets you send logs to Datadog based on LogStash events.