logstash-output-datadog_logs 0.5.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbfe2fc6bf43df61e35e421486e1f7143e4acce4725386decdc05f946249009d
-  data.tar.gz: 36bdf364a1d4ae4d522712914cca7eff722b7f115bf553c6cfeee3d4869a7744
+  metadata.gz: 35f5ce024bffbeaea8a434b00d7aa2e5c1bb2ce2a439bb69dd6f73dd7b92e269
+  data.tar.gz: f5565d500266000af583f7b647e3d33255e64dc8260a7c3d11ce7cb7ed3c2a5d
 SHA512:
-  metadata.gz: 44eb1829c931dda34c65c962084d4be14ae5ef510000b32dec93e5b465a2e12dcc4ac27b7a53cb603f9c4d1e8a36c6c1262279f3c86317fef76d9448fcb0c125
-  data.tar.gz: ee06691a2c65deb5bf743f47072e35cf9615a238dd2a8f40f4db4459ff4e3fed3025908993f4ddf8f4814cd81117961919037c891d53e76b448058f838d77834
+  metadata.gz: 7e3503a6f977e30d581f964825f78348278533007084cbf584c05f8be6f8806c02dd43e5b32b8f7fc1cb9ba4ba47a66f884a95521e46d363e13495c4358e556f
+  data.tar.gz: 27f9fb1522429578e3bcb05575feddb90cf21343de8329d5fe151e25311d0bd91f8a85cac1bae3e4a7cb23429a10dbf735dd3d15321d9ff04bc943311ae98fca

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.5.3
+  - Now is threadsafe: supports shared concurrency.
+  - Increase `DD_MAX_BATCH_LENGTH` to `1000`
+
+## 0.5.2
+  - Now checks if logstash is being shutdown
+
 ## 0.5.1
   - Support using HTTP proxies, adding the `http_proxy` parameter.
 

data/README.md

@@ -5,7 +5,7 @@ DatadogLogs lets you send logs to Datadog based on LogStash events.
 
 ## Requirements
 
-The plugin relies upon the `zlib` library for compressing data.
+The plugin relies upon the `zlib` library for compressing data. Successfully tested with Logstash 6.x, 7.x and 8.x.
 
 ## How to install it?
 
@@ -35,8 +35,8 @@ To enable TCP forwarding, configure your forwarder with:
 output {
     datadog_logs {
         api_key => "<DATADOG_API_KEY>"
-        host => "tcp-intake.logs.datadoghq.com"
-        port => 10516
+        host => "intake.logs.datadoghq.com"
+        port => 443
         use_http => false
     }
 }

data/lib/logstash/outputs/datadog_logs.rb

@@ -14,13 +14,15 @@ require_relative "version"
 # based on LogStash events.
 class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
 
-  # Respect limit documented at https://docs.datadoghq.com/api/?lang=bash#logs
-  DD_MAX_BATCH_LENGTH = 500
+  # Respect limit documented at https://docs.datadoghq.com/api/latest/logs/#send-logs
+  DD_MAX_BATCH_LENGTH = 1000
   DD_MAX_BATCH_SIZE = 5000000
   DD_TRUNCATION_SUFFIX = "...TRUNCATED..."
 
   config_name "datadog_logs"
 
+  concurrency :shared
+
   default :codec, "json"
 
   # Datadog configuration parameters
@@ -65,7 +67,11 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
         end
       end
     rescue => e
-      @logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
+      if e.is_a?(InterruptedError)
+        raise e
+      else
+        @logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
+      end
     end
   end
 
@@ -148,7 +154,7 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   # Build a new transport client
   def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, port, use_compression, force_v1_routes, http_proxy)
     if use_http
-      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes, http_proxy
+      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes, http_proxy, -> { defined?(pipeline_shutdown_requested?) ? pipeline_shutdown_requested? : false }
     else
       DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, port
     end
@@ -157,6 +163,9 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
   class RetryableError < StandardError;
   end
 
+  class InterruptedError < StandardError;
+  end
+
   class DatadogClient
     def send_retries(payload, max_retries, max_backoff)
       backoff = 1
@@ -166,17 +175,36 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
       rescue RetryableError => e
         if retries < max_retries || max_retries < 0
           @logger.warn("Retrying send due to: #{e.message}")
-          sleep backoff
+          interruptableSleep(backoff)
           backoff = 2 * backoff unless backoff > max_backoff
           retries += 1
           retry
         end
-        @logger.error("Max number of retries reached, dropping message. Last exception: #{ex.message}")
+        @logger.error("Max number of retries reached, dropping message. Last exception: #{e.message}")
       rescue => ex
-        @logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
+        if ex.is_a?(InterruptedError)
+          raise ex
+        else
+          @logger.error("Unmanaged exception while sending log to datadog #{ex.message}")
+        end
       end
     end
 
+    def interruptableSleep(duration)
+      amountSlept = 0
+      while amountSlept < duration
+        sleep 1
+        amountSlept += 1
+        if interrupted?
+          raise InterruptedError.new "Interrupted while backing off"
+        end
+      end
+    end
+
+    def interrupted?
+      false
+    end
+
     def send(payload)
       raise NotImplementedError, "Datadog transport client should implement the send method"
     end
@@ -196,7 +224,8 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
         ::Manticore::ResolutionFailure
     ]
 
-    def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes, http_proxy)
+    def initialize(logger, use_ssl, no_ssl_validation, host, port, use_compression, api_key, force_v1_routes, http_proxy, interruptedLambda = nil)
+      @interruptedLambda = interruptedLambda
       @logger = logger
       protocol = use_ssl ? "https" : "http"
 
@@ -224,6 +253,14 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
       @client = Manticore::Client.new(config)
     end
 
+    def interrupted?
+      if @interruptedLambda
+        return @interruptedLambda.call
+      end
+
+      false
+    end
+
     def send(payload)
       begin
         response = @client.post(@url, :body => payload, :headers => @headers).call
@@ -263,6 +300,7 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
       @no_ssl_validation = no_ssl_validation
       @host = host
       @port = port
+      @send_mutex = Mutex.new
     end
 
     def connect
@@ -283,13 +321,15 @@ class LogStash::Outputs::DatadogLogs < LogStash::Outputs::Base
     end
 
     def send(payload)
-      begin
-        @socket ||= connect
-        @socket.puts(payload)
-      rescue => e
-        @socket.close rescue nil
-        @socket = nil
-        raise RetryableError.new "Unable to send payload: #{e.message}."
+      @send_mutex.synchronize do
+        begin
+          @socket ||= connect
+          @socket.puts(payload)
+        rescue => e
+          @socket.close rescue nil
+          @socket = nil
+          raise RetryableError.new "Unable to send payload: #{e.message}."
+        end
       end
     end
 

data/lib/logstash/outputs/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DatadogLogStashPlugin
-  VERSION = '0.5.1'
-end
+  VERSION = '0.5.3'
+end

data/logstash-output-datadog_logs.gemspec

@@ -19,13 +19,22 @@ Gem::Specification.new do |s|
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})
 
-  # Special flag to let us know this is actually a logstash plugin
-  s.metadata = {"logstash_plugin" => "true", "logstash_group" => "output"}
+  s.metadata = {
+    'bug_tracker_uri'   => 'https://github.com/DataDog/logstash-output-datadog_logs/issues',
+    'changelog_uri'     => 'https://github.com/DataDog/logstash-output-datadog_logs/blob/master/CHANGELOG.md',
+    'documentation_uri' => 'https://github.com/DataDog/logstash-output-datadog_logs/blob/master/README.md',
+    'source_code_uri'   => 'https://github.com/DataDog/logstash-output-datadog_logs',
+
+    # Special flag to let us know this is actually a logstash plugin
+    "logstash_plugin" => "true",
+    "logstash_group" => "output",
+  }
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
   s.add_runtime_dependency 'logstash-codec-json'
+  s.add_runtime_dependency 'jar-dependencies', '>= 0.3.12', '< 0.5.0'
 
   s.add_development_dependency 'logstash-devutils'
   s.add_development_dependency 'webmock'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-datadog_logs
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Datadog
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-03-20 00:00:00.000000000 Z
+date: 2025-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -65,6 +65,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
+  name: jar-dependencies
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.12
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.5.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -113,6 +133,10 @@ homepage: https://www.datadoghq.com/
 licenses:
 - Apache-2.0
 metadata:
+  bug_tracker_uri: https://github.com/DataDog/logstash-output-datadog_logs/issues
+  changelog_uri: https://github.com/DataDog/logstash-output-datadog_logs/blob/master/CHANGELOG.md
+  documentation_uri: https://github.com/DataDog/logstash-output-datadog_logs/blob/master/README.md
+  source_code_uri: https://github.com/DataDog/logstash-output-datadog_logs
   logstash_plugin: 'true'
   logstash_group: output
 post_install_message:
@@ -130,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.29
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: DatadogLogs lets you send logs to Datadog based on LogStash events.