fluent-plugin-datadog 0.11.1 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b5d03f1a431fabff4b1578b16530b66af5c4ed9108fafd3149ebff3c3aaeae97
-  data.tar.gz: b0de09ecbf8cd35ffb0b8e5135dc9b362e9a34f8ebec7af1574c54fb331f2e2d
+  metadata.gz: 8907f2de3502bd5d7e4ee138dba441f1d5d0197537b8a9b246d46038f12fa045
+  data.tar.gz: '09e4e69d440b126a68f54bc535c2c2bba8ab1b199e70e306fd0e08ae9fbc5a37'
 SHA512:
-  metadata.gz: 99e3edfafaa3917b04bba0152462cbbafcba16c71b16ea828b46ec3d526d2808c066474eaba0044f37f0af04f94e59a178b35a8a6c8954a59ecd6132f97dfcb7
-  data.tar.gz: 8fb07f8e11898794ef21881694f3f9501d95a8dd80b375fe899ed959b81faa6aa1bd523fe792b315f762c32249b8952e26e332c79f0d240d4ecc9f79f56b12b3
+  metadata.gz: f895096ecb110d7d9a269e882f02c5ba844a4b6593163dcfd82f81f570eb4a3979ffa2f0fedf398f02d22445651005fe6cd9b98172fa91e1ef49b5f55d8c9d11
+  data.tar.gz: ac780383e151377090900d904dc855a161c2c6abaff43bf7ac7b27140fecb1ee461e0cd4bdbcdbc39c9cdf28d569eede8a30634094e7516f0b60d899246846a8

data/.gitignore

@@ -40,5 +40,4 @@ foo/
 *.iml
 .idea/
 
-fluent/
 Gemfile.lock

data/README.md

@@ -1,11 +1,15 @@
 # Fluentd output plugin for Datadog
 
-It mainly contains a proper JSON formatter and a socket handler that
-streams logs directly to Datadog - so no need to use a log shipper
+This output plugin allows sending logs directly from Fluentd to Datadog - so you don't have to use a separate log shipper
 if you don't wan't to.
 
 ## Pre-requirements
 
+| fluent-plugin-datadog | Fluentd    | Ruby   |
+|:--------------------------|:-----------|:-------|
+| \>= 0.12.0               | \>= v1      | \>= 2.4 |
+| < 0.12.0                   | \>= v0.12.0 | \>= 2.1 |
+
 To add the plugin to your fluentd agent, use the following command:
 
     gem install fluent-plugin-datadog
@@ -19,7 +23,7 @@ If you installed the td-agent instead
 
 To match events and send them to Datadog, simply add the following code to your configuration file.
 
-TCP example:
+HTTP example:
 
 ```xml
 # Match events tagged with "datadog.**" and
@@ -35,10 +39,21 @@ TCP example:
   tag_key 'tag'
 
   # Optional parameters
-  dd_source '<INTEGRATION_NAME>' 
-  dd_tags '<KEY1:VALU1>,<KEY2:VALUE2>'
+  dd_source '<INTEGRATION_NAME>'
+  dd_tags '<KEY1:VALUE1>,<KEY2:VALUE2>'
   dd_sourcecategory '<MY_SOURCE_CATEGORY>'
 
+  # Optional http proxy
+  http_proxy 'http://my-proxy.example'
+
+  <buffer>
+          @type memory
+          flush_thread_count 4
+          flush_interval 3s
+          chunk_limit_size 5m
+          chunk_limit_records 500
+  </buffer>
+
 </match>
 ```
 
@@ -72,15 +87,21 @@ As fluent-plugin-datadog is an output_buffer, you can set all output_buffer prop
 | **tag_key** | Where to store the Fluentd tag. | "tag" |
 | **timestamp_key** | Name of the attribute which will contain timestamp of the log event. If nil, timestamp attribute is not added. | "@timestamp" |
 | **use_ssl** | If true, the agent initializes a secure connection to Datadog. In clear TCP otherwise. | true |
-| **ssl_port** | Port used to send logs over a SSL encripted connection to Datadog (use 443 for the EU region) | 10516 |
+| **no_ssl_validation** | Disable SSL validation (useful for proxy forwarding) | false |
+| **ssl_port** | Port used to send logs over a SSL encrypted connection to Datadog. If use_http is disabled, use 10516 for the US region and 443 for the EU region. | 443 |
 | **max_retries** | The number of retries before the output plugin stops. Set to -1 for unlimited retries | -1 |
+| **max_backoff** | The maximum time waited between each retry in seconds | 30 |
+| **use_http** | Enable HTTP forwarding. If you disable it, make sure to change the port to 10514 or ssl_port to 10516 | true |
+| **use_compression** | Enable log compression for HTTP | true |
+| **compression_level** | Set the log compression level for HTTP (1 to 9, 9 being the best ratio) | 6 |
 | **dd_source** | This tells Datadog what integration it is | nil |
 | **dd_sourcecategory** | Multiple value attribute. Can be used to refine the source attribute | nil |
 | **dd_tags** | Custom tags with the following format "key1:value1, key2:value2" | nil |
 | **dd_hostname** | Used by Datadog to identify the host submitting the logs. | `hostname -f` |
 | **service** | Used by Datadog to correlate between logs, traces and metrics. | nil |
-| **port** | Proxy port when logs are not directly forwarded to Datadog and ssl is not used | 10514 |
-| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | intake.logs.datadoghq.com |
+| **port** | Proxy port when logs are not directly forwarded to Datadog and ssl is not used | 80 |
+| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | http-intake.logs.datadoghq.com |
+| **http_proxy** | HTTP proxy, only takes effect if HTTP forwarding is enabled (`use_http`). Defaults to `HTTP_PROXY`/`http_proxy` env vars. | nil |
 
 ### Docker and Kubernetes tags
 
@@ -105,6 +126,24 @@ Configuration example:
 </filter>
 ```
 
+### Encoding
+
+Datadog's API expects log messages to be encoded in UTF-8.
+If some of your logs are encoded with a different encoding, we recommend using the [`record_modifier` filter plugin](https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding)
+to encode these logs to UTF-8.
+
+Configuration example:
+
+```
+# Change encoding of logs tagged with "datadog.**"
+<filter datadog.**>
+  @type record_modifier
+
+  # change the encoding from the '<SOURCE_ENCODING>' of your logs to 'utf-8'
+  char_encoding <SOURCE_ENCODING>:utf-8
+</filter>
+```
+
 ## Build
 
 To build a new version of this plugin and push it to RubyGems:

data/fluent-plugin-datadog.gemspec

@@ -7,21 +7,28 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
+require "fluent/plugin/version.rb"
+
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-datadog"
-  spec.version       = "0.11.1"
+  spec.version       = DatadogFluentPlugin::VERSION
   spec.authors       = ["Datadog Solutions Team"]
   spec.email         = ["support@datadoghq.com"]
   spec.summary       = "Datadog output plugin for Fluent event collector"
   spec.homepage      = "http://datadoghq.com"
-  spec.license       = "Apache License 2.0"
+  spec.license       = "Apache-2.0"
 
-  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/out_datadog.rb"]
+  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_datadog.rb"]
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
-  spec.add_development_dependency "rake"
+  spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
+  spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
+
+  spec.add_development_dependency "bundler", "~> 2.1"
+  spec.add_development_dependency "test-unit", '~> 3.1'
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "yajl-ruby", "~> 1.2"
+  spec.add_development_dependency 'webmock', "~> 3.6.0"
 end

data/lib/fluent/plugin/out_datadog.rb

@@ -3,51 +3,81 @@
 # This product includes software developed at Datadog (https://www.datadoghq.com/).
 # Copyright 2018 Datadog, Inc.
 
-require 'socket'
-require 'openssl'
-require 'yajl'
+require "socket"
+require "openssl"
+require "yajl"
+require "zlib"
+require "fluent/plugin/output"
 
-class Fluent::DatadogOutput < Fluent::BufferedOutput
-  class ConnectionFailure < StandardError; end
+require_relative "version"
+
+class Fluent::DatadogOutput < Fluent::Plugin::Output
+  class RetryableError < StandardError;
+  end
+
+  # Max limits for transport regardless of Fluentd buffer, respecting https://docs.datadoghq.com/api/?lang=bash#logs
+  DD_MAX_BATCH_LENGTH = 500
+  DD_MAX_BATCH_SIZE = 5000000
+  DD_TRUNCATION_SUFFIX = "...TRUNCATED..."
+
+  DD_DEFAULT_HTTP_ENDPOINT = "http-intake.logs.datadoghq.com"
+  DD_DEFAULT_TCP_ENDPOINT = "intake.logs.datadoghq.com"
+
+  helpers :compat_parameters
+
+  DEFAULT_BUFFER_TYPE = "memory"
 
   # Register the plugin
   Fluent::Plugin.register_output('datadog', self)
 
   # Output settings
-  config_param :use_json,           :bool,    :default => true
-  config_param :include_tag_key,    :bool,    :default => false
-  config_param :tag_key,            :string,  :default => 'tag'
-  config_param :timestamp_key,      :string,  :default => '@timestamp'
-  config_param :service,            :string,  :default => nil
-  config_param :dd_sourcecategory,  :string,  :default => nil
-  config_param :dd_source,          :string,  :default => nil
-  config_param :dd_tags,            :string,  :default => nil
-  config_param :dd_hostname,        :string,  :default => nil
+  config_param :include_tag_key, :bool, :default => false
+  config_param :tag_key, :string, :default => 'tag'
+  config_param :timestamp_key, :string, :default => '@timestamp'
+  config_param :service, :string, :default => nil
+  config_param :dd_sourcecategory, :string, :default => nil
+  config_param :dd_source, :string, :default => nil
+  config_param :dd_tags, :string, :default => nil
+  config_param :dd_hostname, :string, :default => nil
 
   # Connection settings
-  config_param :host,           :string,  :default => 'intake.logs.datadoghq.com'
-  config_param :use_ssl,        :bool,    :default => true
-  config_param :port,           :integer, :default => 10514
-  config_param :ssl_port,       :integer, :default => 10516
-  config_param :max_retries,    :integer, :default => -1
-  config_param :tcp_ping_rate,  :integer, :default => 10
+  config_param :host, :string, :default => DD_DEFAULT_HTTP_ENDPOINT
+  config_param :use_ssl, :bool, :default => true
+  config_param :port, :integer, :default => 80
+  config_param :ssl_port, :integer, :default => 443
+  config_param :max_retries, :integer, :default => -1
+  config_param :max_backoff, :integer, :default => 30
+  config_param :use_http, :bool, :default => true
+  config_param :use_compression, :bool, :default => true
+  config_param :compression_level, :integer, :default => 6
+  config_param :no_ssl_validation, :bool, :default => false
+  config_param :http_proxy, :string, :default => nil
+  config_param :force_v1_routes, :bool, :default => false
+
+  # Format settings
+  config_param :use_json, :bool, :default => true
 
   # API Settings
-  config_param :api_key,  :string
+  config_param :api_key, :string, secret: true
 
-  def initialize
-    super
+  config_section :buffer do
+    config_set_default :@type, DEFAULT_BUFFER_TYPE
   end
 
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
+  def initialize
+    super
   end
 
   def configure(conf)
+    compat_parameters_convert(conf, :buffer)
     super
     return if @dd_hostname
 
+    if not @use_http and @host == DD_DEFAULT_HTTP_ENDPOINT
+      @host = DD_DEFAULT_TCP_ENDPOINT
+    end
+
+    # Set dd_hostname if not already set (can be set when using fluentd as aggregator)
     @dd_hostname = %x[hostname -f 2> /dev/null].strip
     @dd_hostname = Socket.gethostname if @dd_hostname.empty?
   end
@@ -56,41 +86,22 @@ class Fluent::DatadogOutput < Fluent::BufferedOutput
     true
   end
 
-  def new_client
-    if @use_ssl
-      context    = OpenSSL::SSL::SSLContext.new
-      socket     = TCPSocket.new @host, @ssl_port
-      ssl_client = OpenSSL::SSL::SSLSocket.new socket, context
-      ssl_client.connect
-      return ssl_client
-    else
-      return TCPSocket.new @host, @port
-    end
+  def formatted_to_msgpack_binary?
+    true
   end
 
   def start
     super
-    @my_mutex = Mutex.new
-    @running = true
-
-    if @tcp_ping_rate > 0
-      @timer = Thread.new do
-        while @running do
-          messages = Array.new
-          messages.push("fp\n")
-          send_to_datadog(messages)
-          sleep(@tcp_ping_rate)
-        end
-      end
-    end
+    @client = new_client(log, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @ssl_port, @port, @http_proxy, @use_compression, @force_v1_routes)
   end
 
   def shutdown
     super
-    @running = false
-    if @client
-      @client.close
-    end
+  end
+
+  def terminate
+    super
+    @client.close if @client
   end
 
   # This method is called when an event reaches Fluentd.
@@ -98,92 +109,296 @@ class Fluent::DatadogOutput < Fluent::BufferedOutput
     # When Fluent::EventTime is msgpack'ed it gets converted to int with seconds
     # precision only. We explicitly convert it to floating point number, which
     # is compatible with Time.at below.
-    return [tag, time.to_f, record].to_msgpack
+    record = enrich_record(tag, time.to_f, record)
+    if @use_http
+      record = Yajl.dump(record)
+    else
+      if @use_json
+        record = "#{api_key} #{Yajl.dump(record)}"
+      else
+        record = "#{api_key} #{record}"
+      end
+    end
+    [record].to_msgpack
   end
 
+
   # NOTE! This method is called by internal thread, not Fluentd's main thread.
   # 'chunk' is a buffer chunk that includes multiple formatted events.
   def write(chunk)
-    messages = Array.new
+    begin
+      if @use_http
+        events = Array.new
+        chunk.msgpack_each do |record|
+          next if record.empty?
+          events.push record[0]
+        end
+        process_http_events(events, @use_compression, @compression_level, @max_retries, @max_backoff, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
+      else
+        chunk.msgpack_each do |record|
+          next if record.empty?
+          process_tcp_event(record[0], @max_retries, @max_backoff, DD_MAX_BATCH_SIZE)
+        end
+      end
+    rescue Exception => e
+      @logger.error("Uncaught processing exception in datadog forwarder #{e.message}")
+    end
+  end
+
+  # Process and send a set of http events. Potentially break down this set of http events in smaller batches
+  def process_http_events(events, use_compression, compression_level, max_retries, max_backoff, max_batch_length, max_batch_size)
+    batches = batch_http_events(events, max_batch_length, max_batch_size)
+    batches.each do |batched_event|
+      formatted_events = format_http_event_batch(batched_event)
+      if use_compression
+        formatted_events = gzip_compress(formatted_events, compression_level)
+      end
+      @client.send_retries(formatted_events, max_retries, max_backoff)
+    end
+  end
 
-    chunk.msgpack_each do |tag, time, record|
-      next unless record.is_a? Hash
-      next if record.empty?
+  # Process and send a single tcp event
+  def process_tcp_event(event, max_retries, max_backoff, max_batch_size)
+    if event.bytesize > max_batch_size
+      event = truncate(event, max_batch_size)
+    end
+    @client.send_retries(event, max_retries, max_backoff)
+  end
 
-      if @dd_sourcecategory
-        record["ddsourcecategory"] ||= @dd_sourcecategory
+  # Group HTTP events in batches
+  def batch_http_events(encoded_events, max_batch_length, max_request_size)
+    batches = []
+    current_batch = []
+    current_batch_size = 0
+    encoded_events.each_with_index do |encoded_event, i|
+      current_event_size = encoded_event.bytesize
+      # If this unique log size is bigger than the request size, truncate it
+      if current_event_size > max_request_size
+        encoded_event = truncate(encoded_event, max_request_size)
+        current_event_size = encoded_event.bytesize
       end
-      if @dd_source
-        record["ddsource"] ||= @dd_source
+
+      if (i > 0 and i % max_batch_length == 0) or (current_batch_size + current_event_size > max_request_size)
+        batches << current_batch
+        current_batch = []
+        current_batch_size = 0
       end
-      if @dd_tags
-        record["ddtags"] ||= @dd_tags
+
+      current_batch_size += encoded_event.bytesize
+      current_batch << encoded_event
+    end
+    batches << current_batch
+    batches
+  end
+
+  # Truncate events over the provided max length, appending a marker when truncated
+  def truncate(event, max_length)
+    if event.length > max_length
+      event = event[0..max_length - 1]
+      event[max(0, max_length - DD_TRUNCATION_SUFFIX.length)..max_length - 1] = DD_TRUNCATION_SUFFIX
+      return event
+    end
+    event
+  end
+
+  def max(a, b)
+    a > b ? a : b
+  end
+
+  # Format batch of http events
+  def format_http_event_batch(events)
+    "[#{events.join(',')}]"
+  end
+
+  # Enrich records with metadata such as service, tags or source
+  def enrich_record(tag, time, record)
+    if @dd_sourcecategory
+      record["ddsourcecategory"] ||= @dd_sourcecategory
+    end
+    if @dd_source
+      record["ddsource"] ||= @dd_source
+    end
+    if @dd_tags
+      record["ddtags"] ||= @dd_tags
+    end
+    if @service
+      record["service"] ||= @service
+    end
+    if @dd_hostname
+      # set the record hostname to the configured dd_hostname only
+      # if the record hostname is empty, ensuring having a hostname set
+      # even if the record doesn't contain any.
+      record["hostname"] ||= @dd_hostname
+    end
+
+    if @include_tag_key
+      record[@tag_key] = tag
+    end
+    # If @timestamp_key already exists, we don't overwrite it.
+    if @timestamp_key and record[@timestamp_key].nil? and time
+      record[@timestamp_key] = Time.at(time).utc.iso8601(3)
+    end
+
+    container_tags = get_container_tags(record)
+    unless container_tags.empty?
+      if record["ddtags"].nil? || record["ddtags"].empty?
+        record["ddtags"] = container_tags
+      else
+        record["ddtags"] = record["ddtags"] + "," + container_tags
       end
-      if @service
-        record["service"] ||= @service
+    end
+    record
+  end
+
+  # Compress logs with GZIP
+  def gzip_compress(payload, compression_level)
+    gz = StringIO.new
+    gz.set_encoding("BINARY")
+    z = Zlib::GzipWriter.new(gz, compression_level)
+    begin
+      z.write(payload)
+    ensure
+      z.close
+    end
+    gz.string
+  end
+
+  # Build a new transport client
+  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, force_v1_routes)
+    if use_http
+      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes
+    else
+      DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port
+    end
+  end
+
+  # Top level class for datadog transport clients, managing retries and backoff
+  class DatadogClient
+    def send_retries(payload, max_retries, max_backoff)
+      backoff = 1
+      retries = 0
+      begin
+        send(payload)
+      rescue RetryableError => e
+        if retries < max_retries || max_retries < 0
+          @logger.warn("Retrying ", :exception => e, :backtrace => e.backtrace)
+          sleep backoff
+          backoff = 2 * backoff unless backoff > max_backoff
+          retries += 1
+          retry
+        end
       end
-      if @dd_hostname
-        record["hostname"] ||= @dd_hostname
+    end
+
+    def send(payload)
+      raise NotImplementedError, "Datadog transport client should implement the send method"
+    end
+
+    def close
+      raise NotImplementedError, "Datadog transport client should implement the close method"
+    end
+  end
+
+  # HTTP datadog client
+  class DatadogHTTPClient < DatadogClient
+    require 'net/http'
+    require 'net/http/persistent'
+
+    def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes = false)
+      @logger = logger
+      protocol = use_ssl ? "https" : "http"
+      port = use_ssl ? ssl_port : port
+      if force_v1_routes
+        @uri = URI("#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}")
+      else
+        @uri = URI("#{protocol}://#{host}:#{port.to_s}/api/v2/logs")
+      end
+      proxy_uri = :ENV
+      if http_proxy
+        proxy_uri = URI.parse(http_proxy)
+      elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
+        logger.info("Using HTTP proxy defined in `HTTP_PROXY`/`http_proxy` env vars")
+      end
+      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
+      @client = Net::HTTP::Persistent.new name: "fluent-plugin-datadog-logcollector", proxy: proxy_uri
+      @client.verify_mode = OpenSSL::SSL::VERIFY_NONE if no_ssl_validation
+      unless force_v1_routes
+        @client.override_headers["DD-API-KEY"] = api_key
+        @client.override_headers["DD-EVP-ORIGIN"] = "fluent"
+        @client.override_headers["DD-EVP-ORIGIN-VERSION"] = DatadogFluentPlugin::VERSION
       end
+      @client.override_headers["Content-Type"] = "application/json"
+      if use_compression
+        @client.override_headers["Content-Encoding"] = "gzip"
+      end
+      if !@client.proxy_uri.nil?
+        # Log the proxy settings as resolved by the HTTP client
+        logger.info("Using HTTP proxy #{@client.proxy_uri.scheme}://#{@client.proxy_uri.host}:#{@client.proxy_uri.port} username: #{@client.proxy_uri.user ? "set" : "unset"}, password: #{@client.proxy_uri.password ? "set" : "unset"}")
+      end
+    end
 
-      if @include_tag_key
-        record[@tag_key] = tag
+    def send(payload)
+      request = Net::HTTP::Post.new @uri.request_uri
+      request.body = payload
+      response = @client.request @uri, request
+      res_code = response.code.to_i
+      # on a backend error or on an http 429, retry with backoff
+      if res_code >= 500 || res_code == 429
+        raise RetryableError.new "Unable to send payload: #{res_code} #{response.message}"
       end
-      # If @timestamp_key already exists, we don't overwrite it.
-      if @timestamp_key and record[@timestamp_key].nil? and time
-        record[@timestamp_key] = Time.at(time).utc.iso8601(3)
+      if res_code >= 400
+        @logger.error("Unable to send payload due to client error: #{res_code} #{response.message}")
       end
+    end
 
-      container_tags = get_container_tags(record)
-      if not container_tags.empty?
-        if record["ddtags"].nil? || record["ddtags"].empty?
-          record["ddtags"] = container_tags
-        else
-          record["ddtags"] = record["ddtags"] + "," + container_tags
+    def close
+      @client.shutdown
+    end
+  end
+
+  # TCP Datadog client
+  class DatadogTCPClient < DatadogClient
+    require "socket"
+
+    def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port)
+      @logger = logger
+      @use_ssl = use_ssl
+      @no_ssl_validation = no_ssl_validation
+      @host = host
+      @port = use_ssl ? ssl_port : port
+    end
+
+    def connect
+      if @use_ssl
+        @logger.info("Starting SSL connection #{@host} #{@port}")
+        socket = TCPSocket.new @host, @port
+        ssl_context = OpenSSL::SSL::SSLContext.new
+        if @no_ssl_validation
+          ssl_context.set_params({:verify_mode => OpenSSL::SSL::VERIFY_NONE})
         end
+        ssl_context = OpenSSL::SSL::SSLSocket.new socket, ssl_context
+        ssl_context.connect
+        ssl_context
+      else
+        @logger.info("Starting plaintext connection #{@host} #{@port}")
+        TCPSocket.new @host, @port
       end
+    end
 
-      if @use_json
-        messages.push "#{api_key} " + Yajl.dump(record) + "\n"
-      else
-        next unless record.has_key? "message"
-        messages.push "#{api_key} " + record["message"].strip + "\n"
-      end
-    end
-    send_to_datadog(messages)
-  end
-
-  def send_to_datadog(events)
-    @my_mutex.synchronize do
-      events.each do |event|
-        log.trace "Datadog plugin: about to send event=#{event}"
-        retries = 0
-        begin
-          log.info "New attempt to Datadog attempt=#{retries}" if retries > 1
-          @client ||= new_client
-          @client.write(event)
-        rescue => e
-          @client.close rescue nil
-          @client = nil
-
-          if retries == 0
-            # immediately retry, in case it's just a server-side close
-            retries += 1
-            retry
-          end
-
-          if retries < @max_retries || @max_retries == -1
-            a_couple_of_seconds = retries ** 2
-            a_couple_of_seconds = 30 unless a_couple_of_seconds < 30
-            retries += 1
-            log.warn "Could not push event to Datadog, attempt=#{retries} max_attempts=#{max_retries} wait=#{a_couple_of_seconds}s error=#{e}"
-            sleep a_couple_of_seconds
-            retry
-          end
-          raise ConnectionFailure, "Could not push event to Datadog after #{retries} retries, #{e}"
-        end
+    def send(payload)
+      begin
+        @socket ||= connect
+        @socket.puts(payload)
+      rescue => e
+        @socket.close rescue nil
+        @socket = nil
+        raise RetryableError.new "Unable to send payload: #{e.message}."
       end
     end
+
+    def close
+      @socket.close rescue nil
+    end
   end
 
   # Collect docker and kubernetes tags for your logs using `filter_kubernetes_metadata` plugin,
@@ -191,9 +406,9 @@ class Fluent::DatadogOutput < Fluent::BufferedOutput
   # https://github.com/fabric8io/fluent-plugin-kubernetes_metadata_filter/blob/master/lib/fluent/plugin/filter_kubernetes_metadata.rb#L265
 
   def get_container_tags(record)
-    return [
-      get_kubernetes_tags(record),
-      get_docker_tags(record)
+    [
+        get_kubernetes_tags(record),
+        get_docker_tags(record)
     ].compact.join(",")
   end
 
@@ -207,7 +422,7 @@ class Fluent::DatadogOutput < Fluent::BufferedOutput
       tags.push("pod_name:" + kubernetes['pod_name']) unless kubernetes['pod_name'].nil?
       return tags.join(",")
     end
-    return nil
+    nil
   end
 
   def get_docker_tags(record)
@@ -217,7 +432,6 @@ class Fluent::DatadogOutput < Fluent::BufferedOutput
       tags.push("container_id:" + docker['container_id']) unless docker['container_id'].nil?
       return tags.join(",")
     end
-    return nil
+    nil
   end
-
 end

data/lib/fluent/plugin/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DatadogFluentPlugin
+  VERSION = '0.14.0'
+end

metadata

@@ -1,43 +1,91 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datadog
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.14.0
 platform: ruby
 authors:
 - Datadog Solutions Team
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-01-22 00:00:00.000000000 Z
+date: 2021-10-18 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.5'
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -52,7 +100,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-description: 
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+description:
 email:
 - support@datadoghq.com
 executables: []
@@ -66,11 +128,12 @@ files:
 - Rakefile
 - fluent-plugin-datadog.gemspec
 - lib/fluent/plugin/out_datadog.rb
+- lib/fluent/plugin/version.rb
 homepage: http://datadoghq.com
 licenses:
-- Apache License 2.0
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -85,9 +148,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.10
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Datadog output plugin for Fluent event collector
 test_files: []