fluent-plugin-datadog 0.12.0 → 0.14.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 361a1fe07f49b0617f1ff79e22ec97ce4fc028a03843c5ead5d2819697ce520e
-  data.tar.gz: 54f91567b59233612522db98ea9baac1735c160ae2154d73bd77b9722cd0636d
+  metadata.gz: 804db6997735d59d4e5cf785c13ac0743462462576931220dbc6b335048cf258
+  data.tar.gz: c128237af27db094d3670979dc71d70f77687d93ad16266fb1831624f70479c1
 SHA512:
-  metadata.gz: 6a75714c43526425cff466d17a6dfc40ca9803a420bdccd7384c02a8fba2dac5b9c6703f0037287221bb1a478dfa0ccd8be293dd109861cb4f4acab46872aa85
-  data.tar.gz: dea0e351eb6a6557662778c72e3f346c42765009cf46dbf9528228292a17c69019fdfc35018c8960198aec0b54c30728d3857f802a413fe0f756408703a5347b
+  metadata.gz: 4e7b4b62f2ef22ebd0fd479cdf272acd4f972ae89466f5da499fbfe3fefef1f9c8e0864f081d1b2e6796f3a9dfde1d4c8b255be8ed899b9457c0a9a774b2d815
+  data.tar.gz: bb3c0f83647dd926ad24dee88728718e2c62cc20661e6da77131a472d7c9b93ceac26f00854c23834be438b3be459c3c489869eeacfc96b69da8d2b480f35987

data/.gitignore

@@ -40,5 +40,4 @@ foo/
 *.iml
 .idea/
 
-fluent/
 Gemfile.lock

data/README.md

@@ -1,7 +1,6 @@
 # Fluentd output plugin for Datadog
 
-It mainly contains a proper JSON formatter and a socket handler that
-streams logs directly to Datadog - so no need to use a log shipper
+This output plugin allows sending logs directly from Fluentd to Datadog - so you don't have to use a separate log shipper
 if you don't wan't to.
 
 ## Pre-requirements
@@ -24,7 +23,7 @@ If you installed the td-agent instead
 
 To match events and send them to Datadog, simply add the following code to your configuration file.
 
-TCP example:
+HTTP example:
 
 ```xml
 # Match events tagged with "datadog.**" and
@@ -40,9 +39,13 @@ TCP example:
   tag_key 'tag'
 
   # Optional parameters
-  dd_source '<INTEGRATION_NAME>' 
-  dd_tags '<KEY1:VALU1>,<KEY2:VALUE2>'
+  dd_source '<INTEGRATION_NAME>'
+  dd_tags '<KEY1:VALUE1>,<KEY2:VALUE2>'
   dd_sourcecategory '<MY_SOURCE_CATEGORY>'
+
+  # Optional http proxy
+  http_proxy 'http://my-proxy.example'
+
   <buffer>
           @type memory
           flush_thread_count 4
@@ -51,7 +54,6 @@ TCP example:
           chunk_limit_records 500
   </buffer>
 
-
 </match>
 ```
 
@@ -98,7 +100,8 @@ As fluent-plugin-datadog is an output_buffer, you can set all output_buffer prop
 | **dd_hostname** | Used by Datadog to identify the host submitting the logs. | `hostname -f` |
 | **service** | Used by Datadog to correlate between logs, traces and metrics. | nil |
 | **port** | Proxy port when logs are not directly forwarded to Datadog and ssl is not used | 80 |
-| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | intake.logs.datadoghq.com |
+| **host** | Proxy endpoint when logs are not directly forwarded to Datadog | http-intake.logs.datadoghq.com |
+| **http_proxy** | HTTP proxy, only takes effect if HTTP forwarding is enabled (`use_http`). Defaults to `HTTP_PROXY`/`http_proxy` env vars. | nil |
 
 ### Docker and Kubernetes tags
 
@@ -123,6 +126,24 @@ Configuration example:
 </filter>
 ```
 
+### Encoding
+
+Datadog's API expects log messages to be encoded in UTF-8.
+If some of your logs are encoded with a different encoding, we recommend using the [`record_modifier` filter plugin](https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding)
+to encode these logs to UTF-8.
+
+Configuration example:
+
+```
+# Change encoding of logs tagged with "datadog.**"
+<filter datadog.**>
+  @type record_modifier
+
+  # change the encoding from the '<SOURCE_ENCODING>' of your logs to 'utf-8'
+  char_encoding <SOURCE_ENCODING>:utf-8
+</filter>
+```
+
 ## Build
 
 To build a new version of this plugin and push it to RubyGems:

data/fluent-plugin-datadog.gemspec

@@ -7,27 +7,28 @@
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
+require "fluent/plugin/version.rb"
+
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-datadog"
-  spec.version       = "0.12.0"
+  spec.version       = DatadogFluentPlugin::VERSION
   spec.authors       = ["Datadog Solutions Team"]
   spec.email         = ["support@datadoghq.com"]
   spec.summary       = "Datadog output plugin for Fluent event collector"
   spec.homepage      = "http://datadoghq.com"
-  spec.license       = "Apache License 2.0"
+  spec.license       = "Apache-2.0"
 
-  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/out_datadog.rb"]
+  spec.files         = [".gitignore", "Gemfile", "LICENSE", "README.md", "Rakefile", "fluent-plugin-datadog.gemspec", "lib/fluent/plugin/version.rb", "lib/fluent/plugin/out_datadog.rb"]
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_runtime_dependency "fluentd", [">= 1", "< 2"]
-  spec.add_runtime_dependency "net-http-persistent", '~> 2.9'
+  spec.add_runtime_dependency "net-http-persistent", '~> 3.1'
 
+  spec.add_development_dependency "bundler", "~> 2.1"
   spec.add_development_dependency "test-unit", '~> 3.1'
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "yajl-ruby", "~> 1.2"
-
-
+  spec.add_development_dependency 'webmock', "~> 3.6.0"
 end

data/lib/fluent/plugin/out_datadog.rb

@@ -9,6 +9,8 @@ require "yajl"
 require "zlib"
 require "fluent/plugin/output"
 
+require_relative "version"
+
 class Fluent::DatadogOutput < Fluent::Plugin::Output
   class RetryableError < StandardError;
   end
@@ -49,12 +51,14 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
   config_param :use_compression, :bool, :default => true
   config_param :compression_level, :integer, :default => 6
   config_param :no_ssl_validation, :bool, :default => false
+  config_param :http_proxy, :string, :default => nil
+  config_param :force_v1_routes, :bool, :default => false
 
   # Format settings
   config_param :use_json, :bool, :default => true
 
   # API Settings
-  config_param :api_key, :string
+  config_param :api_key, :string, secret: true
 
   config_section :buffer do
     config_set_default :@type, DEFAULT_BUFFER_TYPE
@@ -88,7 +92,7 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
 
   def start
     super
-    @client = new_client(log, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @ssl_port, @port, @use_compression)
+    @client = new_client(log, @api_key, @use_http, @use_ssl, @no_ssl_validation, @host, @ssl_port, @port, @http_proxy, @use_compression, @force_v1_routes)
   end
 
   def shutdown
@@ -122,18 +126,22 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
   # NOTE! This method is called by internal thread, not Fluentd's main thread.
   # 'chunk' is a buffer chunk that includes multiple formatted events.
   def write(chunk)
-    if @use_http
-      events = Array.new
-      chunk.msgpack_each do |record|
-        next if record.empty?
-        events.push record[0]
-      end
-      process_http_events(events, @use_compression, @compression_level, @max_retries, @max_backoff, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
-    else
-      chunk.msgpack_each do |record|
-        next if record.empty?
-        process_tcp_event(record[0], @max_retries, @max_backoff, DD_MAX_BATCH_SIZE)
+    begin
+      if @use_http
+        events = Array.new
+        chunk.msgpack_each do |record|
+          next if record.empty?
+          events.push record[0]
+        end
+        process_http_events(events, @use_compression, @compression_level, @max_retries, @max_backoff, DD_MAX_BATCH_LENGTH, DD_MAX_BATCH_SIZE)
+      else
+        chunk.msgpack_each do |record|
+          next if record.empty?
+          process_tcp_event(record[0], @max_retries, @max_backoff, DD_MAX_BATCH_SIZE)
+        end
       end
+    rescue Exception => e
+      log.error("Uncaught processing exception in datadog forwarder #{e.message}")
     end
   end
 
@@ -256,9 +264,9 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
   end
 
   # Build a new transport client
-  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression)
+  def new_client(logger, api_key, use_http, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, force_v1_routes)
     if use_http
-      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression, api_key
+      DatadogHTTPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes
     else
       DatadogTCPClient.new logger, use_ssl, no_ssl_validation, host, ssl_port, port
     end
@@ -296,18 +304,37 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
     require 'net/http'
     require 'net/http/persistent'
 
-    def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, use_compression, api_key)
+    def initialize(logger, use_ssl, no_ssl_validation, host, ssl_port, port, http_proxy, use_compression, api_key, force_v1_routes = false)
       @logger = logger
       protocol = use_ssl ? "https" : "http"
       port = use_ssl ? ssl_port : port
-      @uri = URI("#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}")
-      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled"))
-      @client = Net::HTTP::Persistent.new("fluent-plugin-datadog-logcollector")
+      if force_v1_routes
+        @uri = URI("#{protocol}://#{host}:#{port.to_s}/v1/input/#{api_key}")
+      else
+        @uri = URI("#{protocol}://#{host}:#{port.to_s}/api/v2/logs")
+      end
+      proxy_uri = :ENV
+      if http_proxy
+        proxy_uri = URI.parse(http_proxy)
+      elsif ENV['HTTP_PROXY'] || ENV['http_proxy']
+        logger.info("Using HTTP proxy defined in `HTTP_PROXY`/`http_proxy` env vars")
+      end
+      logger.info("Starting HTTP connection to #{protocol}://#{host}:#{port.to_s} with compression " + (use_compression ? "enabled" : "disabled") + (force_v1_routes ? " using v1 routes" : " using v2 routes"))
+      @client = Net::HTTP::Persistent.new name: "fluent-plugin-datadog-logcollector", proxy: proxy_uri
       @client.verify_mode = OpenSSL::SSL::VERIFY_NONE if no_ssl_validation
+      unless force_v1_routes
+        @client.override_headers["DD-API-KEY"] = api_key
+        @client.override_headers["DD-EVP-ORIGIN"] = "fluent"
+        @client.override_headers["DD-EVP-ORIGIN-VERSION"] = DatadogFluentPlugin::VERSION
+      end
       @client.override_headers["Content-Type"] = "application/json"
       if use_compression
         @client.override_headers["Content-Encoding"] = "gzip"
       end
+      if !@client.proxy_uri.nil?
+        # Log the proxy settings as resolved by the HTTP client
+        logger.info("Using HTTP proxy #{@client.proxy_uri.scheme}://#{@client.proxy_uri.host}:#{@client.proxy_uri.port} username: #{@client.proxy_uri.user ? "set" : "unset"}, password: #{@client.proxy_uri.password ? "set" : "unset"}")
+      end
     end
 
     def send(payload)
@@ -315,7 +342,8 @@ class Fluent::DatadogOutput < Fluent::Plugin::Output
       request.body = payload
       response = @client.request @uri, request
       res_code = response.code.to_i
-      if res_code >= 500
+      # on a backend error or on an http 429, retry with backoff
+      if res_code >= 500 || res_code == 429
         raise RetryableError.new "Unable to send payload: #{res_code} #{response.message}"
       end
       if res_code >= 400

data/lib/fluent/plugin/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module DatadogFluentPlugin
+  VERSION = '0.14.1'
+end

metadata

@@ -1,29 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datadog
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.14.1
 platform: ruby
 authors:
 - Datadog Solutions Team
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-02-20 00:00:00.000000000 Z
+date: 2022-01-03 00:00:00.000000000 Z
 dependencies:
-- !ruby/object:Gem::Dependency
-  name: bundler
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -50,14 +36,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.9'
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
@@ -76,16 +76,16 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
@@ -100,7 +100,21 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
-description: 
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.6.0
+description:
 email:
 - support@datadoghq.com
 executables: []
@@ -114,11 +128,12 @@ files:
 - Rakefile
 - fluent-plugin-datadog.gemspec
 - lib/fluent/plugin/out_datadog.rb
+- lib/fluent/plugin/version.rb
 homepage: http://datadoghq.com
 licenses:
-- Apache License 2.0
+- Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -133,9 +148,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
+rubyforge_project:
 rubygems_version: 2.7.10
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Datadog output plugin for Fluent event collector
 test_files: []