fluent-plugin-datadog-log 0.1.0.rc6 → 0.1.0.rc7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef9ed5f3814abf3c96d3658878c416efae5a5331
-  data.tar.gz: a1db7f7c2c7534cbcc037c7eb1a5e2f3daeab860
+  metadata.gz: aedaaa9c78c6fd346266540ebf04e4f91dcfab96
+  data.tar.gz: 885f73273fd6671a9e191051cc1afff8704af480
 SHA512:
-  metadata.gz: 4620f9e665fcf9e1e5fb218cd6be094cfc22f314e0573f494a93992c666aee61e3678a12d4ad9f408a4c78b46f900f05472f44879260cd501cafb621fc687d1b
-  data.tar.gz: 1435a3d5d3f5470252912fe7a8e00f8d802cc411f6a42e9e97b6f2157f57c0627f3bdb5fab6fddd5a5fb66ce1804fa47224aeae9177759ab0939205cd814a013
+  metadata.gz: 3f0c941f62799dfcac25c8261665e103a14dc81410ebc9376f98fef8b0f216dce35fe53393fd410e29c13923d6bf6cf23571ac3132c564ae7453527750e6b7c2
+  data.tar.gz: 8aade4e0f12e9c07e87be0ccd2e8131b8d247d84c1bcc13ba9a24af0c622cc30ada70a3b336926b89767d3845b5f5a4b4d81b4b36dcf90b02d3599ba54d817a7

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-datadog-log (0.1.0.rc6)
+    fluent-plugin-datadog-log (0.1.0.rc7)
       fluentd (~> 0.14)
       json (~> 1.8)
       net_tcp_client (~> 2.0.1)

data/fluent-plugin-datadog-log.gemspec

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc6'
+  gem.version       = '0.1.0.rc7'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/fluent-plugin-datadog-log.gemspec~

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc3'
+  gem.version       = '0.1.0.rc6'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -21,11 +21,12 @@ eos
   # gem.add_runtime_dependency 'datadog-log-api-client', '~> 0.1'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
-  gem.add_development_dependency 'net_tcp_client', '~> 2.0.1'
+  gem.add_dependency 'net_tcp_client', '~> 2.0.1'
+  gem.add_dependency 'prometheus-client', '~> 0.7.1'
+
   gem.add_development_dependency 'mocha', '~> 1.1'
   gem.add_development_dependency 'rake', '~> 10.3'
   gem.add_development_dependency 'rubocop', '~> 0.35.0'
   gem.add_development_dependency 'webmock', '~> 2.3.1'
   gem.add_development_dependency 'test-unit', '~> 3.0'
-  gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
 end

data/lib/datadog/log.rb

@@ -96,7 +96,9 @@ module Datadog
             )
           )
         )
-        @conn.write(payload)
+        @conn.retry_on_connection_failure do
+          @conn.write(payload)
+        end
         payload
       end
 

data/lib/datadog/log.rb~

@@ -0,0 +1,123 @@
+require 'openssl'
+require 'net/tcp_client'
+require 'socket'
+require 'time'
+
+module Datadog
+  module Log
+    TRUNCATED_MSG = '...TRUNCATED...'
+
+    TRUNCATED_LEN = TRUNCATED_MSG.size
+
+    # MaxMessageLen is the maximum length for any message we send to the intake
+    # see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/constants.go#L9-L10
+    DD_MAX_MESSAGE_LEN = 1 * 1000 * 1000
+
+    MAX_MESSAGE_LEN = DD_MAX_MESSAGE_LEN - TRUNCATED_LEN
+
+    def truncate_message(msg)
+      if msg.size > DD_MAX_MESSAGE_LEN
+        msg.slice(0, MAX_MESSAGE_LEN) + TRUNCATED_MSG
+      else
+        msg
+      end
+    end
+
+    # Given a list of tags, build_tags_payload generates the bytes array
+    # that will be inserted into messages
+    # @see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/integration_config.go#L180
+    def build_tags_payload(config_tags:, source:, source_category:)
+      payload = ''
+
+      payload = "[dd ddsource=\"#{source}\"]" if !source.nil? && source != ''
+
+      if !source_category.nil? && source_category != ''
+        payload = "#{payload}[dd ddsourcecategory=\"#{source_category}\"]"
+      end
+
+      if !config_tags.nil? && config_tags != ''
+        config_tags = config_tags.join(',') if config_tags.is_a? ::Array
+        payload = "#{payload}[dd ddtags=\"#{config_tags}\"]"
+      end
+
+      payload
+    end
+
+    # https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/processor/processor.go#L65
+    def build_extra_content(timestamp:, hostname:, service:, tags_payload:)
+      "<46>0 #{timestamp} #{hostname} #{service} - - #{tags_payload}"
+    end
+
+    def build_api_key_str(api_key:, logset:)
+      if !logset.nil? && logset != ''
+        "#{api_key}/#{logset}"
+      else
+        api_key
+      end
+    end
+
+    # build_payload returns a processed payload from a raw message
+    # @param [String] api_key_str
+    # @param [String] extra_content
+    # @param [String] msg
+    def create_payload(api_key_str:, msg:, extra_content:)
+      "#{api_key_str} #{extra_content} #{msg}\n"
+    end
+
+    class Client
+      include ::Datadog::Log
+
+      def initialize(log_dd_url: 'intake.logs.datadoghq.com', log_dd_port: 10516, api_key:, hostname:, skip_ssl_validation: false)
+        @log_dd_url = log_dd_url
+        @log_dd_port = log_dd_port
+        @api_key = api_key
+        @hostname = hostname
+        @skip_ssl_validation = skip_ssl_validation
+
+        init_api_client
+      end
+
+      def send_payload(logset: 'main', msg:, datetime: nil, service:, source:, source_category:, tags:)
+        datetime = DateTime.now if datetime.nil?
+
+        # new_offset(0) is required. otherwise datadog will silently throws away the log..
+        timestamp_str = datetime.new_offset(0).rfc3339(6)
+        payload = create_payload(
+          api_key_str: build_api_key_str(api_key: @api_key, logset: logset),
+          msg: truncate_message(msg),
+          extra_content: build_extra_content(
+            timestamp: timestamp_str,
+            hostname: @hostname,
+            service: service,
+            tags_payload: build_tags_payload(
+              config_tags: tags,
+              source: source,
+              source_category: source_category
+            )
+          )
+        )
+        @conn.write(payload)
+        payload
+      end
+
+      def shutdown
+        @conn.close unless @conn.nil?
+      end
+
+      class << self
+        def from_env
+          new(api_key: ENV['DD_LOG_API_KEY'], hostname: Socket.gethostname)
+        end
+      end
+
+      private
+
+      def init_api_client
+        ssl = true
+        ssl = { verify_mode: OpenSSL::SSL::VERIFY_NONE } if @skip_ssl_validation
+        server = "#{@log_dd_url}:#{@log_dd_port}"
+        @conn = Net::TCPClient.new(server: server, ssl: ssl)
+      end
+    end
+  end
+end

data/test/plugin/test_out_datadog_log.rb

@@ -79,7 +79,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       conn = StubConn.new
       fluentd_tag = 'mytag'
       Net::TCPClient.stubs(:new)
-        .with(server: ':10516', ssl: true)
+        .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
         .returns(conn)
       d.run(default_tag: fluentd_tag) do
         record = {
@@ -118,7 +118,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       conn = StubConn.new
       fluentd_tag = 'mytag'
       Net::TCPClient.stubs(:new)
-        .with(server: ':10516', ssl: true)
+        .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
         .returns(conn)
       d.run(default_tag: fluentd_tag) do
         record = {
@@ -179,7 +179,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
           EOC
           conn = StubConn.new
           Net::TCPClient.stubs(:new)
-            .with(server: ':10516', ssl: true)
+            .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
             .returns(conn)
           d.run(default_tag: 'mytag') do
             (1..entry_count).each do |i|
@@ -220,6 +220,10 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       @sent = []
     end
 
+    def retry_on_connection_failure
+      yield
+    end
+
     def write(payload)
       @sent << payload
     end

data/test/plugin/test_out_datadog_log.rb~

@@ -35,6 +35,31 @@ class DatadogLogOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_configure_with_env
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      ENV.stubs(:[])
+        .with('DD_API_KEY')
+        .returns('myapikey_from_env')
+
+      ENV.stubs(:[])
+        .with(Not equals 'DD_API_KEY')
+        .returns('')
+        .times(3)
+
+      d = create_driver(<<-EOC)
+        type datadog_log
+        service myservice
+        source mysource
+      EOC
+
+      assert_equal 'myapikey_from_env', d.instance.api_key
+      assert_equal 'myservice', d.instance.service
+      assert_equal 'mysource', d.instance.source
+    end
+  end
+
   def test_write
     new_stub_context do
       setup_ec2_metadata_stubs
@@ -53,18 +78,65 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       EOC
       conn = StubConn.new
       fluentd_tag = 'mytag'
+      Net::TCPClient.stubs(:new)
+        .with(server: ':10516', ssl: true)
+        .returns(conn)
+      d.run(default_tag: fluentd_tag) do
+        record = {
+          'log' => 'mymsg'
+        }
+        d.feed(time, record)
+      end
+
+      # fail d.logs.inspect
+      assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
+      assert_equal(1, conn.sent.size)
+      # rubocop:disable LineLength
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      # rubocop:enable LineLength
+      assert_equal(payload, conn.sent.first)
+    end
+  end
+
+  def test_write_kube
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+        source_category mysourcecategory
+        logset mylogset
+        log_level debug
+        tags ["kube_cluster=MyCluster", "mykey=myval"]
+      EOC
+      conn = StubConn.new
+      fluentd_tag = 'mytag'
       Net::TCPClient.stubs(:new)
         .with(server: ':10516', ssl: true)
         .returns(conn)
       d.run(default_tag: fluentd_tag) do
         record = {
           'log' => 'mymsg',
+          'docker' => {
+            'container_id' => 'myfullcontainerid'
+          },
           'kubernetes' => {
             'namespace' => 'myns',
             'pod_name' => 'mypod',
             'container_name' => 'mycontainer',
             'labels' => {
               'k8s-app' => 'myapp'
+            },
+            'annotations' => {
+              # rubocop:disable LineLength
+              'kubernetes.io/created-by' => '{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"Deployment","namespace":"default","name":"myapp","uid":"d67e8857-c2dc-11e7-aed9-066d23381f8c","apiVersion":"extensions","resourceVersion":"289"}}'
+              # rubocop:enable LineLength
             }
           }
         }
@@ -75,7 +147,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
       assert_equal(1, conn.sent.size)
       # rubocop:disable LineLength
-      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myapp - - [dd ddsource="mypod"][dd ddsourcecategory="mycontainer"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,kube_deployment=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012,kube_cluster=MyCluster,mykey=myval"] mymsg\n)
       # rubocop:enable LineLength
       assert_equal(payload, conn.sent.first)
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datadog-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc6
+  version: 0.1.0.rc7
 platform: ruby
 authors:
 - Yusuke KUOKA
@@ -154,6 +154,7 @@ files:
 - fluent-plugin-datadog-log.gemspec~
 - fluent-plugin-datadog.gemspec~
 - lib/datadog/log.rb
+- lib/datadog/log.rb~
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_datadog_log.rb
 - lib/fluent/plugin/out_datadog_log.rb~
@@ -164,6 +165,7 @@ files:
 - pkg/fluent-plugin-datadog-log-0.1.0.rc3.gem
 - pkg/fluent-plugin-datadog-log-0.1.0.rc4.gem
 - pkg/fluent-plugin-datadog-log-0.1.0.rc5.gem
+- pkg/fluent-plugin-datadog-log-0.1.0.rc6.gem
 - test/helper.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb