fluent-plugin-datadog-log 0.1.0.rc18 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea5367ecfa2d3cfd858c8c3854328f087dc02dc2
-  data.tar.gz: 4a8cb53397ceba0e643919df3b72f0f36fa3f4fc
+  metadata.gz: 7ba2d248887e5b2de08cb5fdb40dd54daf9628fb
+  data.tar.gz: c2dd337e77d4b81cca604116943a22acd60ab0fb
 SHA512:
-  metadata.gz: c54999044447974a62210989f775694e5a75d85377edd57a379072c5ebd652fcc2f06aca44327d491185bd0e478d998e288e52e5a03068cb39974e4b47e4dff1
-  data.tar.gz: 1eeba1a6b4ddbfdee6483288f0ec6cb0d44e3768bf8ce174369ec60e92afb0844025d4840bdeb69bb20d8a1c68e225682a8c8c87e2e0f9905e748e100d57ad61
+  metadata.gz: e2d416a45ca1e3fa96319f75b14d7fc5ef21b18c049483acc754650776c4b50eccc70bb18c2f5ec54bdc74f7cabaaf9c7c9427d04d79429dfa152da0aba406d4
+  data.tar.gz: 50b49c31982537c5aaa6c75cebd2456dc3dfb6d2c92861ea664df55443016010c6274307c76f1a973cd3ffc601c3ec00fce17d071d5fb90e06627a8a7d3d416d

data/Gemfile.lock

@@ -1,12 +1,9 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-datadog-log (0.1.0.rc18)
-      fluentd (~> 1.0.0)
+    fluent-plugin-datadog-log (0.1.0)
+      fluentd (~> 0.14)
       json (~> 1.8)
-      net_tcp_client (~> 2.0.1)
-      prometheus-client (~> 0.7.1)
-      semantic_logger (~> 4.2.0)
 
 GEM
   remote: https://rubygems.org/
@@ -16,11 +13,10 @@ GEM
     ast (2.3.0)
     astrolabe (1.3.1)
       parser (~> 2.2)
-    concurrent-ruby (1.0.5)
-    cool.io (1.5.3)
+    cool.io (1.5.1)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
-    fluentd (1.0.0)
+    fluentd (0.14.22)
       cool.io (>= 1.4.5, < 2.0.0)
       http_parser.rb (>= 0.5.1, < 0.7.0)
       msgpack (>= 0.7.0, < 2.0.0)
@@ -37,15 +33,15 @@ GEM
     metaclass (0.0.4)
     mocha (1.3.0)
       metaclass (~> 0.0.1)
-    msgpack (1.2.0)
+    msgpack (1.1.0)
     net_tcp_client (2.0.1)
-    parser (2.4.0.2)
-      ast (~> 2.3)
+    parser (2.4.0.0)
+      ast (~> 2.2)
     power_assert (1.1.1)
     powerpack (0.1.1)
     prometheus-client (0.7.1)
       quantile (~> 0.2.0)
-    public_suffix (3.0.1)
+    public_suffix (3.0.0)
     quantile (0.2.0)
     rainbow (2.2.2)
       rake
@@ -60,8 +56,6 @@ GEM
     ruby-progressbar (1.9.0)
     ruby_dig (0.0.2)
     safe_yaml (1.0.4)
-    semantic_logger (4.2.0)
-      concurrent-ruby (~> 1.0)
     serverengine (2.0.5)
       sigdump (~> 0.2.2)
     sigdump (0.2.4)
@@ -86,6 +80,8 @@ PLATFORMS
 DEPENDENCIES
   fluent-plugin-datadog-log!
   mocha (~> 1.1)
+  net_tcp_client (~> 2.0.1)
+  prometheus-client (~> 0.7.1)
   rake (~> 10.3)
   rubocop (~> 0.35.0)
   test-unit (~> 3.0)

data/README.rdoc

@@ -37,14 +37,6 @@ of your fluentd configuration file, for example:
 
 Add `log_level debug` to enable debug logging which dumps every payload sent to Datadog.
 
-== Testing Datadog Connectivity within irb
-
-   cd path/to/fluent-plugin-datadog-log
-
-   bundle exec irb -r datadog/lib
-
-   c = Datadog::Log::Client.new(api_key:ENV['DD_LOG_API_KEY'],hostname:'myhost'); pl = c.send_payload(msg: 'foobar', source: 'myapp', source_category: 'stdout', service: 'irb', tags: %w|foo=bar|); puts pl
-
 == Copyright
 
 Copyright:: Copyright 2017 Yusuke KUOKA All rights reserved.

data/fluent-plugin-datadog-log.gemspec

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc18'
+  gem.version       = '0.1.0'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -17,17 +17,15 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '~> 1.0.0'
+  gem.add_runtime_dependency 'fluentd', '~> 0.14'
   # gem.add_runtime_dependency 'datadog-log-api-client', '~> 0.1'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
-  gem.add_dependency 'net_tcp_client', '~> 2.0.1'
-  gem.add_dependency 'prometheus-client', '~> 0.7.1'
-  gem.add_dependency 'semantic_logger', '~> 4.2.0'
-
+  gem.add_development_dependency 'net_tcp_client', '~> 2.0.1'
   gem.add_development_dependency 'mocha', '~> 1.1'
   gem.add_development_dependency 'rake', '~> 10.3'
   gem.add_development_dependency 'rubocop', '~> 0.35.0'
   gem.add_development_dependency 'webmock', '~> 2.3.1'
   gem.add_development_dependency 'test-unit', '~> 3.0'
+  gem.add_development_dependency 'prometheus-client', '~> 0.7.1'
 end

data/lib/fluent/plugin/out_datadog_log.rb

@@ -17,20 +17,18 @@ require 'open-uri'
 require 'socket'
 require 'time'
 require 'yaml'
+require 'net/tcp_client'
 require 'fluent/plugin/output'
 
-require_relative 'datadog_log'
 require_relative 'monitoring'
 
 module Fluent::Plugin
   # fluentd output plugin for the Datadog Log Intake API
   class DatadogOutput < ::Fluent::Plugin::Output
-    Fluent::Plugin.register_output('datadog_log', self)
+    Fluent::Plugin.register_output('datadog', self)
 
     helpers :compat_parameters, :inject
 
-    include ::Datadog::Log
-
     DEFAULT_BUFFER_TYPE = 'memory'
 
     PLUGIN_NAME = 'Fluentd Datadog plugin'
@@ -42,6 +40,16 @@ module Fluent::Plugin
     # Disable this warning to conform to fluentd config_param conventions.
     # rubocop:disable Style/HashSyntax
 
+    TRUNCATED_MSG = '...TRUNCATED...'
+
+    TRUNCATED_LEN = TRUNCATED_MSG.size
+
+    # MaxMessageLen is the maximum length for any message we send to the intake
+    # see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/constants.go#L9-L10
+    DD_MAX_MESSAGE_LEN = 1 * 1000 * 1000
+
+    MAX_MESSAGE_LEN = DD_MAX_MESSAGE_LEN - TRUNCATED_LEN
+
     # see https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/logagent/logsagent.go#L26-L30
     # see https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/config/config.go#L52-L56
     config_param :log_dd_url, :string, default: 'intake.logs.datadoghq.com'
@@ -123,14 +131,63 @@ module Fluent::Plugin
       @log = $log # rubocop:disable Style/GlobalVars
     end
 
+    def truncate_message(msg)
+      if msg.size > DD_MAX_MESSAGE_LEN
+        msg.slice(0, MAX_MESSAGE_LEN) + TRUNCATED_MSG
+      else
+        msg
+      end
+    end
+
+    # Given a list of tags, build_tags_payload generates the bytes array
+    # that will be inserted into messages
+    # @see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/integration_config.go#L180
+    def build_tags_payload(config_tags:, source:, source_category:)
+      payload = ''
+
+      payload = "[dd ddsource=\"#{source}\"]" if !source.nil? && source != ''
+
+      if !source_category.nil? && source_category != ''
+        payload = "#{payload}[dd ddsourcecategory=\"#{source_category}\"]"
+      end
+
+      if !config_tags.nil? && config_tags != ''
+        config_tags = config_tags.join(',') if config_tags.is_a? ::Array
+        payload = "#{payload}[dd ddtags=\"#{config_tags}\"]"
+      end
+
+      payload
+    end
+
+    # https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/processor/processor.go#L65
+    def build_extra_content(timestamp:, hostname:, service:, tags_payload:)
+      "<46>0 #{timestamp} #{hostname} #{service} - - #{tags_payload}"
+    end
+
+    def build_api_key_str(api_key:, logset:)
+      if !logset.nil? && logset != ''
+        "#{api_key}/#{logset}"
+      else
+        api_key
+      end
+    end
+
+    # build_payload returns a processed payload from a raw message
+    # @param [String] api_key_str
+    # @param [String] extra_content
+    # @param [String] msg
+    def build_payload(api_key_str:, msg:, extra_content:)
+      "#{api_key_str} #{extra_content} #{msg}\\n"
+    end
+
     def configure(conf)
       compat_parameters_convert(conf, :buffer, :inject)
       super
 
       if @api_key.size == 0
-        @api_key = ENV['DD_API_KEY']
+        @api_key = ENV['DD_LOG_API_KEY']
         if @api_key == '' || @api_key.nil?
-          error_message = 'Unable to obtain api_key from DD_API_KEY'
+          error_message = 'Unable to obtain api_key from DD_LOG_API_KEY'
           fail Fluent::ConfigError, error_message
         end
       end
@@ -183,7 +240,7 @@ module Fluent::Plugin
 
     def shutdown
       super
-      @conn.shutdown
+      @conn.close unless @conn.nil?
     end
 
     def format(tag, time, record)
@@ -232,10 +289,7 @@ module Fluent::Plugin
         begin
           msg = nil
           %w(log message msg).each do |field|
-            if record.key?(field)
-              msg = record[field]
-              break
-            end
+            msg = record[field] if record.key?(field)
           end
 
           tags = []
@@ -249,31 +303,13 @@ module Fluent::Plugin
           }
 
           mappings.each do |json_key, tag_key|
-            tags << "#{tag_key}:#{kube[json_key]}" if kube.key? json_key
+            tags << "#{tag_key}=#{kube[json_key]}" if kube.key? json_key
           end
 
-          kube_labels = kube['labels']
-          unless kube_labels.nil?
-            kube_labels.each do |k, v|
-              k2 = k.dup
-              k2.gsub!(/[\,\.]/, '_')
-              k2.gsub!(%r{/}, '-')
-              tags << "kube_#{k2}:#{v}"
-            end
-          end
-
-          @log.debug 'Dumping kubernetes metadata', metadata: kube
-
-          annotations = kube['annotations']
-          unless annotations.nil?
-            created_by_str = annotations['kubernetes_io/created-by']
-            unless created_by_str.nil?
-              created_by = JSON.parse(created_by_str)
-              ref = created_by['reference'] unless created_by.nil?
-              kind = ref['kind'] unless ref.nil?
-              name = ref['name'] unless ref.nil?
-              kind = kind.downcase unless kind.nil?
-              tags << "kube_#{kind}:#{name}" if !kind.nil? && !name.nil?
+          if kube.key? 'labels'
+            labels = kube['labels']
+            labels.each do |k, v|
+              tags << "kube_#{k}=#{v}"
             end
           end
 
@@ -285,31 +321,26 @@ module Fluent::Plugin
 
           tags.concat(@default_tags)
 
-          unless kube_labels.nil?
-            service = kube_labels['app'] ||
-                      kube_labels['k8s-app']
-          end
-          source = kube['pod_name']
-          source_category = kube['container_name']
-
-          service = @service if service.nil?
-          source = @source if source.nil?
-          source_category = @source_category if source_category.nil?
-
           datetime = Time.at(Fluent::EventTime.new(time).to_r).utc.to_datetime
-
-          payload =
-            @conn.send_payload(
-              logset: @logset,
-              msg: msg,
-              datetime: datetime,
-              service: service,
-              source: source,
-              source_category: source_category,
-              tags: tags
+          timestamp_str = datetime.rfc3339(6)
+
+          payload = build_payload(
+            api_key_str: build_api_key_str(api_key: @api_key, logset: @logset),
+            msg: truncate_message(msg),
+            extra_content: build_extra_content(
+              timestamp: timestamp_str,
+              hostname: @vm_id,
+              service: @service,
+              tags_payload: build_tags_payload(
+                config_tags: tags,
+                source: @source,
+                source_category: @source_category
+              )
             )
+          )
 
           entries_count = 1
+          @conn.write(payload)
           @log.debug 'Sent payload to Datadog.', payload: payload
           increment_successful_requests_count
           increment_ingested_entries_count(entries_count)
@@ -323,33 +354,17 @@ module Fluent::Plugin
 
         rescue => error
           increment_failed_requests_count
-          if entries_count.nil?
-            increment_dropped_entries_count(1)
-            @log.error 'Not retrying a log message later',
-                       error: error.to_s, error_class: error.class
-          else
-            increment_retried_entries_count(entries_count)
-            # RPC cancelled, so retry via re-raising the error.
-            @log.debug "Retrying #{entries_count} log message(s) later.",
-                       error: error.to_s
-            raise error
-          end
+          increment_retried_entries_count(entries_count)
+          # RPC cancelled, so retry via re-raising the error.
+          @log.debug "Retrying #{entries_count} log message(s) later.",
+                     error: error.to_s
+          raise error
         end
       end
     end
 
     private
 
-    def init_api_client
-      @conn = ::Datadog::Log::Client.new(
-        log_dd_url: @log_dd_url,
-        log_dd_port: @log_dd_port,
-        api_key: @api_key,
-        hostname: @vm_id,
-        skip_ssl_validation: @skip_ssl_validation
-      )
-    end
-
     def parse_json_or_nil(input)
       # Only here to please rubocop...
       return nil if input.nil?
@@ -479,24 +494,18 @@ module Fluent::Plugin
       aws_account_id = ec2_metadata['accountId'] if
           ec2_metadata.key?('accountId')
       # #host:i-09fbfed2672d2c6bf
-      %W(host:#{@vm_id} zone:#{@zone} aws_account_id:#{aws_account_id})
+      %W(host=#{@vm_id} zone=#{@zone} aws_account_id=#{aws_account_id})
         .concat @tags
     end
 
     # Filter out invalid non-Hash entries.
     def each_valid_record(chunk)
       chunk.msgpack_each do |event|
-        unless event.respond_to? :last
-          @log.warn 'Dropping a malformed event: ' \
-                    "'#{event.inspect}'. " \
-                    'An event should be an array.'
-          next
-        end
         record = event.last
         unless record.is_a?(Hash)
           @log.warn 'Dropping log entries with malformed record: ' \
                     "'#{record.inspect}'. " \
-                    "A record should be a hash but was a #{record.class}."
+                    'A log record should be in JSON format.'
           next
         end
         tag = record.first
@@ -524,6 +533,13 @@ module Fluent::Plugin
       tag
     end
 
+    def init_api_client
+      ssl = true
+      ssl = { verify_mode: OpenSSL::SSL::VERIFY_NONE } if @skip_ssl_validation
+      server = "#{@log_dd_url}:#{@log_dd_port}"
+      @conn = Net::TCPClient.new(server: server, ssl: ssl)
+    end
+
     # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
     # non-UTF-8 character would be replaced by the string specified by
     # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any

data/test/plugin/test_out_datadog_log.rb

@@ -35,31 +35,6 @@ class DatadogLogOutputTest < Test::Unit::TestCase
     end
   end
 
-  def test_configure_with_env
-    new_stub_context do
-      setup_ec2_metadata_stubs
-
-      ENV.stubs(:[])
-        .with('DD_API_KEY')
-        .returns('myapikey_from_env')
-
-      ENV.stubs(:[])
-        .with(Not equals 'DD_API_KEY')
-        .returns('')
-        .times(3)
-
-      d = create_driver(<<-EOC)
-        type datadog_log
-        service myservice
-        source mysource
-      EOC
-
-      assert_equal 'myapikey_from_env', d.instance.api_key
-      assert_equal 'myservice', d.instance.service
-      assert_equal 'mysource', d.instance.source
-    end
-  end
-
   def test_write
     new_stub_context do
       setup_ec2_metadata_stubs
@@ -78,66 +53,16 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       EOC
       conn = StubConn.new
       fluentd_tag = 'mytag'
-      Net::TCPClient.stubs(:new)
-        .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
-        .returns(conn)
-      d.run(default_tag: fluentd_tag) do
-        record = {
-          'log' => 'mymsg'
-        }
-        d.feed(time, record)
-      end
-
-      # fail d.logs.inspect
-      assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
-      assert_equal(1, conn.sent.size)
-      # rubocop:disable LineLength
-      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="host:i-81c16767,zone:aws:us-west-2b,aws_account_id:123456789012"] mymsg\n)
-      # rubocop:enable LineLength
-      assert_equal(payload, conn.sent.first)
-    end
-  end
-
-  def test_write_kube
-    new_stub_context do
-      setup_ec2_metadata_stubs
-
-      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
-      t = DateTime.rfc3339(timestamp_str).to_time
-      time = Fluent::EventTime.from_time(t)
-      d = create_driver(<<-EOC)
-        type datadog_log
-        api_key myapikey
-        service myservice
-        source mysource
-        source_category mysourcecategory
-        logset mylogset
-        log_level debug
-        tags ["kube_cluster:MyCluster", "mykey:myval"]
-      EOC
-      conn = StubConn.new
-      fluentd_tag = 'mytag'
-      Net::TCPClient.stubs(:new)
-        .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
-        .returns(conn)
       d.run(default_tag: fluentd_tag) do
+        d.instance.instance_variable_set :@conn, conn
         record = {
           'log' => 'mymsg',
-          'docker' => {
-            'container_id' => 'myfullcontainerid'
-          },
           'kubernetes' => {
             'namespace' => 'myns',
             'pod_name' => 'mypod',
             'container_name' => 'mycontainer',
             'labels' => {
               'k8s-app' => 'myapp'
-            },
-            'annotations' => {
-              # rubocop:disable LineLength
-              # kubernetes.io is translated to kubernetes_io by kubernetes metadata filter
-              'kubernetes_io/created-by' => '{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"Deployment","namespace":"default","name":"myapp","uid":"d67e8857-c2dc-11e7-aed9-066d23381f8c","apiVersion":"extensions","resourceVersion":"289"}}'
-              # rubocop:enable LineLength
             }
           }
         }
@@ -148,7 +73,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
       assert_equal(1, conn.sent.size)
       # rubocop:disable LineLength
-      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myapp - - [dd ddsource="mypod"][dd ddsourcecategory="mycontainer"][dd ddtags="pod_name:mypod,container_name:mycontainer,kube_k8s-app:myapp,kube_deployment:myapp,host:i-81c16767,zone:aws:us-west-2b,aws_account_id:123456789012,kube_cluster:MyCluster,mykey:myval"] mymsg\n)
+      payload = 'myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n'
       # rubocop:enable LineLength
       assert_equal(payload, conn.sent.first)
     end
@@ -178,10 +103,6 @@ class DatadogLogOutputTest < Test::Unit::TestCase
             log_level debug
             enable_monitoring true
           EOC
-          conn = StubConn.new
-          Net::TCPClient.stubs(:new)
-            .with(server: 'intake.logs.datadoghq.com:10516', ssl: true)
-            .returns(conn)
           d.run(default_tag: 'mytag') do
             (1..entry_count).each do |i|
               d.feed time, 'message' => log_entry(i.to_s)
@@ -221,10 +142,6 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       @sent = []
     end
 
-    def retry_on_connection_failure
-      yield
-    end
-
     def write(payload)
       @sent << payload
     end