fluent-plugin-datadog-log 0.1.0.rc1 → 0.1.0.rc2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c5156471f5c6f9fa323d2843a7ae8e291d639cc9
-  data.tar.gz: f9f60fa52dc48eef464da9de7231253f996462b7
+  metadata.gz: 2ec628db3ffd3258f75a1ec766c763f3d4ad70ea
+  data.tar.gz: 9dc3f96c5ff194ab5cdb17e3e7ed1ac7ccfcecfd
 SHA512:
-  metadata.gz: 5b359716b9eb9fbcda4024db5aa0376ba0655081d1def3d9b57a41cf74c35e6e93fcdf0b10d944c2a675f4102b597d2c645aa444d7dece4d92b02af0546c45cb
-  data.tar.gz: 5fcfc226bd44d985616a4bc67e9e603f7ebde31f4ccd468e72cae2ec13dc054c71e5a5785a764f8a0f69ca13e10263e3df2f47b811c72a47568411f1a0a699fc
+  metadata.gz: 5a97cd2159cb172acd76cb1ed9141176cac53758dd2d384b6bcd7c77dacf0f45f37e81edc932b092e2c86847b65c3a552f8a2756057ae456364aacfafc3b1f6a
+  data.tar.gz: e26b6ef4808d0d76fc56f0d63b545f7df3497658ed9353fa7aa1cdb00d20f834227a825bb7e3c45f5dfe127a18bad7add8d55b258cca983c103f70a9fd092329

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-datadog-log (0.1.0.rc1)
+    fluent-plugin-datadog-log (0.1.0.rc2)
       fluentd (~> 0.14)
       json (~> 1.8)
 

data/fluent-plugin-datadog-log.gemspec

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc1'
+  gem.version       = '0.1.0.rc2'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/fluent-plugin-datadog-log.gemspec~

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0'
+  gem.version       = '0.1.0.rc1'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/out_datadog_log.rb

@@ -128,9 +128,9 @@ module Fluent::Plugin
       super
 
       if @api_key.size == 0
-        @api_key = ENV['DD_LOG_API_KEY']
+        @api_key = ENV['DD_API_KEY']
         if @api_key == '' || @api_key.nil?
-          error_message = 'Unable to obtain api_key from DD_LOG_API_KEY'
+          error_message = 'Unable to obtain api_key from DD_API_KEY'
           fail Fluent::ConfigError, error_message
         end
       end
@@ -249,10 +249,26 @@ module Fluent::Plugin
             tags << "#{tag_key}=#{kube[json_key]}" if kube.key? json_key
           end
 
-          if kube.key? 'labels'
-            labels = kube['labels']
-            labels.each do |k, v|
-              tags << "kube_#{k}=#{v}"
+          kube_labels = kube['labels']
+          unless kube_labels.nil?
+            kube_labels.each do |k, v|
+              k2 = k.dup
+              k2.gsub!(/[\,\.]/, '_')
+              k2.gsub!(%r{/}, '-')
+              tags << "kube_#{k2}=#{v}"
+            end
+          end
+
+          if kube.key? 'annotations'
+            annotations = kube['annotations']
+            created_by_str = annotations['kubernetes.io/created-by']
+            unless created_by_str.nil?
+              created_by = JSON.parse(created_by_str)
+              ref = created_by['reference'] unless created_by.nil?
+              kind = ref['kind'] unless ref.nil?
+              name = ref['name'] unless ref.nil?
+              kind = kind.downcase unless kind.nil?
+              tags << "kube_#{kind}=#{name}" if !kind.nil? && !name.nil?
             end
           end
 
@@ -264,6 +280,14 @@ module Fluent::Plugin
 
           tags.concat(@default_tags)
 
+          service = kube_labels['app'] || kube_labels['k8s-app'] unless kube_labels.nil?
+          source = kube['pod_name']
+          source_category = kube['container_name']
+
+          service = @service if service.nil?
+          source = @source if source.nil?
+          source_category = @source_category if source_category.nil?
+
           datetime = Time.at(Fluent::EventTime.new(time).to_r).utc.to_datetime
 
           payload =
@@ -271,9 +295,9 @@ module Fluent::Plugin
               logset: @logset,
               msg: msg,
               datetime: datetime,
-              service: @service,
-              source: @source,
-              source_category: @source_category,
+              service: service,
+              source: source,
+              source_category: source_category,
               tags: tags
             )
 
@@ -290,12 +314,19 @@ module Fluent::Plugin
           end
 
         rescue => error
-          increment_failed_requests_count
-          increment_retried_entries_count(entries_count)
-          # RPC cancelled, so retry via re-raising the error.
-          @log.debug "Retrying #{entries_count} log message(s) later.",
-                     error: error.to_s
           raise error
+          increment_failed_requests_count
+          if entries_count.nil?
+            increment_dropped_entries_count(1)
+            @log.error 'Not retrying a log message later',
+                       error: error.to_s
+          else
+            increment_retried_entries_count(entries_count)
+            # RPC cancelled, so retry via re-raising the error.
+            @log.debug "Retrying #{entries_count} log message(s) later.",
+                       error: error.to_s
+            raise error
+          end
         end
       end
     end

data/lib/fluent/plugin/out_datadog_log.rb~

@@ -0,0 +1,549 @@
+# Copyright 2017 Yusuke KUOKA All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require 'erb'
+require 'json'
+require 'open-uri'
+require 'socket'
+require 'time'
+require 'yaml'
+require 'fluent/plugin/output'
+require 'datadog/log'
+
+require_relative 'monitoring'
+
+module Fluent::Plugin
+  # fluentd output plugin for the Datadog Log Intake API
+  class DatadogOutput < ::Fluent::Plugin::Output
+    Fluent::Plugin.register_output('datadog', self)
+
+    helpers :compat_parameters, :inject
+
+    include ::Datadog::Log
+
+    DEFAULT_BUFFER_TYPE = 'memory'
+
+    PLUGIN_NAME = 'Fluentd Datadog plugin'
+    PLUGIN_VERSION = '0.1.0'
+
+    # Address of the metadata service.
+    METADATA_SERVICE_ADDR = '169.254.169.254'
+
+    # Disable this warning to conform to fluentd config_param conventions.
+    # rubocop:disable Style/HashSyntax
+
+    # see https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/logagent/logsagent.go#L26-L30
+    # see https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/config/config.go#L52-L56
+    config_param :log_dd_url, :string, default: 'intake.logs.datadoghq.com'
+    config_param :log_dd_port, :integer, default: 10516
+    config_param :skip_ssl_validation, default: false
+    config_param :api_key, :string, default: ''
+    config_param :logset, :string, default: 'main'
+
+    # e.g. ['env:prod', 'app:myapp']
+    # see https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/logagent/etc/conf.d/integration.yaml.example
+    config_param :tags, :array, default: [], value_type: :string
+    config_param :service, :string, default: '-'
+    # e.g. 'nginx'
+    config_param :source, :string, default: ''
+    config_param :source_category, :string, default: ''
+
+    config_section :buffer do
+      config_set_default :@type, DEFAULT_BUFFER_TYPE
+    end
+
+    # e.g. 'http_access'
+    # config_param :source_category, :string, default: ''
+
+    # Specify project/instance metadata.
+    #
+    # project_id, zone, and vm_id are required to have valid values, which
+    # can be obtained from the metadata service or set explicitly.
+    # Otherwise, the plugin will fail to initialize.
+    #
+    # Note that while 'project id' properly refers to the alphanumeric name
+    # of the project, the logging service will also accept the project number,
+    # so either one is acceptable in this context.
+    #
+    # Whether to attempt to obtain metadata from the local metadata service.
+    # It is safe to specify 'true' even on platforms with no metadata service.
+    config_param :use_metadata_service, :bool, :default => true
+    # These parameters override any values obtained from the metadata service.
+    config_param :project_id, :string, :default => nil
+    config_param :zone, :string, :default => nil
+    config_param :vm_id, :string, :default => nil
+    config_param :vm_name, :string, :default => nil
+
+    # TODO: Correlate log messages to corresponding Datadog APM spans
+    # config_param :trace_key, :string, :default => DEFAULT_TRACE_KEY
+
+    # Whether to try to detect if the record is a text log entry with JSON
+    # content that needs to be parsed.
+    config_param :detect_json, :bool, :default => false
+
+    # Whether to reject log entries with invalid tags. If this option is set to
+    # false, tags will be made valid by converting any non-string tag to a
+    # string, and sanitizing any non-utf8 or other invalid characters.
+    config_param :require_valid_tags, :bool, :default => false
+
+    # Whether to allow non-UTF-8 characters in user logs. If set to true, any
+    # non-UTF-8 character would be replaced by the string specified by
+    # 'non_utf8_replacement_string'. If set to false, any non-UTF-8 character
+    # would trigger the plugin to error out.
+    config_param :coerce_to_utf8, :bool, :default => true
+
+    # If 'coerce_to_utf8' is set to true, any non-UTF-8 character would be
+    # replaced by the string specified here.
+    config_param :non_utf8_replacement_string, :string, :default => ' '
+
+    # Whether to collect metrics about the plugin usage. The mechanism for
+    # collecting and exposing metrics is controlled by the monitoring_type
+    # parameter.
+    config_param :enable_monitoring, :bool, :default => false
+    config_param :monitoring_type, :string, :default => 'prometheus'
+
+    # rubocop:enable Style/HashSyntax
+
+    attr_reader :zone
+    attr_reader :vm_id
+
+    def initialize
+      super
+      # use the global logger
+      @log = $log # rubocop:disable Style/GlobalVars
+    end
+
+    def configure(conf)
+      compat_parameters_convert(conf, :buffer, :inject)
+      super
+
+      if @api_key.size == 0
+        @api_key = ENV['DD_LOG_API_KEY']
+        if @api_key == '' || @api_key.nil?
+          error_message = 'Unable to obtain api_key from DD_LOG_API_KEY'
+          fail Fluent::ConfigError, error_message
+        end
+      end
+
+      # If monitoring is enabled, register metrics in the default registry
+      # and store metric objects for future use.
+      if @enable_monitoring
+        registry = Monitoring::MonitoringRegistryFactory.create @monitoring_type
+        @successful_requests_count = registry.counter(
+          :datadog_successful_requests_count,
+          'A number of successful requests to the Datadog Log Intake API')
+        @failed_requests_count = registry.counter(
+          :datadog_failed_requests_count,
+          'A number of failed requests to the Datadog Log Intake API,'\
+            ' broken down by the error code')
+        @ingested_entries_count = registry.counter(
+          :datadog_ingested_entries_count,
+          'A number of log entries ingested by Datadog Log Intake')
+        @dropped_entries_count = registry.counter(
+          :datadog_dropped_entries_count,
+          'A number of log entries dropped by the Stackdriver output plugin')
+        @retried_entries_count = registry.counter(
+          :datadog_retried_entries_count,
+          'The number of log entries that failed to be ingested by the'\
+            ' Stackdriver output plugin due to a transient error and were'\
+            ' retried')
+      end
+
+      @platform = detect_platform
+
+      # Set required variables: @project_id, @vm_id, @vm_name and @zone.
+      set_required_metadata_variables
+
+      @default_tags = build_default_tags
+
+      # The resource and labels are now set up; ensure they can't be modified
+      # without first duping them.
+      @default_tags.freeze
+
+      # Log an informational message containing the Logs viewer URL
+      @log.info 'Logs viewer address: https://example.com/logs/'
+    end
+
+    def start
+      super
+      init_api_client
+      @successful_call = false
+      @timenanos_warning = false
+    end
+
+    def shutdown
+      super
+      @conn.shutdown
+    end
+
+    def format(tag, time, record)
+      record = inject_values_to_record(tag, time, record)
+      [tag, time, record].to_msgpack
+    end
+
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def write(chunk)
+      each_valid_record(chunk) do |_tag, time, record|
+        if @detect_json
+          # Save the timestamp and severity if available, then clear it out to
+          # allow for determining whether we should parse the log or message
+          # field.
+          timestamp = record.delete('time')
+          severity = record.delete('severity')
+
+          # If the log is json, we want to export it as a structured log
+          # unless there is additional metadata that would be lost.
+          record_json = nil
+          if record.length == 1
+            %w(log message msg).each do |field|
+              if record.key?(field)
+                record_json = parse_json_or_nil(record[field])
+              end
+            end
+          end
+          record = record_json unless record_json.nil?
+          # Restore timestamp and severity if necessary. Note that we don't
+          # want to override these keys in the JSON we've just parsed.
+          record['time'] ||= timestamp if timestamp
+          record['severity'] ||= severity if severity
+        end
+
+        # TODO: Correlate Datadog APM spans with log messages
+        # fq_trace_id = record.delete(@trace_key)
+        # entry.trace = fq_trace_id if fq_trace_id
+
+        begin
+          msg = nil
+          %w(log message msg).each do |field|
+            msg = record[field] if record.key?(field)
+          end
+
+          tags = []
+
+          kube = record['kubernetes'] || {}
+
+          mappings = {
+            'pod_name' => 'pod_name',
+            'container_name' => 'container_name',
+            'namespace_name' => 'kube_namespace'
+          }
+
+          mappings.each do |json_key, tag_key|
+            tags << "#{tag_key}=#{kube[json_key]}" if kube.key? json_key
+          end
+
+          if kube.key? 'labels'
+            labels = kube['labels']
+            labels.each do |k, v|
+              tags << "kube_#{k}=#{v}"
+            end
+          end
+
+          # TODO: Include K8S tags like
+          # - kube_daemon_set=$daemonset_name
+          # - kube_deployment=$deployment_name
+          # - kube_replica_set=$replicaset_name
+          # -
+
+          tags.concat(@default_tags)
+
+          datetime = Time.at(Fluent::EventTime.new(time).to_r).utc.to_datetime
+
+          payload =
+            @conn.send_payload(
+              logset: @logset,
+              msg: msg,
+              datetime: datetime,
+              service: @service,
+              source: @source,
+              source_category: @source_category,
+              tags: tags
+            )
+
+          entries_count = 1
+          @log.debug 'Sent payload to Datadog.', payload: payload
+          increment_successful_requests_count
+          increment_ingested_entries_count(entries_count)
+
+          # Let the user explicitly know when the first call succeeded, to aid
+          # with verification and troubleshooting.
+          unless @successful_call
+            @successful_call = true
+            @log.info 'Successfully sent to Datadog.'
+          end
+
+        rescue => error
+          increment_failed_requests_count
+          increment_retried_entries_count(entries_count)
+          # RPC cancelled, so retry via re-raising the error.
+          @log.debug "Retrying #{entries_count} log message(s) later.",
+                     error: error.to_s
+          raise error
+        end
+      end
+    end
+
+    private
+
+    def init_api_client
+      @conn = ::Datadog::Log::Client.new(
+        log_dd_url: @log_dd_uri,
+        log_dd_port: @log_dd_port,
+        api_key: @api_key,
+        hostname: @vm_id,
+        skip_ssl_validation: @skip_ssl_validation
+      )
+    end
+
+    def parse_json_or_nil(input)
+      # Only here to please rubocop...
+      return nil if input.nil?
+
+      input.each_codepoint do |c|
+        if c == 123
+          # left curly bracket (U+007B)
+          begin
+            return JSON.parse(input)
+          rescue JSON::ParserError
+            return nil
+          end
+        else
+          # Break (and return nil) unless the current character is whitespace,
+          # in which case we continue to look for a left curly bracket.
+          # Whitespace as per the JSON spec are: tabulation (U+0009),
+          # line feed (U+000A), carriage return (U+000D), and space (U+0020).
+          break unless c == 9 || c == 10 || c == 13 || c == 32
+        end # case
+      end # do
+      nil
+    end
+
+    # "enum" of Platform values
+    module Platform
+      OTHER = 0  # Other/unkown platform
+      GCE = 1    # Google Compute Engine
+      EC2 = 2    # Amazon EC2
+    end
+
+    # Determine what platform we are running on by consulting the metadata
+    # service (unless the user has explicitly disabled using that).
+    def detect_platform
+      unless @use_metadata_service
+        @log.info 'use_metadata_service is false; not detecting platform'
+        return Platform::OTHER
+      end
+
+      begin
+        open('http://' + METADATA_SERVICE_ADDR) do |f|
+          if f.meta['metadata-flavor'] == 'Google'
+            @log.info 'Detected GCE platform'
+            return Platform::GCE
+          end
+          if f.meta['server'] == 'EC2ws'
+            @log.info 'Detected EC2 platform'
+            return Platform::EC2
+          end
+        end
+      rescue StandardError => e
+        @log.error 'Failed to access metadata service: ', error: e
+      end
+
+      @log.info 'Unable to determine platform'
+      Platform::OTHER
+    end
+
+    def fetch_gce_metadata(metadata_path)
+      fail "Called fetch_gce_metadata with platform=#{@platform}" unless
+        @platform == Platform::GCE
+      # See https://cloud.google.com/compute/docs/metadata
+      open('http://' + METADATA_SERVICE_ADDR + '/computeMetadata/v1/' +
+           metadata_path, 'Metadata-Flavor' => 'Google', &:read)
+    end
+
+    # EC2 Metadata server returns everything in one call. Store it after the
+    # first fetch to avoid making multiple calls.
+    def ec2_metadata
+      fail "Called ec2_metadata with platform=#{@platform}" unless
+        @platform == Platform::EC2
+      unless @ec2_metadata
+        # See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
+        open('http://' + METADATA_SERVICE_ADDR +
+             '/latest/dynamic/instance-identity/document') do |f|
+          contents = f.read
+          @ec2_metadata = JSON.parse(contents)
+        end
+      end
+
+      @ec2_metadata
+    end
+
+    # Set required variables like @vm_id, @vm_name and @zone.
+    def set_required_metadata_variables
+      set_vm_id
+      set_vm_name
+      set_zone
+
+      # All metadata parameters must now be set.
+      missing = []
+      missing << 'zone' unless @zone
+      missing << 'vm_id' unless @vm_id
+      missing << 'vm_name' unless @vm_name
+      return if missing.empty?
+      fail Fluent::ConfigError, 'Unable to obtain metadata parameters: ' +
+        missing.join(' ')
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it by calling metadata servers directly.
+    def set_vm_id
+      @vm_id ||= ec2_metadata['instanceId'] if @platform == Platform::EC2
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm_id: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it locally.
+    def set_vm_name
+      @vm_name ||= Socket.gethostname
+    rescue StandardError => e
+      @log.error 'Failed to obtain vm name: ', error: e
+    end
+
+    # 1. Return the value if it is explicitly set in the config already.
+    # 2. If not, try to retrieve it locally.
+    def set_zone
+      @zone ||= 'aws:' + ec2_metadata['availabilityZone'] if
+        @platform == Platform::EC2 && ec2_metadata.key?('availabilityZone')
+    rescue StandardError => e
+      @log.error 'Failed to obtain location: ', error: e
+    end
+
+    # Determine agent level monitored resource labels based on the resource
+    # type. Each resource type has its own labels that need to be filled in.
+    def build_default_tags
+      aws_account_id = ec2_metadata['accountId'] if
+          ec2_metadata.key?('accountId')
+      # #host:i-09fbfed2672d2c6bf
+      %W(host=#{@vm_id} zone=#{@zone} aws_account_id=#{aws_account_id})
+        .concat @tags
+    end
+
+    # Filter out invalid non-Hash entries.
+    def each_valid_record(chunk)
+      chunk.msgpack_each do |event|
+        record = event.last
+        unless record.is_a?(Hash)
+          @log.warn 'Dropping log entries with malformed record: ' \
+                    "'#{record.inspect}'. " \
+                    'A log record should be in JSON format.'
+          next
+        end
+        tag = record.first
+        sanitized_tag = sanitize_tag(tag)
+        if sanitized_tag.nil?
+          @log.warn "Dropping log entries with invalid tag: '#{tag.inspect}'." \
+                    ' A tag should be a string with utf8 characters.'
+          next
+        end
+        yield event
+      end
+    end
+
+    # Given a tag, returns the corresponding valid tag if possible, or nil if
+    # the tag should be rejected. If 'require_valid_tags' is false, non-string
+    # tags are converted to strings, and invalid characters are sanitized;
+    # otherwise such tags are rejected.
+    def sanitize_tag(tag)
+      if @require_valid_tags &&
+         (!tag.is_a?(String) || tag == '' || convert_to_utf8(tag) != tag)
+        return nil
+      end
+      tag = convert_to_utf8(tag.to_s)
+      tag = '_' if tag == ''
+      tag
+    end
+
+    # Encode as UTF-8. If 'coerce_to_utf8' is set to true in the config, any
+    # non-UTF-8 character would be replaced by the string specified by
+    # 'non_utf8_replacement_string'. If 'coerce_to_utf8' is set to false, any
+    # non-UTF-8 character would trigger the plugin to error out.
+    def convert_to_utf8(input)
+      if @coerce_to_utf8
+        input.encode(
+          'utf-8',
+          invalid: :replace,
+          undef: :replace,
+          replace: @non_utf8_replacement_string)
+      else
+        begin
+          input.encode('utf-8')
+        rescue EncodingError
+          @log.error 'Encountered encoding issues potentially due to non ' \
+                     'UTF-8 characters. To allow non-UTF-8 characters and ' \
+                     'replace them with spaces, please set "coerce_to_utf8" ' \
+                     'to true.'
+          raise
+        end
+      end
+    end
+
+    def ensure_array(value)
+      Array.try_convert(value) || (fail JSON::ParserError, "#{value.class}")
+    end
+
+    def ensure_hash(value)
+      Hash.try_convert(value) || (fail JSON::ParserError, "#{value.class}")
+    end
+
+    # Increment the metric for the number of successful requests.
+    def increment_successful_requests_count
+      return unless @successful_requests_count
+      @successful_requests_count.increment
+    end
+
+    # Increment the metric for the number of failed requests, labeled by
+    # the provided status code.
+    def increment_failed_requests_count
+      return unless @failed_requests_count
+      @failed_requests_count.increment
+    end
+
+    # Increment the metric for the number of log entries, successfully
+    # ingested by the Datadog Log Intake API.
+    def increment_ingested_entries_count(count)
+      return unless @ingested_entries_count
+      @ingested_entries_count.increment({}, count)
+    end
+
+    # Increment the metric for the number of log entries that were dropped
+    # and not ingested by the Datadog Log Intake API.
+    def increment_dropped_entries_count(count)
+      return unless @dropped_entries_count
+      @dropped_entries_count.increment({}, count)
+    end
+
+    # Increment the metric for the number of log entries that were dropped
+    # and not ingested by the Datadog Log Intake API.
+    def increment_retried_entries_count(count)
+      return unless @retried_entries_count
+      @retried_entries_count.increment({}, count)
+    end
+  end
+end

data/test/plugin/test_out_datadog_log.rb

@@ -35,7 +35,70 @@ class DatadogLogOutputTest < Test::Unit::TestCase
     end
   end
 
+  def test_configure_with_env
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      ENV.stubs(:[])
+        .with('DD_API_KEY')
+        .returns('myapikey_from_env')
+
+      ENV.stubs(:[])
+        .with(Not equals 'DD_API_KEY')
+        .returns('')
+        .times(3)
+
+      d = create_driver(<<-EOC)
+        type datadog_log
+        service myservice
+        source mysource
+      EOC
+
+      assert_equal 'myapikey_from_env', d.instance.api_key
+      assert_equal 'myservice', d.instance.service
+      assert_equal 'mysource', d.instance.source
+    end
+  end
+
   def test_write
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+        source_category mysourcecategory
+        logset mylogset
+        log_level debug
+      EOC
+      conn = StubConn.new
+      fluentd_tag = 'mytag'
+      Net::TCPClient.stubs(:new)
+        .with(server: ':10516', ssl: true)
+        .returns(conn)
+      d.run(default_tag: fluentd_tag) do
+        record = {
+          'log' => 'mymsg'
+        }
+        d.feed(time, record)
+      end
+
+      # fail d.logs.inspect
+      assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
+      assert_equal(1, conn.sent.size)
+      # rubocop:disable LineLength
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      # rubocop:enable LineLength
+      assert_equal(payload, conn.sent.first)
+    end
+  end
+
+  def test_write_kube
     new_stub_context do
       setup_ec2_metadata_stubs
 
@@ -59,12 +122,20 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       d.run(default_tag: fluentd_tag) do
         record = {
           'log' => 'mymsg',
+          'docker' => {
+            'container_id' => 'myfullcontainerid'
+          },
           'kubernetes' => {
             'namespace' => 'myns',
             'pod_name' => 'mypod',
             'container_name' => 'mycontainer',
             'labels' => {
               'k8s-app' => 'myapp'
+            },
+            'annotations' => {
+              # rubocop:disable LineLength
+              'kubernetes.io/created-by' => '{"kind":"SerializedReference","apiVersion":"v1","reference":{"kind":"Deployment","namespace":"default","name":"myapp","uid":"d67e8857-c2dc-11e7-aed9-066d23381f8c","apiVersion":"extensions","resourceVersion":"289"}}'
+              # rubocop:enable LineLength
             }
           }
         }
@@ -75,7 +146,7 @@ class DatadogLogOutputTest < Test::Unit::TestCase
       assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
       assert_equal(1, conn.sent.size)
       # rubocop:disable LineLength
-      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myapp - - [dd ddsource="mypod"][dd ddsourcecategory="mycontainer"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,kube_deployment=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
       # rubocop:enable LineLength
       assert_equal(payload, conn.sent.first)
     end

data/test/plugin/test_out_datadog_log.rb~

@@ -0,0 +1,206 @@
+# Copyright 2017 Yusuke KUOKA All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require_relative 'base_test'
+
+# Unit tests for Datadog Log plugin
+class DatadogLogOutputTest < Test::Unit::TestCase
+  include BaseTest
+
+  def test_configure
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+      EOC
+
+      assert_equal 'myapikey', d.instance.api_key
+      assert_equal 'myservice', d.instance.service
+      assert_equal 'mysource', d.instance.source
+    end
+  end
+
+  def test_write
+    new_stub_context do
+      setup_ec2_metadata_stubs
+
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      d = create_driver(<<-EOC)
+        type datadog_log
+        api_key myapikey
+        service myservice
+        source mysource
+        source_category mysourcecategory
+        logset mylogset
+        log_level debug
+      EOC
+      conn = StubConn.new
+      fluentd_tag = 'mytag'
+      Net::TCPClient.stubs(:new)
+        .with(server: ':10516', ssl: true)
+        .returns(conn)
+      d.run(default_tag: fluentd_tag) do
+        record = {
+          'log' => 'mymsg',
+          'kubernetes' => {
+            'namespace' => 'myns',
+            'pod_name' => 'mypod',
+            'container_name' => 'mycontainer',
+            'labels' => {
+              'k8s-app' => 'myapp'
+            }
+          }
+        }
+        d.feed(time, record)
+      end
+
+      # fail d.logs.inspect
+      assert_equal(1, d.logs.count { |l| l =~ /Sent payload to Datadog/ })
+      assert_equal(1, conn.sent.size)
+      # rubocop:disable LineLength
+      payload = %(myapikey/mylogset <46>0 2006-01-02T15:04:05.000000+00:00 i-81c16767 myservice - - [dd ddsource="mysource"][dd ddsourcecategory="mysourcecategory"][dd ddtags="pod_name=mypod,container_name=mycontainer,kube_k8s-app=myapp,host=i-81c16767,zone=aws:us-west-2b,aws_account_id=123456789012"] mymsg\n)
+      # rubocop:enable LineLength
+      assert_equal(payload, conn.sent.first)
+    end
+  end
+
+  def test_prometheus_metrics
+    new_stub_context do
+      setup_ec2_metadata_stubs
+      timestamp_str = '2006-01-02T15:04:05.000000+00:00'
+      t = DateTime.rfc3339(timestamp_str).to_time
+      time = Fluent::EventTime.from_time(t)
+      [
+        # Single successful request.
+        [false, 0, 1, 1, [1, 0, 1, 0, 0]],
+        # Several successful requests.
+        [false, 0, 2, 1, [2, 0, 2, 0, 0]]
+      ].each do |_should_fail, _code, request_count, entry_count, metric_values|
+        setup_prometheus
+        (1..request_count).each do
+          d = create_driver(<<-EOC)
+            type datadog_log
+            api_key myapikey
+            service myservice
+            source mysource
+            source_category mysourcecategory
+            logset mylogset
+            log_level debug
+            enable_monitoring true
+          EOC
+          conn = StubConn.new
+          Net::TCPClient.stubs(:new)
+            .with(server: ':10516', ssl: true)
+            .returns(conn)
+          d.run(default_tag: 'mytag') do
+            (1..entry_count).each do |i|
+              d.feed time, 'message' => log_entry(i.to_s)
+            end
+          end
+        end
+        successful_requests_count, failed_requests_count,
+          ingested_entries_count, dropped_entries_count,
+          retried_entries_count = metric_values
+        assert_prometheus_metric_value(:datadog_successful_requests_count,
+                                       successful_requests_count)
+        assert_prometheus_metric_value(:datadog_failed_requests_count,
+                                       failed_requests_count)
+        assert_prometheus_metric_value(:datadog_ingested_entries_count,
+                                       ingested_entries_count)
+        assert_prometheus_metric_value(:datadog_dropped_entries_count,
+                                       dropped_entries_count)
+        assert_prometheus_metric_value(:datadog_retried_entries_count,
+                                       retried_entries_count)
+      end
+    end
+  end
+
+  def test_struct_payload_non_utf8_log
+    # d.emit('msg' => log_entry(0),
+    #        'normal_key' => "test#{non_utf8_character}non utf8",
+    #        "non_utf8#{non_utf8_character}key" => 5000,
+    #        'nested_struct' => { "non_utf8#{non_utf8_character}key" => \
+    #                             "test#{non_utf8_character}non utf8" },
+    #        'null_field' => nil)
+  end
+
+  class StubConn
+    attr_reader :sent
+
+    def initialize
+      @sent = []
+    end
+
+    def write(payload)
+      @sent << payload
+    end
+
+    def close
+    end
+  end
+
+  private
+
+  # Use the right single quotation mark as the sample non-utf8 character.
+  def non_utf8_character
+    [0x92].pack('C*')
+  end
+
+  # For an optional field with default values, Protobuf omits the field when it
+  # is deserialized to json. So we need to add an extra check for gRPC which
+  # uses Protobuf.
+  #
+  # An optional block can be passed in if we need to assert something other than
+  # a plain equal. e.g. assert_in_delta.
+  def assert_equal_with_default(field, expected_value, default_value, entry)
+    if expected_value == default_value
+      assert_nil field
+    elsif block_given?
+      yield
+    else
+      assert_equal expected_value, field, entry
+    end
+  end
+
+  # Get the fields of the payload.
+  def get_fields(payload)
+    payload['fields']
+  end
+
+  # Get the value of a struct field.
+  def get_struct(field)
+    field['structValue']
+  end
+
+  # Get the value of a string field.
+  def get_string(field)
+    field['stringValue']
+  end
+
+  # Get the value of a number field.
+  def get_number(field)
+    field['numberValue']
+  end
+
+  # The null value.
+  def null_value
+    { 'nullValue' => 'NULL_VALUE' }
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datadog-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc1
+  version: 0.1.0.rc2
 platform: ruby
 authors:
 - Yusuke KUOKA
@@ -156,12 +156,16 @@ files:
 - lib/datadog/log.rb
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_datadog_log.rb
+- lib/fluent/plugin/out_datadog_log.rb~
 - pkg/fluent-plugin-datadog-0.1.0.gem
 - pkg/fluent-plugin-datadog-log-0.1.0.gem
+- pkg/fluent-plugin-datadog-log-0.1.0.rc1.gem
+- pkg/fluent-plugin-datadog-log-0.1.0.rc2.gem
 - test/helper.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/test_out_datadog_log.rb
+- test/plugin/test_out_datadog_log.rb~
 homepage: https://github.com/mumoshu/fluent-plugin-datadog-log
 licenses:
 - Apache-2.0
@@ -191,3 +195,4 @@ test_files:
 - test/plugin/base_test.rb
 - test/plugin/constants.rb
 - test/plugin/test_out_datadog_log.rb
+- test/plugin/test_out_datadog_log.rb~