fluent-plugin-datadog-log 0.1.0.rc12 → 0.1.0.rc13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f70d2fad0fa032bae35bc5169d83cd48387e2951
-  data.tar.gz: eb00f95517613fdffff3dfa89112403acd30c023
+  metadata.gz: 3f7fcfdfa0667c08f2d6a9af5609357cc27c43a4
+  data.tar.gz: 1e782d80952ebc9b82339cf1b49b796ee20b407f
 SHA512:
-  metadata.gz: 7a2f3238a1b1fc6f0c69d49da6c82013235d1414117cc7837a9bfdf112a48cb3cf6a57cc6f1f6b91fec8524c9a26aadabfc9fd569d0a282f4490d8fbfb5de690
-  data.tar.gz: 3e900e5766e8b5419af768ec765fd2e075c78b69ac8946fb4f7c62d1c26848d55125160f4684bc5b23e42d9b078a84a3225eadc1e1d91feeb8c0e4a9ad19a753
+  metadata.gz: 76c99159bec7e09eab80546f6b1d67188cc4fc97e1f3c1471d4fcee1c2872e5a1100268ba739ba7e7a474a97a7b5e9e23814011db5fc4d6cdcf8674eee068abe
+  data.tar.gz: 18f6a73b24a7914147b900f94b33f7d60ba9f040d5c5c467d01ad67271d4c944c0c8ef3cbfa2ed0e3c4aded8ded4195ab197410e1193a6439aa5c6a9a82817db

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-datadog-log (0.1.0.rc12)
+    fluent-plugin-datadog-log (0.1.0.rc13)
       fluentd (~> 1.0.0)
       json (~> 1.8)
       net_tcp_client (~> 2.0.1)

data/fluent-plugin-datadog-log.gemspec

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc12'
+  gem.version       = '0.1.0.rc13'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/fluent-plugin-datadog-log.gemspec~

@@ -8,7 +8,7 @@ eos
   gem.homepage      = \
     'https://github.com/mumoshu/fluent-plugin-datadog-log'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.1.0.rc8'
+  gem.version       = '0.1.0.rc12'
   gem.authors       = ['Yusuke KUOKA']
   gem.email         = ['ykuoka@gmail.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -17,12 +17,13 @@ eos
   gem.test_files    = gem.files.grep(/^(test)/)
   gem.require_paths = ['lib']
 
-  gem.add_runtime_dependency 'fluentd', '~> 0.14'
+  gem.add_runtime_dependency 'fluentd', '~> 1.0.0'
   # gem.add_runtime_dependency 'datadog-log-api-client', '~> 0.1'
   gem.add_runtime_dependency 'json', '~> 1.8'
 
   gem.add_dependency 'net_tcp_client', '~> 2.0.1'
   gem.add_dependency 'prometheus-client', '~> 0.7.1'
+  gem.add_dependency 'semantic_logger', '~> 4.2.0'
 
   gem.add_development_dependency 'mocha', '~> 1.1'
   gem.add_development_dependency 'rake', '~> 10.3'

data/lib/fluent/plugin/datadog_log.rb

@@ -0,0 +1,153 @@
+require 'openssl'
+
+require 'semantic_logger'
+
+SemanticLogger.default_level = :warn
+SemanticLogger.add_appender(io: STDOUT, formatter: :json)
+
+require 'net/tcp_client'
+require 'socket'
+require 'time'
+
+# Datadog provides various helpers to programatically access Datadog services
+module Datadog
+  # Log provides various helpers and classes to support programatically
+  # accessing Datadog Log Management
+  module Log
+    TRUNCATED_MSG = '...TRUNCATED...'
+
+    TRUNCATED_LEN = TRUNCATED_MSG.size
+
+    # MaxMessageLen is the maximum length for any message we send to the intake
+    # see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/constants.go#L9-L10
+    DD_MAX_MESSAGE_LEN = 1 * 1000 * 1000
+
+    MAX_MESSAGE_LEN = DD_MAX_MESSAGE_LEN - TRUNCATED_LEN
+
+    def truncate_message(msg)
+      if msg.size > DD_MAX_MESSAGE_LEN
+        msg.slice(0, MAX_MESSAGE_LEN) + TRUNCATED_MSG
+      else
+        msg
+      end
+    end
+
+    # Given a list of tags, build_tags_payload generates the bytes array
+    # that will be inserted into messages
+    # @see https://github.com/DataDog/datadog-log-agent/blob/2394da8c79a6cadbcd1e98d6c89c437becec2732/pkg/config/integration_config.go#L180
+    def build_tags_payload(config_tags:, source:, source_category:)
+      payload = ''
+
+      payload = "[dd ddsource=\"#{source}\"]" if !source.nil? && source != ''
+
+      if !source_category.nil? && source_category != ''
+        payload = "#{payload}[dd ddsourcecategory=\"#{source_category}\"]"
+      end
+
+      if !config_tags.nil? && config_tags != ''
+        config_tags = config_tags.join(',') if config_tags.is_a? ::Array
+        payload = "#{payload}[dd ddtags=\"#{config_tags}\"]"
+      end
+
+      payload
+    end
+
+    # https://github.com/DataDog/datadog-log-agent/blob/db13b53dfdd036d43acfb15089a43eb31548f09f/pkg/processor/processor.go#L65
+    def build_extra_content(timestamp:, hostname:, service:, tags_payload:)
+      "<46>0 #{timestamp} #{hostname} #{service} - - #{tags_payload}"
+    end
+
+    def build_api_key_str(api_key:, logset:)
+      if !logset.nil? && logset != ''
+        "#{api_key}/#{logset}"
+      else
+        api_key
+      end
+    end
+
+    # build_payload returns a processed payload from a raw message
+    # @param [String] api_key_str
+    # @param [String] extra_content
+    # @param [String] msg
+    def create_payload(api_key_str:, msg:, extra_content:)
+      "#{api_key_str} #{extra_content} #{msg}\n"
+    end
+
+    # Client supports building/sending payloads to Datadog Log Management
+    class Client
+      include ::Datadog::Log
+
+      def initialize(
+        log_dd_url: 'intake.logs.datadoghq.com',
+        log_dd_port: 10516,
+        api_key:,
+        hostname:,
+        skip_ssl_validation: false
+      )
+        @log_dd_url = log_dd_url
+        @log_dd_port = log_dd_port
+        @api_key = api_key
+        @hostname = hostname
+        @skip_ssl_validation = skip_ssl_validation
+
+        init_api_client
+      end
+
+      # rubocop:disable Metrics/ParameterLists
+      def send_payload(
+        logset: 'main',
+        msg:,
+        datetime: nil,
+        service:,
+        source:,
+        source_category:,
+        tags:
+      )
+        # rubocop:enable Metrics/ParameterLists
+
+        datetime = DateTime.now if datetime.nil?
+
+        # new_offset(0) is required.
+        # otherwise datadog will silently throws away the log..
+        timestamp_str = datetime.new_offset(0).rfc3339(6)
+        payload = create_payload(
+          api_key_str: build_api_key_str(api_key: @api_key, logset: logset),
+          msg: truncate_message(msg),
+          extra_content: build_extra_content(
+            timestamp: timestamp_str,
+            hostname: @hostname,
+            service: service,
+            tags_payload: build_tags_payload(
+              config_tags: tags,
+              source: source,
+              source_category: source_category
+            )
+          )
+        )
+        @conn.retry_on_connection_failure do
+          @conn.write(payload)
+        end
+        payload
+      end
+
+      def shutdown
+        @conn.close unless @conn.nil?
+      end
+
+      class << self
+        def from_env
+          new(api_key: ENV['DD_LOG_API_KEY'], hostname: Socket.gethostname)
+        end
+      end
+
+      private
+
+      def init_api_client
+        ssl = true
+        ssl = { verify_mode: OpenSSL::SSL::VERIFY_NONE } if @skip_ssl_validation
+        server = "#{@log_dd_url}:#{@log_dd_port}"
+        @conn = Net::TCPClient.new(server: server, ssl: ssl)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_datadog_log.rb

@@ -18,8 +18,8 @@ require 'socket'
 require 'time'
 require 'yaml'
 require 'fluent/plugin/output'
-require 'datadog/log'
 
+require_relative 'datadog_log'
 require_relative 'monitoring'
 
 module Fluent::Plugin

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-datadog-log
 version: !ruby/object:Gem::Version
-  version: 0.1.0.rc12
+  version: 0.1.0.rc13
 platform: ruby
 authors:
 - Yusuke KUOKA
@@ -167,14 +167,12 @@ files:
 - fluent-plugin-datadog-log.gemspec
 - fluent-plugin-datadog-log.gemspec~
 - fluent-plugin-datadog.gemspec~
-- lib/datadog/log.rb
 - lib/datadog/log.rb~
+- lib/fluent/plugin/datadog_log.rb
+- lib/fluent/plugin/datadog_log.rb~
 - lib/fluent/plugin/monitoring.rb
 - lib/fluent/plugin/out_datadog_log.rb
 - lib/fluent/plugin/out_datadog_log.rb~
-- pkg/fluent-plugin-datadog-log-0.1.0.rc10.gem
-- pkg/fluent-plugin-datadog-log-0.1.0.rc11.gem
-- pkg/fluent-plugin-datadog-log-0.1.0.rc12.gem
 - test/helper.rb
 - test/plugin/base_test.rb
 - test/plugin/constants.rb