fluent-plugin-dynatrace-radiant 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9b83e5026a95afbaabec8cd191a2098da1d0b1330183ab2a9ba7fea352466c26
-  data.tar.gz: 3481de5e791b40e8eb1084fe88c3f916e26996f99afda8fb1a25dbb134b4cf56
+  metadata.gz: 5370b16e8925bce1279283e9d76a2fd16b4332f5b5284885e15c30158c60c8af
+  data.tar.gz: 192fbb0a0fe7a8709a0b9c6dd81889493a49d79f4fe6df01a301fc04e6eb1e03
 SHA512:
-  metadata.gz: aef4a985c2839024c4e12dca5db15ef732759ab21a6accd64d32402d5d7442998c8c612b9bfee275eb16e6f88c6f62b32f2b392d71b74592a4513ca3f09b0bb7
-  data.tar.gz: caced0055d07f68dbc44ef4291320f81165e576d30e4ab4cee3d7ffc6887196d3a9c7e02dbecbf7d11fbd9b737ee53724791e10e440870836a8b09043a862d87
+  metadata.gz: 5d11024920be58d54250295d82a09243f06193ac765f31383812c7ab79f653916c78cd7a3a31e2f06f39f65b77e1922e9238b15df2cb186981df8212b54b9390
+  data.tar.gz: dafe657ac421d62d22c61c17503b19faf4120c23032aaea5242e546b97b71c70b8299edad0c3302ab68a6f7cd0a53322510e675ec357f1afeb300420fa3ad102

data/lib/fluent/plugin/dynatrace_radiant_constants.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module DynatraceRadiantConstants
-      VERSION = '0.1.1'
+      VERSION = '0.1.2'
     end
   end
 end

data/lib/fluent/plugin/out_dynatrace_radiant.rb

@@ -7,6 +7,7 @@
 
 require 'fluent/plugin/output'
 require 'net/http'
+require 'json'
 require_relative 'dynatrace_radiant_constants'
 
 module Fluent
@@ -21,6 +22,13 @@ module Fluent
 
       helpers :compat_parameters
 
+      CONTENT_KEYS = %w[content message payload body log _raw].freeze
+      LEVEL_KEYS = %w[loglevel status severity level syslog.severity log_level log.level @l severitytext lvl levelname].freeze
+      TEXT_LEVEL_PATTERN = /\A\[?\d{4}-\d{1,2}-\d{1,2}[T ]
+        \d{1,2}:\d{2}(?::\d{2})?(?:[.,]\d+)?(?:Z|[+-]\d{2}:?\d{2})?\]?\s+
+        (?:\[[^\]\r\n]{0,128}\]\s+)?
+        (TRACE|DEBUG|INFO|NOTICE|WARN|WARNING|ERROR|SEVERE|FATAL|CRITICAL)\b/x.freeze
+
       desc 'The full URL of the Dynatrace log ingestion endpoint'
       config_param :active_gate_url, :string
 
@@ -46,15 +54,32 @@ module Fluent
       config_param :read_timeout, :integer, default: 30
 
       desc 'Inject timestamp into each log message'
-      config_param :inject_timestamp, :bool, default: false
+      config_param :inject_timestamp, :bool, default: true
 
       desc 'Maximum number of records per batch request'
       config_param :batch_size, :integer, default: 5000
 
+      desc 'Maximum serialized bytes per request, except for an oversized single record'
+      config_param :max_request_bytes, :size, default: 4 * 1024 * 1024
+
+      desc 'Generate missing log content as logfmt, JSON, or leave it absent'
+      config_param :content_fallback, :enum, list: [:logfmt, :json, :none], default: :logfmt
+
+      desc 'Exact key names to omit from generated content; @timestamp is always omitted'
+      config_param :content_fallback_exclude_keys, :array, value_type: :string, default: []
+
+      desc 'Derive a missing log level from a timestamped text message'
+      config_param :level_from_message, :bool, default: true
+
+      desc 'LDAP result code field used for a missing log level; empty disables it'
+      config_param :ldap_result_code_key, :string, default: 'RESULTCODE'
+
       config_section :buffer do
         config_set_default :flush_at_shutdown, true
         config_set_default :chunk_limit_size, 5 * 1024 * 1024 # 5MB
         config_set_default :flush_interval, 5
+        config_set_default :total_limit_size, 64 * 1024 * 1024 # 64MB per output
+        config_set_default :overflow_action, :drop_oldest_chunk
       end
 
       def configure(conf)
@@ -62,31 +87,54 @@ module Fluent
         super
 
         raise Fluent::ConfigError, 'api_token is empty' if @api_token.empty?
+        raise Fluent::ConfigError, 'max_request_bytes must be positive' unless @max_request_bytes.positive?
         @uri = parse_and_validate_uri(@active_gate_url)
+        @agents = {}
+        @agents_mutex = Mutex.new
       end
 
-      def start
-        super
-        @agent = create_http_agent
-      end
-
-      def shutdown
-        @agent&.finish if @agent&.started?
+      def after_shutdown
+        # Output stops and joins flush threads here, after the final flush.
         super
+        @agents_mutex.synchronize do
+          @agents.each_value do |agent|
+            begin
+              agent.finish if agent.started?
+            rescue IOError
+              # The peer may already have closed this connection.
+            end
+          end
+          @agents.clear
+        end
       end
 
       def write(chunk)
         records = []
+        parts = []
+        bytes = 3 # Opening bracket, closing bracket and newline.
         chunk.each do |time, record|
+          prepare_record(record)
           record['@timestamp'] = (time.to_f * 1000).to_i if @inject_timestamp
-          records.push(record)
+          record = utf8_value(record)
+          part = JSON.generate(record)
 
-          if records.length >= @batch_size
-            send_batch(records)
+          if !parts.empty? && (records.length >= @batch_size || bytes + 1 + part.bytesize > @max_request_bytes)
+            send_batch(records) { "[#{parts.join(',')}]\n" }
             records = []
+            parts = []
+            bytes = 3
           end
+
+          if part.bytesize + 3 > @max_request_bytes
+            info = { request_bytes: part.bytesize + 3 }
+            info[:log_type] = redact_token(record['log_type'].to_s) if record.key?('log_type')
+            log.warn 'Dynatrace record exceeds max_request_bytes; sending alone', **info
+          end
+          bytes += part.bytesize + (parts.empty? ? 0 : 1)
+          records << record
+          parts << part
         end
-        send_batch(records) unless records.empty?
+        send_batch(records) { "[#{parts.join(',')}]\n" } unless records.empty?
       end
 
       def prefer_buffered_processing
@@ -94,7 +142,7 @@ module Fluent
       end
 
       def multi_workers_ready?
-        true # Thread-safe: each worker gets its own HTTP agent via start()
+        true # Workers are separate processes; each flush thread has its own HTTP agent.
       end
 
       def formatted_to_msgpack_binary?
@@ -103,6 +151,91 @@ module Fluent
 
       private
 
+      def prepare_record(record)
+        present = {}
+        record.each do |key, value|
+          normalized_key = utf8_string(key.to_s).downcase
+          present[normalized_key] = value unless value.nil? || present.key?(normalized_key)
+        end
+        content_key = CONTENT_KEYS.find { |key| present.key?(key) }
+        level_key = LEVEL_KEYS.find { |key| present.key?(key) }
+
+        # Generate from the original fields, before adding a derived log level.
+        content = fallback_content(record) if content_key.nil? && @content_fallback != :none
+        if level_key.nil?
+          level = level_from_text(present[content_key]) if @level_from_message && content_key
+          if level.nil? && !@ldap_result_code_key.empty? && record.key?(@ldap_result_code_key)
+            level = level_from_ldap_result_code(record[@ldap_result_code_key])
+          end
+          record['loglevel'] = level if level
+        end
+        record['content'] = content if content && !content.empty?
+      end
+
+      def level_from_text(value)
+        text = utf8_string(value.to_s.byteslice(0, 256))
+        match = TEXT_LEVEL_PATTERN.match(text)
+        return unless match
+
+        match[1] == 'WARNING' ? 'WARN' : match[1]
+      end
+
+      def level_from_ldap_result_code(value)
+        if value.is_a?(String)
+          value = utf8_string(value).strip
+          return unless /\A[0-9]+\z/.match?(value)
+
+          value = value.to_i
+        end
+        return unless value.is_a?(Integer)
+
+        case value
+        when 0, 5, 6, 10, 14 then 'INFO'
+        when 1, 51, 52, 53, 54, 80 then 'ERROR'
+        else 'WARN'
+        end
+      end
+
+      def fallback_content(record)
+        fields = record.reject do |key, value|
+          key == '@timestamp' || @content_fallback_exclude_keys.include?(key) || value.nil? || value.to_s.empty?
+        end
+        return if fields.empty?
+
+        fields = utf8_value(fields)
+        return JSON.generate(fields) if @content_fallback == :json
+
+        fields.map do |key, value|
+          text = value.is_a?(Hash) || value.is_a?(Array) ? JSON.generate(value) : utf8_string(value.to_s)
+          if /[[:space:]"=\\]|[[:cntrl:]]/.match?(text)
+            text = '"' + text.gsub(/[\\"\n\r\t]/) do |char|
+              { '\\' => '\\\\', '"' => '\\"', "\n" => '\\n', "\r" => '\\r', "\t" => '\\t' }[char]
+            end + '"'
+          end
+          "#{utf8_string(key.to_s)}=#{text}"
+        end.join(' ')
+      end
+
+      def utf8_string(value)
+        if value.encoding == Encoding::BINARY || value.encoding == Encoding::UTF_8
+          value.dup.force_encoding(Encoding::UTF_8).scrub("\uFFFD")
+        else
+          value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "\uFFFD")
+        end
+      end
+
+      def utf8_value(value)
+        case value
+        when String then utf8_string(value)
+        when Hash
+          value.each_with_object({}) do |(key, item), result|
+            result[utf8_value(key)] = utf8_value(item)
+          end
+        when Array then value.map { |item| utf8_value(item) }
+        else value
+        end
+      end
+
       def create_http_agent
         agent = Net::HTTP.new(@uri.host, @uri.port)
         agent.open_timeout = @open_timeout
@@ -120,7 +253,7 @@ module Fluent
 
           if @client_cert && @client_key
             agent.cert = OpenSSL::X509::Certificate.new(File.read(@client_cert))
-            agent.key = OpenSSL::PKey::RSA.new(File.read(@client_key))
+            agent.key = OpenSSL::PKey.read(File.read(@client_key))
           end
         end
 
@@ -128,26 +261,54 @@ module Fluent
       end
 
       def send_batch(records)
-        ensure_connection
+        agent = ensure_connection
 
-        body = "#{records.to_json}\n"
+        # write supplies the body from records serialized once for byte accounting.
+        body = yield
         req = Net::HTTP::Post.new(@uri)
         req['Content-Type'] = 'application/json; charset=utf-8'
         req['Authorization'] = "Api-Token #{@api_token}"
         req['User-Agent'] = "fluent-plugin-dynatrace-radiant/#{DynatraceRadiantConstants::VERSION}"
 
-        response = @agent.request(req, body)
+        response = agent.request(req, body)
 
-        unless response.is_a?(Net::HTTPSuccess)
-          raise "Dynatrace API error: #{response.code} #{response.message} #{response.body}"
+        status = response.code.to_i
+        if status == 200
+          log.warn 'Dynatrace API partial success', status: status, body: response_excerpt(response)
+        elsif status == 400 || status == 413
+          log.error 'Dynatrace API rejected batch; dropping', status: status,
+                    record_count: records.length, body: response_excerpt(response)
+        elsif !(200..299).cover?(status)
+          raise "Dynatrace API error: #{status}"
         end
+      rescue StandardError => error
+        # Transport errors normally contain no token, but never propagate an echo.
+        message = redact_token(error.message)
+        raise if message == error.message
+
+        sanitized = error.exception(message)
+        # Net::ReadTimeout/WriteTimeout append their IO's inspect to message.
+        sanitized.define_singleton_method(:message) { message }
+        sanitized.define_singleton_method(:to_s) { message }
+        raise sanitized, cause: nil
+      end
+
+      def redact_token(text)
+        utf8_string(text).gsub(@api_token, '[REDACTED]')
+      end
+
+      def response_excerpt(response)
+        redact_token(response.body.to_s)[0, 512]
       end
 
       def ensure_connection
-        @agent.start unless @agent.started?
+        agent = @agents_mutex.synchronize { @agents[Thread.current] ||= create_http_agent }
+        agent.start unless agent.started?
+        agent
       rescue IOError
-        @agent = create_http_agent
-        @agent.start
+        agent = @agents_mutex.synchronize { @agents[Thread.current] = create_http_agent }
+        agent.start
+        agent
       end
 
       def parse_and_validate_uri(uri_string)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-dynatrace-radiant
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Radiant Logic, Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-02 00:00:00.000000000 Z
+date: 2026-11-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd