fluent-plugin-opentelemetry-radiant 0.1.0

data/lib/fluent/plugin/opentelemetry/http_input_handler.rb

@@ -0,0 +1,177 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/request"
+require "fluent/plugin/opentelemetry/response"
+require "google/protobuf"
+require "zlib"
+require "stringio"
+
+module Fluent::PluginHelper::HttpServer
+  module Extension
+    refine Request do
+      # This API was introduced at fluentd v1.19.0.
+      # Ref. https://github.com/fluent/fluentd/pull/4903
+      # If we have supported v1.19.0+ only, we can remove this patch.
+      unless method_defined?(:headers)
+        def headers
+          @request.headers
+        end
+      end
+
+      # Workaround for fluentd v1.19.1 or earlier which does not close request body.
+      # Ref. https://github.com/fluent/fluentd/pull/5231
+      unless method_defined?(:close)
+        def close
+          @request.body&.close
+        end
+      end
+
+      unless method_defined?(:body_stream)
+        def body_stream
+          @request.body
+        end
+      end
+    end
+  end
+end
+
+class Fluent::Plugin::Opentelemetry::HttpInputHandler
+  class SizeLimitError < StandardError; end
+
+  using Fluent::PluginHelper::HttpServer::Extension
+
+  def initialize(http_config, logger)
+    @http_config = http_config
+    @logger = logger
+  end
+
+  def logs(req, &block)
+    common(req, Fluent::Plugin::Opentelemetry::Request::Logs, Fluent::Plugin::Opentelemetry::Response::Logs, &block)
+  end
+
+  def metrics(req, &block)
+    common(req, Fluent::Plugin::Opentelemetry::Request::Metrics, Fluent::Plugin::Opentelemetry::Response::Metrics, &block)
+  end
+
+  def traces(req, &block)
+    common(req, Fluent::Plugin::Opentelemetry::Request::Traces, Fluent::Plugin::Opentelemetry::Response::Traces, &block)
+  end
+
+  private
+
+  def common(req, request_class, response_class)
+    content_type = req.headers["content-type"]
+    content_encoding = Array(req.headers["content-encoding"]).first
+
+    content_length = Array(req.headers["content-length"]).first&.to_i
+    if content_length && content_length > @http_config.body_size_limit
+      @logger.warn { "Received too big content length: #{content_length}" }
+      return response_payload_too_large
+    end
+
+    begin
+      body = read_body(req, limit: @http_config.body_size_limit)
+    rescue SizeLimitError
+      @logger.warn { "Received payload exceeding body_size_limit" }
+      return response_payload_too_large
+    end
+
+    return response_unsupported_media_type unless valid_content_type?(content_type)
+    return response_bad_request(content_type) unless valid_content_encoding?(content_encoding)
+
+    if content_encoding == Fluent::Plugin::Opentelemetry::CONTENT_ENCODING_GZIP
+      begin
+        body = decompress(body, limit: @http_config.decompression_size_limit)
+      rescue SizeLimitError
+        @logger.warn { "Decompressed payload exceeding decompression_size_limit" }
+        return response_payload_too_large
+      rescue Zlib::Error, EOFError => e
+        @logger.warn { "Failed to decompress gzip payload: #{e.message}" }
+        return response_bad_request(content_type)
+      end
+    end
+
+    begin
+      record = request_class.new(body).record
+    rescue Google::Protobuf::ParseError => e
+      # The format in request body does not comply with the OpenTelemetry protocol.
+      @logger.warn { "Failed to parse OpenTelemetry payload: #{e.message}" }
+      return response_bad_request(content_type)
+    end
+
+    yield record
+
+    res = response_class.new
+    response(200, content_type, res.body(type: Fluent::Plugin::Opentelemetry::Response.type(content_type)))
+  ensure
+    req.close
+  end
+
+  def read_body(request, limit:)
+    body = +"".b
+    while (chunk = request.body_stream&.read)
+      body << chunk
+      if body.bytesize > limit
+        raise SizeLimitError, "Too large payload"
+      end
+    end
+    body
+  end
+
+  BYTES_TO_READ = 64 * 1024
+
+  def decompress(compressed_data, limit:)
+    io = StringIO.new(compressed_data)
+    out = +"".b
+    loop do
+      reader = Zlib::GzipReader.new(io)
+      while (chunk = reader.read(BYTES_TO_READ))
+        out << chunk
+        if out.bytesize > limit
+          raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+        end
+      end
+
+      unused = reader.unused
+      reader.finish
+      unless unused.nil?
+        adjust = unused.length
+        io.pos -= adjust
+      end
+      break if io.eof?
+    end
+    out
+  end
+
+  def valid_content_type?(content_type)
+    case content_type
+    when Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PROTOBUF, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_JSON
+      true
+    else
+      false
+    end
+  end
+
+  def valid_content_encoding?(content_encoding)
+    return true if content_encoding.nil?
+
+    content_encoding == Fluent::Plugin::Opentelemetry::CONTENT_ENCODING_GZIP
+  end
+
+  def response(code, content_type, body)
+    [code, { Fluent::Plugin::Opentelemetry::CONTENT_TYPE => content_type }, body]
+  end
+
+  def response_unsupported_media_type
+    response(415, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PLAIN, "415 unsupported media type, supported: [application/json, application/x-protobuf]")
+  end
+
+  def response_payload_too_large
+    response(413, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PLAIN, "413 Payload Too Large")
+  end
+
+  def response_bad_request(content_type)
+    response(400, content_type, "") # TODO: fix body message
+  end
+end

data/lib/fluent/plugin/opentelemetry/http_output_handler.rb

@@ -0,0 +1,157 @@
+# frozen_string_literal: true
+
+# Copyright 2025- Shizuo Fujita
+# Copyright 2026 Radiant Logic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#
+# Modified by Radiant Logic, Inc. to export prepared OTLP logs requests while
+# retaining the upstream relay and response-handling paths.
+
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/request"
+require "fluent/plugin/opentelemetry/output_response"
+
+require "excon"
+require "google/protobuf"
+require "stringio"
+require "zlib"
+
+class Fluent::Plugin::Opentelemetry::HttpOutputHandler
+  include Fluent::Plugin::Opentelemetry::OutputResponse
+
+  def initialize(http_config, transport_config, logger, headers = {})
+    @http_config = http_config
+    @logger = logger
+    @headers = headers
+    @endpoint = http_config.endpoint.sub(%r{/+\z}, "")
+
+    tls_settings = {}
+    if transport_config.protocol == :tls
+      tls_settings[:ssl_ca_file] = transport_config.ca_path if transport_config.ca_path
+      tls_settings[:client_cert] = transport_config.cert_path
+      tls_settings[:client_key] = transport_config.private_key_path
+      tls_settings[:client_key_pass] = transport_config.private_key_passphrase
+      tls_settings[:ssl_min_version] = Fluent::Plugin::Opentelemetry::TLS_VERSIONS_MAP[transport_config.min_version]
+      tls_settings[:ssl_max_version] = Fluent::Plugin::Opentelemetry::TLS_VERSIONS_MAP[transport_config.max_version]
+      tls_settings[:ssl_verify_peer] = !transport_config.insecure
+    end
+
+    timeout_settings = {
+      read_timeout: http_config.read_timeout,
+      write_timeout: http_config.write_timeout,
+      connect_timeout: http_config.connect_timeout
+    }
+    @connections = {
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS => http_logs_endpoint,
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_METRICS => http_metrics_endpoint,
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_TRACES => http_traces_endpoint
+    }.transform_values do |uri|
+      Excon.new(uri, proxy: http_config.proxy, persistent: true, **tls_settings, **timeout_settings)
+    end
+  end
+
+  def export(record)
+    uri, headers, body = get_post_data(record)
+    post(uri, @connections.fetch(record["type"]), headers: headers, body: body,
+         logs: record["type"] == Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS)
+  end
+
+  def export_logs(request)
+    headers, body = prepare_body(request.to_proto)
+    post(http_logs_endpoint, @connections.fetch(Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS),
+         headers: headers, body: body, logs: true)
+  end
+
+  private
+
+  def post(uri, connection, **options)
+    logs = options.delete(:logs)
+    response = connection.post(**options)
+    # Read every response, including errors, before reusing the connection.
+    body = response.body
+
+    if (200...300).cover?(response.status)
+      check_logs_response(body, response.headers) if logs && body && !body.empty?
+      return
+    end
+
+    if response.status == 400
+      # The client MUST NOT retry HTTP 400 Bad Request.
+      raise Fluent::UnrecoverableError, "got unrecoverable error response from '#{uri}', response code is #{response.status}"
+    end
+
+    if @http_config.retryable_response_codes&.include?(response.status)
+      raise Fluent::Plugin::OpentelemetryOutput::RetryableResponse, "got retryable error response from '#{uri}', response code is #{response.status}"
+    end
+    if @http_config.error_response_as_unrecoverable
+      raise Fluent::UnrecoverableError, "got unrecoverable error response from '#{uri}', response code is #{response.status}"
+    else
+      @logger.error "got error response from '#{uri}', response code is #{response.status}"
+    end
+  rescue Excon::Error => e
+    message = redact_headers(e.message)
+    raise if message == e.message
+
+    # Fluentd logs retry errors; a backend or proxy may echo request headers.
+    raise Fluent::Plugin::OpentelemetryOutput::RetryableResponse, message, cause: nil
+  end
+
+  def check_logs_response(body, headers)
+    content_type = headers.find { |key, _| key.downcase == "content-type" }&.last.to_s.split(";").first
+    response_class = Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceResponse
+    response = if content_type == Fluent::Plugin::Opentelemetry::CONTENT_TYPE_JSON
+                 response_class.decode_json(body)
+               else
+                 response_class.decode(body)
+               end
+    warn_partial_success(response)
+  rescue Google::Protobuf::ParseError
+    # A successful HTTP status acknowledges delivery even if its body is invalid.
+    @logger.warn "Could not decode successful OTLP logs response; delivery will not be retried"
+  end
+
+  def http_logs_endpoint
+    "#{@endpoint}/v1/logs"
+  end
+
+  def http_metrics_endpoint
+    "#{@endpoint}/v1/metrics"
+  end
+
+  def http_traces_endpoint
+    "#{@endpoint}/v1/traces"
+  end
+
+  def get_post_data(record)
+    uri, request_class = case record["type"]
+                         when Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS
+                           [http_logs_endpoint, Fluent::Plugin::Opentelemetry::Request::Logs]
+                         when Fluent::Plugin::Opentelemetry::RECORD_TYPE_METRICS
+                           [http_metrics_endpoint, Fluent::Plugin::Opentelemetry::Request::Metrics]
+                         when Fluent::Plugin::Opentelemetry::RECORD_TYPE_TRACES
+                           [http_traces_endpoint, Fluent::Plugin::Opentelemetry::Request::Traces]
+                         else
+                           raise Fluent::UnrecoverableError, "got unknown record type '#{record['type']}'"
+                         end
+    headers, body = prepare_body(request_class.new(record["message"]).body)
+    [uri, headers, body]
+  rescue Google::Protobuf::ParseError => e
+    raise Fluent::UnrecoverableError, e.message
+  end
+
+  def prepare_body(body)
+    # Header names are case-insensitive. Neither reserved header is user-settable,
+    # including Content-Encoding when compression is disabled.
+    headers = @headers.reject { |key, _| %w[content-type content-encoding].include?(key.downcase) }
+    headers[Fluent::Plugin::Opentelemetry::CONTENT_TYPE] = Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PROTOBUF
+    if @http_config.compress == :gzip
+      headers[Fluent::Plugin::Opentelemetry::CONTENT_ENCODING] = Fluent::Plugin::Opentelemetry::CONTENT_ENCODING_GZIP
+      gz = Zlib::GzipWriter.new(StringIO.new)
+      gz << body
+      body = gz.close.string
+    end
+    [headers, body]
+  end
+end

data/lib/fluent/plugin/opentelemetry/log_record_builder.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+# Copyright 2025- Shizuo Fujita
+# Copyright 2026 Radiant Logic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#
+# This file was added by Radiant Logic, Inc. to convert ordinary Fluentd
+# records into OpenTelemetry Protocol log export requests.
+
+require "fluent/plugin/opentelemetry/constant"
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+
+require "json"
+
+class Fluent::Plugin::Opentelemetry::LogRecordBuilder
+  SEVERITY_NUMBERS = {
+    "TRACE" => 1,
+    "DEBUG" => 5,
+    "INFO" => 9,
+    "WARN" => 13,
+    "ERROR" => 17,
+    "FATAL" => 21
+  }.freeze
+
+  INT64_RANGE = (-(2**63)..((2**63) - 1)).freeze
+
+  def initialize(body_key:, body_format:, severity_key:, attributes_from_record:, exclude_keys:,
+                 resource_attributes:, resource_attributes_from_record:, scope_name:, clock: nil)
+    @body_key = body_key
+    @body_format = body_format
+    @severity_key = severity_key
+    @attributes_from_record = attributes_from_record
+    @exclude_keys = exclude_keys.map(&:to_s).freeze
+    @resource_attributes = resource_attributes
+    @resource_attributes_from_record = resource_attributes_from_record
+    @scope_name = scope_name
+    @clock = clock || method(:current_time_unix_nano)
+  end
+
+  def build(tag, events)
+    raise ArgumentError, "events must not be empty" if events.empty?
+
+    observed_time_unix_nano = @clock.call
+    log_records = events.map do |time, record|
+      build_log_record(tag, time, record, observed_time_unix_nano)
+    end
+
+    collector_logs_v1::ExportLogsServiceRequest.new(
+      resource_logs: [
+        logs_v1::ResourceLogs.new(
+          resource: resource_v1::Resource.new(attributes: resource_attributes(events.first[1])),
+          scope_logs: [
+            logs_v1::ScopeLogs.new(
+              scope: common_v1::InstrumentationScope.new(name: utf8_string(@scope_name)),
+              log_records: log_records
+            )
+          ]
+        )
+      ]
+    )
+  end
+
+  private
+
+  def build_log_record(tag, time, record, observed_time_unix_nano)
+    values = {
+      time_unix_nano: event_time_unix_nano(time),
+      observed_time_unix_nano: observed_time_unix_nano,
+      body: body(record),
+      attributes: log_attributes(tag, record)
+    }
+
+    severity_present, severity = record_value(record, @severity_key)
+    if severity_present
+      values[:severity_text] = utf8_string(severity.to_s)
+      values[:severity_number] = SEVERITY_NUMBERS.fetch(values[:severity_text].upcase, 0)
+    end
+
+    logs_v1::LogRecord.new(**values)
+  end
+
+  def body(record)
+    if @body_format == :kvlist
+      return common_v1::AnyValue.new(
+        kvlist_value: common_v1::KeyValueList.new(values: key_values(record))
+      )
+    end
+
+    body_present, body_value = record_value(record, @body_key)
+    text = body_present ? body_value.to_s : JSON.generate(normalize_json_strings(record))
+    common_v1::AnyValue.new(string_value: utf8_string(text))
+  end
+
+  def log_attributes(tag, record)
+    attributes = {}
+    if @attributes_from_record
+      record.each do |key, value|
+        attribute_name = key.to_s
+        next if attribute_name == @body_key
+        next if attribute_name == @severity_key
+        next if @exclude_keys.include?(attribute_name)
+        next if attribute_name == "fluent.tag"
+
+        attributes[attribute_name] = value
+      end
+    end
+    attributes["fluent.tag"] = tag
+    key_values(attributes)
+  end
+
+  def resource_attributes(first_record)
+    attributes = @resource_attributes.each_with_object({}) do |(key, value), result|
+      result[key.to_s] = value
+    end
+
+    @resource_attributes_from_record.each do |attribute_name, record_key|
+      present, value = record_value(first_record, record_key.to_s)
+      attributes[attribute_name.to_s] = value if present
+    end
+
+    key_values(attributes)
+  end
+
+  def key_values(values)
+    values.map do |key, value|
+      common_v1::KeyValue.new(key: utf8_string(key.to_s), value: any_value(value))
+    end
+  end
+
+  def any_value(value)
+    case value
+    when String
+      common_v1::AnyValue.new(string_value: utf8_string(value))
+    when Integer
+      if INT64_RANGE.cover?(value)
+        common_v1::AnyValue.new(int_value: value)
+      else
+        common_v1::AnyValue.new(string_value: utf8_string(value.to_s))
+      end
+    when Float
+      common_v1::AnyValue.new(double_value: value)
+    when TrueClass, FalseClass
+      common_v1::AnyValue.new(bool_value: value)
+    when Hash
+      common_v1::AnyValue.new(
+        kvlist_value: common_v1::KeyValueList.new(values: key_values(value))
+      )
+    when Array
+      common_v1::AnyValue.new(
+        array_value: common_v1::ArrayValue.new(values: value.map { |item| any_value(item) })
+      )
+    when NilClass
+      common_v1::AnyValue.new(string_value: "null")
+    else
+      common_v1::AnyValue.new(string_value: utf8_string(value.to_s))
+    end
+  end
+
+  def utf8_string(string)
+    return string if string.encoding == Encoding::UTF_8 && string.valid_encoding?
+
+    string = if string.encoding == Encoding::ASCII_8BIT
+               string.dup.force_encoding(Encoding::UTF_8)
+             elsif string.encoding != Encoding::UTF_8
+               string.encode(Encoding::UTF_8, invalid: :replace, undef: :replace, replace: "\uFFFD")
+             else
+               string
+             end
+    string.scrub("\uFFFD")
+  end
+
+  def normalize_json_strings(value)
+    case value
+    when String
+      utf8_string(value)
+    when Hash
+      value.each_with_object({}) do |(key, item), result|
+        normalized_key = key.is_a?(String) ? utf8_string(key) : key
+        result[normalized_key] = normalize_json_strings(item)
+      end
+    when Array
+      value.map { |item| normalize_json_strings(item) }
+    else
+      value
+    end
+  end
+
+  def record_value(record, key)
+    return [true, record[key]] if record.key?(key)
+
+    symbol_key = key.to_sym
+    return [true, record[symbol_key]] if record.key?(symbol_key)
+
+    [false, nil]
+  end
+
+  def event_time_unix_nano(time)
+    if time.is_a?(Time)
+      (time.to_i * 1_000_000_000) + time.nsec
+    elsif time.respond_to?(:sec) && time.respond_to?(:nsec)
+      (time.sec * 1_000_000_000) + time.nsec
+    else
+      time.to_i * 1_000_000_000
+    end
+  end
+
+  def current_time_unix_nano
+    now = Time.now
+    (now.to_i * 1_000_000_000) + now.nsec
+  end
+
+  def collector_logs_v1
+    ::Opentelemetry::Proto::Collector::Logs::V1
+  end
+
+  def logs_v1
+    ::Opentelemetry::Proto::Logs::V1
+  end
+
+  def common_v1
+    ::Opentelemetry::Proto::Common::V1
+  end
+
+  def resource_v1
+    ::Opentelemetry::Proto::Resource::V1
+  end
+end

data/lib/fluent/plugin/opentelemetry/output_response.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/response"
+
+module Fluent::Plugin::Opentelemetry::OutputResponse
+  private
+
+  def redact_headers(message)
+    @headers.values.reject(&:empty?).sort_by { |value| -value.length }.reduce(message.to_s) do |text, value|
+      text.gsub(value, "[REDACTED]")
+    end
+  end
+
+  def warn_partial_success(response)
+    partial = response&.partial_success
+    return unless partial && partial.rejected_log_records > 0
+
+    @logger.warn "OTLP logs partially accepted: rejected_log_records=#{partial.rejected_log_records}, " \
+                 "error_message=#{redact_headers(partial.error_message)}"
+  end
+end

data/lib/fluent/plugin/opentelemetry/request.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/constant"
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_pb"
+
+require "google/protobuf"
+
+class Fluent::Plugin::Opentelemetry::Request
+  class Logs
+    def initialize(body, ignore_unknown_fields: true)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
+        else
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+
+  class Metrics
+    def initialize(body, ignore_unknown_fields: true)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
+        else
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+
+  class Traces
+    def initialize(body, ignore_unknown_fields: true)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
+        else
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+end