fluent-plugin-opentelemetry-radiant 0.1.0

data/lib/fluent/plugin/in_opentelemetry_metrics.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/input"
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/version"
+require "fluent/plugin_helper/timer"
+require "fluent/version"
+
+require "json"
+require "socket"
+
+module Fluent::Plugin
+  class OpentelemetryMetricsInput < Input
+    Fluent::Plugin.register_input("opentelemetry_metrics", self)
+
+    helpers :timer
+
+    desc "Determine the rate to emit internal metrics as events."
+    config_param :emit_interval, :time, default: 60
+
+    desc "The tag of the event."
+    config_param :tag, :string
+
+    desc "The prefix of metric name."
+    config_param :metric_name_prefix, :string, default: "fluentd_"
+
+    def start
+      super
+
+      @metrics = Metrics.new(metric_name_prefix: @metric_name_prefix)
+      timer_execute(:in_opentelemetry_metrics, @emit_interval) do
+        router.emit(@tag, Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_METRICS, "message" => @metrics.record })
+      end
+    end
+
+    module Extension
+      refine Time do
+        def to_nano_sec
+          (to_i * 1_000_000_000) + nsec
+        end
+      end
+    end
+
+    class Metrics
+      using Extension
+
+      def initialize(metric_name_prefix:)
+        @start_time_unix_nano = Time.now.to_nano_sec
+        @metric_name_prefix = metric_name_prefix.to_s
+        @hostname = Socket.gethostname
+        @monitor_info = MonitorInfo.new
+      end
+
+      def record
+        values.to_json
+      end
+
+      def values
+        metrics_data
+      end
+
+      private
+
+      def metrics_data
+        {
+          "resourceMetrics" => [
+            {
+              "resource" => {
+                "attributes" => [
+                  string_value_attribute("service.name", "fluentd"),
+                  string_value_attribute("service.version", Fluent::VERSION),
+                  string_value_attribute("host.name", @hostname),
+                  string_value_attribute("process.runtime.name", "ruby"),
+                  string_value_attribute("process.runtime.version", RUBY_VERSION),
+                  int_value_attribute("process.pid", Process.pid)
+                ]
+              },
+              "scopeMetrics" => scope_metrics
+            }
+          ]
+        }
+      end
+
+      def scope_metrics
+        [
+          {
+            "scope" => {
+              "name" => "fluent-plugin-opentelemetry",
+              "version" => Fluent::Plugin::Opentelemetry::VERSION
+            },
+            "metrics" => metrics
+          }
+        ]
+      end
+
+      def metrics
+        time_nano_sec = Time.now.to_nano_sec
+        metrics = []
+
+        @monitor_info.plugins_info_all.each do |record|
+          attributes = {
+            plugin_id: record["plugin_id"],
+            plugin: plugin_name(record["plugin_category"], record["type"]),
+            plugin_category: record["plugin_category"],
+            plugin_type: record["type"]
+          }.map { |k, v| string_value_attribute(k, v) }
+
+          record.each do |key, value|
+            next unless value.is_a?(Numeric)
+
+            metrics << {
+              "name" => @metric_name_prefix + key.to_s,
+              "unit" => "1",
+              # TODO: "description"
+              "gauge" => {
+                "dataPoints" => [
+                  {
+                    "startTimeUnixNano" => @start_time_unix_nano,
+                    "timeUnixNano" => time_nano_sec,
+                    "asDouble" => value,
+                    "attributes" => attributes
+                  }
+                ]
+              }
+            }
+          end
+        end
+
+        metrics
+      end
+
+      def plugin_name(category, type)
+        prefix =
+          case category
+          when "input"
+            "in"
+          when "output"
+            "out"
+          else
+            category
+          end
+
+        "#{prefix}_#{type}"
+      end
+
+      def string_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "stringValue" => value.to_s
+          }
+        }
+      end
+
+      def int_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "intValue" => value
+          }
+        }
+      end
+    end
+
+    # Imported from Fluent::Plugin::MonitorAgentInput
+    class MonitorInfo
+      # They are deprecated but remain for compatibiscripts/pluginslity
+      MONITOR_INFO = {
+        "output_plugin" => -> { is_a?(::Fluent::Plugin::Output) },
+        "buffer_queue_length" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.queue.size
+        },
+        "buffer_timekeys" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.timekeys
+        },
+        "buffer_total_queued_size" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.stage_size + @buffer.queue_size
+        },
+        "retry_count" => -> { respond_to?(:num_errors) ? num_errors : nil }
+      }.freeze
+
+      def all_plugins
+        array = []
+
+        # get all input plugins
+        array.concat Fluent::Engine.root_agent.inputs
+
+        # get all output plugins
+        array.concat Fluent::Engine.root_agent.outputs
+
+        # get all filter plugins
+        array.concat Fluent::Engine.root_agent.filters
+
+        Fluent::Engine.root_agent.labels.each_value do |l|
+          # TODO: Add label name to outputs / filters for identifying plugins
+          array.concat l.outputs
+          array.concat l.filters
+        end
+
+        array
+      end
+
+      def plugin_category(pe)
+        case pe
+        when Fluent::Plugin::Input
+          "input"
+        when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
+          "output"
+        when Fluent::Plugin::Filter
+          "filter"
+        else
+          "unknown"
+        end
+      end
+
+      def plugins_info_all(opts = {})
+        all_plugins.map do |pe|
+          get_monitor_info(pe, opts)
+        end
+      end
+
+      IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config).freeze
+
+      # get monitor info from the plugin `pe` and return a hash object
+      def get_monitor_info(pe, opts = {})
+        obj = {}
+
+        # Common plugin information
+        obj["plugin_id"] = pe.plugin_id
+        obj["plugin_category"] = plugin_category(pe)
+        obj["type"] = pe.config["@type"]
+        obj["config"] = pe.config if opts[:with_config]
+
+        # run MONITOR_INFO in plugins' instance context and store the info to obj
+        MONITOR_INFO.each_pair do |key, code|
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue NoMethodError => e
+          unless @first_warn
+            log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
+            log.error_backtrace
+            @first_warn = true
+          end
+        rescue StandardError => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
+        end
+
+        if pe.respond_to?(:statistics)
+          obj.merge!(pe.statistics["output"] || {})
+          obj.merge!(pe.statistics["filter"] || {})
+          obj.merge!(pe.statistics["input"] || {})
+        end
+
+        obj["retry"] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
+
+        # include all instance variables if :with_debug_info is set
+        if opts[:with_debug_info]
+          iv = {}
+          pe.instance_eval do
+            instance_variables.each do |sym|
+              next if IGNORE_ATTRIBUTES.include?(sym)
+
+              key = sym.to_s[1..] # removes first '@'
+              iv[key] = instance_variable_get(sym)
+            end
+          end
+          obj["instance_variables"] = iv
+        elsif (ivars = opts[:ivars])
+          iv = {}
+          ivars.each do |name|
+            iname = "@#{name}"
+            iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
+          end
+          obj["instance_variables"] = iv
+        end
+
+        obj
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/constant.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "fluent/plugin"
+require "openssl"
+
+module Fluent::Plugin::Opentelemetry
+  CONTENT_TYPE = "Content-Type"
+  CONTENT_TYPE_PLAIN = "text/plain"
+  CONTENT_TYPE_PAIN = CONTENT_TYPE_PLAIN
+  CONTENT_TYPE_PROTOBUF = "application/x-protobuf"
+  CONTENT_TYPE_JSON = "application/json"
+
+  CONTENT_ENCODING = "Content-Encoding"
+  CONTENT_ENCODING_GZIP = "gzip"
+
+  RECORD_TYPE_LOGS = "opentelemetry_logs"
+  RECORD_TYPE_METRICS = "opentelemetry_metrics"
+  RECORD_TYPE_TRACES = "opentelemetry_traces"
+
+  PLACEHOLDER_TYPE = "${type}"
+  PLACEHOLDER_TYPE_LOGS = "logs"
+  PLACEHOLDER_TYPE_METRICS = "metrics"
+  PLACEHOLDER_TYPE_TRACES = "traces"
+
+  TLS_VERSIONS_MAP =
+    begin
+      map = {
+        TLSv1: OpenSSL::SSL::TLS1_VERSION,
+        TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+        TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+      }
+      map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+      map.freeze
+    end
+end

data/lib/fluent/plugin/opentelemetry/grpc_input_handler.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/response"
+require "opentelemetry/proto/collector/logs/v1/logs_service_services_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_services_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_services_pb"
+
+module Fluent::Plugin::Opentelemetry
+  class GrpcInputHandler
+    class ServiceHandler
+      class Logs < Opentelemetry::Proto::Collector::Logs::V1::LogsService::Service
+        def callback=(block)
+          @callback = block
+        end
+
+        def export(req, _call)
+          @callback.call(req)
+        end
+      end
+
+      class Metrics < Opentelemetry::Proto::Collector::Metrics::V1::MetricsService::Service
+        def callback=(block)
+          @callback = block
+        end
+
+        def export(req, _call)
+          @callback.call(req)
+        end
+      end
+
+      class Traces < Opentelemetry::Proto::Collector::Trace::V1::TraceService::Service
+        def callback=(block)
+          @callback = block
+        end
+
+        def export(req, _call)
+          @callback.call(req)
+        end
+      end
+    end
+
+    class ExceptionInterceptor < GRPC::ServerInterceptor
+      def request_response(request:, call:, method:)
+        # call actual service
+        yield
+      rescue StandardError => e
+        puts "[#{method}] Error: #{e.message}"
+        raise
+      end
+    end
+
+    def initialize(grpc_config, logger)
+      @grpc_config = grpc_config
+      @logger = logger
+    end
+
+    def run(logs:, metrics:, traces:)
+      @server = GRPC::RpcServer.new(interceptors: [ExceptionInterceptor.new])
+      @server.add_http2_port("#{@grpc_config.bind}:#{@grpc_config.port}", :this_port_is_insecure)
+
+      logs_handler = ServiceHandler::Logs.new
+      logs_handler.callback = lambda { |request|
+        logs.call(request.to_json)
+        Fluent::Plugin::Opentelemetry::Response::Logs.build
+      }
+      @server.handle(logs_handler)
+
+      metrics_handler = ServiceHandler::Metrics.new
+      metrics_handler.callback = lambda { |request|
+        metrics.call(request.to_json)
+        Fluent::Plugin::Opentelemetry::Response::Metrics.build
+      }
+      @server.handle(metrics_handler)
+
+      traces_handler = ServiceHandler::Traces.new
+      traces_handler.callback = lambda { |request|
+        traces.call(request.to_json)
+        Fluent::Plugin::Opentelemetry::Response::Traces.build
+      }
+      @server.handle(traces_handler)
+
+      @server.run_till_terminated
+    end
+
+    def stop
+      @server.stop
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/grpc_output_handler.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+# Copyright 2025- Shizuo Fujita
+# Copyright 2026 Radiant Logic, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+#
+# Modified by Radiant Logic, Inc. to export prepared OTLP logs requests while
+# retaining the upstream relay path.
+
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+require "opentelemetry/proto/collector/logs/v1/logs_service_services_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_services_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_services_pb"
+
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/output_response"
+require "google/protobuf"
+
+class Fluent::Plugin::Opentelemetry::GrpcOutputHandler
+  include Fluent::Plugin::Opentelemetry::OutputResponse
+
+  RETRYABLE_CODES = %i[CANCELLED DEADLINE_EXCEEDED ABORTED OUT_OF_RANGE UNAVAILABLE DATA_LOSS RESOURCE_EXHAUSTED].map do |name|
+    GRPC::Core::StatusCodes.const_get(name)
+  end.freeze
+  CODE_NAMES = GRPC::Core::StatusCodes.constants.to_h do |name|
+    [GRPC::Core::StatusCodes.const_get(name), name.to_s]
+  end.freeze
+
+  class ServiceStub
+    class Logs
+      def initialize(host, creds, **kw)
+        @stub = Opentelemetry::Proto::Collector::Logs::V1::LogsService::Stub.new(host, creds, **kw)
+      end
+
+      def export(json, **kw)
+        message = Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(json)
+        export_request(message, **kw)
+      end
+
+      def export_request(message, **kw)
+        @stub.export(message, **kw)
+      end
+    end
+
+    class Metrics
+      def initialize(host, creds, **kw)
+        @stub = Opentelemetry::Proto::Collector::Metrics::V1::MetricsService::Stub.new(host, creds, **kw)
+      end
+
+      def export(json, **kw)
+        message = Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(json)
+        @stub.export(message, **kw)
+      end
+    end
+
+    class Traces
+      def initialize(host, creds, **kw)
+        @stub = Opentelemetry::Proto::Collector::Trace::V1::TraceService::Stub.new(host, creds, **kw)
+      end
+
+      def export(json, **kw)
+        message = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(json)
+        @stub.export(message, **kw)
+      end
+    end
+  end
+
+  def initialize(grpc_config, transport_config, logger, headers = {})
+    @grpc_config = grpc_config
+    @logger = logger
+    @headers = headers
+    @metadata = headers.transform_keys(&:downcase).reject { |key, _| %w[content-type content-encoding].include?(key) }
+
+    unless grpc_config.endpoint.match?(%r{\A(?:\[[^\]]+\]|[^:/\s]+):[0-9]+\z})
+      raise Fluent::ConfigError, "gRPC endpoint must be host:port (without ://); use <transport tls> for TLS"
+    end
+    if transport_config&.insecure
+      raise Fluent::ConfigError, "certificate verification cannot be turned off for gRPC; set ca_path to the server's CA instead"
+    end
+
+    credential = channel_credentials(transport_config)
+    channel_args = {}
+    if grpc_config.compress == :gzip
+      channel_args = GRPC::Core::CompressionOptions.new(default_algorithm: :gzip).to_channel_arg_hash
+    end
+    channel_args["grpc.keepalive_time_ms"] = grpc_config.keepalive_time * 1000
+    channel_args["grpc.keepalive_timeout_ms"] = grpc_config.keepalive_timeout * 1000
+    channel_args["grpc.keepalive_permit_without_calls"] = 1
+    @services = {
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS => ServiceStub::Logs.new(grpc_config.endpoint, credential, channel_args: channel_args),
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_METRICS => ServiceStub::Metrics.new(grpc_config.endpoint, credential, channel_args: channel_args),
+      Fluent::Plugin::Opentelemetry::RECORD_TYPE_TRACES => ServiceStub::Traces.new(grpc_config.endpoint, credential, channel_args: channel_args)
+    }
+  end
+
+  def export(record)
+    service = @services[record["type"]]
+    raise Fluent::UnrecoverableError, "got unknown record type '#{record['type']}'" unless service
+
+    response = service.export(record["message"], **call_options)
+    warn_partial_success(response) if record["type"] == Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS
+  rescue Google::Protobuf::ParseError => e
+    raise Fluent::UnrecoverableError, e.message
+  rescue GRPC::BadStatus => e
+    # Preserve upstream relay retry behavior without exposing echoed credentials.
+    raise GRPC::BadStatus.new_status_exception(e.code, redact_headers(e.details)), cause: nil
+  end
+
+  def export_logs(request)
+    response = logs_service.export_request(request, **call_options)
+    warn_partial_success(response)
+  rescue GRPC::BadStatus => e
+    code_name = CODE_NAMES.fetch(e.code, e.code.to_s)
+    message = "OTLP gRPC export failed: #{code_name}: #{redact_headers(e.details)}"
+    error_class = RETRYABLE_CODES.include?(e.code) ? Fluent::Plugin::OpentelemetryOutput::RetryableResponse : Fluent::UnrecoverableError
+    # Fluentd logs errors when retrying chunks. Do not retain a cause containing
+    # unredacted server details or response metadata.
+    raise error_class, message, cause: nil
+  end
+
+  private
+
+  def logs_service
+    @services.fetch(Fluent::Plugin::Opentelemetry::RECORD_TYPE_LOGS)
+  end
+
+  def call_options
+    { deadline: Time.now + @grpc_config.timeout, metadata: @metadata }
+  end
+
+  def channel_credentials(transport)
+    return :this_channel_is_insecure unless transport&.protocol == :tls
+
+    ca_pem = read_pem(transport.ca_path)
+    key_pem = read_pem(transport.private_key_path)
+    cert_pem = read_pem(transport.cert_path)
+    if key_pem.nil? != cert_pem.nil?
+      raise Fluent::ConfigError, "gRPC TLS requires both cert_path and private_key_path for a client certificate"
+    end
+    if key_pem && transport.private_key_passphrase
+      begin
+        key_pem = OpenSSL::PKey.read(key_pem, transport.private_key_passphrase).to_pem
+      rescue OpenSSL::OpenSSLError, ArgumentError
+        raise Fluent::ConfigError, "could not decrypt gRPC TLS private_key_path '#{transport.private_key_path}'", cause: nil
+      end
+    end
+    GRPC::Core::ChannelCredentials.new(ca_pem, key_pem, cert_pem)
+  end
+
+  def read_pem(path)
+    return nil unless path
+
+    File.read(path)
+  rescue SystemCallError, IOError
+    raise Fluent::ConfigError, "could not read gRPC TLS file '#{path}'", cause: nil
+  end
+end