fluent-plugin-opentelemetry 0.1.0

data/lib/fluent/plugin/in_opentelemetry.rb

@@ -0,0 +1,211 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/input"
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/request"
+require "fluent/plugin/opentelemetry/response"
+require "fluent/plugin/opentelemetry/service_handler"
+require "fluent/plugin_helper/http_server"
+require "fluent/plugin_helper/thread"
+
+require "zlib"
+
+unless Fluent::PluginHelper::HttpServer::Request.method_defined?(:headers)
+  # This API was introduced at fluentd v1.19.0.
+  # Ref. https://github.com/fluent/fluentd/pull/4903
+  # If we have supported v1.19.0+ only, we can remove this patch.
+  module Fluent::PluginHelper::HttpServer
+    module Extension
+      refine Request do
+        def headers
+          @request.headers
+        end
+      end
+    end
+  end
+
+  using Fluent::PluginHelper::HttpServer::Extension
+end
+
+module Fluent::Plugin
+  class OpentelemetryInput < Input
+    Fluent::Plugin.register_input("opentelemetry", self)
+
+    helpers :thread, :http_server
+
+    desc "The tag of the event."
+    config_param :tag, :string
+
+    config_section :http, required: false, multi: false, init: false, param_name: :http_config do
+      desc "The address to bind to."
+      config_param :bind, :string, default: "0.0.0.0"
+      desc "The port to listen to."
+      config_param :port, :integer, default: 4318
+    end
+
+    config_section :grpc, required: false, multi: false, init: false, param_name: :grpc_config do
+      desc "The address to bind to."
+      config_param :bind, :string, default: "0.0.0.0"
+      desc "The port to listen to."
+      config_param :port, :integer, default: 4317
+    end
+
+    config_section :transport, required: false, multi: false, init: false, param_name: :transport_config do
+      config_argument :protocol, :enum, list: [:tls], default: nil
+    end
+
+    def configure(conf)
+      super
+
+      unless [@http_config, @grpc_config].any?
+        raise Fluent::ConfigError, "Please configure either <http> or <grpc> section, or both."
+      end
+    end
+
+    def start
+      super
+
+      if @http_config
+        http_handler = HttpHandler.new
+        http_server_create_http_server(:in_opentelemetry_http_server, addr: @http_config.bind, port: @http_config.port, logger: log) do |serv|
+          serv.post("/v1/logs") do |req|
+            http_handler.logs(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_LOGS, message: record }) }
+          end
+          serv.post("/v1/metrics") do |req|
+            http_handler.metrics(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_METRICS, message: record }) }
+          end
+          serv.post("/v1/traces") do |req|
+            http_handler.traces(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_TRACES, message: record }) }
+          end
+        end
+      end
+
+      if @grpc_config
+        thread_create(:in_opentelemetry_grpc_server) do
+          grpc_handler = GrpcHandler.new(@grpc_config, log)
+          grpc_handler.run(
+            logs: lambda { |record|
+              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_LOGS, message: record })
+            },
+            metrics: lambda { |record|
+              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_METRICS, message: record })
+            },
+            traces: lambda { |record|
+              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_TRACES, message: record })
+            }
+          )
+        end
+      end
+    end
+
+    class HttpHandler
+      def logs(req, &block)
+        common(req, Opentelemetry::Request::Logs, Opentelemetry::Response::Logs, &block)
+      end
+
+      def metrics(req, &block)
+        common(req, Opentelemetry::Request::Metrics, Opentelemetry::Response::Metrics, &block)
+      end
+
+      def traces(req, &block)
+        common(req, Opentelemetry::Request::Traces, Opentelemetry::Response::Traces, &block)
+      end
+
+      private
+
+      def common(req, request_class, response_class)
+        content_type = req.headers["content-type"]
+        content_encoding = req.headers["content-encoding"]&.first
+        return response_unsupported_media_type unless valid_content_type?(content_type)
+        return response_bad_request(content_type) unless valid_content_encoding?(content_encoding)
+
+        body = req.body
+        body = Zlib::GzipReader.new(StringIO.new(body)).read if content_encoding == Opentelemetry::CONTENT_ENCODING_GZIP
+
+        begin
+          record = request_class.new(body).record
+        rescue Google::Protobuf::ParseError
+          # The format in request body does not comply with the OpenTelemetry protocol.
+          return response_bad_request(content_type)
+        end
+
+        yield record
+
+        res = response_class.new
+        response(200, content_type, res.body(type: Opentelemetry::Response.type(content_type)))
+      end
+
+      def valid_content_type?(content_type)
+        case content_type
+        when Opentelemetry::CONTENT_TYPE_PROTOBUF, Opentelemetry::CONTENT_TYPE_JSON
+          true
+        else
+          false
+        end
+      end
+
+      def valid_content_encoding?(content_encoding)
+        return true if content_encoding.nil?
+
+        content_encoding == Opentelemetry::CONTENT_ENCODING_GZIP
+      end
+
+      def response(code, content_type, body)
+        [code, { Opentelemetry::CONTENT_TYPE => content_type }, body]
+      end
+
+      def response_unsupported_media_type
+        response(415, Opentelemetry::CONTENT_TYPE_PAIN, "415 unsupported media type, supported: [application/json, application/x-protobuf]")
+      end
+
+      def response_bad_request(content_type)
+        response(400, content_type, "") # TODO: fix body message
+      end
+    end
+
+    class GrpcHandler
+      class ExceptionInterceptor < GRPC::ServerInterceptor
+        def request_response(request:, call:, method:)
+          # call actual service
+          yield
+        rescue StandardError => e
+          puts "[#{method}] Error: #{e.message}"
+          raise
+        end
+      end
+
+      def initialize(grpc_config, logger)
+        @grpc_config = grpc_config
+        @logger = logger
+      end
+
+      def run(logs:, metrics:, traces:)
+        server = GRPC::RpcServer.new(interceptors: [ExceptionInterceptor.new])
+        server.add_http2_port("#{@grpc_config.bind}:#{@grpc_config.port}", :this_port_is_insecure)
+
+        logs_handler = Opentelemetry::ServiceHandler::Logs.new
+        logs_handler.callback = lambda { |request|
+          logs.call(request.to_json)
+          Opentelemetry::Response::Logs.build
+        }
+        server.handle(logs_handler)
+
+        metrics_handler = Opentelemetry::ServiceHandler::Metrics.new
+        metrics_handler.callback = lambda { |request|
+          metrics.call(request.to_json)
+          Opentelemetry::Response::Metrics.build
+        }
+        server.handle(metrics_handler)
+
+        traces_handler = Opentelemetry::ServiceHandler::Traces.new
+        traces_handler.callback = lambda { |request|
+          traces.call(request.to_json)
+          Opentelemetry::Response::Traces.build
+        }
+        server.handle(traces_handler)
+
+        server.run_till_terminated
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/constant.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require "fluent/plugin"
+require "openssl"
+
+module Fluent::Plugin::Opentelemetry
+  CONTENT_TYPE = "Content-Type"
+  CONTENT_TYPE_PAIN = "text/plain"
+  CONTENT_TYPE_PROTOBUF = "application/x-protobuf"
+  CONTENT_TYPE_JSON = "application/json"
+
+  CONTENT_ENCODING = "Content-Encoding"
+  CONTENT_ENCODING_GZIP = "gzip"
+
+  RECORD_TYPE_LOGS = "opentelemetry_logs"
+  RECORD_TYPE_METRICS = "opentelemetry_metrics"
+  RECORD_TYPE_TRACES = "opentelemetry_traces"
+
+  TLS_VERSIONS_MAP =
+    begin
+      map = {
+        TLSv1: OpenSSL::SSL::TLS1_VERSION,
+        TLSv1_1: OpenSSL::SSL::TLS1_1_VERSION,
+        TLSv1_2: OpenSSL::SSL::TLS1_2_VERSION
+      }
+      map[:TLSv1_3] = OpenSSL::SSL::TLS1_3_VERSION if defined?(OpenSSL::SSL::TLS1_3_VERSION)
+      map.freeze
+    end
+end

data/lib/fluent/plugin/opentelemetry/request.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/constant"
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_pb"
+
+require "google/protobuf"
+
+class Fluent::Plugin::Opentelemetry::Request
+  class Logs
+    def initialize(body)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+        else
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+
+  class Metrics
+    def initialize(body)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+        else
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+
+  class Traces
+    def initialize(body)
+      @request =
+        if body.start_with?("{")
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: true)
+        else
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(body)
+        end
+    end
+
+    def body
+      @request.to_proto
+    end
+
+    def record
+      @request.to_json
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/response.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/constant"
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_pb"
+
+require "google/protobuf"
+
+module Fluent::Plugin::Opentelemetry::Response
+  def self.type(content_type)
+    case content_type
+    when Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PROTOBUF
+      :protobuf
+    when Fluent::Plugin::Opentelemetry::CONTENT_TYPE_JSON
+      :json
+    else
+      raise "unknown content-type: #{content_type}"
+    end
+  end
+
+  class Logs
+    def self.build(rejected: 0, error: "")
+      Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceResponse.new(
+        partial_success: Opentelemetry::Proto::Collector::Logs::V1::ExportLogsPartialSuccess.new(
+          rejected_log_records: rejected,
+          error_message: error
+        )
+      )
+    end
+
+    def initialize(rejected: 0, error: "")
+      @response = Logs.build(rejected: rejected, error: error)
+    end
+
+    def body(type:)
+      if type == :protobuf
+        @response.to_proto
+      else
+        @response.to_json
+      end
+    end
+  end
+
+  class Metrics
+    def self.build(rejected: 0, error: "")
+      Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceResponse.new(
+        partial_success: Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsPartialSuccess.new(
+          rejected_data_points: rejected,
+          error_message: error
+        )
+      )
+    end
+
+    def initialize(rejected: 0, error: "")
+      @response = Metrics.build(rejected: rejected, error: error)
+    end
+
+    def body(type:)
+      if type == :protobuf
+        @response.to_proto
+      else
+        @response.to_json
+      end
+    end
+  end
+
+  class Traces
+    def self.build(rejected: 0, error: "")
+      Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceResponse.new(
+        partial_success: Opentelemetry::Proto::Collector::Trace::V1::ExportTracePartialSuccess.new(
+          rejected_spans: rejected,
+          error_message: error
+        )
+      )
+    end
+
+    def initialize(rejected: 0, error: "")
+      @response = Traces.build(rejected: rejected, error: error)
+    end
+
+    def body(type:)
+      if type == :protobuf
+        @response.to_proto
+      else
+        @response.to_json
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/service_handler.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "opentelemetry/proto/collector/logs/v1/logs_service_services_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_services_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_services_pb"
+
+class Fluent::Plugin::Opentelemetry::ServiceHandler
+  class Logs < Opentelemetry::Proto::Collector::Logs::V1::LogsService::Service
+    def callback=(block)
+      @callback = block
+    end
+
+    def export(req, _call)
+      @callback.call(req)
+    end
+  end
+
+  class Metrics < Opentelemetry::Proto::Collector::Metrics::V1::MetricsService::Service
+    def callback=(block)
+      @callback = block
+    end
+
+    def export(req, _call)
+      @callback.call(req)
+    end
+  end
+
+  class Traces < Opentelemetry::Proto::Collector::Trace::V1::TraceService::Service
+    def callback=(block)
+      @callback = block
+    end
+
+    def export(req, _call)
+      @callback.call(req)
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/service_stub.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "opentelemetry/proto/collector/logs/v1/logs_service_pb"
+require "opentelemetry/proto/collector/logs/v1/logs_service_services_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_pb"
+require "opentelemetry/proto/collector/metrics/v1/metrics_service_services_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_pb"
+require "opentelemetry/proto/collector/trace/v1/trace_service_services_pb"
+
+require "grpc"
+
+class Fluent::Plugin::Opentelemetry::ServiceStub
+  class Logs
+    def initialize(host, creds, **kw)
+      @stub = Opentelemetry::Proto::Collector::Logs::V1::LogsService::Stub.new(host, creds, **kw)
+    end
+
+    def export(json)
+      message = Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(json)
+      @stub.export(message)
+    end
+  end
+
+  class Metrics
+    def initialize(host, creds, **kw)
+      @stub = Opentelemetry::Proto::Collector::Metrics::V1::MetricsService::Stub.new(host, creds, **kw)
+    end
+
+    def export(json)
+      message = Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(json)
+      @stub.export(message)
+    end
+  end
+
+  class Traces
+    def initialize(host, creds, **kw)
+      @stub = Opentelemetry::Proto::Collector::Trace::V1::TraceService::Stub.new(host, creds, **kw)
+    end
+
+    def export(json)
+      message = Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(json)
+      @stub.export(message)
+    end
+  end
+end

data/lib/fluent/plugin/out_opentelemetry.rb

@@ -0,0 +1,207 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/request"
+require "fluent/plugin/opentelemetry/service_stub"
+require "fluent/plugin/output"
+
+require "excon"
+require "grpc"
+require "json"
+require "stringio"
+require "zlib"
+
+module Fluent::Plugin
+  class OpentelemetryOutput < Output
+    class RetryableResponse < StandardError; end
+
+    Fluent::Plugin.register_output("opentelemetry", self)
+
+    helpers :server
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ["tag"]
+    end
+
+    config_section :http, required: false, multi: false, init: false, param_name: :http_config do
+      desc "The endpoint"
+      config_param :endpoint, :string, default: "http://127.0.0.1:4318"
+      desc "The proxy for HTTP request"
+      config_param :proxy, :string, default: ENV["HTTP_PROXY"] || ENV["http_proxy"]
+
+      desc "Raise UnrecoverableError when the response is non success, 4xx/5xx"
+      config_param :error_response_as_unrecoverable, :bool, default: true
+      desc "The list of retryable response code"
+      config_param :retryable_response_codes, :array, value_type: :integer, default: [429, 502, 503, 504]
+
+      desc "Compress request body"
+      config_param :compress, :enum, list: %i[text gzip], default: :text
+
+      desc "The read timeout in seconds"
+      config_param :read_timeout, :integer, default: 60
+      desc "The write timeout in seconds"
+      config_param :write_timeout, :integer, default: 60
+      desc "The connect timeout in seconds"
+      config_param :connect_timeout, :integer, default: 60
+    end
+
+    config_section :grpc, required: false, multi: false, init: false, param_name: :grpc_config do
+      desc "The endpoint"
+      config_param :endpoint, :string, default: "127.0.0.1:4317"
+    end
+
+    config_section :transport, required: false, multi: false, init: false, param_name: :transport_config do
+      config_argument :protocol, :enum, list: [:tls], default: nil
+    end
+
+    def configure(conf)
+      super
+
+      unless [@http_config, @grpc_config].one?
+        raise Fluent::ConfigError, "Please configure either <http> or <grpc> section."
+      end
+
+      @http_handler = HttpHandler.new(@http_config, @transport_config, log) if @http_config
+      @grpc_handler = GrpcHandler.new(@grpc_config, @transport_config, log) if @grpc_config
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def format(tag, time, record)
+      JSON.generate(record)
+    end
+
+    def write(chunk)
+      record = JSON.parse(chunk.read)
+
+      if @grpc_handler
+        @grpc_handler.export(record)
+      else
+        @http_handler.export(record)
+      end
+    end
+
+    class HttpHandler
+      def initialize(http_config, transport_config, logger)
+        @http_config = http_config
+        @transport_config = transport_config
+        @logger = logger
+
+        @tls_settings = {}
+        if @transport_config.protocol == :tls
+          @tls_settings[:client_cert] = @transport_config.cert_path
+          @tls_settings[:client_key] = @transport_config.private_key_path
+          @tls_settings[:client_key_pass] = @transport_config.private_key_passphrase
+          @tls_settings[:ssl_min_version] = Opentelemetry::TLS_VERSIONS_MAP[@transport_config.min_version]
+          @tls_settings[:ssl_max_version] = Opentelemetry::TLS_VERSIONS_MAP[@transport_config.max_version]
+        end
+
+        @timeout_settings = {
+          read_timeout: http_config.read_timeout,
+          write_timeout: http_config.write_timeout,
+          connect_timeout: http_config.connect_timeout
+        }
+      end
+
+      def export(record)
+        uri, connection = create_http_connection(record)
+        response = connection.post
+
+        if response.status != 200
+          if response.status == 400
+            # The client MUST NOT retry the request when it receives HTTP 400 Bad Request response.
+            raise Fluent::UnrecoverableError, "got unrecoverable error response from '#{uri}', response code is #{response.status}"
+          end
+
+          if @http_config.retryable_response_codes&.include?(response.status)
+            raise RetryableResponse, "got retryable error response from '#{uri}', response code is #{response.status}"
+          end
+          if @http_config.error_response_as_unrecoverable
+            raise Fluent::UnrecoverableError, "got unrecoverable error response from '#{uri}', response code is #{response.status}"
+          else
+            @logger.error "got error response from '#{uri}', response code is #{response.status}"
+          end
+        end
+      end
+
+      private
+
+      def http_logs_endpoint
+        "#{@http_config.endpoint}/v1/logs"
+      end
+
+      def http_metrics_endpoint
+        "#{@http_config.endpoint}/v1/metrics"
+      end
+
+      def http_traces_endpoint
+        "#{@http_config.endpoint}/v1/traces"
+      end
+
+      def create_http_connection(record)
+        msg = record["message"]
+
+        begin
+          case record["type"]
+          when Opentelemetry::RECORD_TYPE_LOGS
+            uri = http_logs_endpoint
+            body = Opentelemetry::Request::Logs.new(msg).body
+          when Opentelemetry::RECORD_TYPE_METRICS
+            uri = http_metrics_endpoint
+            body = Opentelemetry::Request::Metrics.new(msg).body
+          when Opentelemetry::RECORD_TYPE_TRACES
+            uri = http_traces_endpoint
+            body = Opentelemetry::Request::Traces.new(msg).body
+          end
+        rescue Google::Protobuf::ParseError => e
+          # The message format does not comply with the OpenTelemetry protocol.
+          raise ::Fluent::UnrecoverableError, e.message
+        end
+
+        headers = { Opentelemetry::CONTENT_TYPE => Opentelemetry::CONTENT_TYPE_PROTOBUF }
+        if @http_config.compress == :gzip
+          headers[Opentelemetry::CONTENT_ENCODING] = Opentelemetry::CONTENT_ENCODING_GZIP
+          gz = Zlib::GzipWriter.new(StringIO.new)
+          gz << body
+          body = gz.close.string
+        end
+
+        Excon.defaults[:ssl_verify_peer] = false if @transport_config.insecure
+        connection = Excon.new(uri, body: body, headers: headers, proxy: @http_config.proxy, persistent: true, **@tls_settings, **@timeout_settings)
+        [uri, connection]
+      end
+    end
+
+    class GrpcHandler
+      def initialize(grpc_config, transport_config, logger)
+        @grpc_config = grpc_config
+        @transport_config = transport_config
+        @logger = logger
+      end
+
+      def export(record)
+        msg = record["message"]
+
+        credential = :this_channel_is_insecure
+
+        case record["type"]
+        when Opentelemetry::RECORD_TYPE_LOGS
+          service = Opentelemetry::ServiceStub::Logs.new(@grpc_config.endpoint, credential)
+        when Opentelemetry::RECORD_TYPE_METRICS
+          service = Opentelemetry::ServiceStub::Metrics.new(@grpc_config.endpoint, credential)
+        when Opentelemetry::RECORD_TYPE_TRACES
+          service = Opentelemetry::ServiceStub::Traces.new(@grpc_config.endpoint, credential)
+        end
+
+        begin
+          service.export(msg)
+        rescue Google::Protobuf::ParseError => e
+          # The message format does not comply with the OpenTelemetry protocol.
+          raise ::Fluent::UnrecoverableError, e.message
+        end
+      end
+    end
+  end
+end