fluent-plugin-opentelemetry 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa3bba6bc0556816b7f7fba2e7be82ffc2d629e484778042c7aab5113df8f514
-  data.tar.gz: f6bf4718017e97e2976ba2f2d74dc60fd85b2c7806087300a1e61788578ad528
+  metadata.gz: 1fce288ab5297db7255a6f89bd709731863d5a8952a08a40276f2f06381bfdf6
+  data.tar.gz: df4713f998552481517d0af45085b99551751e89600d9c6f46e541349173aaf3
 SHA512:
-  metadata.gz: 5a5b6d29a34477e583bcde8d81a174df7e8b55ec20ee93b7bf054882626621c2e800b610bd5025a0a8a96b80f6d29d4067a4f958f5f250e398ac2cf368e1439d
-  data.tar.gz: 8f514226a7c08fe6b890bae1a8004ec5f3cc58b267883339219937baa7f7c4aaade44fab72b791bcbcf246e83b5fdcf06544420371beb2b2521e3b6c437d9ae9
+  metadata.gz: 8e1d534c9ed6045031b9a40e58751c9c4a1c77e519d37a06dffa9d9d9e97eacccb8e0f90b3931ef925b20268dd97163423aaf05d4013f4b73192fbd46f322f82
+  data.tar.gz: e11de5fbc7b9282a205a653b05f7ed6e6554ac5760ef02b3e9f143875cc771720609095012f343ea5bc0a02daeba6e85baafb7e2a1ffe5fdcb0a7756f40e12a0

data/.editorconfig

@@ -0,0 +1,14 @@
+# EditorConfig helps developers define and maintain consistent coding styles between different editors and IDEs
+# @see http://editorconfig.org
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+tab_width = 4
+
+[*.{rb,yml,yaml,md,conf}]
+indent_size = 2

data/.rubocop.yml

@@ -1,3 +1,7 @@
+plugins:
+  - rubocop-performance
+  - rubocop-fluentd
+
 AllCops:
   Exclude:
     - 'lib/opentelemetry/**/*'
@@ -6,6 +10,18 @@ AllCops:
   SuggestExtensions: false
   TargetRubyVersion: 3.2
 
+# rubocop-fluentd
+Lint/FluentdPluginLogScope:
+  AssumeConfigLogLevel: 'info'
+  Enabled: true
+Lint/FluentdPluginConfigParamDefaultTime:
+  Enabled: true
+Lint/FluentdPluginIgnoreStandardError:
+  Enabled: true
+Performance/FluentdPluginLogStringInterpolation:
+  Enabled: true
+
+# rubocop
 Gemspec/DevelopmentDependencies:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 ## [Unreleased]
 
+## [0.4.0] - 2025-10-10
+
+- in_opentelemetry_metrics: Add plugin to support fluentd metrics export (#10)
+
+## [0.3.0] - 2025-07-23
+
+- in_opentelemetry: add ${type} placeholder support in tag parameter (#8)
+- in_opentelemetry: use String for record keys (#6)
+
 ## [0.2.0] - 2025-06-26
 
 - Make gRPC sending and receiving optional (#4)

data/README.md

@@ -24,7 +24,7 @@ $ bundle
 
 ## Configuration
 
-### Input plugin
+### Input `opentelemetry` plugin
 
 To receive data, this plugin requires `<http>` or `<grpc>` section, or both.
 
@@ -34,6 +34,33 @@ To receive data, this plugin requires `<http>` or `<grpc>` section, or both.
 |-----------|--------|----------------------|----------|
 | tag       | string | The tag of the event | required |
 
+`tag` parameter supports `${type}` placeholder and it will be expanded with data type.
+
+* log data
+  * `${type}` will be replaced to `logs`
+* metric data
+  * `${type}` will be replaced to `metrics`
+* trace data
+  * `${type}` will be replaced to `traces`
+
+This can be used to change the output destination for each message type.
+
+Example:
+
+```
+<source>
+  @type opentelemetry
+
+  # Expand to opentelemetry.logs, opentelemetry.metrics, opentelemetry.traces according to received data.
+  tag opentelemetry.${type}
+
+  <http>
+    bind 127.0.0.1
+    port 4318
+  </http>
+</source>
+```
+
 #### `<http>` section
 
 This requires to receive data via HTTP/HTTPS.
@@ -79,7 +106,40 @@ Refer [Config: Transport Section](https://docs.fluentd.org/configuration/transpo
 </source>
 ```
 
-### Output plugin
+### Input `opentelemetry_metrics` plugin
+
+This plugin emits Fluentd's metric data that conforms to the OpenTelemetry Protocol.
+To output the data, it requires to use output `opentelemetry` plugin.
+
+#### Root section
+
+| parameter          | type   | description                                           | default     |
+|--------------------|--------|-------------------------------------------------------|-------------|
+| tag                | string | The tag of the event                                  | required    |
+| emit_interval      | time   | Determine the rate to emit internal metrics as events | `60`        |
+| metric_name_prefix | string | The prefix of metric name                             | `fluentd_`  |
+
+#### Example
+
+```
+# Emit Fluentd metrics
+<source>
+  @type opentelemetry_metrics
+  tag opentelemetry.fluentd.metrics
+  emit_interval 300s
+</source>
+
+# Send Fluentd metrics to OpenTelemetry Collector
+<match opentelemetry.fluentd.metrics>
+  @type opentelemetry
+
+  <http>
+    endpoint "https://127.0.0.1:4318"
+  </http>
+</match>
+```
+
+### Output `opentelemetry` plugin
 
 To send data, this plugin requires `<http>` or `<grpc>` section.
 

data/lib/fluent/plugin/in_opentelemetry.rb

@@ -43,6 +43,8 @@ module Fluent::Plugin
     def configure(conf)
       super
 
+      expand_tag_placeholders(@tag)
+
       if @grpc_config && !defined?(GRPC)
         raise Fluent::ConfigError, "To use gRPC feature, please install grpc gem such as 'fluent-gem install grpc'."
       end
@@ -59,13 +61,13 @@ module Fluent::Plugin
         http_handler = Opentelemetry::HttpInputHandler.new
         http_server_create_http_server(:in_opentelemetry_http_server, addr: @http_config.bind, port: @http_config.port, logger: log) do |serv|
           serv.post("/v1/logs") do |req|
-            http_handler.logs(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_LOGS, message: record }) }
+            http_handler.logs(req) { |record| router.emit(tag_for(Opentelemetry::RECORD_TYPE_LOGS), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_LOGS, "message" => record }) }
           end
           serv.post("/v1/metrics") do |req|
-            http_handler.metrics(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_METRICS, message: record }) }
+            http_handler.metrics(req) { |record| router.emit(tag_for(Opentelemetry::RECORD_TYPE_METRICS), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_METRICS, "message" => record }) }
           end
           serv.post("/v1/traces") do |req|
-            http_handler.traces(req) { |record| router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_TRACES, message: record }) }
+            http_handler.traces(req) { |record| router.emit(tag_for(Opentelemetry::RECORD_TYPE_TRACES), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_TRACES, "message" => record }) }
           end
         end
       end
@@ -75,17 +77,31 @@ module Fluent::Plugin
           grpc_handler = Opentelemetry::GrpcInputHandler.new(@grpc_config, log)
           grpc_handler.run(
             logs: lambda { |record|
-              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_LOGS, message: record })
+              router.emit(tag_for(Opentelemetry::RECORD_TYPE_LOGS), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_LOGS, "message" => record })
             },
             metrics: lambda { |record|
-              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_METRICS, message: record })
+              router.emit(tag_for(Opentelemetry::RECORD_TYPE_METRICS), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_METRICS, "message" => record })
             },
             traces: lambda { |record|
-              router.emit(@tag, Fluent::EventTime.now, { type: Opentelemetry::RECORD_TYPE_TRACES, message: record })
+              router.emit(tag_for(Opentelemetry::RECORD_TYPE_TRACES), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_TRACES, "message" => record })
             }
           )
         end
       end
     end
+
+    private
+
+    def expand_tag_placeholders(tag)
+      @expand_tag_placeholders ||= {
+        Opentelemetry::RECORD_TYPE_LOGS => tag.gsub(Opentelemetry::PLACEHOLDER_TYPE, Opentelemetry::PLACEHOLDER_TYPE_LOGS),
+        Opentelemetry::RECORD_TYPE_METRICS => tag.gsub(Opentelemetry::PLACEHOLDER_TYPE, Opentelemetry::PLACEHOLDER_TYPE_METRICS),
+        Opentelemetry::RECORD_TYPE_TRACES => tag.gsub(Opentelemetry::PLACEHOLDER_TYPE, Opentelemetry::PLACEHOLDER_TYPE_TRACES)
+      }
+    end
+
+    def tag_for(type)
+      @expand_tag_placeholders[type]
+    end
   end
 end

data/lib/fluent/plugin/in_opentelemetry_metrics.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/input"
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/version"
+require "fluent/plugin_helper/timer"
+require "fluent/version"
+
+require "json"
+require "socket"
+
+module Fluent::Plugin
+  class OpentelemetryMetricsInput < Input
+    Fluent::Plugin.register_input("opentelemetry_metrics", self)
+
+    helpers :timer
+
+    desc "Determine the rate to emit internal metrics as events."
+    config_param :emit_interval, :time, default: 60
+
+    desc "The tag of the event."
+    config_param :tag, :string
+
+    desc "The prefix of metric name."
+    config_param :metric_name_prefix, :string, default: "fluentd_"
+
+    def start
+      super
+
+      @metrics = Metrics.new(metric_name_prefix: @metric_name_prefix)
+      timer_execute(:in_opentelemetry_metrics, @emit_interval) do
+        router.emit(@tag, Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_METRICS, "message" => @metrics.record })
+      end
+    end
+
+    module Extension
+      refine Time do
+        def to_nano_sec
+          (to_i * 1_000_000_000) + nsec
+        end
+      end
+    end
+
+    class Metrics
+      using Extension
+
+      def initialize(metric_name_prefix:)
+        @start_time_unix_nano = Time.now.to_nano_sec
+        @metric_name_prefix = metric_name_prefix.to_s
+        @hostname = Socket.gethostname
+        @monitor_info = MonitorInfo.new
+      end
+
+      def record
+        values.to_json
+      end
+
+      def values
+        metrics_data
+      end
+
+      private
+
+      def metrics_data
+        {
+          "resourceMetrics" => [
+            {
+              "resource" => {
+                "attributes" => [
+                  string_value_attribute("service.name", "fluentd"),
+                  string_value_attribute("service.version", Fluent::VERSION),
+                  string_value_attribute("host.name", @hostname),
+                  string_value_attribute("process.runtime.name", "ruby"),
+                  string_value_attribute("process.runtime.version", RUBY_VERSION),
+                  int_value_attribute("process.pid", Process.pid)
+                ]
+              },
+              "scopeMetrics" => scope_metrics
+            }
+          ]
+        }
+      end
+
+      def scope_metrics
+        [
+          {
+            "scope" => {
+              "name" => "fluent-plugin-opentelemetry",
+              "version" => Fluent::Plugin::Opentelemetry::VERSION
+            },
+            "metrics" => metrics
+          }
+        ]
+      end
+
+      def metrics
+        time_nano_sec = Time.now.to_nano_sec
+        metrics = []
+
+        @monitor_info.plugins_info_all.each do |record|
+          attributes = {
+            plugin_id: record["plugin_id"],
+            plugin: plugin_name(record["plugin_category"], record["type"]),
+            plugin_category: record["plugin_category"],
+            plugin_type: record["type"]
+          }.map { |k, v| string_value_attribute(k, v) }
+
+          record.each do |key, value|
+            next unless value.is_a?(Numeric)
+
+            metrics << {
+              "name" => @metric_name_prefix + key.to_s,
+              "unit" => "1",
+              # TODO: "description"
+              "gauge" => {
+                "dataPoints" => [
+                  {
+                    "startTimeUnixNano" => @start_time_unix_nano,
+                    "timeUnixNano" => time_nano_sec,
+                    "asDouble" => value,
+                    "attributes" => attributes
+                  }
+                ]
+              }
+            }
+          end
+        end
+
+        metrics
+      end
+
+      def plugin_name(category, type)
+        prefix =
+          case category
+          when "input"
+            "in"
+          when "output"
+            "out"
+          else
+            category
+          end
+
+        "#{prefix}_#{type}"
+      end
+
+      def string_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "stringValue" => value.to_s
+          }
+        }
+      end
+
+      def int_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "intValue" => value
+          }
+        }
+      end
+    end
+
+    # Imported from Fluent::Plugin::MonitorAgentInput
+    class MonitorInfo
+      # They are deprecated but remain for compatibiscripts/pluginslity
+      MONITOR_INFO = {
+        "output_plugin" => -> { is_a?(::Fluent::Plugin::Output) },
+        "buffer_queue_length" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.queue.size
+        },
+        "buffer_timekeys" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.timekeys
+        },
+        "buffer_total_queued_size" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.stage_size + @buffer.queue_size
+        },
+        "retry_count" => -> { respond_to?(:num_errors) ? num_errors : nil }
+      }.freeze
+
+      def all_plugins
+        array = []
+
+        # get all input plugins
+        array.concat Fluent::Engine.root_agent.inputs
+
+        # get all output plugins
+        array.concat Fluent::Engine.root_agent.outputs
+
+        # get all filter plugins
+        array.concat Fluent::Engine.root_agent.filters
+
+        Fluent::Engine.root_agent.labels.each_value do |l|
+          # TODO: Add label name to outputs / filters for identifying plugins
+          array.concat l.outputs
+          array.concat l.filters
+        end
+
+        array
+      end
+
+      def plugin_category(pe)
+        case pe
+        when Fluent::Plugin::Input
+          "input"
+        when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
+          "output"
+        when Fluent::Plugin::Filter
+          "filter"
+        else
+          "unknown"
+        end
+      end
+
+      def plugins_info_all(opts = {})
+        all_plugins.map do |pe|
+          get_monitor_info(pe, opts)
+        end
+      end
+
+      IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config).freeze
+
+      # get monitor info from the plugin `pe` and return a hash object
+      def get_monitor_info(pe, opts = {})
+        obj = {}
+
+        # Common plugin information
+        obj["plugin_id"] = pe.plugin_id
+        obj["plugin_category"] = plugin_category(pe)
+        obj["type"] = pe.config["@type"]
+        obj["config"] = pe.config if opts[:with_config]
+
+        # run MONITOR_INFO in plugins' instance context and store the info to obj
+        MONITOR_INFO.each_pair do |key, code|
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue NoMethodError => e
+          unless @first_warn
+            log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
+            log.error_backtrace
+            @first_warn = true
+          end
+        rescue StandardError => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
+        end
+
+        if pe.respond_to?(:statistics)
+          obj.merge!(pe.statistics["output"] || {})
+          obj.merge!(pe.statistics["filter"] || {})
+          obj.merge!(pe.statistics["input"] || {})
+        end
+
+        obj["retry"] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
+
+        # include all instance variables if :with_debug_info is set
+        if opts[:with_debug_info]
+          iv = {}
+          pe.instance_eval do
+            instance_variables.each do |sym|
+              next if IGNORE_ATTRIBUTES.include?(sym)
+
+              key = sym.to_s[1..] # removes first '@'
+              iv[key] = instance_variable_get(sym)
+            end
+          end
+          obj["instance_variables"] = iv
+        elsif (ivars = opts[:ivars])
+          iv = {}
+          ivars.each do |name|
+            iname = "@#{name}"
+            iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
+          end
+          obj["instance_variables"] = iv
+        end
+
+        obj
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/constant.rb

@@ -16,6 +16,11 @@ module Fluent::Plugin::Opentelemetry
   RECORD_TYPE_METRICS = "opentelemetry_metrics"
   RECORD_TYPE_TRACES = "opentelemetry_traces"
 
+  PLACEHOLDER_TYPE = "${type}"
+  PLACEHOLDER_TYPE_LOGS = "logs"
+  PLACEHOLDER_TYPE_METRICS = "metrics"
+  PLACEHOLDER_TYPE_TRACES = "traces"
+
   TLS_VERSIONS_MAP =
     begin
       map = {

data/lib/fluent/plugin/opentelemetry/request.rb

@@ -9,10 +9,10 @@ require "google/protobuf"
 
 class Fluent::Plugin::Opentelemetry::Request
   class Logs
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode(body)
         end
@@ -28,10 +28,10 @@ class Fluent::Plugin::Opentelemetry::Request
   end
 
   class Metrics
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode(body)
         end
@@ -47,10 +47,10 @@ class Fluent::Plugin::Opentelemetry::Request
   end
 
   class Traces
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(body)
         end

data/lib/fluent/plugin/opentelemetry/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module Opentelemetry
+      VERSION = "0.4.0"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opentelemetry
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Shizuo Fujita
@@ -72,6 +72,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".rubocop.yml"
 - CHANGELOG.md
 - LICENSE
@@ -79,6 +80,7 @@ files:
 - Rakefile
 - TODO.md
 - lib/fluent/plugin/in_opentelemetry.rb
+- lib/fluent/plugin/in_opentelemetry_metrics.rb
 - lib/fluent/plugin/opentelemetry/constant.rb
 - lib/fluent/plugin/opentelemetry/grpc_input_handler.rb
 - lib/fluent/plugin/opentelemetry/grpc_output_handler.rb
@@ -86,6 +88,7 @@ files:
 - lib/fluent/plugin/opentelemetry/http_output_handler.rb
 - lib/fluent/plugin/opentelemetry/request.rb
 - lib/fluent/plugin/opentelemetry/response.rb
+- lib/fluent/plugin/opentelemetry/version.rb
 - lib/fluent/plugin/out_opentelemetry.rb
 - lib/opentelemetry/proto/collector/logs/v1/logs_service_pb.rb
 - lib/opentelemetry/proto/collector/logs/v1/logs_service_services_pb.rb
@@ -122,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Fluentd input/output plugin to forward OpenTelemetry Protocol data.
 test_files: []