fluent-plugin-opentelemetry 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7e88abfcc25a1079994f2801f2b32665bb39bb32408cff6efb91125ef9b60149
-  data.tar.gz: 2b64e708b7ea494e4c35e4218c4f33e2ca15f04ac55672d95dc9d7456435303c
+  metadata.gz: 1fce288ab5297db7255a6f89bd709731863d5a8952a08a40276f2f06381bfdf6
+  data.tar.gz: df4713f998552481517d0af45085b99551751e89600d9c6f46e541349173aaf3
 SHA512:
-  metadata.gz: 26fde29bef629246d797440d306b863ed6ed8cd57d0871f6c66b52b0c3109b453e6d836cb5b99d5b545d778cbb9f4893fdc37166499baf61e92de5bf478f60c5
-  data.tar.gz: 1510198977d2f3b71a58a7a85e9a948b40716bf036c84968b8bb5c1507d63553fd1a06dbc70633d788d110dbc510ef898710eaaf2727d76adfd8baba9b5c9e81
+  metadata.gz: 8e1d534c9ed6045031b9a40e58751c9c4a1c77e519d37a06dffa9d9d9e97eacccb8e0f90b3931ef925b20268dd97163423aaf05d4013f4b73192fbd46f322f82
+  data.tar.gz: e11de5fbc7b9282a205a653b05f7ed6e6554ac5760ef02b3e9f143875cc771720609095012f343ea5bc0a02daeba6e85baafb7e2a1ffe5fdcb0a7756f40e12a0

data/.editorconfig

@@ -0,0 +1,14 @@
+# EditorConfig helps developers define and maintain consistent coding styles between different editors and IDEs
+# @see http://editorconfig.org
+root = true
+
+[*]
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+indent_style = space
+tab_width = 4
+
+[*.{rb,yml,yaml,md,conf}]
+indent_size = 2

data/.rubocop.yml

@@ -12,9 +12,12 @@ AllCops:
 
 # rubocop-fluentd
 Lint/FluentdPluginLogScope:
+  AssumeConfigLogLevel: 'info'
   Enabled: true
 Lint/FluentdPluginConfigParamDefaultTime:
   Enabled: true
+Lint/FluentdPluginIgnoreStandardError:
+  Enabled: true
 Performance/FluentdPluginLogStringInterpolation:
   Enabled: true
 

data/CHANGELOG.md

@@ -1,5 +1,9 @@
 ## [Unreleased]
 
+## [0.4.0] - 2025-10-10
+
+- in_opentelemetry_metrics: Add plugin to support fluentd metrics export (#10)
+
 ## [0.3.0] - 2025-07-23
 
 - in_opentelemetry: add ${type} placeholder support in tag parameter (#8)

data/README.md

@@ -24,7 +24,7 @@ $ bundle
 
 ## Configuration
 
-### Input plugin
+### Input `opentelemetry` plugin
 
 To receive data, this plugin requires `<http>` or `<grpc>` section, or both.
 
@@ -106,7 +106,40 @@ Refer [Config: Transport Section](https://docs.fluentd.org/configuration/transpo
 </source>
 ```
 
-### Output plugin
+### Input `opentelemetry_metrics` plugin
+
+This plugin emits Fluentd's metric data that conforms to the OpenTelemetry Protocol.
+To output the data, it requires to use output `opentelemetry` plugin.
+
+#### Root section
+
+| parameter          | type   | description                                           | default     |
+|--------------------|--------|-------------------------------------------------------|-------------|
+| tag                | string | The tag of the event                                  | required    |
+| emit_interval      | time   | Determine the rate to emit internal metrics as events | `60`        |
+| metric_name_prefix | string | The prefix of metric name                             | `fluentd_`  |
+
+#### Example
+
+```
+# Emit Fluentd metrics
+<source>
+  @type opentelemetry_metrics
+  tag opentelemetry.fluentd.metrics
+  emit_interval 300s
+</source>
+
+# Send Fluentd metrics to OpenTelemetry Collector
+<match opentelemetry.fluentd.metrics>
+  @type opentelemetry
+
+  <http>
+    endpoint "https://127.0.0.1:4318"
+  </http>
+</match>
+```
+
+### Output `opentelemetry` plugin
 
 To send data, this plugin requires `<http>` or `<grpc>` section.
 

data/lib/fluent/plugin/in_opentelemetry_metrics.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require "fluent/plugin/input"
+require "fluent/plugin/opentelemetry/constant"
+require "fluent/plugin/opentelemetry/version"
+require "fluent/plugin_helper/timer"
+require "fluent/version"
+
+require "json"
+require "socket"
+
+module Fluent::Plugin
+  class OpentelemetryMetricsInput < Input
+    Fluent::Plugin.register_input("opentelemetry_metrics", self)
+
+    helpers :timer
+
+    desc "Determine the rate to emit internal metrics as events."
+    config_param :emit_interval, :time, default: 60
+
+    desc "The tag of the event."
+    config_param :tag, :string
+
+    desc "The prefix of metric name."
+    config_param :metric_name_prefix, :string, default: "fluentd_"
+
+    def start
+      super
+
+      @metrics = Metrics.new(metric_name_prefix: @metric_name_prefix)
+      timer_execute(:in_opentelemetry_metrics, @emit_interval) do
+        router.emit(@tag, Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_METRICS, "message" => @metrics.record })
+      end
+    end
+
+    module Extension
+      refine Time do
+        def to_nano_sec
+          (to_i * 1_000_000_000) + nsec
+        end
+      end
+    end
+
+    class Metrics
+      using Extension
+
+      def initialize(metric_name_prefix:)
+        @start_time_unix_nano = Time.now.to_nano_sec
+        @metric_name_prefix = metric_name_prefix.to_s
+        @hostname = Socket.gethostname
+        @monitor_info = MonitorInfo.new
+      end
+
+      def record
+        values.to_json
+      end
+
+      def values
+        metrics_data
+      end
+
+      private
+
+      def metrics_data
+        {
+          "resourceMetrics" => [
+            {
+              "resource" => {
+                "attributes" => [
+                  string_value_attribute("service.name", "fluentd"),
+                  string_value_attribute("service.version", Fluent::VERSION),
+                  string_value_attribute("host.name", @hostname),
+                  string_value_attribute("process.runtime.name", "ruby"),
+                  string_value_attribute("process.runtime.version", RUBY_VERSION),
+                  int_value_attribute("process.pid", Process.pid)
+                ]
+              },
+              "scopeMetrics" => scope_metrics
+            }
+          ]
+        }
+      end
+
+      def scope_metrics
+        [
+          {
+            "scope" => {
+              "name" => "fluent-plugin-opentelemetry",
+              "version" => Fluent::Plugin::Opentelemetry::VERSION
+            },
+            "metrics" => metrics
+          }
+        ]
+      end
+
+      def metrics
+        time_nano_sec = Time.now.to_nano_sec
+        metrics = []
+
+        @monitor_info.plugins_info_all.each do |record|
+          attributes = {
+            plugin_id: record["plugin_id"],
+            plugin: plugin_name(record["plugin_category"], record["type"]),
+            plugin_category: record["plugin_category"],
+            plugin_type: record["type"]
+          }.map { |k, v| string_value_attribute(k, v) }
+
+          record.each do |key, value|
+            next unless value.is_a?(Numeric)
+
+            metrics << {
+              "name" => @metric_name_prefix + key.to_s,
+              "unit" => "1",
+              # TODO: "description"
+              "gauge" => {
+                "dataPoints" => [
+                  {
+                    "startTimeUnixNano" => @start_time_unix_nano,
+                    "timeUnixNano" => time_nano_sec,
+                    "asDouble" => value,
+                    "attributes" => attributes
+                  }
+                ]
+              }
+            }
+          end
+        end
+
+        metrics
+      end
+
+      def plugin_name(category, type)
+        prefix =
+          case category
+          when "input"
+            "in"
+          when "output"
+            "out"
+          else
+            category
+          end
+
+        "#{prefix}_#{type}"
+      end
+
+      def string_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "stringValue" => value.to_s
+          }
+        }
+      end
+
+      def int_value_attribute(key, value)
+        {
+          "key" => key.to_s,
+          "value" => {
+            "intValue" => value
+          }
+        }
+      end
+    end
+
+    # Imported from Fluent::Plugin::MonitorAgentInput
+    class MonitorInfo
+      # They are deprecated but remain for compatibiscripts/pluginslity
+      MONITOR_INFO = {
+        "output_plugin" => -> { is_a?(::Fluent::Plugin::Output) },
+        "buffer_queue_length" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.queue.size
+        },
+        "buffer_timekeys" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.timekeys
+        },
+        "buffer_total_queued_size" => lambda {
+          throw(:skip) unless instance_variable_defined?(:@buffer) && !@buffer.nil? && @buffer.is_a?(::Fluent::Plugin::Buffer)
+          @buffer.stage_size + @buffer.queue_size
+        },
+        "retry_count" => -> { respond_to?(:num_errors) ? num_errors : nil }
+      }.freeze
+
+      def all_plugins
+        array = []
+
+        # get all input plugins
+        array.concat Fluent::Engine.root_agent.inputs
+
+        # get all output plugins
+        array.concat Fluent::Engine.root_agent.outputs
+
+        # get all filter plugins
+        array.concat Fluent::Engine.root_agent.filters
+
+        Fluent::Engine.root_agent.labels.each_value do |l|
+          # TODO: Add label name to outputs / filters for identifying plugins
+          array.concat l.outputs
+          array.concat l.filters
+        end
+
+        array
+      end
+
+      def plugin_category(pe)
+        case pe
+        when Fluent::Plugin::Input
+          "input"
+        when Fluent::Plugin::Output, Fluent::Plugin::MultiOutput, Fluent::Plugin::BareOutput
+          "output"
+        when Fluent::Plugin::Filter
+          "filter"
+        else
+          "unknown"
+        end
+      end
+
+      def plugins_info_all(opts = {})
+        all_plugins.map do |pe|
+          get_monitor_info(pe, opts)
+        end
+      end
+
+      IGNORE_ATTRIBUTES = %i(@config_root_section @config @masked_config).freeze
+
+      # get monitor info from the plugin `pe` and return a hash object
+      def get_monitor_info(pe, opts = {})
+        obj = {}
+
+        # Common plugin information
+        obj["plugin_id"] = pe.plugin_id
+        obj["plugin_category"] = plugin_category(pe)
+        obj["type"] = pe.config["@type"]
+        obj["config"] = pe.config if opts[:with_config]
+
+        # run MONITOR_INFO in plugins' instance context and store the info to obj
+        MONITOR_INFO.each_pair do |key, code|
+          catch(:skip) do
+            obj[key] = pe.instance_exec(&code)
+          end
+        rescue NoMethodError => e
+          unless @first_warn
+            log.error "NoMethodError in monitoring plugins", key: key, plugin: pe.class, error: e
+            log.error_backtrace
+            @first_warn = true
+          end
+        rescue StandardError => e
+          log.warn "unexpected error in monitoring plugins", key: key, plugin: pe.class, error: e
+        end
+
+        if pe.respond_to?(:statistics)
+          obj.merge!(pe.statistics["output"] || {})
+          obj.merge!(pe.statistics["filter"] || {})
+          obj.merge!(pe.statistics["input"] || {})
+        end
+
+        obj["retry"] = get_retry_info(pe.retry) if opts[:with_retry] && pe.instance_variable_defined?(:@retry)
+
+        # include all instance variables if :with_debug_info is set
+        if opts[:with_debug_info]
+          iv = {}
+          pe.instance_eval do
+            instance_variables.each do |sym|
+              next if IGNORE_ATTRIBUTES.include?(sym)
+
+              key = sym.to_s[1..] # removes first '@'
+              iv[key] = instance_variable_get(sym)
+            end
+          end
+          obj["instance_variables"] = iv
+        elsif (ivars = opts[:ivars])
+          iv = {}
+          ivars.each do |name|
+            iname = "@#{name}"
+            iv[name] = pe.instance_variable_get(iname) if pe.instance_variable_defined?(iname)
+          end
+          obj["instance_variables"] = iv
+        end
+
+        obj
+      end
+    end
+  end
+end

data/lib/fluent/plugin/opentelemetry/request.rb

@@ -9,10 +9,10 @@ require "google/protobuf"
 
 class Fluent::Plugin::Opentelemetry::Request
   class Logs
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Logs::V1::ExportLogsServiceRequest.decode(body)
         end
@@ -28,10 +28,10 @@ class Fluent::Plugin::Opentelemetry::Request
   end
 
   class Metrics
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Metrics::V1::ExportMetricsServiceRequest.decode(body)
         end
@@ -47,10 +47,10 @@ class Fluent::Plugin::Opentelemetry::Request
   end
 
   class Traces
-    def initialize(body)
+    def initialize(body, ignore_unknown_fields: true)
       @request =
         if body.start_with?("{")
-          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: true)
+          Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode_json(body, ignore_unknown_fields: ignore_unknown_fields)
         else
           Opentelemetry::Proto::Collector::Trace::V1::ExportTraceServiceRequest.decode(body)
         end

data/lib/fluent/plugin/opentelemetry/version.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Fluent
+  module Plugin
+    module Opentelemetry
+      VERSION = "0.4.0"
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opentelemetry
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Shizuo Fujita
@@ -72,6 +72,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
 - ".rubocop.yml"
 - CHANGELOG.md
 - LICENSE
@@ -79,6 +80,7 @@ files:
 - Rakefile
 - TODO.md
 - lib/fluent/plugin/in_opentelemetry.rb
+- lib/fluent/plugin/in_opentelemetry_metrics.rb
 - lib/fluent/plugin/opentelemetry/constant.rb
 - lib/fluent/plugin/opentelemetry/grpc_input_handler.rb
 - lib/fluent/plugin/opentelemetry/grpc_output_handler.rb
@@ -86,6 +88,7 @@ files:
 - lib/fluent/plugin/opentelemetry/http_output_handler.rb
 - lib/fluent/plugin/opentelemetry/request.rb
 - lib/fluent/plugin/opentelemetry/response.rb
+- lib/fluent/plugin/opentelemetry/version.rb
 - lib/fluent/plugin/out_opentelemetry.rb
 - lib/opentelemetry/proto/collector/logs/v1/logs_service_pb.rb
 - lib/opentelemetry/proto/collector/logs/v1/logs_service_services_pb.rb
@@ -122,7 +125,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.7.0
+rubygems_version: 3.7.2
 specification_version: 4
 summary: Fluentd input/output plugin to forward OpenTelemetry Protocol data.
 test_files: []