fluent-plugin-opentelemetry 0.5.1 → 0.5.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe7bce47ce8bec3c95c596ccdd5c444f3d4d069e6a7e4b865145d8fde923acb7
-  data.tar.gz: e6afb3a4ceb6b5b9bd469cac2474e87dffe65a2fdb6b16c5b918382ed5b854ab
+  metadata.gz: 2e1331e92688a03b2da70e3712257f4dfd4ae1cec7c5da354084f6fc883ae5cf
+  data.tar.gz: 403339bc82d1c09823afc7a5c2b16199cafab92300a0e56eb446fb33e55e1407
 SHA512:
-  metadata.gz: be3ade279fc65f4850918f9c477d3b0d912c641fe657ffa9c554d7afe73da902688328199e33c352f2f804687568e2263149acefb904ff53ef863779083a067b
-  data.tar.gz: f8ffa9756821bb1ef81155ed2e7283f6af734363336cd8229a6278d532ffebd4ad9cc88b38420aeedc591449bab8a2efbf5e9f4ed99714c4aef6a3fe7be35e38
+  metadata.gz: d3cc4c75562d9698c668b02e172d8eeb2c15de4233cb1ebe55607e6daeb18efe19df1bd20f2b0c586f79e07d63301bcf880221d904ebcf11c34047b7a00fd12d
+  data.tar.gz: 83d8d1e66e74c93a23f46d7580591b72d1bd3dfd064365ce262057890cb8aff0377cae4cf7c88ad810ed94842a5eb7c91e526109efd50fcff7d131c6afb4f3a7

data/.rubocop.yml

@@ -78,6 +78,9 @@ Style/MixinUsage:
 Style/NumericLiterals:
   Enabled: false
 
+Style/OneClassPerFile:
+  Enabled: false
+
 Style/PercentLiteralDelimiters:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,5 +1,17 @@
 ## [Unreleased]
 
+## [0.5.3] - 2026-06-25
+
+Fixes:
+
+- in_opentelemetry: enforce size limits on incoming payloads
+
+## [0.5.2] - 2026-03-15
+
+Improvements:
+
+- out_opentelemetry: add BatchProcessor for bulk transfer (#33)
+
 ## [0.5.1] - 2026-02-09
 
 Fixes:

data/README.md

@@ -65,10 +65,12 @@ Example:
 
 This requires to receive data via HTTP/HTTPS.
 
-| parameter | type    | description            | default   |
-|-----------|---------|------------------------|-----------|
-| bind      | string  | The address to bind to | `0.0.0.0` |
-| port      | integer | The port to listen to  | `4318`    |
+| parameter                | type    | description                                                                                                               | default   |
+|--------------------------|---------|---------------------------------------------------------------------------------------------------------------------------|-----------|
+| bind                     | string  | The address to bind to                                                                                                    | `0.0.0.0` |
+| port                     | integer | The port to listen to                                                                                                     | `4318`    |
+| body_size_limit          | size    | The size limit of the POSTed element. This value should be larger than the 'chunk_limit_size' in out_opentelemetry plugin | `32M` (32MiB)     |
+| decompression_size_limit | size    | The size limit of the decompressed element                                                                                | `256M`  (256MiB)   |
 
 #### `<grpc>` section
 
@@ -144,6 +146,19 @@ To output the data, it requires to use output `opentelemetry` plugin.
 >
 > **PromQL Example:** `rate(fluentd_process_cpu_time_seconds_total[$__rate_interval])`
 
+> [!NOTE]
+> **Process Metrics (CPU / Memory)**
+>
+> Fluentd operates on a multi-process architecture consisting of a Supervisor process and one or more Worker processes.
+>
+> Because this plugin runs inside a specific worker process, the process-level metrics it collects (such as CPU time and memory usage) represent **only the footprint of that single worker process**, not the total resource consumption of the entire Fluentd instance.
+>
+> If you are using a multi-worker configuration (`<system> workers N </system>`), be aware that the actual total CPU and memory usage of Fluentd will be higher than what this plugin reports.
+>
+> For accurate, instance-wide resource monitoring, we strongly recommend using external OS-level monitoring tools rather than relying on application-internal metrics:
+> * Container Environments (Kubernetes/Docker): Use tools like `cAdvisor` or `kube-state-metrics`, etc.
+> * VM/Bare-metal Environments: Use `node_exporter` combined with `process-exporter`, etc.
+
 ### Output `opentelemetry` plugin
 
 To send data, this plugin requires `<http>` or `<grpc>` section.
@@ -202,9 +217,15 @@ Refer [Config: Transport Section](https://docs.fluentd.org/configuration/transpo
 
 #### `<buffer>` section
 
-| parameter  | type   | description                                               | default |
-|------------|--------|-----------------------------------------------------------|---------|
-| chunk_keys | array  | Overwrites the default `chunk_keys` value in this plugin. | `tag`   |
+| parameter        | type    | description                                                     | default    |
+|------------------|---------|-----------------------------------------------------------------|------------|
+| chunk_keys       | array   | Overwrites the default `chunk_keys` value in this plugin.       | `tag`      |
+| chunk_limit_size | integer | Overwrites the default `chunk_limit_size` value in this plugin. | `8M` (8MB) |
+
+**Note on chunk_limit_size:**
+Currently, this plugin is reducing I/O load by combining data within a chunk and sending it as a single unit.
+Depending on the configuration of your OpenTelemetry Collector or downstream backend (e.g., maximum request body size limits), an 8MB chunk might be rejected with a `413 Payload Too Large` error.
+If you encounter this error in your Fluentd logs, please adjust (decrease) the `chunk_limit_size` in your `<buffer>` section.
 
 Refer [Config: Buffer Section](https://docs.fluentd.org/configuration/buffer-section)
 
@@ -225,4 +246,3 @@ Refer [Config: Buffer Section](https://docs.fluentd.org/configuration/buffer-sec
 * Copyright(c) 2025- Shizuo Fujita
 * License
   * Apache License, Version 2.0
-

data/TODO.md

@@ -8,10 +8,13 @@ Ref. https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specif
   - [x] [MUST] Gzip compression, denoted by gzip.
 
 ### OTLP/gRPC Concurrent Requests
-- [ ] [SHOULD] The implementations that need to achieve high throughput SHOULD support concurrent Unary calls to achieve higher throughput.
+- [x] [SHOULD] The implementations that need to achieve high throughput SHOULD support concurrent Unary calls to achieve higher throughput.
+   * NOTE: Supported by `flush_thread_count` settings in `<buffer>`.
 - [ ] [SHOULD] The client SHOULD send new requests without waiting for the response to the earlier sent requests, essentially creating a pipeline of requests that are currently in flight that are not acknowledged.
-- [ ] [SHOULD] The number of concurrent requests SHOULD be configurable.
-- [ ] [SHOULD] The client implementation SHOULD expose an option to turn on and off the waiting during a shutdown.
+- [x] [SHOULD] The number of concurrent requests SHOULD be configurable.
+   * NOTE: Supported by `flush_thread_count` settings in `<buffer>`.
+- [x] [SHOULD] The client implementation SHOULD expose an option to turn on and off the waiting during a shutdown.
+   * NOTE: Supported by `flush_at_shutdown` settings in `<buffer>`.
 
 ### OTLP/gRPC Response
 - Full Success
@@ -30,7 +33,8 @@ Ref. https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specif
   - [ ] [SHOULD] Retryable errors indicate that telemetry data processing failed, and the client SHOULD record the error and may retry exporting the same data.
   - [ ] [SHOULD] The client SHOULD maintain a counter of such dropped data.
   - [ ] [SHOULD] The client SHOULD interpret gRPC status codes as retryable or not-retryable according to the following table.
-  - [ ] [SHOULD] When retrying, the client SHOULD implement an exponential backoff strategy.
+  - [x] [SHOULD] When retrying, the client SHOULD implement an exponential backoff strategy.
+     * NOTE: Fluentd's retry mechanism has `retry_type: exponential_backoff` by default.
   - [ ] [SHOULD] The client SHOULD interpret RESOURCE_EXHAUSTED code as retryable only if the server signals that the recovery from resource exhaustion is possible.
 
 ### OTLP/gRPC Throttling
@@ -91,18 +95,21 @@ Ref. https://github.com/open-telemetry/opentelemetry-proto/blob/main/docs/specif
 - OTLP/HTTP Throttling
   - [ ] [SHOULD] If the server receives more requests than the client is allowed or the server is overloaded, the server SHOULD respond with HTTP 429 Too Many Requests or HTTP 503 Service Unavailable.
   - [ ] [SHOULD] The client SHOULD honour the waiting interval specified in the "Retry-After" header if it is present.
-  - [ ] [SHOULD] The "Retry-After" header is not present in the response, then the client SHOULD implement an exponential backoff strategy between retries.
+  - [x] [SHOULD] The "Retry-After" header is not present in the response, then the client SHOULD implement an exponential backoff strategy between retries.
+    * NOTE: Fluentd's retry mechanism has `retry_type: exponential_backoff` by default.
 - All Other Responses
   - [x] [SHOULD] If the server disconnects without returning a response, the client SHOULD retry and send the same request.
   - [x] [SHOULD] The client SHOULD implement an exponential backoff strategy between retries to avoid overwhelming the server.
 
 ### OTLP/HTTP Connection
-- [ ] [SHOULD] If the client cannot connect to the server, the client SHOULD retry the connection using an exponential backoff strategy between retries.
-- [ ] [SHOULD] The client SHOULD keep the connection alive between requests.
+- [x] [SHOULD] If the client cannot connect to the server, the client SHOULD retry the connection using an exponential backoff strategy between retries.
+  * NOTE: Fluentd's retry mechanism has `retry_type: exponential_backoff` by default.
+- [x] [SHOULD] The client SHOULD keep the connection alive between requests.
 - [ ] [SHOULD] Server implementations SHOULD accept OTLP/HTTP with binary-encoded Protobuf payload and OTLP/HTTP with JSON-encoded Protobuf payload requests on the same port and multiplex the requests to the corresponding payload decoder based on the "Content-Type" request header.
 
 ### OTLP/HTTP Concurrent Requests
-- [ ] [SHOULD] To achieve higher total throughput, the client MAY send requests using several parallel HTTP connections. In that case, the maximum number of parallel connections SHOULD be configurable.
+- [x] [SHOULD] To achieve higher total throughput, the client MAY send requests using several parallel HTTP connections. In that case, the maximum number of parallel connections SHOULD be configurable.
+  * NOTE: Supported by `flush_thread_count` settings in `<buffer>`.
 
 # Future Versions and Interoperability
 - [ ] [MUST] When possible, the interoperability MUST be ensured between all versions of OTLP that are not declared obsolete.

data/lib/fluent/plugin/in_opentelemetry.rb

@@ -27,6 +27,11 @@ module Fluent::Plugin
       config_param :bind, :string, default: "0.0.0.0"
       desc "The port to listen to."
       config_param :port, :integer, default: 4318
+
+      desc "The size limit of the POSTed element. This value should be larger than the 'chunk_limit_size' in out_opentelemetry plugin."
+      config_param :body_size_limit, :size, default: 32 * 1024 * 1024
+      desc "The size limit of the decompressed element."
+      config_param :decompression_size_limit, :size, default: 256 * 1024 * 1024
     end
 
     config_section :grpc, required: false, multi: false, init: false, param_name: :grpc_config do
@@ -58,7 +63,7 @@ module Fluent::Plugin
       super
 
       if @http_config
-        http_handler = Opentelemetry::HttpInputHandler.new
+        http_handler = Opentelemetry::HttpInputHandler.new(@http_config, log)
         http_server_create_http_server(:in_opentelemetry_http_server, addr: @http_config.bind, port: @http_config.port, logger: log) do |serv|
           serv.post("/v1/logs") do |req|
             http_handler.logs(req) { |record| router.emit(tag_for(Opentelemetry::RECORD_TYPE_LOGS), Fluent::EventTime.now, { "type" => Opentelemetry::RECORD_TYPE_LOGS, "message" => record }) }

data/lib/fluent/plugin/opentelemetry/constant.rb

@@ -5,7 +5,7 @@ require "openssl"
 
 module Fluent::Plugin::Opentelemetry
   CONTENT_TYPE = "Content-Type"
-  CONTENT_TYPE_PAIN = "text/plain"
+  CONTENT_TYPE_PLAIN = "text/plain"
   CONTENT_TYPE_PROTOBUF = "application/x-protobuf"
   CONTENT_TYPE_JSON = "application/json"
 

data/lib/fluent/plugin/opentelemetry/http_input_handler.rb

@@ -25,13 +25,26 @@ module Fluent::PluginHelper::HttpServer
           @request.body&.close
         end
       end
+
+      unless method_defined?(:body_stream)
+        def body_stream
+          @request.body
+        end
+      end
     end
   end
 end
 
 class Fluent::Plugin::Opentelemetry::HttpInputHandler
+  class SizeLimitError < StandardError; end
+
   using Fluent::PluginHelper::HttpServer::Extension
 
+  def initialize(http_config, logger)
+    @http_config = http_config
+    @logger = logger
+  end
+
   def logs(req, &block)
     common(req, Fluent::Plugin::Opentelemetry::Request::Logs, Fluent::Plugin::Opentelemetry::Response::Logs, &block)
   end
@@ -49,16 +62,40 @@ class Fluent::Plugin::Opentelemetry::HttpInputHandler
   def common(req, request_class, response_class)
     content_type = req.headers["content-type"]
     content_encoding = req.headers["content-encoding"]&.first
-    body = req.body
+
+    content_length = req.headers["content-length"]&.first&.to_i
+    if content_length && content_length >= @http_config.body_size_limit
+      @logger.warn { "Received too big content length: #{content_length}" }
+      return response_payload_too_large
+    end
+
+    begin
+      body = read_body(req, limit: @http_config.body_size_limit)
+    rescue SizeLimitError
+      @logger.warn { "Received payload exceeding body_size_limit" }
+      return response_payload_too_large
+    end
+
     return response_unsupported_media_type unless valid_content_type?(content_type)
     return response_bad_request(content_type) unless valid_content_encoding?(content_encoding)
 
-    body = Zlib::GzipReader.new(StringIO.new(body)).read if content_encoding == Fluent::Plugin::Opentelemetry::CONTENT_ENCODING_GZIP
+    if content_encoding == Fluent::Plugin::Opentelemetry::CONTENT_ENCODING_GZIP
+      begin
+        body = decompress(body, limit: @http_config.decompression_size_limit)
+      rescue SizeLimitError
+        @logger.warn { "Decompressed payload exceeding decompression_size_limit" }
+        return response_payload_too_large
+      rescue Zlib::Error => e
+        @logger.warn { "Failed to decompress gzip payload: #{e.message}" }
+        return response_bad_request(content_type)
+      end
+    end
 
     begin
       record = request_class.new(body).record
-    rescue Google::Protobuf::ParseError
+    rescue Google::Protobuf::ParseError => e
       # The format in request body does not comply with the OpenTelemetry protocol.
+      @logger.warn { "Failed to parse OpenTelemetry payload: #{e.message}" }
       return response_bad_request(content_type)
     end
 
@@ -70,6 +107,42 @@ class Fluent::Plugin::Opentelemetry::HttpInputHandler
     req.close
   end
 
+  def read_body(request, limit:)
+    body = +""
+    while (chunk = request.body_stream&.read)
+      body << chunk
+      if body.bytesize > limit
+        raise SizeLimitError, "Too large payload"
+      end
+    end
+    body
+  end
+
+  BYTES_TO_READ = 64 * 1024
+
+  def decompress(compressed_data, limit:)
+    io = StringIO.new(compressed_data)
+    out = +""
+    loop do
+      reader = Zlib::GzipReader.new(io)
+      while (chunk = reader.read(BYTES_TO_READ))
+        out << chunk
+        if out.bytesize > limit
+          raise SizeLimitError, "Decompressed data exceeds limit of #{limit} bytes"
+        end
+      end
+
+      unused = reader.unused
+      reader.finish
+      unless unused.nil?
+        adjust = unused.length
+        io.pos -= adjust
+      end
+      break if io.eof?
+    end
+    out
+  end
+
   def valid_content_type?(content_type)
     case content_type
     when Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PROTOBUF, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_JSON
@@ -90,7 +163,11 @@ class Fluent::Plugin::Opentelemetry::HttpInputHandler
   end
 
   def response_unsupported_media_type
-    response(415, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PAIN, "415 unsupported media type, supported: [application/json, application/x-protobuf]")
+    response(415, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PLAIN, "415 unsupported media type, supported: [application/json, application/x-protobuf]")
+  end
+
+  def response_payload_too_large
+    response(413, Fluent::Plugin::Opentelemetry::CONTENT_TYPE_PLAIN, "413 Payload Too Large")
   end
 
   def response_bad_request(content_type)

data/lib/fluent/plugin/opentelemetry/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module Opentelemetry
-      VERSION = "0.5.1"
+      VERSION = "0.5.3"
     end
   end
 end

data/lib/fluent/plugin/out_opentelemetry.rb

@@ -21,8 +21,10 @@ module Fluent::Plugin
 
     helpers :server
 
+    DEFAULT_CHUNK_LIMIT_SIZE = 8 * 1024 * 1024 # 8MB
     config_section :buffer do
       config_set_default :chunk_keys, ["tag"]
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
     end
 
     config_section :http, required: false, multi: false, init: false, param_name: :http_config do
@@ -88,13 +90,47 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      chunk.each do |_, record| # rubocop:disable Style/HashEachMethods
+      BatchProcessor.build_export_requests(chunk).each do |export_request|
         if @grpc_handler
-          @grpc_handler.export(record)
+          @grpc_handler.export(export_request)
         else
-          @http_handler.export(record)
+          @http_handler.export(export_request)
         end
       end
     end
+
+    class BatchProcessor
+      RESOURCE_KEY_MAP = {
+        Opentelemetry::RECORD_TYPE_LOGS => "resourceLogs",
+        Opentelemetry::RECORD_TYPE_METRICS => "resourceMetrics",
+        Opentelemetry::RECORD_TYPE_TRACES => "resourceSpans"
+      }.freeze
+
+      def self.build_export_requests(chunk)
+        requests = {
+          Opentelemetry::RECORD_TYPE_LOGS => {},
+          Opentelemetry::RECORD_TYPE_METRICS => {},
+          Opentelemetry::RECORD_TYPE_TRACES => {}
+        }
+
+        chunk.each do |_, record| # rubocop:disable Style/HashEachMethods
+          record_type = record["type"]
+          resource_key = RESOURCE_KEY_MAP[record_type]
+          record["message"] = JSON.parse(record["message"])
+          resource_hash = record["message"][resource_key][0]["resource"].hash
+          if requests[record_type][resource_hash].nil?
+            requests[record_type][resource_hash] = record
+          else
+            requests[record_type][resource_hash]["message"][resource_key].concat(record["message"][resource_key])
+          end
+        end
+
+        merged_records = requests.values.flat_map(&:values)
+        merged_records.each do |record|
+          record["message"] = record["message"].to_json
+        end
+        merged_records
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opentelemetry
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.3
 platform: ruby
 authors:
 - Shizuo Fujita
@@ -138,6 +138,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/fluent-plugins-nursery/fluent-plugin-opentelemetry
   source_code_uri: https://github.com/fluent-plugins-nursery/fluent-plugin-opentelemetry
+  changelog_uri: https://github.com/fluent-plugins-nursery/fluent-plugin-opentelemetry/blob/main/CHANGELOG.md
   rubygems_mfa_required: 'true'
 rdoc_options: []
 require_paths:
@@ -153,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Fluentd input/output plugin to forward OpenTelemetry Protocol data.
 test_files: []