fluent-plugin-opentelemetry 0.5.1 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe7bce47ce8bec3c95c596ccdd5c444f3d4d069e6a7e4b865145d8fde923acb7
-  data.tar.gz: e6afb3a4ceb6b5b9bd469cac2474e87dffe65a2fdb6b16c5b918382ed5b854ab
+  metadata.gz: 7caa8c736a332ab38da184793751d6760bc1d8bf59c996bffcbe324dd5da7ab5
+  data.tar.gz: d4c07bffd3ccdd29e8b151a134e5bbe77e1686d8e0ad9a84e5ecd432224d1dd1
 SHA512:
-  metadata.gz: be3ade279fc65f4850918f9c477d3b0d912c641fe657ffa9c554d7afe73da902688328199e33c352f2f804687568e2263149acefb904ff53ef863779083a067b
-  data.tar.gz: f8ffa9756821bb1ef81155ed2e7283f6af734363336cd8229a6278d532ffebd4ad9cc88b38420aeedc591449bab8a2efbf5e9f4ed99714c4aef6a3fe7be35e38
+  metadata.gz: 48d509ee79d59085508bd591c051d7ba8cc33efc3e0cb1d64e8ad04f28ed60444fdd4f322e65d4e41ffc84c3d0e7c7770bf7f3cfe1d1076df692ba5dfb8dddcc
+  data.tar.gz: 9c21c80065796cdadc5fb8aad7c4c1b810cff27a1581d5e83a21d4de920ff34f9318d52f39337c599da8148bbbb6f4f1ac6e365459373079d906cd6812397c77

data/.rubocop.yml

@@ -78,6 +78,9 @@ Style/MixinUsage:
 Style/NumericLiterals:
   Enabled: false
 
+Style/OneClassPerFile:
+  Enabled: false
+
 Style/PercentLiteralDelimiters:
   Enabled: false
 

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 ## [Unreleased]
 
+## [0.5.2] - 2026-03-15
+
+Improvements:
+
+- out_opentelemetry: add BatchProcessor for bulk transfer (#33)
+
 ## [0.5.1] - 2026-02-09
 
 Fixes:

data/README.md

@@ -202,9 +202,15 @@ Refer [Config: Transport Section](https://docs.fluentd.org/configuration/transpo
 
 #### `<buffer>` section
 
-| parameter  | type   | description                                               | default |
-|------------|--------|-----------------------------------------------------------|---------|
-| chunk_keys | array  | Overwrites the default `chunk_keys` value in this plugin. | `tag`   |
+| parameter        | type    | description                                                     | default    |
+|------------------|---------|-----------------------------------------------------------------|------------|
+| chunk_keys       | array   | Overwrites the default `chunk_keys` value in this plugin.       | `tag`      |
+| chunk_limit_size | integer | Overwrites the default `chunk_limit_size` value in this plugin. | `8M` (8MB) |
+
+**Note on chunk_limit_size:**
+Currently, this plugin is reducing I/O load by combining data within a chunk and sending it as a single unit.
+Depending on the configuration of your OpenTelemetry Collector or downstream backend (e.g., maximum request body size limits), an 8MB chunk might be rejected with a `413 Payload Too Large` error.
+If you encounter this error in your Fluentd logs, please adjust (decrease) the `chunk_limit_size` in your `<buffer>` section.
 
 Refer [Config: Buffer Section](https://docs.fluentd.org/configuration/buffer-section)
 

data/lib/fluent/plugin/opentelemetry/version.rb

@@ -3,7 +3,7 @@
 module Fluent
   module Plugin
     module Opentelemetry
-      VERSION = "0.5.1"
+      VERSION = "0.5.2"
     end
   end
 end

data/lib/fluent/plugin/out_opentelemetry.rb

@@ -21,8 +21,10 @@ module Fluent::Plugin
 
     helpers :server
 
+    DEFAULT_CHUNK_LIMIT_SIZE = 8 * 1024 * 1024 # 8MB
     config_section :buffer do
       config_set_default :chunk_keys, ["tag"]
+      config_set_default :chunk_limit_size, DEFAULT_CHUNK_LIMIT_SIZE
     end
 
     config_section :http, required: false, multi: false, init: false, param_name: :http_config do
@@ -88,13 +90,47 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      chunk.each do |_, record| # rubocop:disable Style/HashEachMethods
+      BatchProcessor.build_export_requests(chunk).each do |export_request|
         if @grpc_handler
-          @grpc_handler.export(record)
+          @grpc_handler.export(export_request)
         else
-          @http_handler.export(record)
+          @http_handler.export(export_request)
         end
       end
     end
+
+    class BatchProcessor
+      RESOURCE_KEY_MAP = {
+        Opentelemetry::RECORD_TYPE_LOGS => "resourceLogs",
+        Opentelemetry::RECORD_TYPE_METRICS => "resourceMetrics",
+        Opentelemetry::RECORD_TYPE_TRACES => "resourceSpans"
+      }.freeze
+
+      def self.build_export_requests(chunk)
+        requests = {
+          Opentelemetry::RECORD_TYPE_LOGS => {},
+          Opentelemetry::RECORD_TYPE_METRICS => {},
+          Opentelemetry::RECORD_TYPE_TRACES => {}
+        }
+
+        chunk.each do |_, record| # rubocop:disable Style/HashEachMethods
+          record_type = record["type"]
+          resource_key = RESOURCE_KEY_MAP[record_type]
+          record["message"] = JSON.parse(record["message"])
+          resource_hash = record["message"][resource_key][0]["resource"].hash
+          if requests[record_type][resource_hash].nil?
+            requests[record_type][resource_hash] = record
+          else
+            requests[record_type][resource_hash]["message"][resource_key].concat(record["message"][resource_key])
+          end
+        end
+
+        merged_records = requests.values.flat_map(&:values)
+        merged_records.each do |record|
+          record["message"] = record["message"].to_json
+        end
+        merged_records
+      end
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-opentelemetry
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.5.2
 platform: ruby
 authors:
 - Shizuo Fujita
@@ -153,7 +153,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Fluentd input/output plugin to forward OpenTelemetry Protocol data.
 test_files: []