fluent-plugin-record-demux 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 04b8dcca3813f2024f2b47c2cde9dbbc131c3ac9d312cef5810189ce9d672228
-  data.tar.gz: 74485fe4f4d05eae9e13f55bf92a6eb9ea9648d309bbe697c69ee2aae0e59f3e
+  metadata.gz: 3c3c56b3996c80096423cb21c058f778c52ef98f6235cd4ea0ba1ba47236e918
+  data.tar.gz: 04006b6d00d241abeea57d56041816405dc17a88284e4e8409eee57668842caa
 SHA512:
-  metadata.gz: c499af008b748a39f9e85eae54ce319ef34c18d69876c4e3188c67d87a6eb513044e56fcb47a4aa3f4bc04f6bf63b83938c12f95f80d86c986ceb28d46e94b7c
-  data.tar.gz: 63639df945ee52df7e5443cd426aff84641889f6d258f5f49298b8cfd2841f6382642f2d66489ec08f9e49f9f9a8a4452fa098b449e25d46860bf3b7a56985e1
+  metadata.gz: 28ea048eb175624647b564df623109a0d380de38ff4a4c8125a6b6212ff2c7ad38b80c8922d7af133a009b2acdb807cc9184423723168ccf797fd0fededa27d8
+  data.tar.gz: f9ebf69ccb5fa1c9d8793db3c35fcc7f656039ca3debcf7dcce0d40ae2ba98b5c8df43f080f00cf2789ae55fdbee6704ea2c5a3bab2d1ad50ee217ee57262cde

data/.rubocop.yml

@@ -34,5 +34,8 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Exclude: []
 
+Naming/MethodParameterName:
+  MinNameLength: 2
+
 Style/Documentation:
   Enabled: false

data/README.md

@@ -1,8 +1,17 @@
 # fluent-plugin-record-demux
 
-[Fluentd](https://fluentd.org/) plugin to dmux records.
+[Fluentd](https://fluentd.org/) plugin to demux records.
 
-## plugin - out - record_dmux
+
+## Table of Contents
+* [plugin - out - record_demux](#plugin---out---record_demux)
+* [plugin - out - record_demux_picker](#plugin---out---record_demux_picker)
+* [plugin - filter - record_array demux](#plugin---filter---record_array_demux)
+* [Installation](#installation)
+* [Contribution](#contribution)
+
+
+## plugin - out - record_demux
 
 ### config
 
@@ -76,11 +85,12 @@ into events:
 { "c": "data_c", "tags_1": "data_1", "tags_2": "data_2" }
 ```
 
-## plugin - out - record_dmux_picker
+
+## plugin - out - record_demux_picker
 
 Demux record/event, by selecting nested fields for shared and demux parts.
 
-It transforms 1 record/event to multiple record/event.
+It transforms 1 record/event to multiple records/events.
 
 ### params
 
@@ -97,7 +107,7 @@ It transforms 1 record/event to multiple record/event.
 
 ### example 1
 
-Give this event :
+Given this event :
 
 ``` text
 {
@@ -136,6 +146,49 @@ It will produce :
 
 ```
 
+
+## plugin - filter - record_array_demux
+
+Demux record/event, by specified fields when they are arrays.
+
+It transforms 1 record/event to multiple records/events, one for each element of the arrays.
+
+| setting     | type            | default | description                             |
+|-------------|-----------------|---------|-----------------------------------------|
+| demux_keys  | array of string | *nil*   | arrays keys to demux, computed when nil |
+| shared_keys | array of string | []      | keys to be shared in new record         |
+| remove_keys | array of string | []      | keys to remove                          |
+
+### example 1
+
+Given this event :
+
+``` text
+{
+    "source": ["source1", "source2", "source3"],
+    "status": "active",
+    "timestamp": "2026-01-02T03:04:05.678Z"
+}
+```
+
+With this conf :
+
+``` text
+<filter data>
+  @type record_array_demux
+  demux_keys source
+</filter>
+```
+
+It will produce :
+
+``` text
+{"source":"source1","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+{"source":"source2","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+{"source":"source3","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+```
+
+
 ## Installation
 
 Manual install, by executing:
@@ -146,9 +199,13 @@ Add to Gemfile with:
 
     $ bundle add fluent-plugin-record-demux
 
+## Contribution
+
+PR WELCOME !
+
 
 ## Copyright
 
-* Copyright(c) 2024-2025 Thomas Tych
+* Copyright(c) 2024-2026 Thomas Tych
 * License
   * Apache License, Version 2.0

data/fluent-plugin-record-demux.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-record-demux'
-  spec.version = '0.3.0'
+  spec.version = '0.4.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/filter_record_array_demux.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+#
+# Copyright 2026- Thomas Tych
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class RecordArrayDemuxFilter < Fluent::Plugin::Filter
+      NAME = 'record_array_demux'
+      Fluent::Plugin.register_filter(NAME, self)
+
+      helpers :event_emitter, :timer
+
+      desc 'list of arrays keys to demux'
+      config_param :demux_keys, :array, value_type: :string, default: nil
+      desc 'list of keys to be shared in all new records'
+      config_param :shared_keys, :array, value_type: :string, default: nil
+      desc 'list of keys to be removed'
+      config_param :remove_keys, :array, value_type: :string, default: []
+
+      def configure(conf)
+        super
+
+        if demux_keys.nil? && shared_keys.nil?
+          raise Fluent::ConfigError, 'specify demux_keys or shared_keys'
+        end
+
+        true
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def filter_stream(_tag, es)
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          new_records = process_record(time, record)
+          new_records.each { |new_record| new_es.add(time, new_record) }
+        end
+        new_es
+      end
+
+      def dispatch_record_keys(record)
+        record_keys = record.keys - remove_keys
+
+        record_shared_keys = if shared_keys.nil?
+                               record_keys - demux_keys
+                             else
+                               record_keys.intersection(shared_keys)
+                             end
+
+        record_demux_keys = if demux_keys.nil?
+                              record_keys - record_shared_keys
+                            else
+                              record_keys.intersection(demux_keys)
+                            end
+        real_record_demux_keys, non_record_demux_keys = record_demux_keys.partition do |key|
+          record[key].is_a?(Array)
+        end
+        record_shared_keys += non_record_demux_keys
+
+        [real_record_demux_keys, record_shared_keys]
+      end
+
+      def process_record(_time, record)
+        record_demux_keys, record_shared_keys = dispatch_record_keys(record)
+
+        return [record] if record_demux_keys.empty?
+
+        record_shared = record.slice(*record_shared_keys)
+
+        demux_arrays = record_demux_keys.map do |key|
+          array_demux_value = record[key]
+          array_demux_value.size.positive? ? array_demux_value : [nil]
+        end
+
+        demux_combinations = demux_arrays.first.product(*demux_arrays[1..])
+        demux_combinations.map do |combination|
+          new_record = record_shared.dup
+          record_demux_keys.each_with_index do |key, index|
+            new_record[key] = combination[index]
+          end
+          new_record
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-demux
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Thomas Tych
 bindir: bin
 cert_chain: []
-date: 2025-04-07 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -238,6 +238,7 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-record-demux.gemspec
+- lib/fluent/plugin/filter_record_array_demux.rb
 - lib/fluent/plugin/out_record_demux.rb
 - lib/fluent/plugin/out_record_demux_picker.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-record-demux
@@ -259,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.6
+rubygems_version: 4.0.3
 specification_version: 4
 summary: fluentd plugin to demux record by keys.
 test_files: []