fluent-plugin-record-demux 0.2.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9da92e93a04e15799949041055f91a9d9ff9327627028ce85322343778311bf
-  data.tar.gz: 7eb377c975bdd3bcdbb192dd43a89af9995292dadc5bcfad3174f8f1f580ca09
+  metadata.gz: 3c3c56b3996c80096423cb21c058f778c52ef98f6235cd4ea0ba1ba47236e918
+  data.tar.gz: 04006b6d00d241abeea57d56041816405dc17a88284e4e8409eee57668842caa
 SHA512:
-  metadata.gz: a04985f89183aa35c8dbd0b4e72fbc430a5160a71b63eb539a25e720b11d51376f82cb87a07b6317fe89a001876a53161c78adda0151b6f51ca4b9536f826b7d
-  data.tar.gz: 586b8c3ffc1b22d553a056fd0e3f96bf9ccada5e7d4c27a5e2b1e6ff714cc376d8c347f96278410b02dbc86c160d4abf94b05a4aa4abccc2f3da6ddb389d6181
+  metadata.gz: 28ea048eb175624647b564df623109a0d380de38ff4a4c8125a6b6212ff2c7ad38b80c8922d7af133a009b2acdb807cc9184423723168ccf797fd0fededa27d8
+  data.tar.gz: f9ebf69ccb5fa1c9d8793db3c35fcc7f656039ca3debcf7dcce0d40ae2ba98b5c8df43f080f00cf2789ae55fdbee6704ea2c5a3bab2d1ad50ee217ee57262cde

data/.rubocop.yml

@@ -34,5 +34,8 @@ Metrics/MethodLength:
 Metrics/ParameterLists:
   Exclude: []
 
+Naming/MethodParameterName:
+  MinNameLength: 2
+
 Style/Documentation:
   Enabled: false

data/README.md

@@ -1,8 +1,17 @@
 # fluent-plugin-record-demux
 
-[Fluentd](https://fluentd.org/) plugin to dmux records.
+[Fluentd](https://fluentd.org/) plugin to demux records.
 
-## plugin - out - record_dmux
+
+## Table of Contents
+* [plugin - out - record_demux](#plugin---out---record_demux)
+* [plugin - out - record_demux_picker](#plugin---out---record_demux_picker)
+* [plugin - filter - record_array demux](#plugin---filter---record_array_demux)
+* [Installation](#installation)
+* [Contribution](#contribution)
+
+
+## plugin - out - record_demux
 
 ### config
 
@@ -76,6 +85,110 @@ into events:
 { "c": "data_c", "tags_1": "data_1", "tags_2": "data_2" }
 ```
 
+
+## plugin - out - record_demux_picker
+
+Demux record/event, by selecting nested fields for shared and demux parts.
+
+It transforms 1 record/event to multiple records/events.
+
+### params
+
+| setting                        | type                    | default | description                                    |
+|--------------------------------|-------------------------|---------|------------------------------------------------|
+| tags                           | string                  |         | tag to emit demux events on                    |
+| demux_keys                     | hash                    |         | nested key to demux                            |
+| shared_keys                    | hash                    | {}      | shared key to be present in each new event     |
+| demux_key_normalize            | bool                    | false   | format key to demux, in a key: , value: format |
+| demux_key_normalize_key_name   | string                  | key     | when demux_key_normalize, key name to use      |
+| demux_key_normalize_value_name | string                  | value   | when demux_key_normalize, value name to use    |
+| timestamp_key                  | string                  | nil     | key for timestamp field (nil means skip)       |
+| timestamp_format               | enum (iso, epochmillis) | iso     | format of the time value                       |
+
+### example 1
+
+Given this event :
+
+``` text
+{
+    "label1": "value_label1",
+    "label2": "value_label2",
+    "label3": "value_label3",
+    "common": {
+        "metadata1": "value_metadata1",
+        "metadata2": "value_metadata2",
+        "metadata3": "value_metadata3",
+        "metadata4": "value_metadata4",
+        "metadata5": "value_metadata5",
+        "metadata6": "value_metadata6"
+    }
+}
+
+```
+
+With this conf :
+
+``` text
+<match data>
+  @type record_demux_picker
+  tag demuxed
+  demux_keys $.label1:data1 , $.label2:data2 , $.label3:data3
+  shared_keys $.common.metadata1:description1 , $.common.metadata2:description2 , $.common.metadata3:description3
+</match>
+```
+
+It will produce :
+
+``` text
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data1":"value_label1"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data2":"value_label2"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data3":"value_label3"}
+
+```
+
+
+## plugin - filter - record_array_demux
+
+Demux record/event, by specified fields when they are arrays.
+
+It transforms 1 record/event to multiple records/events, one for each element of the arrays.
+
+| setting     | type            | default | description                             |
+|-------------|-----------------|---------|-----------------------------------------|
+| demux_keys  | array of string | *nil*   | arrays keys to demux, computed when nil |
+| shared_keys | array of string | []      | keys to be shared in new record         |
+| remove_keys | array of string | []      | keys to remove                          |
+
+### example 1
+
+Given this event :
+
+``` text
+{
+    "source": ["source1", "source2", "source3"],
+    "status": "active",
+    "timestamp": "2026-01-02T03:04:05.678Z"
+}
+```
+
+With this conf :
+
+``` text
+<filter data>
+  @type record_array_demux
+  demux_keys source
+</filter>
+```
+
+It will produce :
+
+``` text
+{"source":"source1","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+{"source":"source2","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+{"source":"source3","status":"active","timestamp": "2026-01-02T03:04:05.678Z"}
+```
+
+
 ## Installation
 
 Manual install, by executing:
@@ -86,9 +199,13 @@ Add to Gemfile with:
 
     $ bundle add fluent-plugin-record-demux
 
+## Contribution
+
+PR WELCOME !
+
 
 ## Copyright
 
-* Copyright(c) 2024-2025 Thomas Tych
+* Copyright(c) 2024-2026 Thomas Tych
 * License
   * Apache License, Version 2.0

data/Rakefile

@@ -6,9 +6,15 @@ Bundler::GemHelper.install_tasks
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |t|
   t.libs.push('lib', 'test')
-  t.test_files = FileList['test/**/test_*.rb', 'test/**/*_test.rb',]
-  t.verbose = true
-  t.warning = true
+
+  t.test_files = if ENV['TEST_FILE']
+                   FileList["#{ENV['TEST_FILE']}*",
+                            "test/**/#{ENV['TEST_FILE']}*"]
+                 else
+                   FileList['test/**/test_*.rb', 'test/**/*_test.rb']
+                 end
+  t.verbose = ENV.fetch('VERBOSE', false)
+  t.warning = ENV.fetch('WARNING', false)
 end
 
 require 'rubocop/rake_task'

data/fluent-plugin-record-demux.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-record-demux'
-  spec.version = '0.2.0'
+  spec.version = '0.4.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.10'
   spec.add_development_dependency 'bundler', '~> 2.6', '>= 2.6.5'
   spec.add_development_dependency 'byebug', '~> 11.1', '>= 11.1.3'
+  spec.add_development_dependency 'mocha', '~> 2.7', '>= 2.7.1'
   spec.add_development_dependency 'rake', '~> 13.2', '>= 13.2.1'
   spec.add_development_dependency 'reek', '~> 6.4'
   spec.add_development_dependency 'rubocop', '~> 1.74'

data/lib/fluent/plugin/filter_record_array_demux.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+#
+# Copyright 2026- Thomas Tych
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/filter'
+
+module Fluent
+  module Plugin
+    class RecordArrayDemuxFilter < Fluent::Plugin::Filter
+      NAME = 'record_array_demux'
+      Fluent::Plugin.register_filter(NAME, self)
+
+      helpers :event_emitter, :timer
+
+      desc 'list of arrays keys to demux'
+      config_param :demux_keys, :array, value_type: :string, default: nil
+      desc 'list of keys to be shared in all new records'
+      config_param :shared_keys, :array, value_type: :string, default: nil
+      desc 'list of keys to be removed'
+      config_param :remove_keys, :array, value_type: :string, default: []
+
+      def configure(conf)
+        super
+
+        if demux_keys.nil? && shared_keys.nil?
+          raise Fluent::ConfigError, 'specify demux_keys or shared_keys'
+        end
+
+        true
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def filter_stream(_tag, es)
+        new_es = Fluent::MultiEventStream.new
+        es.each do |time, record|
+          new_records = process_record(time, record)
+          new_records.each { |new_record| new_es.add(time, new_record) }
+        end
+        new_es
+      end
+
+      def dispatch_record_keys(record)
+        record_keys = record.keys - remove_keys
+
+        record_shared_keys = if shared_keys.nil?
+                               record_keys - demux_keys
+                             else
+                               record_keys.intersection(shared_keys)
+                             end
+
+        record_demux_keys = if demux_keys.nil?
+                              record_keys - record_shared_keys
+                            else
+                              record_keys.intersection(demux_keys)
+                            end
+        real_record_demux_keys, non_record_demux_keys = record_demux_keys.partition do |key|
+          record[key].is_a?(Array)
+        end
+        record_shared_keys += non_record_demux_keys
+
+        [real_record_demux_keys, record_shared_keys]
+      end
+
+      def process_record(_time, record)
+        record_demux_keys, record_shared_keys = dispatch_record_keys(record)
+
+        return [record] if record_demux_keys.empty?
+
+        record_shared = record.slice(*record_shared_keys)
+
+        demux_arrays = record_demux_keys.map do |key|
+          array_demux_value = record[key]
+          array_demux_value.size.positive? ? array_demux_value : [nil]
+        end
+
+        demux_combinations = demux_arrays.first.product(*demux_arrays[1..])
+        demux_combinations.map do |combination|
+          new_record = record_shared.dup
+          record_demux_keys.each_with_index do |key, index|
+            new_record[key] = combination[index]
+          end
+          new_record
+        end
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_record_demux.rb

@@ -26,7 +26,7 @@ module Fluent
       helpers :event_emitter, :timer
 
       desc 'tag to emit events on'
-      config_param :tag, :string, default: nil
+      config_param :tag, :string
 
       desc 'list of keys to demux'
       config_param :demux_keys, :array, value_type: :string, default: nil

data/lib/fluent/plugin/out_record_demux_picker.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+#
+# Copyright 2024- Thomas Tych
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/output'
+
+module Fluent
+  module Plugin
+    class RecordDemuxPickerOutput < Fluent::Plugin::Output
+      NAME = 'record_demux_picker'
+      Fluent::Plugin.register_output(NAME, self)
+
+      DEMUX_KEY_NORMALIZE_KEY_NAME = 'key'
+      DEMUX_KEY_NORMALIZE_VALUE_NAME = 'value'
+
+      helpers :event_emitter, :timer, :record_accessor
+
+      desc 'tag to emit events on'
+      config_param :tag, :string
+
+      desc 'list of keys to demux'
+      config_param :demux_keys, :hash
+      desc 'list of keys to be shared in all new events'
+      config_param :shared_keys, :hash, default: {}
+
+      desc 'normalize demux key format'
+      config_param :demux_key_normalize, :bool, default: false
+      desc 'demux key normalize key name'
+      config_param :demux_key_normalize_key_name, :string, default: DEMUX_KEY_NORMALIZE_KEY_NAME
+      desc 'demux key normalize value name'
+      config_param :demux_key_normalize_value_name, :string, default: DEMUX_KEY_NORMALIZE_VALUE_NAME
+
+      desc 'timestamp key'
+      config_param :timestamp_key, :string, default: nil
+      desc 'timestamp format'
+      config_param :timestamp_format, :enum, list: %i[iso epochmillis], default: :iso
+
+      def configure(conf)
+        super
+
+        @demux_keys_mappers = @demux_keys.map { |key, target| KeyMapper.new(key, target) }
+        @shared_keys_mappers = @shared_keys.map { |key, target| KeyMapper.new(key, target) }
+
+        true
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def process(_events_tag, events)
+        demux_events = MultiEventStream.new
+        events.each do |time, event|
+          new_events = process_event(time, event)
+          new_events.each { |new_event| demux_events.add(time, new_event) }
+        end
+        router.emit_stream(tag, demux_events)
+      end
+
+      def process_event(time, event)
+        shared_event = extract_shared_event(event)
+
+        @demux_keys_mappers.map do |mapper|
+          value = mapper.accessor.call(event)
+          shared_event
+            .merge(format_demux_key(mapper.target, value))
+            .merge(format_time(time))
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          nil
+        end.compact
+      end
+
+      private
+
+      def extract_shared_event(event)
+        @shared_keys_mappers.each_with_object({}) do |mapper, new_event|
+          value = mapper.accessor.call(event)
+          new_event[mapper.target] = value
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          next
+        end
+      end
+
+      def format_demux_key(key, value)
+        if demux_key_normalize
+          { demux_key_normalize_key_name => key,
+            demux_key_normalize_value_name => value }
+        else
+          { key => value }
+        end
+      end
+
+      def format_time(time)
+        return {} unless timestamp_key
+
+        { timestamp_key => format_timestamp(time) }
+      end
+
+      def format_timestamp(time)
+        return (time.to_time.utc.to_f * 1000).to_i if @timestamp_format == :epochmillis
+
+        time.to_time.utc.iso8601(3)
+      end
+
+      class KeyMapper
+        include Fluent::PluginHelper::RecordAccessor
+
+        attr_reader :key, :accessor, :target, :setter
+
+        def initialize(key, target)
+          @key = key
+          @accessor = record_accessor_create(key)
+          @target = target
+          @target ||= @accessor.keys.is_a?(Array) ? @accessor.keys.last : @accessor.keys
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-demux
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Thomas Tych
 bindir: bin
 cert_chain: []
-date: 2025-03-14 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -63,6 +63,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 11.1.3
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -218,7 +238,9 @@ files:
 - README.md
 - Rakefile
 - fluent-plugin-record-demux.gemspec
+- lib/fluent/plugin/filter_record_array_demux.rb
 - lib/fluent/plugin/out_record_demux.rb
+- lib/fluent/plugin/out_record_demux_picker.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-record-demux
 licenses:
 - Apache-2.0
@@ -238,7 +260,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.5
+rubygems_version: 4.0.3
 specification_version: 4
 summary: fluentd plugin to demux record by keys.
 test_files: []