fluent-plugin-record-demux 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3af2e3acf245f6e85007a6832dec1cdeb7fb7e54fc91afc2312e5c9da4be9be
-  data.tar.gz: 8296a752f7a84b20f900a214abc55bea52af25b4e9a06d1d079b445921331e39
+  metadata.gz: 04b8dcca3813f2024f2b47c2cde9dbbc131c3ac9d312cef5810189ce9d672228
+  data.tar.gz: 74485fe4f4d05eae9e13f55bf92a6eb9ea9648d309bbe697c69ee2aae0e59f3e
 SHA512:
-  metadata.gz: 2d99ca520a2a8e281769c3dcc508937e9ec34e4c925e54c848590aa4f59ed4ed0a773730847b697ef3e3d511d2437077531f31bd3b1841e4ed0b3539f6f55943
-  data.tar.gz: 6170f58d6f984e9ea7ac87f9bc4b71e7e69e6274d1b848998dd3c53ebb2de6bf6e50fa6246553ce513f028698561f7c93816f8ea2b6f224a2ca02acb10beac0e
+  metadata.gz: c499af008b748a39f9e85eae54ce319ef34c18d69876c4e3188c67d87a6eb513044e56fcb47a4aa3f4bc04f6bf63b83938c12f95f80d86c986ceb28d46e94b7c
+  data.tar.gz: 63639df945ee52df7e5443cd426aff84641889f6d258f5f49298b8cfd2841f6382642f2d66489ec08f9e49f9f9a8a4452fa098b449e25d46860bf3b7a56985e1

data/.gitignore

@@ -1,6 +1,5 @@
-# tmp files
+/pkg
 /tmp
-.byebug_history
+/.byebug_history
 /coverage
-
 /Gemfile.lock

data/.rubocop.yml

@@ -14,7 +14,7 @@ Layout/LineLength:
   Max: 100
 
 Metrics/AbcSize:
-  Max: 25
+  Max: 30
 
 Metrics/BlockLength:
   Exclude:

data/README.md

@@ -18,9 +18,9 @@
 | timestamp_key        | string          | *nil*   | add timestamp key when defined, computed from time of the event            |
 | timestamp_format     | enum            | iso     | can be iso for iso8601(3) or epochmillis for epoch with millisecond format |
 
-### example
+### example 1
 
-Example of configuration
+Example of configuration:
 
 ``` text
 <match *>
@@ -34,6 +34,107 @@ Example of configuration
 </match>
 ```
 
+### example 2
+
+With configuration:
+
+``` text
+<match *>
+  @type record_demux
+
+  tag data.demux
+
+  shared_keys tags_1, tags_2
+  remove_keys tmp
+</match>
+```
+
+or with configuration:
+
+``` text
+<match *>
+  @type record_demux
+
+  tag data.demux
+
+  demux_keys a, b, c
+  remove_keys tmp
+</match>
+```
+
+It will transform event like:
+
+``` text
+{ "a": "data_a", "b": "data_b", "c": "data_c", "tags_1": "data_1", "tags_2": "data_2", "tmp": "data_tmp" }
+```
+
+into events:
+
+``` text
+{ "a": "data_a", "tags_1": "data_1", "tags_2": "data_2" }
+{ "b": "data_b", "tags_1": "data_1", "tags_2": "data_2" }
+{ "c": "data_c", "tags_1": "data_1", "tags_2": "data_2" }
+```
+
+## plugin - out - record_dmux_picker
+
+Demux record/event, by selecting nested fields for shared and demux parts.
+
+It transforms 1 record/event to multiple record/event.
+
+### params
+
+| setting                        | type                    | default | description                                    |
+|--------------------------------|-------------------------|---------|------------------------------------------------|
+| tags                           | string                  |         | tag to emit demux events on                    |
+| demux_keys                     | hash                    |         | nested key to demux                            |
+| shared_keys                    | hash                    | {}      | shared key to be present in each new event     |
+| demux_key_normalize            | bool                    | false   | format key to demux, in a key: , value: format |
+| demux_key_normalize_key_name   | string                  | key     | when demux_key_normalize, key name to use      |
+| demux_key_normalize_value_name | string                  | value   | when demux_key_normalize, value name to use    |
+| timestamp_key                  | string                  | nil     | key for timestamp field (nil means skip)       |
+| timestamp_format               | enum (iso, epochmillis) | iso     | format of the time value                       |
+
+### example 1
+
+Give this event :
+
+``` text
+{
+    "label1": "value_label1",
+    "label2": "value_label2",
+    "label3": "value_label3",
+    "common": {
+        "metadata1": "value_metadata1",
+        "metadata2": "value_metadata2",
+        "metadata3": "value_metadata3",
+        "metadata4": "value_metadata4",
+        "metadata5": "value_metadata5",
+        "metadata6": "value_metadata6"
+    }
+}
+
+```
+
+With this conf :
+
+``` text
+<match data>
+  @type record_demux_picker
+  tag demuxed
+  demux_keys $.label1:data1 , $.label2:data2 , $.label3:data3
+  shared_keys $.common.metadata1:description1 , $.common.metadata2:description2 , $.common.metadata3:description3
+</match>
+```
+
+It will produce :
+
+``` text
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data1":"value_label1"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data2":"value_label2"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data3":"value_label3"}
+
+```
 
 ## Installation
 

data/Rakefile

@@ -6,9 +6,15 @@ Bundler::GemHelper.install_tasks
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |t|
   t.libs.push('lib', 'test')
-  t.test_files = FileList['test/**/test_*.rb', 'test/**/*_test.rb',]
-  t.verbose = true
-  t.warning = true
+
+  t.test_files = if ENV['TEST_FILE']
+                   FileList["#{ENV['TEST_FILE']}*",
+                            "test/**/#{ENV['TEST_FILE']}*"]
+                 else
+                   FileList['test/**/test_*.rb', 'test/**/*_test.rb']
+                 end
+  t.verbose = ENV.fetch('VERBOSE', false)
+  t.warning = ENV.fetch('WARNING', false)
 end
 
 require 'rubocop/rake_task'

data/fluent-plugin-record-demux.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-record-demux'
-  spec.version = '0.1.1'
+  spec.version = '0.3.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.10'
   spec.add_development_dependency 'bundler', '~> 2.6', '>= 2.6.5'
   spec.add_development_dependency 'byebug', '~> 11.1', '>= 11.1.3'
+  spec.add_development_dependency 'mocha', '~> 2.7', '>= 2.7.1'
   spec.add_development_dependency 'rake', '~> 13.2', '>= 13.2.1'
   spec.add_development_dependency 'reek', '~> 6.4'
   spec.add_development_dependency 'rubocop', '~> 1.74'

data/lib/fluent/plugin/out_record_demux.rb

@@ -26,12 +26,12 @@ module Fluent
       helpers :event_emitter, :timer
 
       desc 'tag to emit events on'
-      config_param :tag, :string, default: nil
+      config_param :tag, :string
 
       desc 'list of keys to demux'
       config_param :demux_keys, :array, value_type: :string, default: nil
       desc 'list of keys to be shared in all new records'
-      config_param :shared_keys, :array, value_type: :string, default: []
+      config_param :shared_keys, :array, value_type: :string, default: nil
       desc 'list of keys to be removed'
       config_param :remove_keys, :array, value_type: :string, default: []
 
@@ -51,9 +51,13 @@ module Fluent
       def configure(conf)
         super
 
-        return unless @tag.nil?
+        raise Fluent::ConfigError, "#{NAME}: `tag` must be specified" if tag.nil? || tag.empty?
 
-        raise Fluent::ConfigError, "#{NAME}: `tag` must be specified"
+        if demux_keys.nil? && shared_keys.nil?
+          raise Fluent::ConfigError, 'specify demux_keys or shared_keys'
+        end
+
+        true
       end
 
       def multi_workers_ready?
@@ -61,17 +65,27 @@ module Fluent
       end
 
       def process(_events_tag, events)
-        demux_events = Fluent::EventStream.new
+        demux_events = MultiEventStream.new
         events.each do |time, record|
           record_keys = record.keys - remove_keys
-          record_shared_keys = record_keys.intersection(shared_keys)
-          record.slice(*record_shared_keys)
-          record_demux_keys = record_keys - record_shared_keys if !demux_keys || demux_keys.empty?
+
+          record_shared_keys = if shared_keys.nil?
+                                 record_keys - demux_keys
+                               else
+                                 record_keys.intersection(shared_keys)
+                               end
+          record_shared = record.slice(*record_shared_keys)
+
+          record_demux_keys = if demux_keys.nil?
+                                record_keys - record_shared_keys
+                              else
+                                record_keys.intersection(demux_keys)
+                              end
 
           record_demux_keys.each do |key|
             next unless record.key?(key)
 
-            new_record = format(time, key, record[key], shared)
+            new_record = format(time, key, record[key], record_shared)
             demux_events.add(time, new_record)
           end
         end

data/lib/fluent/plugin/out_record_demux_picker.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+#
+# Copyright 2024- Thomas Tych
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/output'
+
+module Fluent
+  module Plugin
+    class RecordDemuxPickerOutput < Fluent::Plugin::Output
+      NAME = 'record_demux_picker'
+      Fluent::Plugin.register_output(NAME, self)
+
+      DEMUX_KEY_NORMALIZE_KEY_NAME = 'key'
+      DEMUX_KEY_NORMALIZE_VALUE_NAME = 'value'
+
+      helpers :event_emitter, :timer, :record_accessor
+
+      desc 'tag to emit events on'
+      config_param :tag, :string
+
+      desc 'list of keys to demux'
+      config_param :demux_keys, :hash
+      desc 'list of keys to be shared in all new events'
+      config_param :shared_keys, :hash, default: {}
+
+      desc 'normalize demux key format'
+      config_param :demux_key_normalize, :bool, default: false
+      desc 'demux key normalize key name'
+      config_param :demux_key_normalize_key_name, :string, default: DEMUX_KEY_NORMALIZE_KEY_NAME
+      desc 'demux key normalize value name'
+      config_param :demux_key_normalize_value_name, :string, default: DEMUX_KEY_NORMALIZE_VALUE_NAME
+
+      desc 'timestamp key'
+      config_param :timestamp_key, :string, default: nil
+      desc 'timestamp format'
+      config_param :timestamp_format, :enum, list: %i[iso epochmillis], default: :iso
+
+      def configure(conf)
+        super
+
+        @demux_keys_mappers = @demux_keys.map { |key, target| KeyMapper.new(key, target) }
+        @shared_keys_mappers = @shared_keys.map { |key, target| KeyMapper.new(key, target) }
+
+        true
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def process(_events_tag, events)
+        demux_events = MultiEventStream.new
+        events.each do |time, event|
+          new_events = process_event(time, event)
+          new_events.each { |new_event| demux_events.add(time, new_event) }
+        end
+        router.emit_stream(tag, demux_events)
+      end
+
+      def process_event(time, event)
+        shared_event = extract_shared_event(event)
+
+        @demux_keys_mappers.map do |mapper|
+          value = mapper.accessor.call(event)
+          shared_event
+            .merge(format_demux_key(mapper.target, value))
+            .merge(format_time(time))
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          nil
+        end.compact
+      end
+
+      private
+
+      def extract_shared_event(event)
+        @shared_keys_mappers.each_with_object({}) do |mapper, new_event|
+          value = mapper.accessor.call(event)
+          new_event[mapper.target] = value
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          next
+        end
+      end
+
+      def format_demux_key(key, value)
+        if demux_key_normalize
+          { demux_key_normalize_key_name => key,
+            demux_key_normalize_value_name => value }
+        else
+          { key => value }
+        end
+      end
+
+      def format_time(time)
+        return {} unless timestamp_key
+
+        { timestamp_key => format_timestamp(time) }
+      end
+
+      def format_timestamp(time)
+        return (time.to_time.utc.to_f * 1000).to_i if @timestamp_format == :epochmillis
+
+        time.to_time.utc.iso8601(3)
+      end
+
+      class KeyMapper
+        include Fluent::PluginHelper::RecordAccessor
+
+        attr_reader :key, :accessor, :target, :setter
+
+        def initialize(key, target)
+          @key = key
+          @accessor = record_accessor_create(key)
+          @target = target
+          @target ||= @accessor.keys.is_a?(Array) ? @accessor.keys.last : @accessor.keys
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-demux
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Thomas Tych
 bindir: bin
 cert_chain: []
-date: 2025-03-14 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -63,6 +63,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 11.1.3
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -219,6 +239,7 @@ files:
 - Rakefile
 - fluent-plugin-record-demux.gemspec
 - lib/fluent/plugin/out_record_demux.rb
+- lib/fluent/plugin/out_record_demux_picker.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-record-demux
 licenses:
 - Apache-2.0
@@ -238,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.5
+rubygems_version: 3.6.6
 specification_version: 4
 summary: fluentd plugin to demux record by keys.
 test_files: []