fluent-plugin-record-demux 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c9da92e93a04e15799949041055f91a9d9ff9327627028ce85322343778311bf
-  data.tar.gz: 7eb377c975bdd3bcdbb192dd43a89af9995292dadc5bcfad3174f8f1f580ca09
+  metadata.gz: 04b8dcca3813f2024f2b47c2cde9dbbc131c3ac9d312cef5810189ce9d672228
+  data.tar.gz: 74485fe4f4d05eae9e13f55bf92a6eb9ea9648d309bbe697c69ee2aae0e59f3e
 SHA512:
-  metadata.gz: a04985f89183aa35c8dbd0b4e72fbc430a5160a71b63eb539a25e720b11d51376f82cb87a07b6317fe89a001876a53161c78adda0151b6f51ca4b9536f826b7d
-  data.tar.gz: 586b8c3ffc1b22d553a056fd0e3f96bf9ccada5e7d4c27a5e2b1e6ff714cc376d8c347f96278410b02dbc86c160d4abf94b05a4aa4abccc2f3da6ddb389d6181
+  metadata.gz: c499af008b748a39f9e85eae54ce319ef34c18d69876c4e3188c67d87a6eb513044e56fcb47a4aa3f4bc04f6bf63b83938c12f95f80d86c986ceb28d46e94b7c
+  data.tar.gz: 63639df945ee52df7e5443cd426aff84641889f6d258f5f49298b8cfd2841f6382642f2d66489ec08f9e49f9f9a8a4452fa098b449e25d46860bf3b7a56985e1

data/README.md

@@ -76,6 +76,66 @@ into events:
 { "c": "data_c", "tags_1": "data_1", "tags_2": "data_2" }
 ```
 
+## plugin - out - record_dmux_picker
+
+Demux record/event, by selecting nested fields for shared and demux parts.
+
+It transforms 1 record/event to multiple record/event.
+
+### params
+
+| setting                        | type                    | default | description                                    |
+|--------------------------------|-------------------------|---------|------------------------------------------------|
+| tags                           | string                  |         | tag to emit demux events on                    |
+| demux_keys                     | hash                    |         | nested key to demux                            |
+| shared_keys                    | hash                    | {}      | shared key to be present in each new event     |
+| demux_key_normalize            | bool                    | false   | format key to demux, in a key: , value: format |
+| demux_key_normalize_key_name   | string                  | key     | when demux_key_normalize, key name to use      |
+| demux_key_normalize_value_name | string                  | value   | when demux_key_normalize, value name to use    |
+| timestamp_key                  | string                  | nil     | key for timestamp field (nil means skip)       |
+| timestamp_format               | enum (iso, epochmillis) | iso     | format of the time value                       |
+
+### example 1
+
+Give this event :
+
+``` text
+{
+    "label1": "value_label1",
+    "label2": "value_label2",
+    "label3": "value_label3",
+    "common": {
+        "metadata1": "value_metadata1",
+        "metadata2": "value_metadata2",
+        "metadata3": "value_metadata3",
+        "metadata4": "value_metadata4",
+        "metadata5": "value_metadata5",
+        "metadata6": "value_metadata6"
+    }
+}
+
+```
+
+With this conf :
+
+``` text
+<match data>
+  @type record_demux_picker
+  tag demuxed
+  demux_keys $.label1:data1 , $.label2:data2 , $.label3:data3
+  shared_keys $.common.metadata1:description1 , $.common.metadata2:description2 , $.common.metadata3:description3
+</match>
+```
+
+It will produce :
+
+``` text
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data1":"value_label1"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data2":"value_label2"}
+{"description1":"value_metadata1","description2":"value_metadata2","description3":"value_metadata3","data3":"value_label3"}
+
+```
+
 ## Installation
 
 Manual install, by executing:

data/Rakefile

@@ -6,9 +6,15 @@ Bundler::GemHelper.install_tasks
 require 'rake/testtask'
 Rake::TestTask.new(:test) do |t|
   t.libs.push('lib', 'test')
-  t.test_files = FileList['test/**/test_*.rb', 'test/**/*_test.rb',]
-  t.verbose = true
-  t.warning = true
+
+  t.test_files = if ENV['TEST_FILE']
+                   FileList["#{ENV['TEST_FILE']}*",
+                            "test/**/#{ENV['TEST_FILE']}*"]
+                 else
+                   FileList['test/**/test_*.rb', 'test/**/*_test.rb']
+                 end
+  t.verbose = ENV.fetch('VERBOSE', false)
+  t.warning = ENV.fetch('WARNING', false)
 end
 
 require 'rubocop/rake_task'

data/fluent-plugin-record-demux.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-record-demux'
-  spec.version = '0.2.0'
+  spec.version = '0.3.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'bump', '~> 0.10'
   spec.add_development_dependency 'bundler', '~> 2.6', '>= 2.6.5'
   spec.add_development_dependency 'byebug', '~> 11.1', '>= 11.1.3'
+  spec.add_development_dependency 'mocha', '~> 2.7', '>= 2.7.1'
   spec.add_development_dependency 'rake', '~> 13.2', '>= 13.2.1'
   spec.add_development_dependency 'reek', '~> 6.4'
   spec.add_development_dependency 'rubocop', '~> 1.74'

data/lib/fluent/plugin/out_record_demux.rb

@@ -26,7 +26,7 @@ module Fluent
       helpers :event_emitter, :timer
 
       desc 'tag to emit events on'
-      config_param :tag, :string, default: nil
+      config_param :tag, :string
 
       desc 'list of keys to demux'
       config_param :demux_keys, :array, value_type: :string, default: nil

data/lib/fluent/plugin/out_record_demux_picker.rb

@@ -0,0 +1,134 @@
+# frozen_string_literal: true
+
+#
+# Copyright 2024- Thomas Tych
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/output'
+
+module Fluent
+  module Plugin
+    class RecordDemuxPickerOutput < Fluent::Plugin::Output
+      NAME = 'record_demux_picker'
+      Fluent::Plugin.register_output(NAME, self)
+
+      DEMUX_KEY_NORMALIZE_KEY_NAME = 'key'
+      DEMUX_KEY_NORMALIZE_VALUE_NAME = 'value'
+
+      helpers :event_emitter, :timer, :record_accessor
+
+      desc 'tag to emit events on'
+      config_param :tag, :string
+
+      desc 'list of keys to demux'
+      config_param :demux_keys, :hash
+      desc 'list of keys to be shared in all new events'
+      config_param :shared_keys, :hash, default: {}
+
+      desc 'normalize demux key format'
+      config_param :demux_key_normalize, :bool, default: false
+      desc 'demux key normalize key name'
+      config_param :demux_key_normalize_key_name, :string, default: DEMUX_KEY_NORMALIZE_KEY_NAME
+      desc 'demux key normalize value name'
+      config_param :demux_key_normalize_value_name, :string, default: DEMUX_KEY_NORMALIZE_VALUE_NAME
+
+      desc 'timestamp key'
+      config_param :timestamp_key, :string, default: nil
+      desc 'timestamp format'
+      config_param :timestamp_format, :enum, list: %i[iso epochmillis], default: :iso
+
+      def configure(conf)
+        super
+
+        @demux_keys_mappers = @demux_keys.map { |key, target| KeyMapper.new(key, target) }
+        @shared_keys_mappers = @shared_keys.map { |key, target| KeyMapper.new(key, target) }
+
+        true
+      end
+
+      def multi_workers_ready?
+        true
+      end
+
+      def process(_events_tag, events)
+        demux_events = MultiEventStream.new
+        events.each do |time, event|
+          new_events = process_event(time, event)
+          new_events.each { |new_event| demux_events.add(time, new_event) }
+        end
+        router.emit_stream(tag, demux_events)
+      end
+
+      def process_event(time, event)
+        shared_event = extract_shared_event(event)
+
+        @demux_keys_mappers.map do |mapper|
+          value = mapper.accessor.call(event)
+          shared_event
+            .merge(format_demux_key(mapper.target, value))
+            .merge(format_time(time))
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          nil
+        end.compact
+      end
+
+      private
+
+      def extract_shared_event(event)
+        @shared_keys_mappers.each_with_object({}) do |mapper, new_event|
+          value = mapper.accessor.call(event)
+          new_event[mapper.target] = value
+        rescue StandardError => e
+          log.warn "#{NAME} : failure while processing event : #{e}"
+          next
+        end
+      end
+
+      def format_demux_key(key, value)
+        if demux_key_normalize
+          { demux_key_normalize_key_name => key,
+            demux_key_normalize_value_name => value }
+        else
+          { key => value }
+        end
+      end
+
+      def format_time(time)
+        return {} unless timestamp_key
+
+        { timestamp_key => format_timestamp(time) }
+      end
+
+      def format_timestamp(time)
+        return (time.to_time.utc.to_f * 1000).to_i if @timestamp_format == :epochmillis
+
+        time.to_time.utc.iso8601(3)
+      end
+
+      class KeyMapper
+        include Fluent::PluginHelper::RecordAccessor
+
+        attr_reader :key, :accessor, :target, :setter
+
+        def initialize(key, target)
+          @key = key
+          @accessor = record_accessor_create(key)
+          @target = target
+          @target ||= @accessor.keys.is_a?(Array) ? @accessor.keys.last : @accessor.keys
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-demux
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Thomas Tych
 bindir: bin
 cert_chain: []
-date: 2025-03-14 00:00:00.000000000 Z
+date: 2025-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bump
@@ -63,6 +63,26 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 11.1.3
+- !ruby/object:Gem::Dependency
+  name: mocha
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.7'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.7.1
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -219,6 +239,7 @@ files:
 - Rakefile
 - fluent-plugin-record-demux.gemspec
 - lib/fluent/plugin/out_record_demux.rb
+- lib/fluent/plugin/out_record_demux_picker.rb
 homepage: https://gitlab.com/ttych/fluent-plugin-record-demux
 licenses:
 - Apache-2.0
@@ -238,7 +259,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.5
+rubygems_version: 3.6.6
 specification_version: 4
 summary: fluentd plugin to demux record by keys.
 test_files: []