fluent-plugin-record-demux 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3af2e3acf245f6e85007a6832dec1cdeb7fb7e54fc91afc2312e5c9da4be9be
-  data.tar.gz: 8296a752f7a84b20f900a214abc55bea52af25b4e9a06d1d079b445921331e39
+  metadata.gz: c9da92e93a04e15799949041055f91a9d9ff9327627028ce85322343778311bf
+  data.tar.gz: 7eb377c975bdd3bcdbb192dd43a89af9995292dadc5bcfad3174f8f1f580ca09
 SHA512:
-  metadata.gz: 2d99ca520a2a8e281769c3dcc508937e9ec34e4c925e54c848590aa4f59ed4ed0a773730847b697ef3e3d511d2437077531f31bd3b1841e4ed0b3539f6f55943
-  data.tar.gz: 6170f58d6f984e9ea7ac87f9bc4b71e7e69e6274d1b848998dd3c53ebb2de6bf6e50fa6246553ce513f028698561f7c93816f8ea2b6f224a2ca02acb10beac0e
+  metadata.gz: a04985f89183aa35c8dbd0b4e72fbc430a5160a71b63eb539a25e720b11d51376f82cb87a07b6317fe89a001876a53161c78adda0151b6f51ca4b9536f826b7d
+  data.tar.gz: 586b8c3ffc1b22d553a056fd0e3f96bf9ccada5e7d4c27a5e2b1e6ff714cc376d8c347f96278410b02dbc86c160d4abf94b05a4aa4abccc2f3da6ddb389d6181

data/.gitignore

@@ -1,6 +1,5 @@
-# tmp files
+/pkg
 /tmp
-.byebug_history
+/.byebug_history
 /coverage
-
 /Gemfile.lock

data/.rubocop.yml

@@ -14,7 +14,7 @@ Layout/LineLength:
   Max: 100
 
 Metrics/AbcSize:
-  Max: 25
+  Max: 30
 
 Metrics/BlockLength:
   Exclude:

data/README.md

@@ -18,9 +18,9 @@
 | timestamp_key        | string          | *nil*   | add timestamp key when defined, computed from time of the event            |
 | timestamp_format     | enum            | iso     | can be iso for iso8601(3) or epochmillis for epoch with millisecond format |
 
-### example
+### example 1
 
-Example of configuration
+Example of configuration:
 
 ``` text
 <match *>
@@ -34,6 +34,47 @@ Example of configuration
 </match>
 ```
 
+### example 2
+
+With configuration:
+
+``` text
+<match *>
+  @type record_demux
+
+  tag data.demux
+
+  shared_keys tags_1, tags_2
+  remove_keys tmp
+</match>
+```
+
+or with configuration:
+
+``` text
+<match *>
+  @type record_demux
+
+  tag data.demux
+
+  demux_keys a, b, c
+  remove_keys tmp
+</match>
+```
+
+It will transform event like:
+
+``` text
+{ "a": "data_a", "b": "data_b", "c": "data_c", "tags_1": "data_1", "tags_2": "data_2", "tmp": "data_tmp" }
+```
+
+into events:
+
+``` text
+{ "a": "data_a", "tags_1": "data_1", "tags_2": "data_2" }
+{ "b": "data_b", "tags_1": "data_1", "tags_2": "data_2" }
+{ "c": "data_c", "tags_1": "data_1", "tags_2": "data_2" }
+```
 
 ## Installation
 

data/fluent-plugin-record-demux.gemspec

@@ -5,7 +5,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-record-demux'
-  spec.version = '0.1.1'
+  spec.version = '0.2.0'
   spec.authors = ['Thomas Tych']
   spec.email   = ['thomas.tych@gmail.com']
 

data/lib/fluent/plugin/out_record_demux.rb

@@ -31,7 +31,7 @@ module Fluent
       desc 'list of keys to demux'
       config_param :demux_keys, :array, value_type: :string, default: nil
       desc 'list of keys to be shared in all new records'
-      config_param :shared_keys, :array, value_type: :string, default: []
+      config_param :shared_keys, :array, value_type: :string, default: nil
       desc 'list of keys to be removed'
       config_param :remove_keys, :array, value_type: :string, default: []
 
@@ -51,9 +51,13 @@ module Fluent
       def configure(conf)
         super
 
-        return unless @tag.nil?
+        raise Fluent::ConfigError, "#{NAME}: `tag` must be specified" if tag.nil? || tag.empty?
 
-        raise Fluent::ConfigError, "#{NAME}: `tag` must be specified"
+        if demux_keys.nil? && shared_keys.nil?
+          raise Fluent::ConfigError, 'specify demux_keys or shared_keys'
+        end
+
+        true
       end
 
       def multi_workers_ready?
@@ -61,17 +65,27 @@ module Fluent
       end
 
       def process(_events_tag, events)
-        demux_events = Fluent::EventStream.new
+        demux_events = MultiEventStream.new
         events.each do |time, record|
           record_keys = record.keys - remove_keys
-          record_shared_keys = record_keys.intersection(shared_keys)
-          record.slice(*record_shared_keys)
-          record_demux_keys = record_keys - record_shared_keys if !demux_keys || demux_keys.empty?
+
+          record_shared_keys = if shared_keys.nil?
+                                 record_keys - demux_keys
+                               else
+                                 record_keys.intersection(shared_keys)
+                               end
+          record_shared = record.slice(*record_shared_keys)
+
+          record_demux_keys = if demux_keys.nil?
+                                record_keys - record_shared_keys
+                              else
+                                record_keys.intersection(demux_keys)
+                              end
 
           record_demux_keys.each do |key|
             next unless record.key?(key)
 
-            new_record = format(time, key, record[key], shared)
+            new_record = format(time, key, record[key], record_shared)
             demux_events.add(time, new_record)
           end
         end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-demux
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Thomas Tych