fluent-plugin-records-merger 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5c802733686b3da60f4482bac6f209bbb8852f21a93c8adacf770c24e10cc8ab
-  data.tar.gz: 4c6157e2bc485d422e2debe7a988169682375af12a6e48df6a906ff361cafe77
+  metadata.gz: 8eb7fbf8e02b7ce0c3f820c4ddbaa27526d6d43e409912d14f19ce5807220787
+  data.tar.gz: 192ade91d03b5233185f84dae69a206efb4a211ecc9bb87ed8c728d34ccfdeea
 SHA512:
-  metadata.gz: 0d2078ded6a41d434291ae41468fa76784c09f64a57123148c96452a7f73cc6ccfcab53e7d7e0997df12aa7bce8812236a98dccf03a37a3e2992569ffa568a97
-  data.tar.gz: 3ce73da1351c592ee3a7f1cb6d7a3a8c2ae598aaee31337d251331f32ccd1233c21273a87bf954fb60068aaf7e6812b3a8c7e5c62704a510c20a9b6d951c7f4e
+  metadata.gz: cc2af7b4a0929a32fce8b0fbc296428cdc81da53153e31ba92928865698e9dcfd58fb11780b836899ff00d01e1e8a5189093ceeab203674420ac4203ebc50688
+  data.tar.gz: 5c0f1ea3554704a93b4987767b51b88c8141c6463266fb644b766fc094b55caa35122d379119278895299df6ca12c9ef5bb0d125e3e6ed92d1188cd0c91141de

data/.rubocop.yml

@@ -0,0 +1,18 @@
+Style/AsciiComments:
+  Enabled: false
+Metrics/LineLength:
+  Max: 140
+Naming/MethodParameterName:
+  Enabled: false
+Metrics/AbcSize:
+  Max: 30
+Metrics/MethodLength:
+  Max: 30
+Metrics/PerceivedComplexity:
+  Max: 10
+Metrics/CyclomaticComplexity:
+  Max: 10
+Metrics/BlockLength:
+  Max: 45
+Metrics/ClassLength:
+  Max: 250

data/example/README.md

@@ -0,0 +1,22 @@
+## 動かす
+
+
+```cassandraql
+fluentd -c example/fluent.conf -p lib/fluent/plugin 
+```
+
+## 動作を確認する
+
+```cassandraql
+cat worw2.json | fluent-cat w.sub_word1
+cat word1.json | fluent-cat w.main_word
+
+# 条件にマッチするのでemitされる
+
+
+cat worw3.json | fluent-cat w.sub_word1
+cat word1.json | fluent-cat w.main_word
+
+# 条件にマッチしないのでemitされない
+
+```

data/example/fluent.conf

@@ -0,0 +1,22 @@
+<source>
+  @type forward
+  port 24224
+</source>
+
+<match w.**>
+  @type records_merger
+  tag result
+  merge_timing before
+  main_tag w.main_word
+  sub_tag1 w.sub_word1
+  condition sub1["word"] == "hello"
+  <record>
+    main  ${main["word"]}
+    sub   ${sub1["word"]}
+  </record>
+</match>
+
+
+<match result>
+  @type stdout
+</match>

data/example/word1.json

@@ -0,0 +1 @@
+{"word": "aaaaaaaaa"}

data/example/word2.json

@@ -0,0 +1 @@
+{"word": "hello"}

data/example/word3.json

@@ -0,0 +1 @@
+{"word": "not_hello"}

data/fluent-plugin-records-merger.gemspec

@@ -5,13 +5,13 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = 'fluent-plugin-records-merger'
-  spec.version = '0.1.0'
+  spec.version = '0.1.1'
   spec.authors = ['Nobuyuki Oishi', 'WallyNegima']
   spec.email   = ['u.str.gm@gmail.com']
 
   spec.summary       = 'this is fluentd plugin. some records merge and out.'
   spec.description   = 'Parallels data is merged and out.'
-  spec.homepage      = "https://github.com/WallyNegima/fluent-plugin-records-merger"
+  spec.homepage      = 'https://github.com/WallyNegima/fluent-plugin-records-merger'
   spec.license       = 'Apache-2.0'
 
   test_files, files  = `git ls-files -z`.split("\x0").partition do |f|

data/lib/fluent/plugin/out_records_merger.rb

@@ -19,6 +19,7 @@ require 'fluent/plugin/output'
 
 module Fluent
   module Plugin
+    # fluentd records merger
     class RecordsMergerOutput < Fluent::Plugin::Output
       Fluent::Plugin.register_output('records_merger', self)
       helpers :event_emitter
@@ -28,67 +29,56 @@ module Fluent
                                   desc: 'Specify the output tag'
       config_param :main_tag, :string,
                    desc: 'Specify main_tag name'
-      config_param :sub_tag1, :string, # string: NAME REGEXP
-                   desc: 'Specify sub_tag name'
-      (2..PATTERN_MAX_NUM).each do |i|
+      (1..PATTERN_MAX_NUM).each do |i|
         config_param ('sub_tag' + i.to_s).to_sym, :string, default: nil, # NAME REGEXP
                                                            desc: 'Specify sub_tag name'
       end
       config_param :auto_typecast, :bool, default: true, # false for lower version compatibility
                                           desc: 'Automatically cast the field types.'
-      config_param :merge_timing, :enum, list: %i[before after simple], default: 'before',
+      config_param :merge_timing, :enum, list: %i[before after simple], default: 'simple',
                                          desc: 'Choose merge timing before/after'
       # ↓ これ、なんかエラー出てるっぽい。修正する。
       config_param :tolerable_time_range, :integer, default: nil,
                                                     desc: 'Specify tolerable time range for merging'
+
+      config_param :condition, :string, default: nil, desc: 'emit if condition is true '
+
       # 以下、未実装
       config_param :force_emit, :bool, default: false,
                                        desc: 'Specify to emit or not when exceeded the tolerable_time_range'
 
       def configure(conf)
         super
-        # p 'in configure()'
-        # p conf
         # conf: {"@type"=>"records_merger", "tag"=>"merged1", "main_tag"=>"accesslog.1.main", "sub_tag1"=>"accesslog.1.sub1", "sub_tag2"=>"accesslog.1.sub2"}, []
         @tag = conf['tag'] # これも，結局 ${tag} とかを有効にする関係で使わなくなりそう．
         # set the tags set by the user
+        @last_records = {}
         @main_tag = conf['main_tag']
+
         @sub_tags = []
         (1..PATTERN_MAX_NUM).each do |i|
           next unless conf["sub_tag#{i}"]
 
           @sub_tags.push(conf["sub_tag#{i}"])
         end
+
         if conf['merge_timing']
           @merge_timing = conf['merge_timing']
-          p '@merge_timing is ' + @merge_timing.to_s
-        else
-          p 'Specify merge_timing before/after'
         end
 
         @tolerable_time_range = conf['tolerable_time_range'].to_i
 
-        @last_records = {} # main を流した後に削除する? (未実装)
-
-        p 'size of @sub_tags is ' + @sub_tags.size.to_s
         @flags4merge = Array.new(@sub_tags.size + 1, 0)
-        # p @flags4merge # [0, 0,..., 0]
-
         @force_emit = conf['force_emit']
 
         # from out-record-reformer
         map = {}
-        # load <record></record> directive
         conf.elements.select { |element| element.name == 'record' }.each do |element|
           element.each_pair do |k, v|
-            # p k
-            # p v
             element.key?(k) # to suppress unread configuration warning
             map[k] = parse_value(v)
           end
         end
-        # p 'print map below'
-        # p map
 
         placeholder_expander_params = {
           log: log,
@@ -98,6 +88,7 @@ module Fluent
         @map = @placeholder_expander.preprocess_map(map)
         @tag = @placeholder_expander.preprocess_map(@tag)
         @hostname = Socket.gethostname
+        @condition = conf['condition']
       end
 
       def start
@@ -109,18 +100,12 @@ module Fluent
       end
 
       def process(tag, es)
-        # p 'in process(), merge_timing is '+@merge_timing.to_s
         if @merge_timing == 'before'
-          # p 'merge timing is before'
           if @main_tag == tag
-            # p 'main tag has come'
             store_to_lastrecords(tag, es)
             @flags4merge[0] = 1
-            # p 'store_to_lastrecords() seems to have no problems'
             check_timegap if @tolerable_time_range > 0
-            # pp @flags4merge
             if @flags4merge.all? { |flag| flag == 1 }
-              # pp 'Great! Now you can omit merged event!'
               emit_new_event(es)
             else
               (0..@flags4merge.length - 1).each do |i|
@@ -131,53 +116,35 @@ module Fluent
             store_to_lastrecords(tag, es)
             @flags4merge[@sub_tags.index(tag) + 1] = 1
             check_timegap if @tolerable_time_range > 0
-            # p @flags4merge
           end
         elsif @merge_timing == 'after'
-          # p 'merge timing is after'
           if @main_tag == tag
             store_to_lastrecords(tag, es)
             @flags4merge[0] = 1
             check_timegap if @tolerable_time_range > 0
-            # pp @flags4merge
           elsif @sub_tags.include?(tag)
             if @flags4merge[0] == 1
               store_to_lastrecords(tag, es)
               @flags4merge[@sub_tags.index(tag) + 1] = 1
               check_timegap if @tolerable_time_range > 0
-              # p @flags4merge
-              if @flags4merge.all? { |flag| flag == 1 }
-                # pp 'Great! Now you can omit merged event!'
-                emit_new_event(es)
-              end
+              emit_new_event(es) if @flags4merge.all? {|flag| flag == 1}
             else
               (0..@flags4merge.length - 1).each do |i|
                 @flags4merge[i] = 0
               end
-              # p @flags4merge
             end
           else
             p 'something else has come! check it!' + tag.to_s
           end
         elsif @merge_timing == 'simple'
-          # p 'merge timing is simple'
           if @main_tag == tag
-            # p "main_tag_came"
             store_to_lastrecords(tag, es)
-            # p 'after store_to_lastrecords()'
             @flags4merge[0] = 1
             check_timegap if @tolerable_time_range > 0
-            # pp @flags4merge
-            if @flags4merge.all? { |flag| flag == 1 }
-              # p 'before emit_new event()'
-              emit_new_event(es)
-              # p 'after emit_new event()'
-
-            end
+            emit_new_event(es) if @flags4merge.all? { |flag| flag == 1 }
           elsif @sub_tags.include?(tag)
             store_to_lastrecords(tag, es)
             @flags4merge[@sub_tags.index(tag) + 1] = 1
-            # p @flags4merge
             check_timegap if @tolerable_time_range > 0
             emit_new_event(es) if @flags4merge.all? { |flag| flag == 1 }
           else
@@ -186,7 +153,7 @@ module Fluent
         end
       rescue StandardError => e
         log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}"
-        log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_records} placeholder_values:#{placeholder_values}"
+        log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{@last_records} placeholder_values"
       end
 
       # そのまま持ってきてしまっているので，rewriteしたほうが良さそう?
@@ -195,26 +162,17 @@ module Fluent
 
       def store_to_lastrecords(tag, es)
         es.each do |time, record|
-          # p 'in store_to_lastrecords()'
-          # print 'time: '
-          # p time
-          # print 'record: '
-          # p record
           record.store('time', time)
           @last_records.store(tag, record)
-          # これで，last_recordsにはsub_tagの情報がそのまま入ってる．ちなみに，storeはkey, valueの形でハッシュに値を入れるために使う．
-          # ここはそのままでも問題なくて，mainが来たときにまとめて調整すれば良い．
+          # これで，last_recordsにはsub_tagの情報がそのまま入ってる．ちなみに，storeはkey, valueの形でハッシュに値を入れるために使う．´
         end
       end
 
       def generate_placeholders
         # here, tag can be used directly only because it is the main tag
         main_tag_parts = @main_tag.split('.')
-        # p tag_parts
         main_tag_prefix = tag_prefix(main_tag_parts)
-        # p tag_prefix
         main_tag_suffix = tag_suffix(main_tag_parts)
-        # p tag_suffix
 
         placeholder_values = {
           'main' => @last_records[@main_tag],
@@ -248,46 +206,43 @@ module Fluent
       end
 
       def emit_new_event(es)
-        # p "in emit_new_event"
         es.each do |time, _record|
           placeholder_values = generate_placeholders
           message = @last_records
-          # print "message:"
-          # p @last_records
           next if message.keys.empty?
 
           new_tag, new_record = reform(@tag, @last_records, placeholder_values)
           next unless new_tag
 
+          unless @condition.nil?
+            # conditionがOKじゃなければemiしない
+            next unless condition_is_ok?(placeholder_values)
+          end
+
           router.emit(new_tag, time, new_record)
           # flagの初期化
           (0..@flags4merge.length - 1).each do |i|
             @flags4merge[i] = 0
           end
-          # p @flags4merge
-          # router.emit(@tag, time, message)
         end
       end
 
+      def condition_is_ok?(placeholder_values)
+        placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
+        instance_eval(@condition.gsub(/((main\["\w+"\])|(sub[0-9]{1,2}\["\w+"\]))/, 'placeholders[\'${\1}\']'))
+      end
+
       def check_timegap
-        p @last_records.size
         tag_time_hash = {}
         if @last_records.size > 1
           @last_records.each do |k, v|
-            # p k
-            # p v['time'].sec
-            # p v['time'].nsec # ナノセカンドでの比較は余裕があったら実装する
             tag_time_hash.store(k, v['time'].sec)
           end
           # ここでtag_nameと一緒に保持しておきたい
           latest = tag_time_hash.max { |a, b| a[1] <=> b[1] }
-          # p latest
           oldest = tag_time_hash.min { |a, b| a[1] <=> b[1] }
-          # p oldest
-          # p @last_records
 
           if latest[1] - oldest[1] > @tolerable_time_range
-            p oldest[0].to_s + ' is too old! so deleted'
             @last_records.delete(oldest[0])
             if oldest[0] == @main_tag
               @flags4merge[0] = 0
@@ -310,37 +265,21 @@ module Fluent
       end
 
       def reform(tag, _record, placeholder_values)
-        # p 'in reform()'
-        # p 'tag: '
-        # pp tag
-        # p 'record'
-        # pp record
         placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
-        # p 'placeholders'
-        # pp placeholders
 
         new_tag = expand_placeholders(tag, placeholders) # 新しいtagを生成してる
-        # p 'new_tag'
-        # p new_tag
 
         # 基本的に，全部ゼロから作るからここでdupする必要はない．
         # new_record = @renew_record ? {} : record.dup # dupは新しいのを作ってコピーするやつ
         # @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
 
         new_record = {}
-        # p '@map' # ここには，<record></record>で指定したものが入ってる．
-        # pp @map
         new_record.merge!(expand_placeholders(@map, placeholders))
-        # p 'new_record'
-        # pp new_record
 
         [new_tag, new_record]
       end
 
       def expand_placeholders(value, placeholders)
-        # p 'in expand_placeholders()'
-        # pp value
-        # pp placeholders
         if value.is_a?(String)
           new_value = @placeholder_expander.expand(value, placeholders)
         elsif value.is_a?(Hash)
@@ -403,11 +342,9 @@ module Fluent
         end
 
         def prepare_placeholders(placeholder_values)
-          # p 'in prepare_placeholders()'
           placeholders = {}
 
           placeholder_values.each do |key, value|
-            # p key
             if value.is_a?(Array) # tag_parts, etc
               size = value.size
               value.each_with_index do |v, idx|
@@ -416,17 +353,13 @@ module Fluent
               end
             elsif value.is_a?(Hash) # record, etc
               value.each do |k, v|
-                unless placeholder_values.key?(k) # prevent overwriting the reserved keys such as tag
-                  placeholders.store("${#{k}}", v)
-                end
+                placeholders.store("${#{k}}", v) unless placeholder_values.key?(k) # prevent overwriting the reserved keys such as tag
                 placeholders.store(%(${#{key}["#{k}"]}), v) # record["foo"]
               end
             else # string, interger, float, and others?
               placeholders.store("${#{key}}", value)
             end
           end
-          # pp placeholders
-          # p 'end of prepare_placeholders()'
           placeholders
         end
 
@@ -451,9 +384,7 @@ module Fluent
         private
 
         def log_if_unknown_placeholder(placeholder, placeholders)
-          unless placeholders.include?(placeholder)
-            log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
-          end
+          log.warn "record_reformer: unknown placeholder `#{placeholder}` found" unless placeholders.include?(placeholder)
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-records-merger
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Nobuyuki Oishi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-28 00:00:00.000000000 Z
+date: 2020-03-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,10 +81,16 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rubocop.yml"
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- example/README.md
+- example/fluent.conf
+- example/word1.json
+- example/word2.json
+- example/word3.json
 - fluent-plugin-records-merger.gemspec
 - lib/fluent/plugin/out_records_merger.rb
 - test/helper.rb