fluent-plugin-concat 0.2.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ab2756baf7697fc5f22d31a25bc1d93206dbc7e3
-  data.tar.gz: de0b4dfce0995316546e180ad3454453c2af476f
+  metadata.gz: 7b95595a2254d2e9f69c9962e78a80fc180a823d
+  data.tar.gz: f66346b750f623cf21619185a3ab786b486d02f3
 SHA512:
-  metadata.gz: a5e9dfa61b84279df61639b4698fb0a762301dc427c2ee95a6d314a7e6804fdc799a3c35cd7988781c8ab1c24607b2617bac18976c74634fa8f3a31d8f669581
-  data.tar.gz: 05b1ba5590e8581db137768bc91bcc34f9b8358d4741758d8283c26d1b7b40b0d6b0b33ad4a218977dc50a47eb1cd6be6e8c76408f3e33afcd99cf92352c0c4e
+  metadata.gz: a16c79ae6dd8c57f033ffe9f82624987e9eeaf6a4ff291e30c1c73be72f38ddc862804bc67fd557da82d3496f9743de9bfa1c965fae0f60f6d32eaecd0199647
+  data.tar.gz: f4bd47413c823f36775d6c5ecc2a533dd801ece990b2a39da86b74308c873ad664b1478656bd48c8e8b9a194417223741f01128380eeff082710bcd5e17cb18a

data/README.md

@@ -41,6 +41,10 @@ This is exclusive with `multiline_start_regex`.
 The regexp to match beginning of multiline.
 This is exclusive with `n_lines.`
 
+**multiline\_end\_regexp**
+
+The regexp to match ending of multiline.
+
 **stream\_identity\_key**
 
 The key to determine which stream an event belongs to.

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "0.2.1"
+  spec.version       = "0.3.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -10,6 +10,8 @@ module Fluent
     config_param :n_lines, :integer, default: nil
     desc "The regexp to match beginning of multiline"
     config_param :multiline_start_regexp, :string, default: nil
+    desc "The regexp to match ending of multiline"
+    config_param :multiline_end_regexp, :string, default: nil
     desc "The key to determine which stream an event belongs to"
     config_param :stream_identity_key, :string, default: nil
 
@@ -34,16 +36,24 @@ module Fluent
       when @n_lines
         @mode = :line
       when @multiline_start_regexp
-        @multiline_start_regexp = Regexp.compile(@multiline_start_regexp[1..-2])
         @mode = :regexp
+        @multiline_start_regexp = Regexp.compile(@multiline_start_regexp[1..-2])
+        if @multiline_end_regexp
+          @multiline_end_regexp = Regexp.compile(@multiline_end_regexp[1..-2])
+        end
       end
     end
 
+    def shutdown
+      super
+      flush_all_buffer
+    end
+
     def filter_stream(tag, es)
       new_es = MultiEventStream.new
       es.each do |time, record|
         begin
-          new_record = process(record)
+          new_record = process(tag, time, record)
           new_es.add(time, record.merge(new_record)) if new_record
         rescue => e
           router.emit_error_event(tag, time, record, e)
@@ -54,7 +64,7 @@ module Fluent
 
     private
 
-    def process(record)
+    def process(tag, time, record)
       if @stream_identity_key
         stream_identity = record[@stream_identity_key]
       else
@@ -62,19 +72,23 @@ module Fluent
       end
       case @mode
       when :line
-        @buffer[stream_identity] << record[@key]
-        if @n_lines && @buffer[stream_identity].size >= @n_lines
+        @buffer[stream_identity] << [tag, time, record]
+        if @buffer[stream_identity].size >= @n_lines
           return flush_buffer(stream_identity)
         end
       when :regexp
-        if firstline?(record[@key])
+        case
+        when firstline?(record[@key])
           if @buffer[stream_identity].empty?
-            @buffer[stream_identity] << record[@key]
+            @buffer[stream_identity] << [tag, time, record]
           else
-            return flush_buffer(stream_identity, record[@key])
+            return flush_buffer(stream_identity, [tag, time, record])
           end
+        when lastline?(record[@key])
+          @buffer[stream_identity] << [tag, time, record]
+          return flush_buffer(stream_identity)
         else
-          @buffer[stream_identity] << record[@key]
+          @buffer[stream_identity] << [tag, time, record]
         end
       end
       nil
@@ -84,12 +98,33 @@ module Fluent
       !!@multiline_start_regexp.match(text)
     end
 
-    def flush_buffer(stream_identity, new_message = nil)
-      new_record = {}
-      new_record[@key] = @buffer[stream_identity].join(@separator)
+    def lastline?(text)
+      @multiline_end_regexp && !!@multiline_end_regexp.match(text)
+    end
+
+    def flush_buffer(stream_identity, new_element = nil)
+      lines = @buffer[stream_identity].map{|_tag, _time, record| record[@key] }
+      new_record = {
+        @key => lines.join(@separator)
+      }
       @buffer[stream_identity] = []
-      @buffer[stream_identity] << new_message if new_message
+      @buffer[stream_identity] << new_element if new_element
       new_record
     end
+
+    def flush_all_buffer
+      @buffer.each do |stream_identity, elements|
+        next if elements.empty?
+        es = MultiEventStream.new
+        lines = elements.map{|_tag, _time, record| record[@key] }
+        new_record = {
+          @key => lines.join(@separator)
+        }
+        tag, time, record = elements.last
+        es.add(time, record.merge(new_record))
+        router.emit_stream(tag, es)
+      end
+      @buffer.clear
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Kenji Okimoto