fluent-plugin-concat 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e28946b6269c174d18d658dbee72cb77e74ebb2d
-  data.tar.gz: e5e0ff498b6594a702c9e2413d77be387488fcc4
+  metadata.gz: 4dd0443193203c8fba9648470a1deca9cd5f66a3
+  data.tar.gz: b6a9917305e5a1bfe26a95dae7e3902743814f55
 SHA512:
-  metadata.gz: 135d4d421b3e4f3960416825d7c3f42f556d55fb51e0c60a4ba4f2c5e9a759a24a475d3e139affe1cfaaad21fa63e11efd9be6309442e055f92f12ccbfe57903
-  data.tar.gz: 906220dbab6d6b9912654614efe0681ae00538a46d86d51f2c7db51af4304d63e128086bdeee623933e982a247e3be8adabe6ab137e4477d3e8702bfaa023957
+  metadata.gz: 61803699c453e5f67fb7e44b6f5b50c225319f1389f63a85e77dd63dd44f49ca4b157c93d27c6e7dfe9937f1eef6f57419d94e88969ebbb4d53f4a736819fcc2
+  data.tar.gz: 50411a7ffd2b30f774b8d0556ff5f6fe2465a3620b1ea83b0bf7544a70a80004c51a3fd44ceec2bd72a84fbfe9c84b82e99e8e7a77cf2d8fc92ebddcbf7a67e4

data/NEWS.md

@@ -1,5 +1,9 @@
 # ChangeLog
 
+## v2.0.0
+
+* Use Fluentd v0.14 API and drop Fluentd v0.12 or earlier support
+
 ## v0.6.2
 
 ### Fixes

data/README.md

@@ -4,6 +4,13 @@
 
 Fluentd Filter plugin to concatenate multiline log separated in multiple events.
 
+## Requirements
+
+| fluent-plugin-concat | fluentd    | ruby   |
+|----------------------|------------|--------|
+| >= 2.0.0             | >= v0.14.0 | >= 2.1 |
+| < 2.0.0              | >= v0.12.0 | >= 1.9 |
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -45,6 +52,10 @@ This is exclusive with `n_lines.`
 
 The regexp to match ending of multiline.
 
+**continuous\_line_\_regexp**
+
+The regexp to match continuous lines.
+
 **stream\_identity\_key**
 
 The key to determine which stream an event belongs to.
@@ -91,6 +102,29 @@ You can handle timeout events and remaining buffers on shutdown this plugin.
 </label>
 ```
 
+Handle timeout log lines the same as normal logs.
+
+```aconf
+<filter **>
+  @type concat
+  key message
+  multiline_start_regexp /^Start/
+  flush_interval 5
+  timeout_label @NORMAL
+</filter>
+
+<match **>
+  @type relabel
+  @label @NORMAL
+</match>
+
+<label @NORMAL>
+  <match **>
+    @type stdout
+  </match>
+</label>
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "2.0.1"
+  spec.version       = "2.1.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -1,4 +1,4 @@
-require 'fluent/plugin/filter'
+require "fluent/plugin/filter"
 
 module Fluent::Plugin
   class ConcatFilter < Filter
@@ -16,6 +16,8 @@ module Fluent::Plugin
     config_param :multiline_start_regexp, :string, default: nil
     desc "The regexp to match ending of multiline"
     config_param :multiline_end_regexp, :string, default: nil
+    desc "The regexp to match continuous lines"
+    config_param :continuous_line_regexp, :string, default: nil
     desc "The key to determine which stream an event belongs to"
     config_param :stream_identity_key, :string, default: nil
     desc "The interval between data flushes, 0 means disable timeout"
@@ -55,6 +57,9 @@ module Fluent::Plugin
         if @multiline_end_regexp
           @multiline_end_regexp = Regexp.compile(@multiline_end_regexp[1..-2])
         end
+        if @continuous_line_regexp
+          @continuous_line_regexp = Regexp.compile(@continuous_line_regexp[1..-2])
+        end
       end
     end
 
@@ -101,7 +106,6 @@ module Fluent::Plugin
     end
 
     def process(tag, time, record)
-      new_es = Fluent::MultiEventStream.new
       if @stream_identity_key
         stream_identity = "#{tag}:#{record[@stream_identity_key]}"
       else
@@ -110,47 +114,64 @@ module Fluent::Plugin
       @timeout_map[stream_identity] = Fluent::Engine.now
       case @mode
       when :line
-        @buffer[stream_identity] << [tag, time, record]
-        if @buffer[stream_identity].size >= @n_lines
-          new_time, new_record = flush_buffer(stream_identity)
-          time = new_time if @use_first_timestamp
-          new_es.add(time, new_record)
-          return new_es
-        end
+        process_line(stream_identity, tag, time, record)
       when :regexp
-        case
-        when firstline?(record[@key])
-          if @buffer[stream_identity].empty?
-            @buffer[stream_identity] << [tag, time, record]
-            if lastline?(record[@key])
-              new_time, new_record = flush_buffer(stream_identity)
-              time = new_time if @use_first_timestamp
-              new_es.add(time, new_record)
-            end
-          else
-            new_time, new_record = flush_buffer(stream_identity, [tag, time, record])
+        process_regexp(stream_identity, tag, time, record)
+      end
+    end
+
+    def process_line(stream_identity, tag, time, record)
+      new_es = Fluent::MultiEventStream.new
+      @buffer[stream_identity] << [tag, time, record]
+      if @buffer[stream_identity].size >= @n_lines
+        new_time, new_record = flush_buffer(stream_identity)
+        time = new_time if @use_first_timestamp
+        new_es.add(time, new_record)
+      end
+      new_es
+    end
+
+    def process_regexp(stream_identity, tag, time, record)
+      new_es = Fluent::MultiEventStream.new
+      case
+      when firstline?(record[@key])
+        if @buffer[stream_identity].empty?
+          @buffer[stream_identity] << [tag, time, record]
+          if lastline?(record[@key])
+            new_time, new_record = flush_buffer(stream_identity)
             time = new_time if @use_first_timestamp
             new_es.add(time, new_record)
-            if lastline?(record[@key])
-              new_time, new_record = flush_buffer(stream_identity)
-              time = new_time if @use_first_timestamp
-              new_es.add(time, new_record)
-            end
-            return new_es
           end
-        when lastline?(record[@key])
-          @buffer[stream_identity] << [tag, time, record]
-          new_time, new_record = flush_buffer(stream_identity)
+        else
+          new_time, new_record = flush_buffer(stream_identity, [tag, time, record])
           time = new_time if @use_first_timestamp
           new_es.add(time, new_record)
+          if lastline?(record[@key])
+            new_time, new_record = flush_buffer(stream_identity)
+            time = new_time if @use_first_timestamp
+            new_es.add(time, new_record)
+          end
+          return new_es
+        end
+      when lastline?(record[@key])
+        @buffer[stream_identity] << [tag, time, record]
+        new_time, new_record = flush_buffer(stream_identity)
+        time = new_time if @use_first_timestamp
+        new_es.add(time, new_record)
+        return new_es
+      else
+        if @buffer[stream_identity].empty?
+          new_es.add(time, record)
           return new_es
         else
-          if @buffer[stream_identity].empty?
-            new_es.add(time, record)
-            return new_es
-          else
+          if continuous_line?(record[@key])
             # Continuation of the previous line
             @buffer[stream_identity] << [tag, time, record]
+          else
+            new_time, new_record = flush_buffer(stream_identity)
+            time = new_time if @use_first_timestamp
+            new_es.add(time, new_record)
+            new_es.add(time, record)
           end
         end
       end
@@ -165,6 +186,14 @@ module Fluent::Plugin
       @multiline_end_regexp && !!@multiline_end_regexp.match(text)
     end
 
+    def continuous_line?(text)
+      if @continuous_line_regexp
+        !!@continuous_line_regexp.match(text)
+      else
+        true
+      end
+    end
+
     def flush_buffer(stream_identity, new_element = nil)
       lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
       _tag, time, first_record = @buffer[stream_identity].first

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 2.0.1
+  version: 2.1.0
 platform: ruby
 authors:
 - Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-01 00:00:00.000000000 Z
+date: 2017-03-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd