fluent-plugin-concat 0.5.0 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e0e8f69b86fd07cc91f4045b25ac65cb7355130
-  data.tar.gz: 3927ef6dd92afbe5807ce1dbf34c77a190e723b4
+  metadata.gz: aba37ca7a793288cddef2c986f31caec1c492b4f
+  data.tar.gz: beaa009f9c3bc35bc38e39390918bdd29e53361b
 SHA512:
-  metadata.gz: 3f2e2eeaa5321ab37aaa86cc2974e1239121e84dd9c8fd4a80aa809ac0f9483307cacb50ff003b6ab749c21fc56ff6d00c0913367fd4294b2ff82f493609c4ee
-  data.tar.gz: c27abd17cef7c40df3cd09d63e8ca61a5d2d87a6d465cd5f4df50e5246b9c933a593a987fba967ecf51132098f5857fc8128bc7e026a03b96496d5fbb7463acb
+  metadata.gz: f62bffeed9583c683da11a8b6b7f58f4ab45fd1d39905e46a8cfa11747e96a185f5464933c9cc989aa466dd199215855b4ca2cb5a4510d9b54b13b51f6f1eb59
+  data.tar.gz: 569e6b2393a83a0303cab98319ddcd412f6c11245ac26dde9524ff5bf12cbd7c82fae9765e901a84cfa834b8514978d0219f726ea5b8b34418b2b26bfeea4ea5

data/README.md

@@ -52,6 +52,11 @@ The key to determine which stream an event belongs to.
 **flush\_interval**
 
 The number of seconds after which the last received event log will be flushed.
+If specified 0, wait for next line forever.
+
+**use\_first\_timestamp**
+
+Use timestamp of first record when buffer is flushed.
 
 ## Usage
 

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "0.5.0"
+  spec.version       = "0.6.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 

data/lib/fluent/plugin/filter_concat.rb

@@ -14,10 +14,12 @@ module Fluent
     config_param :multiline_end_regexp, :string, default: nil
     desc "The key to determine which stream an event belongs to"
     config_param :stream_identity_key, :string, default: nil
-    desc "The interval between data flushes"
+    desc "The interval between data flushes, 0 means disable timeout"
     config_param :flush_interval, :time, default: 60
     desc "The label name to handle timeout"
     config_param :timeout_label, :string, default: nil
+    desc "Use timestamp of first record when buffer is flushed"
+    config_param :use_first_timestamp, :bool, default: false
 
     class TimeoutError < StandardError
     end
@@ -74,8 +76,11 @@ module Fluent
       new_es = MultiEventStream.new
       es.each do |time, record|
         begin
-          new_record = process(tag, time, record)
-          new_es.add(time, record.merge(new_record)) if new_record
+          new_time, new_record = process(tag, time, record)
+          if new_record
+            time = new_time if @use_first_timestamp
+            new_es.add(time, record.merge(new_record))
+          end
         rescue => e
           router.emit_error_event(tag, time, record, e)
         end
@@ -86,6 +91,7 @@ module Fluent
     private
 
     def on_timer
+      return if @flush_interval <= 0
       return if @finished
       flush_timeout_buffer
     end
@@ -104,19 +110,20 @@ module Fluent
           return flush_buffer(stream_identity)
         end
       when :regexp
-        case
-        when firstline?(record[@key])
+        if firstline?(record[@key])
           if @buffer[stream_identity].empty?
             @buffer[stream_identity] << [tag, time, record]
           else
             return flush_buffer(stream_identity, [tag, time, record])
           end
-        when lastline?(record[@key])
-          @buffer[stream_identity] << [tag, time, record]
+        end
+        if lastline?(record[@key])
+          @buffer[stream_identity] << [tag, time, record] unless firstline?(record[@key])
           return flush_buffer(stream_identity)
-        else
+        end
+        if !firstline?(record[@key]) && !lastline?(record[@key])
           if @buffer[stream_identity].empty?
-            return record
+            return [time, record]
           else
             # Continuation of the previous line
             @buffer[stream_identity] << [tag, time, record]
@@ -127,7 +134,7 @@ module Fluent
     end
 
     def firstline?(text)
-      !!@multiline_start_regexp.match(text)
+      @multiline_start_regexp && !!@multiline_start_regexp.match(text)
     end
 
     def lastline?(text)
@@ -136,13 +143,13 @@ module Fluent
 
     def flush_buffer(stream_identity, new_element = nil)
       lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
-      _tag, _time, last_record = @buffer[stream_identity].last
+      _tag, time, first_record = @buffer[stream_identity].first
       new_record = {
         @key => lines.join(@separator)
       }
       @buffer[stream_identity] = []
       @buffer[stream_identity] << new_element if new_element
-      last_record.merge(new_record)
+      [time, first_record.merge(new_record)]
     end
 
     def flush_timeout_buffer
@@ -150,11 +157,11 @@ module Fluent
       timeout_stream_identities = []
       @timeout_map.each do |stream_identity, previous_timestamp|
         next if @flush_interval > (now - previous_timestamp)
-        flushed_record = flush_buffer(stream_identity)
+        time, flushed_record = flush_buffer(stream_identity)
         timeout_stream_identities << stream_identity
         tag = stream_identity.split(":").first
         message = "Timeout flush: #{stream_identity}"
-        handle_timeout_error(tag, now, flushed_record, message)
+        handle_timeout_error(tag, @use_first_timestamp ? time : now, flushed_record, message)
         log.info(message)
       end
       @timeout_map.reject! do |stream_identity, _|
@@ -170,7 +177,7 @@ module Fluent
         new_record = {
           @key => lines.join(@separator)
         }
-        tag, time, record = elements.last
+        tag, time, record = elements.first
         message = "Flush remaining buffer: #{stream_identity}"
         handle_timeout_error(tag, time, record.merge(new_record), message)
         log.info(message)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-06-23 00:00:00.000000000 Z
+date: 2016-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -139,7 +139,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.4
 signing_key: 
 specification_version: 4
 summary: Fluentd Filter plugin to concat multiple event messages