fluent-plugin-concat 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5d138482d1de0924a22f567fd46e0bb9ecefb777
-  data.tar.gz: 6e6ba49ac63a58b262e7cefd011fda2f30bbdb9c
+  metadata.gz: 53cd1a90ba28d88202ca99243dc4df5aaff6662a
+  data.tar.gz: 4cba9d7d4c63ef8429c304f4255cbb239c87c8e6
 SHA512:
-  metadata.gz: 75e64de92bb7a93eb5603872794bf24147627fd5a8f9bb6ceb3879e50339737c96a39674e7d05319038e3c326279d4a82ef40dc0de00dd8f2638d1c17c3c34ec
-  data.tar.gz: 2a1a2d24bd9945a4c9dba0f1bf400a7fc61c2276b955dbef5323f4d0f0bd98a29603d4acd32958b3531d2775ce3c85310f14773482cc24d6a5cf6d021028f6bd
+  metadata.gz: 9161c7af7c16cadd1a991a3e120d87ff681aadfe30b9c493764d993a50d73c00fe3be1b5ac84a712a1484b5d94706c51d9c95939eed510a3ea4fce19675e41f7
+  data.tar.gz: 1521cff3d1a0b54dc22d5e207f679216db987651fe981234a39f43e9d58d8c99342468679e8d0f3a992396721e4dfc1ec919887e5bf92b3d5359acee8876cd44

data/README.md

@@ -71,6 +71,17 @@ Specify first line of multiline by regular expression.
 </filter>
 ```
 
+You can handle timeout events and remaining buffers on shutdown this plugin.
+
+```aconf
+<label @ERROR>
+  <match docker.log>
+    @type file
+    path /path/to/error.log
+  </match>
+</label>
+```
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-concat.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-concat"
-  spec.version       = "0.3.1"
+  spec.version       = "0.4.0"
   spec.authors       = ["Kenji Okimoto"]
   spec.email         = ["okimoto@clear-code.com"]
 
@@ -20,4 +20,5 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.11"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "test-unit", ">= 3.1.0"
+  spec.add_development_dependency "test-unit-rr"
 end

data/lib/fluent/plugin/filter_concat.rb

@@ -14,11 +14,17 @@ module Fluent
     config_param :multiline_end_regexp, :string, default: nil
     desc "The key to determine which stream an event belongs to"
     config_param :stream_identity_key, :string, default: nil
+    desc "The interval between data flushes"
+    config_param :flush_interval, :time, default: 60
+
+    class TimeoutError < StandardError
+    end
 
     def initialize
       super
 
-      @buffer = Hash.new{|h, k| h[k] = [] }
+      @buffer = Hash.new {|h, k| h[k] = [] }
+      @timeout_map = Hash.new {|h, k| h[k] = Fluent::Engine.now }
     end
 
     def configure(conf)
@@ -44,9 +50,22 @@ module Fluent
       end
     end
 
+    def start
+      super
+      @finished = false
+      @loop = Coolio::Loop.new
+      timer = TimeoutTimer.new(1, method(:on_timer))
+      @loop.attach(timer)
+      @thread = Thread.new(@loop, &:run)
+    end
+
     def shutdown
       super
-      flush_all_buffer
+      @finished = true
+      @loop.watchers.each(&:detach)
+      @loop.stop
+      @thread.join
+      flush_remaining_buffer
     end
 
     def filter_stream(tag, es)
@@ -64,12 +83,18 @@ module Fluent
 
     private
 
+    def on_timer
+      return if @finished
+      flush_timeout_buffer
+    end
+
     def process(tag, time, record)
       if @stream_identity_key
-        stream_identity = record[@stream_identity_key]
+        stream_identity = "#{tag}:#{record[@stream_identity_key]}"
       else
-        stream_identity = "default"
+        stream_identity = "#{tag}:default"
       end
+      @timeout_map[stream_identity] = Fluent::Engine.now
       case @mode
       when :line
         @buffer[stream_identity] << [tag, time, record]
@@ -91,6 +116,7 @@ module Fluent
           if @buffer[stream_identity].empty?
             return record
           else
+            # Continuation of the previous line
             @buffer[stream_identity] << [tag, time, record]
           end
         end
@@ -107,7 +133,7 @@ module Fluent
     end
 
     def flush_buffer(stream_identity, new_element = nil)
-      lines = @buffer[stream_identity].map{|_tag, _time, record| record[@key] }
+      lines = @buffer[stream_identity].map {|_tag, _time, record| record[@key] }
       new_record = {
         @key => lines.join(@separator)
       }
@@ -116,19 +142,47 @@ module Fluent
       new_record
     end
 
-    def flush_all_buffer
+    def flush_timeout_buffer
+      now = Fluent::Engine.now
+      timeout_stream_identities = []
+      @timeout_map.each do |stream_identity, previous_timestamp|
+        next unless @flush_interval > (now - previous_timestamp)
+        timeout_stream_identities << stream_identity
+        flushed_record = flush_buffer(stream_identity)
+        tag = stream_identity.split(":").first
+        message = "Timeout flush: #{stream_identity}"
+        router.emit_error_event(tag, now, flushed_record, TimeoutError.new(message))
+        log.info message
+      end
+      @timeout_map.reject! do |stream_identity, _|
+        timeout_stream_identities.include?(stream_identity)
+      end
+    end
+
+    def flush_remaining_buffer
       @buffer.each do |stream_identity, elements|
         next if elements.empty?
-        es = MultiEventStream.new
-        lines = elements.map{|_tag, _time, record| record[@key] }
+
+        lines = elements.map {|_tag, _time, record| record[@key] }
         new_record = {
           @key => lines.join(@separator)
         }
         tag, time, record = elements.last
-        es.add(time, record.merge(new_record))
-        router.emit_stream(tag, es)
+        message = "Flush remaining buffer: #{stream_identity}"
+        router.emit_error_event(tag, time, record.merge(new_record), TimeoutError.new(message))
       end
       @buffer.clear
     end
+
+    class TimeoutTimer < Coolio::TimerWatcher
+      def initialize(interval, callback)
+        super(interval, true)
+        @callback = callback
+      end
+
+      def on_timer
+        @callback.call
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-concat
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Kenji Okimoto
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-13 00:00:00.000000000 Z
+date: 2016-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,6 +66,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd Filter plugin to concat multiple event messages
 email:
 - okimoto@clear-code.com