fluent-plugin-detect-exceptions 0.0.3 → 0.0.4

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -22,7 +22,7 @@ module Fluent
   # an exception stack trace, they forwarded as a single, combined JSON
   # object. Otherwise, the input log data is forwarded as is.
   class DetectExceptionsOutput < Output
-    desc 'The field which contains the raw message text in the input json data.'
+    desc 'The field which contains the raw message text in the input JSON data.'
     config_param :message, :string, default: ''
     desc 'The prefix to be removed from the input tag when outputting a record.'
     config_param :remove_tag_prefix, :string, default: ''
@@ -34,6 +34,8 @@ module Fluent
     config_param :max_lines, :integer, default: 1000
     desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
     config_param :max_bytes, :integer, default: 0
+    desc 'Separate log streams by this field in the input JSON data.'
+    config_param :stream, :string, default: ''
 
     Fluent::Plugin.register_output('detect_exceptions', self)
 
@@ -69,7 +71,7 @@ module Fluent
       # Before shutdown is not available in older fluentd versions.
       # Hence, we make sure that we flush the buffers here as well.
       flush_buffers
-      @thread.join if multiline_flush_interval
+      @thread.join if @multiline_flush_interval
       super
     end
 
@@ -84,17 +86,19 @@ module Fluent
 
     def process_record(tag, time_sec, record)
       synchronize do
-        unless @accumulators.key?(tag)
-          out_tag = tag.sub(/^#{Regexp.escape(remove_tag_prefix)}\./, '')
-          @accumulators[tag] =
-            Fluent::TraceAccumulator.new(message, @languages,
-                                         max_lines: max_lines,
-                                         max_bytes: max_bytes) do |t, r|
+        log_id = [tag]
+        log_id.push(record.fetch(@stream, '')) unless @stream.empty?
+        unless @accumulators.key?(log_id)
+          out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+          @accumulators[log_id] =
+            Fluent::TraceAccumulator.new(@message, @languages,
+                                         max_lines: @max_lines,
+                                         max_bytes: @max_bytes) do |t, r|
               router.emit(out_tag, t, r)
             end
         end
 
-        @accumulators[tag].push(time_sec, record)
+        @accumulators[log_id].push(time_sec, record)
       end
     end
 
@@ -112,8 +116,8 @@ module Fluent
           now = Time.now
           break if @stop_check
           @accumulators.each_value do |acc|
-            needs_flush = now - acc.buffer_start_time > multiline_flush_interval
-            acc.force_flush if needs_flush
+            acc.force_flush if now - acc.buffer_start_time >
+                               @multiline_flush_interval
           end
         end
       end
@@ -123,7 +127,7 @@ module Fluent
     end
 
     def synchronize(&block)
-      if multiline_flush_interval
+      if @multiline_flush_interval
         @flush_buffer_mutex.synchronize(&block)
       else
         yield

data/measure_memory.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+DELAY=0.1
+
+while true
+do
+	start_timestamp=`date +%s.%N`
+
+	paths=`find /sys/fs/cgroup/memory/docker/* -type d`
+	for path in $paths
+	do
+		id=${path##*/}
+		memory=`cat /sys/fs/cgroup/memory/docker/${id}/memory.usage_in_bytes`
+		echo -ne "${id}\t${memory} "
+	done
+	if [[ "$paths" != "" ]]
+	then
+		echo
+	fi
+
+	end_timestamp=`date +%s.%N`
+	to_wait=`perl -e "print ${DELAY} - ${end_timestamp} + ${start_timestamp}"`
+
+	if [[ `echo ${to_wait}'>'0 | bc -l` -ne 0 ]]
+	then
+		sleep $to_wait
+	fi
+done

data/test/plugin/test_out_detect_exceptions.rb

@@ -46,11 +46,17 @@ END
     d
   end
 
-  def feed_lines(driver, t, *messages)
+  def log_entry(message, count, stream)
+    log_entry = { 'message' => message, 'count' => count }
+    log_entry['stream'] = stream unless stream.nil?
+    log_entry
+  end
+
+  def feed_lines(driver, t, *messages, stream: nil)
     count = 0
     messages.each do |m|
       m.each_line do |line|
-        driver.emit({ 'message' => line, 'count' => count }, t + count)
+        driver.emit(log_entry(line, count, stream), t + count)
         count += 1
       end
     end
@@ -63,11 +69,11 @@ END
     end
   end
 
-  def make_logs(t, *messages)
+  def make_logs(t, *messages, stream: nil)
     count = 0
     logs = []
     messages.each do |m|
-      logs << [t + count, { 'message' => m, 'count' => count }]
+      logs << [t + count, log_entry(m, count, stream)]
       count += m.lines.count
     end
     logs
@@ -175,4 +181,27 @@ END
       [PYTHON_EXC.lines[0..1].join] + PYTHON_EXC.lines[2..-1] + [JAVA_EXC]
     assert_equal(make_logs(t, *expected), d.events)
   end
+
+  def test_separate_streams
+    cfg = 'stream stream'
+    d = create_driver(cfg)
+    t = Time.now.to_i
+    d.run do
+      feed_lines(d, t, JAVA_EXC.lines[0], stream: 'java')
+      feed_lines(d, t, PYTHON_EXC.lines[0..1].join, stream: 'python')
+      feed_lines(d, t, JAVA_EXC.lines[1..-1].join, stream: 'java')
+      feed_lines(d, t, JAVA_EXC, stream: 'java')
+      feed_lines(d, t, PYTHON_EXC.lines[2..-1].join, stream: 'python')
+      feed_lines(d, t, 'something else', stream: 'java')
+    end
+    # Expected: the Python and the Java exceptions are handled separately
+    # because they belong to different streams.
+    # Note that the Java exception is only detected when 'something else'
+    # is processed.
+    expected = make_logs(t, JAVA_EXC, stream: 'java') +
+               make_logs(t, PYTHON_EXC, stream: 'python') +
+               make_logs(t, JAVA_EXC, stream: 'java') +
+               make_logs(t, 'something else', stream: 'java')
+    assert_equal(expected, d.events)
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-detect-exceptions
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Thomas Schickinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-09 00:00:00.000000000 Z
+date: 2016-12-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -90,8 +90,11 @@ files:
 - README.rdoc
 - Rakefile
 - fluent-plugin-detect-exceptions.gemspec
+- fluent-plugin-detect-exceptions.sublime-project
+- fluent-plugin-detect-exceptions.sublime-workspace
 - lib/fluent/plugin/exception_detector.rb
 - lib/fluent/plugin/out_detect_exceptions.rb
+- measure_memory.sh
 - test/helper.rb
 - test/plugin/bench_exception_detector.rb
 - test/plugin/test_exception_detector.rb