fluentd 0.12.30 → 0.12.31

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 63a646de1e8fe70118ebe431f68ab2492486c094
-  data.tar.gz: 2f5d57af5036d8197aa5bcb92061498d273031e2
+  metadata.gz: 0aa9f8f4d0ebedebdba427af533a3b6b67eb7bcd
+  data.tar.gz: f69ae9d212c29b4c68303460aa10a9c21aa2080a
 SHA512:
-  metadata.gz: b5a6242bf2ee47e9aa32ad27bb9af3c67643a9d6b41decc801afaee842758a2dfd18b2bd32fabe2e21eadb293d370bd2a06c0b05e3d609f0ba97bed6325c42b4
-  data.tar.gz: 0bfabdc2a87e25ca1106b6df6d1193ebe2bd6dd7c47ac8b32050877d3ab5fe9ba7b9d94222f6cd8b0002c1a2dcf37fbaf3d541f385386b9d60237f582b735fd9
+  metadata.gz: 4a89d78f39a8f3d5ced5460274db3e0c8ac7923fe68e9dc0d119225f912aa30a37a067f5d7c403f156f12491d9bd3440eeb82913cf7e3092ce49199b9c7ddb4b
+  data.tar.gz: a3009318c51d0f8dcd774521c67b0935c6d619d5cd8b506e126cd3bf2e55c24c39c8150fca88cfb0eb25eaa0fae4fbf778f482d6fd26e52ea03d990c49c6fddb

data/ChangeLog

@@ -1,5 +1,16 @@
 # v0.12
 
+## Release 0.12.31 - 2016/12/14
+
+### New features / Enhancement
+
+* output: Add slow_flush_log_threshold parameter
+  https://github.com/fluent/fluentd/pull/1366
+* formatter_csv: Change fields parameter to required. Now accepts both a,b and ["a", "b"]
+  https://github.com/fluent/fluentd/pull/1361
+* in_syslog: Add "priority_key" and "facility_key" parameters
+  https://github.com/fluent/fluentd/pull/1351
+
 ## Release 0.12.30 - 2016/12/01
 
 ### New features / Enhancement

data/fluentd.gemspec

@@ -30,12 +30,12 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency("string-scrub", [">= 0.0.3", "<= 0.0.5"])
 
   gem.add_development_dependency("rake", [">= 0.9.2"])
-  gem.add_development_dependency("flexmock", ["~> 1.3.3"])
-  gem.add_development_dependency("parallel_tests", [">= 0.15.3"])
-  gem.add_development_dependency("simplecov", ["~> 0.6.4"])
-  gem.add_development_dependency("rr", [">= 1.0.0"])
-  gem.add_development_dependency("timecop", [">= 0.3.0"])
-  gem.add_development_dependency("test-unit", ["~> 3.1.4"])
-  gem.add_development_dependency("test-unit-rr", ["~> 1.0.3"])
+  gem.add_development_dependency("flexmock", ["~> 2.0"])
+  gem.add_development_dependency("parallel_tests", ["~> 0.15.3"])
+  gem.add_development_dependency("simplecov", ["~> 0.7"])
+  gem.add_development_dependency("rr", ["~> 1.0"])
+  gem.add_development_dependency("timecop", ["~> 0.3"])
+  gem.add_development_dependency("test-unit", ["~> 3.2"])
+  gem.add_development_dependency("test-unit-rr", ["~> 1.0"])
   gem.add_development_dependency("oj", ["~> 2.14"])
 end

data/lib/fluent/formatter.rb

@@ -196,26 +196,28 @@ module Fluent
         ['\t', 'TAB'].include?(val) ? "\t" : val
       end
       config_param :force_quotes, :bool, default: true
-      config_param :fields, default: [] do |val|
-        val.split(',').map do |f|
-          f.strip!
-          f.size > 0 ? f : nil
-        end.compact
-      end
+      config_param :fields, :array, value_type: :string
 
       def initialize
         super
         require 'csv'
       end
 
+      def configure(conf)
+        super
+        @fields = fields.select { |f| !f.empty? }
+        raise ConfigError, "empty value is specified in fields parameter" if @fields.empty?
+
+        @generate_opts = {col_sep: @delimiter, force_quotes: @force_quotes}
+      end
+
       def format(tag, time, record)
         filter_record(tag, time, record)
         row = @fields.inject([]) do |memo, key|
             memo << record[key]
             memo
         end
-        CSV.generate_line(row, col_sep: @delimiter,
-                          force_quotes: @force_quotes)
+        CSV.generate_line(row, @generate_opts)
       end
     end
 

data/lib/fluent/output.rb

@@ -203,6 +203,8 @@ module Fluent
     config_param :max_retry_wait, :time, default: nil
     desc 'The number of threads to flush the buffer.'
     config_param :num_threads, :integer, default: 1
+    desc 'The threshold to show slow flush logs'
+    config_param :slow_flush_log_threshold, :float, default: 20.0
     desc 'The interval between data flushes for queued chunk.'
     config_param :queued_chunk_flush_interval, :time, default: 1
 
@@ -332,12 +334,20 @@ module Fluent
           end
         end
 
+        chunk_write_start = Time.now
+
         if @secondary && !@disable_retry_limit && @num_errors > @retry_limit
           has_next = flush_secondary(@secondary)
         else
           has_next = @buffer.pop(self)
         end
 
+        elapsed_time = Time.now - chunk_write_start
+        if elapsed_time > @slow_flush_log_threshold
+          $log.warn "buffer flush took longer time than slow_flush_log_threshold:",
+                    plugin_id: plugin_id, elapsed_time: elapsed_time, slow_flush_log_threshold: @slow_flush_log_threshold
+        end
+
         # success
         if retrying
           @num_errors = 0
@@ -500,6 +510,7 @@ module Fluent
     config_set_default :buffer_type, 'file'  # overwrite default buffer_type
     config_set_default :buffer_chunk_limit, 256*1024*1024  # overwrite default buffer_chunk_limit
     config_set_default :flush_interval, nil
+    config_set_default :slow_flush_log_threshold, 40.0
 
     attr_accessor :localtime
     attr_reader :time_slicer # for test

data/lib/fluent/plugin/in_syslog.rb

@@ -91,6 +91,10 @@ module Fluent
     config_param :include_source_host, :bool, default: false
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
+    desc 'The field name of the priority.'
+    config_param :priority_key, :string, default: nil
+    desc 'The field name of the facility.'
+    config_param :facility_key, :string, default: nil
     config_param :blocking_timeout, :time, default: 0.5
     config_param :message_length_limit, :size, default: 2048
 
@@ -153,8 +157,15 @@ module Fluent
           return
         end
 
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
       }
     rescue => e
       log.error data.dump, error: e.to_s
@@ -168,9 +179,16 @@ module Fluent
           return
         end
 
-        pri = record.delete('pri')
+        pri = record.delete('pri'.freeze)
+        facility = FACILITY_MAP[pri >> 3]
+        priority = PRIORITY_MAP[pri & 0b111]
+
+        record[@priority_key] = priority if @priority_key
+        record[@facility_key] = facility if @facility_key
         record[@source_host_key] = addr[2] if @include_source_host
-        emit(pri, time, record)
+
+        tag = "#{@tag}.#{facility}.#{priority}"
+        emit(tag, time, record)
       }
     rescue => e
       log.error data.dump, error: e.to_s
@@ -191,12 +209,7 @@ module Fluent
       end
     end
 
-    def emit(pri, time, record)
-      facility = FACILITY_MAP[pri >> 3]
-      priority = PRIORITY_MAP[pri & 0b111]
-
-      tag = "#{@tag}.#{facility}.#{priority}"
-
+    def emit(tag, time, record)
       router.emit(tag, time, record)
     rescue => e
       log.error "syslog failed to emit", error: e.to_s, error_class: e.class.to_s, tag: tag, record: Yajl.dump(record)

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.30'
+  VERSION = '0.12.31'
 
 end

data/test/plugin/test_in_syslog.rb

@@ -57,6 +57,7 @@ class SyslogInputTest < Test::Unit::TestCase
       end
 
       emits = d.emits
+      assert_equal 2, emits.size
       emits.each_index {|i|
         assert_equal(tests[i]['expected'], emits[i][1])
       }
@@ -76,6 +77,7 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -94,6 +96,7 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
+    assert_equal 3, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -110,6 +113,7 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -126,6 +130,7 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -146,6 +151,7 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
+    assert_equal 2, d.emits.size
     compare_test_result(d.emits, tests)
   end
 
@@ -164,7 +170,44 @@ class SyslogInputTest < Test::Unit::TestCase
       sleep 1
     end
 
-    compare_test_result(d.emits, tests, host)
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {host: host})
+  end
+
+  def test_msg_size_with_include_priority
+    d = create_driver([CONFIG, 'priority_key priority'].join("\n"))
+    tests = create_test_case
+
+    priority = 'info'
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {priority: priority})
+  end
+
+  def test_msg_size_with_include_facility
+    d = create_driver([CONFIG, 'facility_key facility'].join("\n"))
+    tests = create_test_case
+
+    facility = 'kern'
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    assert_equal 2, d.emits.size
+    compare_test_result(d.emits, tests, {facility: facility})
   end
 
   def create_test_case(large_message = false)
@@ -183,11 +226,13 @@ class SyslogInputTest < Test::Unit::TestCase
     end
   end
 
-  def compare_test_result(emits, tests, host = nil)
-    emits.each_index { |i|
-      assert_equal('syslog.kern.info', emits[0][0]) # <6> means kern.info
-      assert_equal(tests[i]['expected'], emits[i][2]['message'])
-      assert_equal(host, emits[i][2]['source_host']) if host
+  def compare_test_result(events, tests, options = {})
+    events.each_index { |i|
+      assert_equal('syslog.kern.info', events[i][0]) # <6> means kern.info
+      assert_equal(tests[i]['expected'], events[i][2]['message'])
+      assert_equal(options[:host], events[i][2]['source_host']) if options[:host]
+      assert_equal(options[:priority], events[i][2]['priority']) if options[:priority]
+      assert_equal(options[:facility], events[i][2]['facility']) if options[:facility]
     }
   end
 end

data/test/test_formatter.rb

@@ -289,7 +289,15 @@ module FormatterTest
     def test_config_params
       assert_equal ',', @formatter.delimiter
       assert_equal true, @formatter.force_quotes
-      assert_equal [], @formatter.fields
+      assert_nil @formatter.fields
+    end
+
+    data('empty array' => [],
+         'array including empty string' => ['', ''])
+    def test_empty_fields(param)
+      assert_raise ConfigError do
+        @formatter.configure('fields' => param)
+      end
     end
 
     data(
@@ -298,7 +306,7 @@ module FormatterTest
       'pipe' => ['|', '|'])
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
-      @formatter.configure('delimiter' => target)
+      @formatter.configure('fields' => 'f1', 'delimiter' => target)
       assert_equal expected, @formatter.delimiter
     end
 

data/test/test_output.rb

@@ -47,6 +47,7 @@ module FluentOutputTest
       assert_equal nil, d.instance.max_retry_wait
       assert_equal 1.0, d.instance.retry_wait
       assert_equal 1, d.instance.num_threads
+      assert_equal 20.0, d.instance.slow_flush_log_threshold
       assert_equal 1, d.instance.queued_chunk_flush_interval
 
       # max_retry_wait
@@ -201,6 +202,68 @@ module FluentOutputTest
         10.times { sleep 0.05 }
       end
     end
+
+    sub_test_case "test slow_flush_log_threshold" do
+      def create_slow_driver(conf, sleep_time)
+        d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
+          attr_accessor :sleep_time
+
+          def initialize
+            super
+            @_written_check_mutex = Mutex.new
+            @_written_check = false
+          end
+
+          def written_check
+            @_written_check_mutex.synchronize { @_written_check }
+          end
+
+          def write(chunk)
+            sleep @sleep_time
+          end
+
+          def try_flush
+            super
+            @_written_check_mutex.synchronize { @_written_check = true }
+          end
+        end.configure(conf)
+        d.instance.sleep_time = sleep_time
+        d
+      end
+
+      def setup
+        # output#try_flush uses $log, so it should be replaced
+        @orig_log = $log
+        $log = Fluent::Test::TestLogger.new
+      end
+
+      def teardown
+        $log = @orig_log
+      end
+
+      test "flush took longer time than slow_flush_log_threshold" do
+        d = create_slow_driver(%[
+          slow_flush_log_threshold 0.5
+        ], 1.5)
+        run_driver(d)
+        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
+      end
+
+      test "flush took lower time than slow_flush_log_threshold. No logs" do
+        d = create_slow_driver(%[
+          slow_flush_log_threshold 5.0
+        ], 0)
+        run_driver(d)
+        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
+      end
+
+      def run_driver(d)
+        d.instance.start
+        d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
+        d.instance.force_flush
+        until d.instance.written_check; end
+      end
+    end
   end
 
   class TimeSlicedOutputTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.30
+  version: 0.12.31
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-01 00:00:00.000000000 Z
+date: 2016-12-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -180,26 +180,26 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.3
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.3
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: parallel_tests
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.15.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.15.3
 - !ruby/object:Gem::Dependency
@@ -208,70 +208,70 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: '0.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.6.4
+        version: '0.7'
 - !ruby/object:Gem::Dependency
   name: rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.0
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.4
+        version: '3.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 3.1.4
+        version: '3.2'
 - !ruby/object:Gem::Dependency
   name: test-unit-rr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.3
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement