fluentd 0.12.26 → 0.12.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8ecb5a38a503480766ad28bd9c90d8e650e857a7
-  data.tar.gz: 9e033240aed3098e153ccf552987a72acfa2ab0b
+  metadata.gz: f6f3ea036e68e5a0a46afbe518b05590fec7addf
+  data.tar.gz: 1e73ba803fdbceec16d3e7d39f80744b696d271b
 SHA512:
-  metadata.gz: 71605065f90941ffc682fe7811f19fe88e2899e312fdf7e0633fb3b552e00e53fc8ba68f41114ce2ca885fae825d457d86b48c0266093f55699a1571d280a4d2
-  data.tar.gz: c9f4609a941738a2bf2e99a9a123c074a0444c6aa9b8fdc6584e9a9d50492d47b98ad0c57e96e5440a3f94f42cac19f39ba8242604863dbf51d8454e7f588e7c
+  metadata.gz: 8ea885c9bbd62870782c04c8da7e3a29e78fa28c2ecc944d57454cc271238dc11b7d0a9de56c77aaa922871151eba2520133399c48826711d6a2b6d3154c4910
+  data.tar.gz: 189f245592e68f78d61cda44c7253d7e5ec90a01369adf90e05a463a2a2461e6f26f838635d939b626f317af864d9c13e421db3a3190b6f2d59efd86c9a11567

data/ChangeLog

@@ -1,5 +1,29 @@
 # v0.12
 
+## Release 0.12.27 - 2016/08/03
+
+### New features / Enhancement
+
+* config: Add simplified syntax for hash and array
+  https://github.com/fluent/fluentd/pull/939
+* config: Add include support for Ruby DSL
+  https://github.com/fluent/fluentd/pull/950
+* in_http: Add support of X-Forwarded-For header
+  https://github.com/fluent/fluentd/pull/1051
+* in_syslog: Add message_length_limit option to receive longer udp syslog messages
+  https://github.com/fluent/fluentd/pull/1128
+
+### Bug fixes
+
+* test: Fix test for TimeSlicedOutput
+  https://github.com/fluent/fluentd/pull/941
+* example: Use `@type` in example confs
+  https://github.com/fluent/fluentd/pull/1043
+* Fix regression of fluent-debug command
+  https://github.com/fluent/fluentd/pull/1046
+* config: Fix wrong message for unsupported parameter
+  https://github.com/fluent/fluentd/pull/1132
+
 ## Release 0.12.26 - 2016/05/30
 
 ### Bug fixes

data/example/filter_stdout.conf

@@ -1,22 +1,22 @@
 <source>
-  type dummy
+  @type dummy
   tag dummy
 </source>
 
 <filter **>
-  type stdout
+  @type stdout
 </filter>
 
 <filter **>
-  type stdout
+  @type stdout
   output_type hash
 </filter>
 
 <filter **>
-  type stdout
+  @type stdout
   format ltsv
 </filter>
 
 <match **>
-  type null
+  @type null
 </match>

data/example/in_forward.conf

@@ -1,7 +1,7 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/in_http.conf

@@ -1,5 +1,5 @@
 <source>
-  type http
+  @type http
   bind 0.0.0.0
   port 9880
   body_size_limit 32MB
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/in_syslog.conf

@@ -1,5 +1,5 @@
 <source>
-  type syslog
+  @type syslog
   bind 0.0.0.0
   port 5140
   tag test
@@ -11,5 +11,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/in_tail.conf

@@ -1,5 +1,5 @@
 <source>
-  type tail
+  @type tail
   format none
   path /var/log/fluentd_test.log
   pos_file /var/log/fluentd_test.pos
@@ -10,5 +10,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/in_tcp.conf

@@ -1,5 +1,5 @@
 <source>
-  type tcp
+  @type tcp
   format none
   bind 0.0.0.0
   port 5170
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/in_udp.conf

@@ -1,5 +1,5 @@
 <source>
-  type udp
+  @type udp
   format none
   bind 0.0.0.0
   port 5160
@@ -9,5 +9,5 @@
 </source>
 
 <match test>
-  type stdout
+  @type stdout
 </match>

data/example/out_copy.conf

@@ -1,15 +1,15 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type copy
+  @type copy
   deep_copy false
   <store>
-    type stdout
+    @type stdout
   </store>
   <store>
-    type file
+    @type file
     path /var/log/fluentd/out_file_test
     format json
     buffer_type memory

data/example/out_file.conf

@@ -1,9 +1,9 @@
 <source>
-  type forward
+  @type forward
 </source>
 
 <match test>
-  type file
+  @type file
   path /var/log/fluentd/out_file_test
   format json
   buffer_type memory

data/example/out_forward.conf

@@ -1,10 +1,10 @@
 <source>
-  type dummy
+  @type dummy
   tag test
 </source>
 
 <match test>
-  type forward
+  @type forward
 
   <server>
     # first server

data/example/v0_12_filter.conf

@@ -9,7 +9,7 @@
 # $ echo '{"message":"hello world"}' | fluent-cat foo
 
 <source>
-  type forward
+  @type forward
   port 24224
 </source>
 
@@ -21,13 +21,13 @@
 # - {"messag22":"keep this please"} is filtered out.
 
 <filter foo>
-  type grep
+  @type grep
   regexp1  message keep this
 </filter>
 
 # Matches the events that was kept by the above filter
 <match foo>
-  type stdout
+  @type stdout
 </match>
 
 # For all events with the tag "bar", add the machine's hostname with
@@ -35,7 +35,7 @@
 # at 123.4.2.4:24224.
 
 <filter bar>
-  type record_transformer
+  @type record_transformer
   <record>
     hostname ${hostname}
   </record>
@@ -44,7 +44,7 @@
 # By the time it is getting matched here, the event has
 # the "hostname" field.
 <match bar>
-  type forward
+  @type forward
   <server>
     host 123.4.2.4
     port 24225
@@ -61,12 +61,12 @@
 # - {"last_name":"FURUHASHI"} throws an error because it has no field called "name"
 
 <filter foo.bar>
-  type grep
+  @type grep
   exclude1 hello .
 </filter>
 
 <filter foo.bar>
-  type record_transformer
+  @type record_transformer
   enable_ruby true
   <record>
     name ${name.downcase}
@@ -74,5 +74,5 @@
 </filter>
 
 <match foo.bar>
-  type stdout
+  @type stdout
 </match>

data/lib/fluent/command/debug.rb

@@ -62,12 +62,13 @@ end
 
 require 'fluent/log'
 require 'fluent/engine'
+require 'fluent/config'
 require 'fluent/system_config'
 
-include Fluent::SystemConfig::Mixin
-
 $log = Fluent::Log.new(STDERR, Fluent::Log::LEVEL_TRACE)
-Fluent::Engine.init(system_config)
+conf = Fluent::Config.parse('', 'fluent-debug.conf', '', true)
+system_conf = Fluent::SystemConfig.create(conf)
+Fluent::Engine.init(system_conf)
 
 DRb::DRbObject.class_eval do
   undef_method :methods

data/lib/fluent/config/dsl.rb

@@ -35,14 +35,19 @@ module Fluent
       end
 
       class Proxy
-        def initialize(name, arg)
+        def initialize(name, arg, include_basepath = Dir.pwd)
           @element = Element.new(name, arg, self)
+          @include_basepath = include_basepath
         end
 
         def element
           @element
         end
 
+        def include_basepath
+          @include_basepath
+        end
+
         def eval(source, source_path)
           @element.instance_eval(source, source_path)
           self
@@ -96,6 +101,18 @@ module Fluent
           self
         end
 
+        def include(*args)
+          ::Kernel.raise ::ArgumentError, "#{name} block requires arguments for include path" if args.nil? || args.size != 1
+          if args.first =~ /\.rb$/
+            path = File.expand_path(args.first)
+            data = File.read(path)
+            self.instance_eval(data, path)
+          else
+            ss = StringScanner.new('')
+            Config::V1Parser.new(ss, @proxy.include_basepath, '', nil).eval_include(@attrs, @elements, args.first)
+          end
+        end
+
         def source(&block)
           @proxy.add_element('source', nil, block)
         end

data/lib/fluent/config/section.rb

@@ -151,6 +151,10 @@ module Fluent
         elems = conf.respond_to?(:elements) ? conf.elements : []
         elems.each { |e|
           next if plugin_class.nil? && Fluent::Config::V1Parser::ELEM_SYMBOLS.include?(e.name) # skip pre-defined non-plugin elements because it doens't have proxy section
+          # In v0.12, buffer and output parameters are defined in same place.
+          # It causes same configuration is used in different buffer / output
+          # and buffer plugin's sections are always empty. It should be skipped.
+          next if proxy.sections.empty?
 
           unless proxy.sections.any? { |name, subproxy| e.name == subproxy.name.to_s || e.name == subproxy.alias.to_s }
             parent_name = if conf.arg.empty?

data/lib/fluent/config/types.rb

@@ -80,19 +80,58 @@ module Fluent
     SIZE_TYPE = Proc.new { |val, opts| Config.size_value(val) }
     BOOL_TYPE = Proc.new { |val, opts| Config.bool_value(val) }
     TIME_TYPE = Proc.new { |val, opts| Config.time_value(val) }
+
+    REFORMAT_VALUE = ->(type, value) {
+      if value.nil?
+        value
+      else
+        case type
+        when :string  then value.to_s
+        when :integer then value.to_i
+        when :float   then value.to_f
+        when :size then Config.size_value(value)
+        when :bool then Config.bool_value(value)
+        when :time then Config.time_value(value)
+        else
+          raise "unknown type in REFORMAT: #{type}"
+        end
+      end
+    }
+
     HASH_TYPE = Proc.new { |val, opts|
-      param = val.is_a?(String) ? JSON.load(val) : val
+      param = if val.is_a?(String)
+                val.start_with?('{') ? JSON.load(val) : Hash[val.strip.split(/\s*,\s*/).map{|v| v.split(':', 2)}]
+              else
+                val
+              end
       if param.class != Hash
         raise ConfigError, "hash required but got #{val.inspect}"
       end
-      param
+      if opts.empty?
+        param
+      else
+        newparam = {}
+        param.each_pair do |key, value|
+          new_key = opts[:symbolize_keys] ? key.to_sym : key
+          newparam[new_key] = opts[:value_type] ? REFORMAT_VALUE.call(opts[:value_type], value) : value
+        end
+        newparam
+      end
     }
     ARRAY_TYPE = Proc.new { |val, opts|
-      param = val.is_a?(String) ? JSON.load(val) : val
+      param = if val.is_a?(String)
+                val.start_with?('[') ? JSON.load(val) : val.strip.split(/\s*,\s*/)
+              else
+                val
+              end
       if param.class != Array
         raise ConfigError, "array required but got #{val.inspect}"
       end
-      param
+      if opts[:value_type]
+        param.map{|v| REFORMAT_VALUE.call(opts[:value_type], v) }
+      else
+        param
+      end
     }
   end
 

data/lib/fluent/config/v1_parser.rb

@@ -148,7 +148,9 @@ module Fluent
 
       def eval_include(attrs, elems, uri)
         u = URI.parse(uri)
-        if u.scheme == 'file' || u.path == uri  # file path
+        if u.scheme == 'file' || (!u.scheme.nil? && u.scheme.length == 1) || u.path == uri # file path
+          # When the Windows absolute path then u.scheme.length == 1
+          # e.g. C:
           path = u.path
           if path[0] != ?/
             pattern = File.expand_path("#{@include_basepath}/#{path}")
@@ -156,10 +158,10 @@ module Fluent
             pattern = path
           end
 
-          Dir.glob(pattern).sort.each { |path|
-            basepath = File.dirname(path)
-            fname = File.basename(path)
-            data = File.read(path)
+          Dir.glob(pattern).sort.each { |entry|
+            basepath = File.dirname(entry)
+            fname = File.basename(entry)
+            data = File.read(entry)
             data.force_encoding('UTF-8')
             ss = StringScanner.new(data)
             V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)

data/lib/fluent/plugin/in_http.rb

@@ -300,6 +300,8 @@ module Fluent
             end
           when /Origin/i
             @origin  = v
+          when /X-Forwarded-For/i
+            @remote_addr = v.split(",").first
           end
         }
         if expect

data/lib/fluent/plugin/in_syslog.rb

@@ -92,6 +92,7 @@ module Fluent
     desc 'Specify key of source host when include_source_host is true.'
     config_param :source_host_key, :string, default: 'source_host'.freeze
     config_param :blocking_timeout, :time, default: 0.5
+    config_param :message_length_limit, :size, default: 2048
 
     def configure(conf)
       super
@@ -183,7 +184,7 @@ module Fluent
       if @protocol_type == :udp
         @usock = SocketUtil.create_udp_socket(@bind)
         @usock.bind(@bind, @port)
-        SocketUtil::UdpHandler.new(@usock, log, 2048, callback)
+        SocketUtil::UdpHandler.new(@usock, log, @message_length_limit, callback)
       else
         # syslog family add "\n" to each message and this seems only way to split messages in tcp stream
         Coolio::TCPServer.new(@bind, @port, SocketUtil::TcpHandler, log, "\n", callback)

data/lib/fluent/test/output_test.rb

@@ -145,17 +145,19 @@ module Fluent
             assert_equal(@expected_buffer, buffer)
           end
 
-          chunks = @instance.instance_eval {
-            @buffer.instance_eval {
-              chunks = []
-              @map.keys.each {|key|
-                chunks.push(@map.delete(key))
-              }
-              chunks
-            }
-          }
+          chunks = []
+          @instance.instance_eval do
+            @buffer.instance_eval{ @map.keys }.each do |key|
+              @buffer.push(key)
+              chunks << @buffer.instance_eval{ @queue.pop }
+            end
+          end
           chunks.each { |chunk|
-            result.push(@instance.write(chunk))
+            begin
+              result.push(@instance.write(chunk))
+            ensure
+              chunk.purge
+            end
           }
         }
         result

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.26'
+  VERSION = '0.12.27'
 
 end

data/test/config/test_dsl.rb

@@ -2,6 +2,66 @@ require_relative '../helper'
 require 'fluent/config/element'
 require "fluent/config/dsl"
 
+TMP_DIR = File.dirname(__FILE__) + "/tmp/config_dsl#{ENV['TEST_ENV_NUMBER']}"
+def write_config(path, data)
+  FileUtils.mkdir_p(File.dirname(path))
+  File.open(path, "w") {|f| f.write data }
+end
+
+def prepare_config1
+  write_config "#{TMP_DIR}/config_test_1.conf", %[
+  k1 root_config
+  include dir/config_test_2.conf  #
+  @include #{TMP_DIR}/config_test_4.conf
+  include file://#{TMP_DIR}/config_test_5.conf
+  @include config.d/*.conf
+]
+  write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+  k2 relative_path_include
+  @include ../config_test_3.conf
+]
+  write_config "#{TMP_DIR}/config_test_3.conf", %[
+  k3 relative_include_in_included_file
+]
+  write_config "#{TMP_DIR}/config_test_4.conf", %[
+  k4 absolute_path_include
+]
+  write_config "#{TMP_DIR}/config_test_5.conf", %[
+  k5 uri_include
+]
+  write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+  k6 wildcard_include_1
+  <elem1 name>
+    include normal_parameter
+  </elem1>
+]
+  write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+  k7 wildcard_include_2
+]
+  write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+  <elem2 name>
+    @include ../dir/config_test_9.conf
+  </elem2>
+]
+  write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+  k9 embeded
+  <elem3 name>
+    nested nested_value
+    include hoge
+  </elem3>
+]
+  write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+  k8 wildcard_include_3
+  <elem4 name>
+    include normal_parameter
+  </elem4>
+]
+end
+
+def prepare_config2
+  write_config "#{TMP_DIR}/config_test_1.rb", DSL_CONFIG_EXAMPLE
+end
+
 DSL_CONFIG_EXAMPLE = %q[
 worker {
   hostname = "myhostname"
@@ -57,6 +117,14 @@ source {
 }
 ]
 
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF = %q[
+include "#{TMP_DIR}/config_test_1.conf"
+]
+
+DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB = %q[
+include "#{TMP_DIR}/config_test_1.rb"
+]
+
 DSL_CONFIG_RETURNS_NON_ELEMENT = %q[
 worker {
 }
@@ -74,6 +142,9 @@ match('aa','bb'){
 DSL_CONFIG_WRONG_SYNTAX3 = %q[
 match('aa','bb')
 ]
+DSL_CONFIG_WRONG_SYNTAX4 = %q[
+include
+]
 
 module Fluent::Config
   class TestDSLParser < ::Test::Unit::TestCase
@@ -158,6 +229,112 @@ module Fluent::Config
       end
     end
 
+    sub_test_case 'with include conf' do
+      def setup
+        prepare_config1
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_CONF, 'dsl_config_for_include.conf')
+      end
+      test 'include config' do
+        assert_equal('root_config', @root['k1'])
+        assert_equal('relative_path_include', @root['k2'])
+        assert_equal('relative_include_in_included_file', @root['k3'])
+        assert_equal('absolute_path_include', @root['k4'])
+        assert_equal('uri_include', @root['k5'])
+        assert_equal('wildcard_include_1', @root['k6'])
+        assert_equal('wildcard_include_2', @root['k7'])
+        assert_equal('wildcard_include_3', @root['k8'])
+        assert_equal([
+            'k1',
+            'k2',
+            'k3',
+            'k4',
+            'k5',
+            'k8', # Because of the file name this comes first.
+            'k6',
+            'k7',
+          ], @root.keys)
+
+        elem1 = @root.elements.find { |e| e.name == 'elem1' }
+        assert(elem1)
+        assert_equal('name', elem1.arg)
+        assert_equal('normal_parameter', elem1['include'])
+
+        elem2 = @root.elements.find { |e| e.name == 'elem2' }
+        assert(elem2)
+        assert_equal('name', elem2.arg)
+        assert_equal('embeded', elem2['k9'])
+        assert_not_include(elem2, 'include')
+
+        elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+        assert(elem3)
+        assert_equal('nested_value', elem3['nested'])
+        assert_equal('hoge', elem3['include'])
+      end
+
+      # TODO: Add uri based include spec
+    end
+
+    sub_test_case 'with include rb' do
+      def setup
+        prepare_config2
+        @root = Fluent::Config::DSL::Parser.parse(DSL_CONFIG_EXAMPLE_FOR_INCLUDE_RB, 'dsl_config_for_include.rb')
+      end
+      sub_test_case '.parse' do
+        test 'makes root element' do
+          assert_equal('ROOT', @root.name)
+          assert_predicate(@root.arg, :empty?)
+          assert_equal(0, @root.keys.size)
+        end
+
+        test 'makes worker element for worker tag' do
+          assert_equal(1, @root.elements.size)
+
+          worker = @root.elements.first
+          assert_equal('worker', worker.name)
+          assert_predicate(worker.arg, :empty?)
+          assert_equal(0, worker.keys.size)
+          assert_equal(10, worker.elements.size)
+        end
+
+        test 'makes subsections for blocks, with variable substitution' do
+          ele4 = @root.elements.first.elements[4]
+
+          assert_equal('source', ele4.name)
+          assert_predicate(ele4.arg, :empty?)
+          assert_equal(2, ele4.keys.size)
+          assert_equal('tail', ele4['type'])
+          assert_equal("/var/log/httpd/access.part4.log", ele4['path'])
+        end
+
+        test 'makes user-defined sections with blocks' do
+          filter0 = @root.elements.first.elements[4].elements.first
+
+          assert_equal('filter', filter0.name)
+          assert_equal('bar.**', filter0.arg)
+          assert_equal('hoge', filter0['type'])
+          assert_equal('moge', filter0['val1'])
+          assert_equal(JSON.dump(['foo', 'bar', 'baz']), filter0['val2'])
+          assert_equal('10', filter0['val3'])
+          assert_equal('hoge', filter0['id'])
+
+          assert_equal(2, filter0.elements.size)
+          assert_equal('subsection', filter0.elements[0].name)
+          assert_equal('bar', filter0.elements[0]['foo'])
+          assert_equal('subsection', filter0.elements[1].name)
+          assert_equal('baz', filter0.elements[1]['foo'])
+        end
+
+        test 'makes values with user-assigned variable substitutions' do
+          match0 = @root.elements.first.elements[4].elements.last
+
+          assert_equal('match', match0.name)
+          assert_equal('{foo,bar}.**', match0.arg)
+          assert_equal('file', match0['type'])
+          assert_equal('/var/log/httpd/access.myhostname.4.log', match0['path'])
+        end
+      end
+    end
+
     sub_test_case 'with configuration that returns non element on top' do
       sub_test_case '.parse' do
         test 'does not crash' do
@@ -170,8 +347,9 @@ module Fluent::Config
       sub_test_case '.parse' do
         test 'raises ArgumentError correctly' do
           assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX1, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax1') }
-          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax1') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX2, 'dsl_config_wrong_syntax2') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX3, 'dsl_config_wrong_syntax3') }
+          assert_raise(ArgumentError) { Fluent::Config::DSL::Parser.parse(DSL_CONFIG_WRONG_SYNTAX4, 'dsl_config_wrong_syntax4') }
         end
       end
     end

data/test/config/test_types.rb

@@ -124,10 +124,44 @@ class TestConfigTypes < ::Test::Unit::TestCase
 
     test 'hash' do
       assert_equal({"x"=>"v","k"=>1}, Config::HASH_TYPE.call('{"x":"v","k":1}', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v,k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v, k:1', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call(' x:v, k:1 ', {}))
+      assert_equal({"x"=>"v","k"=>"1"}, Config::HASH_TYPE.call('x:v , k:1 ', {}))
+
+      assert_equal({"x"=>"v:v","k"=>"1"}, Config::HASH_TYPE.call('x:v:v, k:1', {}))
+
+      assert_equal({x: "v", k: 1},   Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('x:v,k:1',         {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: "v", k: "1"}, Config::HASH_TYPE.call('{"x":"v","k":1}', {symbolize_keys: true, value_type: :string}))
+      assert_equal({x: 0, k: 1},     Config::HASH_TYPE.call('x:0,k:1',         {symbolize_keys: true, value_type: :integer}))
+
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('{"x":"1s","y":"1m","z":"1h"}', {value_type: :time}))
+      assert_equal({"x"=>1,"y"=>60,"z"=>3600}, Config::HASH_TYPE.call('x:1s,y:1m,z:1h',               {value_type: :time}))
     end
 
     test 'array' do
       assert_equal(["1","2",1], Config::ARRAY_TYPE.call('["1","2",1]', {}))
+      assert_equal(["1","2","1"], Config::ARRAY_TYPE.call('1,2,1', {}))
+
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('["a","b","c"]', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a,b,c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a, b, c', {}))
+      assert_equal(["a","b","c"], Config::ARRAY_TYPE.call('a , b , c', {}))
+
+      assert_equal(["a a","b,b"," c "], Config::ARRAY_TYPE.call('["a a","b,b"," c "]', {}))
+
+      assert_equal(["a a","b","c"], Config::ARRAY_TYPE.call('a a,b,c', {}))
+
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('[1,2,1]', {}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('["1","2","1"]', {value_type: :integer}))
+      assert_equal([1,2,1], Config::ARRAY_TYPE.call('1,2,1', {value_type: :integer}))
+
+      array_options = {
+        default: [],
+      }
+      assert_equal(["1","2"], Config::ARRAY_TYPE.call('["1","2"]', array_options))
+      assert_equal(["3"], Config::ARRAY_TYPE.call('["3"]', array_options))
     end
   end
 end

data/test/plugin/test_in_dummy.rb

@@ -42,6 +42,10 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
+    def json_error_messages_regexp
+      /JSON::ParserError|got incomplete JSON|0th element of dummy, foo, is not a hash/
+    end
+
     test 'dummy' do
       # hash is okay
       d = create_driver(config + %[dummy {"foo":"bar"}])
@@ -51,7 +55,7 @@ class DummyTest < Test::Unit::TestCase
       d = create_driver(config + %[dummy [{"foo":"bar"}]])
       assert_equal [{"foo"=>"bar"}], d.instance.dummy
 
-      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
+      assert_raise_message(json_error_messages_regexp) do
         create_driver(config + %[dummy "foo"])
       end
 

data/test/plugin/test_in_http.rb

@@ -119,6 +119,39 @@ class HttpInputTest < Test::Unit::TestCase
 
   end
 
+  def test_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag2", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+        assert_equal "200", res.code
+      }
+    end
+
+  end
+
+  def test_multi_json_with_add_remote_addr_given_x_forwarded_for
+    d = create_driver(CONFIG + "add_remote_addr true")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>1}
+    d.expect_emit "tag1", time, {"REMOTE_ADDR"=>"129.78.138.66", "a"=>2}
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s}, {"X-Forwarded-For"=>"129.78.138.66, 127.0.0.1"})
+      assert_equal "200", res.code
+    end
+
+  end
+
   def test_multi_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
 
@@ -274,50 +307,33 @@ class HttpInputTest < Test::Unit::TestCase
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_allowed = nil
-    acao = nil
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://foo.com"})
-          res = http.request(req)
+    d.expect_emit "tag1", time, {"a"=>1}
+    d.expect_emit "tag2", time, {"a"=>1}
 
-          acao = res["Access-Control-Allow-Origin"]
-        end
-      end
-      test_in_http_cros_allowed = true
-    rescue
-      test_in_http_cros_allowed = false
+    d.run do
+      d.expected_emits.each {|tag,_time,record|
+        res = post("/#{tag}", {"json"=>record.to_json, "time"=>_time.to_s}, {"Origin"=>"http://foo.com"})
+        assert_equal "200", res.code
+        assert_equal "http://foo.com", res["Access-Control-Allow-Origin"]
+      }
     end
-
-    assert_equal true, test_in_http_cros_allowed
-    assert_equal "http://foo.com", acao
   end
 
   def test_cors_disallowed
     d = create_driver(CONFIG + "cors_allow_origins [\"http://foo.com\"]")
     assert_equal ["http://foo.com"], d.instance.cors_allow_origins
 
-    test_in_http_cros_disallowed = nil
-    response_code = nil
-
-    begin
-      d.run do
-        Net::HTTP.start("127.0.0.1", PORT) do |http|
-          req = Net::HTTP::Post.new("/foo/bar", {"Origin" => "http://bar.com"})
-          res = http.request(req)
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-          response_code = res.code
-        end
-      end
-      test_in_http_cros_disallowed = true
-    rescue
-      test_in_http_cros_disallowed = false
+    d.expected_emits_length = 0
+    d.run do
+      res = post("/tag1", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
+      res = post("/tag2", {"json"=>{"a"=>1}.to_json, "time"=>time.to_s}, {"Origin"=>"http://bar.com"})
+      assert_equal "403", res.code
     end
-
-    assert_equal true, test_in_http_cros_disallowed
-    assert_equal "403", response_code
   end
 
   $test_in_http_connection_object_ids = []

data/test/plugin/test_in_syslog.rb

@@ -79,6 +79,24 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests)
   end
 
+  def test_msg_size_udp_for_large_msg
+    d = create_driver(CONFIG + %[
+      message_length_limit 5k
+    ])
+    tests = create_test_case(true)
+
+    d.run do
+      u = UDPSocket.new
+      u.connect('127.0.0.1', PORT)
+      tests.each {|test|
+        u.send(test['msg'], 0)
+      }
+      sleep 1
+    end
+
+    compare_test_result(d.emits, tests)
+  end
+
   def test_msg_size_with_tcp
     d = create_driver([CONFIG, 'protocol_type tcp'].join("\n"))
     tests = create_test_case
@@ -149,12 +167,20 @@ class SyslogInputTest < Test::Unit::TestCase
     compare_test_result(d.emits, tests, host)
   end
 
-  def create_test_case
+  def create_test_case(large_message = false)
     # actual syslog message has "\n"
-    [
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
-      {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
-    ]
+    if large_message
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 4096 + "\n", 'expected' => 'x' * 4096},
+      ]
+    else
+      [
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 100 + "\n", 'expected' => 'x' * 100},
+        {'msg' => '<6>Sep 10 00:00:00 localhost logger: ' + 'x' * 1024 + "\n", 'expected' => 'x' * 1024},
+      ]
+    end
   end
 
   def compare_test_result(emits, tests, host = nil)

data/test/plugin/test_out_file.rb

@@ -20,7 +20,7 @@ class FileOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::BufferedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
+    Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::FileOutput).configure(conf)
   end
 
   def with_timezone(timezone = 'UTC', &block)
@@ -143,11 +143,11 @@ class FileOutputTest < Test::Unit::TestCase
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    expect_path = "#{TMP_DIR}/out_file_test.20110102_0.log.gz"
-    assert_equal expect_path, path
+    paths = d.run
+    expect_paths = ["#{TMP_DIR}/out_file_test.20110102_0.log.gz"]
+    assert_equal expect_paths, paths
 
-    check_gzipped_result(path, %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+    check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
   end
 
   def test_write_with_format_json
@@ -158,8 +158,8 @@ class FileOutputTest < Test::Unit::TestCase
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
   end
 
   def test_write_with_format_ltsv
@@ -170,8 +170,8 @@ class FileOutputTest < Test::Unit::TestCase
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[a:1\ttime:2011-01-02T13:14:15Z\n] + %[a:2\ttime:2011-01-02T13:14:15Z\n])
   end
 
   def test_write_with_format_single_value
@@ -182,53 +182,66 @@ class FileOutputTest < Test::Unit::TestCase
     d.emit({"a"=>2}, time)
 
     # FileOutput#write returns path
-    path = d.run
-    check_gzipped_result(path, %[1\n] + %[2\n])
+    paths = d.run
+    check_gzipped_result(paths[0], %[1\n] + %[2\n])
   end
 
   def test_write_path_increment
-    d = create_driver
-
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
 
+    write_once = ->(){
+      d = create_driver
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+      d.run
+    }
+
+    assert !File.exist?("#{TMP_DIR}/out_file_test.20110102_0.log.gz")
+
     # FileOutput#write returns path
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_0.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_1.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102_2.log.gz", path
-    check_gzipped_result(path, formatted_lines)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_0.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 1, Dir.glob("#{TMP_DIR}/out_file_test.*").size
+
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_1.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 2, Dir.glob("#{TMP_DIR}/out_file_test.*").size
+
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102_2.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
   def test_write_with_append
-    d = create_driver %[
-      path #{TMP_DIR}/out_file_test
-      compress gz
-      utc
-      append true
-    ]
-
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
-    d.emit({"a"=>1}, time)
-    d.emit({"a"=>2}, time)
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
 
+    write_once = ->(){
+      d = create_driver %[
+        path #{TMP_DIR}/out_file_test
+        compress gz
+        utc
+        append true
+      ]
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+      d.run
+    }
+
     # FileOutput#write returns path
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
-    path = d.run
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines * 2)
+    paths = write_once.call
+    assert_equal ["#{TMP_DIR}/out_file_test.20110102.log.gz"], paths
+    check_gzipped_result(paths[0], formatted_lines * 3)
   end
 
   def test_write_with_symlink
@@ -270,8 +283,8 @@ class FileOutputTest < Test::Unit::TestCase
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
       # FileOutput#write returns path
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.log", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13_0.log"], paths
     end
 
     test 'normal with append' do
@@ -283,8 +296,8 @@ class FileOutputTest < Test::Unit::TestCase
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.log", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13.log"], paths
     end
 
     test '*' do
@@ -295,8 +308,8 @@ class FileOutputTest < Test::Unit::TestCase
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13_0.txt"], paths
     end
 
     test '* with append' do
@@ -308,8 +321,8 @@ class FileOutputTest < Test::Unit::TestCase
       ])
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       d.emit({"a"=>1}, time)
-      path = d.run
-      assert_equal "#{TMP_DIR}/out_file_test.2011-01-02-13.txt", path
+      paths = d.run
+      assert_equal ["#{TMP_DIR}/out_file_test.2011-01-02-13.txt"], paths
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.26
+  version: 0.12.27
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-05-30 00:00:00.000000000 Z
+date: 2016-08-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack