fluentd 0.12.8 → 0.12.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 67639a7dc51e74c0269b78897487a90bd0acf29b
-  data.tar.gz: 43fd091c294a06a0d88eea360a7efd5b215d5577
+  metadata.gz: 389d806680686ce59ee87528aeb56c6bb3eabe92
+  data.tar.gz: 7756ef8d28122b38f3ded3413a1735f664926117
 SHA512:
-  metadata.gz: 1357f9b5a372172f9e6441dafdb106d8fa2e6756d3bb23c54b0d16c1e8888fc97e75b5ebbf94aac7ae589d48d42af6b08968653de0309cc8a9efe6b2a9ade81d
-  data.tar.gz: 00d88127b76af84ba10b1830c0be4861047a571cb953dfd63c7887d145ca8752fb8a4a978b586159918356ee57f6b7b00190295d9a6509467bd41791e8b63dfb
+  metadata.gz: a1e17c1773eb0e805352862b6bc76a051aaa6ff257c5acef32fa244de245d9d67e261824c939187a76b22146c87a288494c9b31e92df176a4f910008f6ef198c
+  data.tar.gz: 07c21ddcdbf4d140e6151d60187b951a909e9269a48f037e18e7f5f43dc5f45cc2a42ea55ac538af1d36d42d0a273acdeed01f4835342163cd6893d7922d1d3b

data/ChangeLog

@@ -1,5 +1,25 @@
 # v0.12
 
+## Release 0.12.9 - 2015/05/19
+
+### New features / Enhancement
+
+* in_tail: Add read_lines_limit parameter to control chunk size
+  https://github.com/fluent/fluentd/pull/593
+* filter: add filter_stdout plugin
+  https://github.com/fluent/fluentd/pull/586
+* parser: add keep_time_key option
+  https://github.com/fluent/fluentd/pull/587
+* parser: keys parameter accepts json array configuration
+  https://github.com/fluent/fluentd/pull/592
+* Implement RPC server for better instance management
+  https://github.com/fluent/fluentd/pull/585
+
+### Bug fixes
+
+* out_file: Fix out_file can create directory recursively
+  https://github.com/fluent/fluentd/pull/595
+
 ## Release 0.12.8 - 2015/04/22
 
 ### New features / Enhancement

data/example/filter_stdout.conf

@@ -0,0 +1,22 @@
+<source>
+  type dummy
+  tag dummy
+</source>
+
+<filter **>
+  type stdout
+</filter>
+
+<filter **>
+  type stdout
+  output_type hash
+</filter>
+
+<filter **>
+  type stdout
+  format ltsv
+</filter>
+
+<match **>
+  type null
+</match>

data/lib/fluent/formatter.rb

@@ -84,6 +84,22 @@ module Fluent
       end
     end
 
+    class StdoutFormatter < Formatter
+      config_param :output_type, :string, :default => 'json'
+
+      def configure(conf)
+        super
+
+        @formatter = Plugin.new_formatter(@output_type)
+        @formatter.configure(conf)
+      end
+
+      def format(tag, time, record)
+        header = "#{Time.now.localtime} #{tag}: "
+        "#{header}#{@formatter.format(tag, time, record)}"
+      end
+    end
+
     module StructuredFormatMixin
       def self.included(klass)
         klass.instance_eval {
@@ -120,6 +136,15 @@ module Fluent
       end
     end
 
+    class HashFormatter < Formatter
+      include HandleTagAndTimeMixin
+      include StructuredFormatMixin
+
+      def format_record(record)
+        "#{record.to_s}\n"
+      end
+    end
+
     class MessagePackFormatter < Formatter
       include HandleTagAndTimeMixin
       include StructuredFormatMixin
@@ -203,7 +228,9 @@ module Fluent
     TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
     {
       'out_file' => Proc.new { OutFileFormatter.new },
+      'stdout' => Proc.new { StdoutFormatter.new },
       'json' => Proc.new { JSONFormatter.new },
+      'hash' => Proc.new { HashFormatter.new },
       'msgpack' => Proc.new { MessagePackFormatter.new },
       'ltsv' => Proc.new { LabeledTSVFormatter.new },
       'csv' => Proc.new { CsvFormatter.new },

data/lib/fluent/load.rb

@@ -21,6 +21,7 @@ require 'fluent/log'
 require 'fluent/status'
 require 'fluent/config'
 require 'fluent/engine'
+require 'fluent/rpc'
 require 'fluent/mixin'
 require 'fluent/process'
 require 'fluent/plugin'

data/lib/fluent/parser.rb

@@ -27,6 +27,8 @@ module Fluent
     # 'configure()' may raise errors for unexpected configurations
     attr_accessor :estimate_current_event
 
+    config_param :keep_time_key, :bool, :default => false
+
     def initialize
       super
       @estimate_current_event = true
@@ -201,6 +203,13 @@ module Fluent
             case name
             when "time"
               time = @mutex.synchronize { @time_parser.parse(value) }
+              if @keep_time_key
+                record[name] = if @type_converters.nil?
+                                 value
+                               else
+                                 convert_type(name, value)
+                               end
+              end
             else
               record[name] = if @type_converters.nil?
                                value
@@ -239,7 +248,8 @@ module Fluent
       def parse(text)
         record = Yajl.load(text)
 
-        if value = record.delete(@time_key)
+        value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
+        if value
           if @time_format
             time = @mutex.synchronize { @time_parser.parse(value) }
           else
@@ -274,15 +284,19 @@ module Fluent
     class ValuesParser < Parser
       include TypeConverter
 
-      config_param :keys, :string
+      config_param :keys, :default => [] do |val|
+        if val.start_with?('[') # This check is enough because keys parameter is simple. No '[' started column name.
+          JSON.load(val)
+        else
+          val.split(",")
+        end
+      end
       config_param :time_key, :string, :default => nil
       config_param :time_format, :string, :default => nil
 
       def configure(conf)
         super
 
-        @keys = @keys.split(",")
-
         if @time_key && !@keys.include?(@time_key) && @estimate_current_event
           raise ConfigError, "time_key (#{@time_key.inspect}) is not included in keys (#{@keys.inspect})"
         end
@@ -299,7 +313,7 @@ module Fluent
         record = Hash[keys.zip(values)]
 
         if @time_key
-          value = record.delete(@time_key)
+          value = @keep_time_key ? record[@time_key] : record.delete(@time_key)
           time = if value.nil?
                    if @estimate_current_event
                      Engine.now
@@ -465,6 +479,7 @@ module Fluent
           "referer" => referer,
           "agent" => agent,
         }
+        record["time"] = m['time'] if @keep_time_key
 
         if block_given?
           yield time, record
@@ -520,6 +535,7 @@ module Fluent
               record['pri'] = value.to_i
             when "time"
               time = @mutex.synchronize { @time_parser.parse(value.gsub(/ +/, ' ')) }
+              record[name] = value if @keep_time_key
             else
               record[name] = value
             end

data/lib/fluent/plugin/file_util.rb

@@ -22,9 +22,31 @@ module Fluent
     # @param [String] path File path
     # @return [Boolean] file is writable or not
     def writable?(path)
-      path = File.exist?(path) ? path : File.dirname(path)
-      File.writable?(path)
+      return false if File.directory?(path)
+      return File.writable?(path) if File.exist?(path)
+
+      dirname = File.dirname(path)
+      return false if !File.directory?(dirname)
+      File.writable?(dirname)
     end
     module_function :writable?
+
+    # Check file is writable in conjunction wtih mkdir_p(dirname(path))
+    #
+    # @param [String] path File path
+    # @return [Boolean] file writable or not
+    def writable_p?(path)
+      return false if File.directory?(path)
+      return File.writable?(path) if File.exist?(path)
+
+      dirname = File.dirname(path)
+      until File.exist?(dirname)
+        dirname = File.dirname(dirname)
+      end
+
+      return false if !File.directory?(dirname)
+      File.writable?(dirname)
+    end
+    module_function :writable_p?
   end
 end

data/lib/fluent/plugin/filter_stdout.rb

@@ -0,0 +1,46 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  class StdoutFilter < Filter
+    Plugin.register_filter('stdout', self)
+
+    # for tests
+    attr_reader :formatter
+
+    config_param :format, :string, :default => 'stdout'
+    # config_param :output_type, :string, :default => 'json' (StdoutFormatter defines this)
+
+    def configure(conf)
+      super
+
+      @formatter = Plugin.new_formatter(@format)
+      @formatter.configure(conf)
+    end
+
+    def filter_stream(tag, es)
+      es.each { |time, record|
+        begin
+          log.write @formatter.format(tag, time, record)
+        rescue => e
+          router.emit_error_event(tag, time, record, e)
+        end
+      }
+      log.flush
+      es
+    end
+  end
+end

data/lib/fluent/plugin/in_tail.rb

@@ -31,6 +31,7 @@ module Fluent
     config_param :pos_file, :string, :default => nil
     config_param :read_from_head, :bool, :default => false
     config_param :refresh_interval, :time, :default => 60
+    config_param :read_lines_limit, :integer, :default => 1000
 
     attr_reader :paths
 
@@ -131,7 +132,7 @@ module Fluent
     end
 
     def setup_watcher(path, pe)
-      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, method(:update_watcher), &method(:receive_lines))
+      tw = TailWatcher.new(path, @rotate_wait, pe, log, @read_from_head, @read_lines_limit, method(:update_watcher), &method(:receive_lines))
       tw.attach(@loop)
       tw
     end
@@ -292,11 +293,12 @@ module Fluent
     end
 
     class TailWatcher
-      def initialize(path, rotate_wait, pe, log, read_from_head, update_watcher, &receive_lines)
+      def initialize(path, rotate_wait, pe, log, read_from_head, read_lines_limit, update_watcher, &receive_lines)
         @path = path
         @rotate_wait = rotate_wait
         @pe = pe || MemoryPositionEntry.new
         @read_from_head = read_from_head
+        @read_lines_limit = read_lines_limit
         @receive_lines = receive_lines
         @update_watcher = update_watcher
 
@@ -377,7 +379,7 @@ module Fluent
             end
             io.seek(pos)
 
-            @io_handler = IOHandler.new(io, @pe, @log, &method(:wrap_receive_lines))
+            @io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
           else
             @io_handler = NullIOHandler.new
           end
@@ -391,12 +393,12 @@ module Fluent
             inode = stat.ino
             if inode == @pe.read_inode # truncated
               @pe.update_pos(stat.size)
-              io_handler = IOHandler.new(io, @pe, @log, &method(:wrap_receive_lines))
+              io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
               @io_handler.close
               @io_handler = io_handler
             elsif @io_handler.io.nil? # There is no previous file. Reuse TailWatcher
               @pe.update(inode, io.pos)
-              io_handler = IOHandler.new(io, @pe, @log, &method(:wrap_receive_lines))
+              io_handler = IOHandler.new(io, @pe, @log, @read_lines_limit, &method(:wrap_receive_lines))
               @io_handler = io_handler
             else # file is rotated and new file found
               @update_watcher.call(@path, swap_state(@pe))
@@ -469,14 +471,13 @@ module Fluent
         end
       end
 
-      MAX_LINES_AT_ONCE = 1000
-
       class IOHandler
-        def initialize(io, pe, log, first = true, &receive_lines)
+        def initialize(io, pe, log, read_lines_limit, first = true, &receive_lines)
           @log = log
           @log.info "following tail of #{io.path}" if first
           @io = io
           @pe = pe
+          @read_lines_limit = read_lines_limit
           @receive_lines = receive_lines
           @buffer = ''.force_encoding('ASCII-8BIT')
           @iobuf = ''.force_encoding('ASCII-8BIT')
@@ -500,7 +501,7 @@ module Fluent
                 while line = @buffer.slice!(/.*?\n/m)
                   lines << line
                 end
-                if lines.size >= MAX_LINES_AT_ONCE
+                if lines.size >= @read_lines_limit
                   # not to use too much memory in case the file is very large
                   read_more = true
                   break

data/lib/fluent/plugin/out_file.rb

@@ -61,7 +61,7 @@ module Fluent
       end
 
       test_path = generate_path(Time.now.strftime(@time_slice_format))
-      unless ::Fluent::FileUtil.writable?(test_path)
+      unless ::Fluent::FileUtil.writable_p?(test_path)
         raise ConfigError, "out_file: `#{test_path}` is not writable"
       end
 

data/lib/fluent/rpc.rb

@@ -0,0 +1,94 @@
+#
+# Fluentd
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+
+module Fluent
+  module RPC
+    require 'webrick'
+
+    class Server
+      def initialize(endpoint, log)
+        bind, port = endpoint.split(':')
+        @bind = bind
+        @port = port
+        @log = log
+
+        @server = WEBrick::HTTPServer.new(
+          :BindAddress => @bind,
+          :Port => @port,
+          :Logger => WEBrick::Log.new(STDERR, WEBrick::Log::FATAL),
+          :AccessLog => [],
+        )
+      end
+
+      def mount(path, servlet, *args)
+        @server.mount(path, servlet, *args)
+        @log.debug "register #{path} RPC servlet"
+      end
+
+      def mount_proc(path, &block)
+        @server.mount_proc(path) { |req, res|
+          begin
+            code, header, body = block.call(req, res)
+          rescue => e
+            @log.warn "failed to handle RPC request", :path => path, :error => e.to_s
+            @log.warn_backtrace e.backtrace
+
+            code = 500
+            body = {
+              'message '=> 'Internal Server Error',
+              'error' => "#{e}",
+              'backtrace'=> e.backtrace,
+            }
+          end
+
+          code = 200 if code.nil?
+          header = {'Content-Type' => 'application/json'} if header.nil?
+          body = if body.nil?
+                   '{"ok":true}'
+                 else
+                   body['ok'] = code == 200
+                   body.to_json
+                 end
+
+          res.status = code
+          header.each_pair { |k, v|
+            res[k] = v
+          }
+          res.body = body
+        }
+        @log.debug "register #{path} RPC handler"
+      end
+
+      def start
+        @log.debug "listening RPC http server on http://#{@bind}:#{@port}/"
+        @thread = Thread.new {
+          @server.start
+        }
+      end
+
+      def shutdown
+        if @server
+          @server.shutdown
+          @server = nil
+        end
+        if @thread
+          @thread.join
+          @thread = nil
+        end
+      end
+    end
+  end
+end

data/lib/fluent/supervisor.rb

@@ -109,6 +109,7 @@ module Fluent
       @plugin_dirs = opt[:plugin_dirs]
       @chgroup = opt[:chgroup]
       @chuser = opt[:chuser]
+      @rpc_server = nil
 
       @log_level = opt[:log_level]
       @suppress_interval = opt[:suppress_interval]
@@ -128,8 +129,10 @@ module Fluent
 
       dry_run if @dry_run
       start_daemonize if @daemonize
+      setup_rpc_server if @rpc_endpoint
       if @supervise
         install_supervisor_signal_handlers
+        run_rpc_server if @rpc_endpoint
         until @finished
           supervise do
             change_privilege
@@ -144,6 +147,7 @@ module Fluent
         end
       else
         $log.info "starting fluentd-#{Fluent::VERSION} without supervision"
+        run_rpc_server if @rpc_endpoint
         main_process do
           change_privilege
           init_engine
@@ -154,6 +158,7 @@ module Fluent
           exit 0
         end
       end
+      stop_rpc_server if @rpc_endpoint
     end
 
     def options
@@ -233,6 +238,41 @@ module Fluent
       end
     end
 
+    def setup_rpc_server
+      @rpc_server = RPC::Server.new(@rpc_endpoint, $log)
+
+      # built-in RPC for signals
+      @rpc_server.mount_proc('/api/processes.interruptWorkers') { |req, res|
+        $log.debug "fluentd RPC got /api/processes.interruptWorkers request"
+        supervisor_sigint_handler
+        nil
+      }
+      @rpc_server.mount_proc('/api/processes.killWorkers') { |req, res|
+        $log.debug "fluentd RPC got /api/processes.killWorkers request"
+        supervisor_sigterm_handler
+        nil
+      }
+      @rpc_server.mount_proc('/api/plugins.flushBuffers') { |req, res|
+        $log.debug "fluentd RPC got /api/plugins.flushBuffers request"
+        supervisor_sigusr1_handler
+        nil
+      }
+      @rpc_server.mount_proc('/api/config.reload') { |req, res|
+        $log.debug "fluentd RPC got /api/config.reload request"
+        $log.info "restarting"
+        supervisor_sighup_handler
+        nil
+      }
+    end
+
+    def run_rpc_server
+      @rpc_server.start
+    end
+
+    def stop_rpc_server
+      @rpc_server.shutdown
+    end
+
     def supervise(&block)
       start_time = Time.now
 
@@ -290,56 +330,72 @@ module Fluent
     def install_supervisor_signal_handlers
       trap :INT do
         $log.debug "fluentd supervisor process get SIGINT"
-        @finished = true
-        if pid = @main_pid
-          # kill processes only still exists
-          unless Process.waitpid(pid, Process::WNOHANG)
-            begin
-              Process.kill(:INT, pid)
-            rescue Errno::ESRCH
-              # ignore processes already died
-            end
-          end
-        end
+        supervisor_sigint_handler
       end
 
       trap :TERM do
         $log.debug "fluentd supervisor process get SIGTERM"
-        @finished = true
-        if pid = @main_pid
-          # kill processes only still exists
-          unless Process.waitpid(pid, Process::WNOHANG)
-            begin
-              Process.kill(:TERM, pid)
-            rescue Errno::ESRCH
-              # ignore processes already died
-            end
-          end
-        end
+        supervisor_sigterm_handler
       end
 
       trap :HUP do
         $log.debug "fluentd supervisor process get SIGHUP"
         $log.info "restarting"
-        # Creating new thread due to mutex can't lock
-        # in main thread during trap context
-        Thread.new {
-          read_config
-          apply_system_config
-          if pid = @main_pid
-            Process.kill(:TERM, pid)
-            # don't resuce Erro::ESRSH here (invalid status)
-          end
-        }.run
+        supervisor_sighup_handler
       end
 
       trap :USR1 do
         $log.debug "fluentd supervisor process get SIGUSR1"
-        @log.reopen!
+        supervisor_sigusr1_handler
+      end
+    end
+
+    def supervisor_sigint_handler
+      @finished = true
+      if pid = @main_pid
+        # kill processes only still exists
+        unless Process.waitpid(pid, Process::WNOHANG)
+          begin
+            Process.kill(:INT, pid)
+          rescue Errno::ESRCH
+            # ignore processes already died
+          end
+        end
+      end
+    end
+
+    def supervisor_sigterm_handler
+      @finished = true
+      if pid = @main_pid
+        # kill processes only still exists
+        unless Process.waitpid(pid, Process::WNOHANG)
+          begin
+            Process.kill(:TERM, pid)
+          rescue Errno::ESRCH
+            # ignore processes already died
+          end
+        end
+      end
+    end
+
+    def supervisor_sighup_handler
+      # Creating new thread due to mutex can't lock
+      # in main thread during trap context
+      Thread.new {
+        read_config
+        apply_system_config
         if pid = @main_pid
-          Process.kill(:USR1, pid)
+          Process.kill(:TERM, pid)
           # don't resuce Erro::ESRSH here (invalid status)
         end
+      }.run
+    end
+
+    def supervisor_sigusr1_handler
+      @log.reopen!
+      if pid = @main_pid
+        Process.kill(:USR1, pid)
+        # don't resuce Erro::ESRSH here (invalid status)
       end
     end
 
@@ -366,6 +422,7 @@ module Fluent
       config_param :emit_error_log_interval, :time, :default => nil
       config_param :suppress_config_dump, :bool, :default => nil
       config_param :without_source, :bool, :default => nil
+      config_param :rpc_endpoint, :string, :default => nil
 
       def initialize(conf)
         super()
@@ -380,6 +437,7 @@ module Fluent
           @suppress_config_dump = system.suppress_config_dump unless system.suppress_config_dump.nil?
           @suppress_repeated_stacktrace = system.suppress_repeated_stacktrace unless system.suppress_repeated_stacktrace.nil?
           @without_source = system.without_source unless system.without_source.nil?
+          @rpc_endpoint = system.rpc_endpoint unless system.rpc_endpoint.nil?
         }
       end
     end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '0.12.8'
+  VERSION = '0.12.9'
 
 end

data/test/plugin/test_file_util.rb

@@ -22,18 +22,75 @@ class FileUtilTest < Test::Unit::TestCase
       assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
     end
 
-    test 'file does not exist and directory is writable' do
-      assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    test 'directory exists' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir")
     end
 
-    test 'file does not exist and directory is not writable' do
-      File.chmod(0444, TEST_DIR)
-      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    test 'file does not exist and parent directory is writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      assert_true Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file")
     end
 
-    test 'directory does not exist' do
-      FileUtils.rm_rf(TEST_DIR)
-      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file")
+    test 'file does not exist and parent directory is not writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      File.chmod(0444, "#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file")
+    end
+
+    test 'parent directory does not exist' do
+      FileUtils.rm_rf("#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_dir/test_file")
+    end
+
+    test 'parent file (not directory) exists' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      assert_false Fluent::FileUtil.writable?("#{TEST_DIR}/test_file/foo")
+    end
+  end
+
+  sub_test_case 'writable_p?' do
+    test 'file exists and writable' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file")
+    end
+
+    test 'file exists and not writable' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      File.chmod(0444, "#{TEST_DIR}/test_file")
+      assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file")
+    end
+
+    test 'directory exists' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir")
+    end
+
+    test 'parent directory exists and writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/test_file")
+    end
+
+    test 'parent directory exists and not writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      File.chmod(0555, "#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/test_file")
+    end
+
+    test 'parent of parent (of parent ...) directory exists and writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      assert_true Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/foo/bar/baz")
+    end
+
+    test 'parent of parent (of parent ...) directory exists and not writable' do
+      FileUtils.mkdir_p("#{TEST_DIR}/test_dir")
+      File.chmod(0555, "#{TEST_DIR}/test_dir")
+      assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_dir/foo/bar/baz")
+    end
+
+    test 'parent of parent (of parent ...) file (not directory) exists' do
+      FileUtils.touch("#{TEST_DIR}/test_file")
+      assert_false Fluent::FileUtil.writable_p?("#{TEST_DIR}/test_file/foo/bar/baz")
     end
   end
 end

data/test/plugin/test_filter_stdout.rb

@@ -0,0 +1,112 @@
+require_relative '../helper'
+require 'fluent/plugin/filter_stdout'
+require 'timecop'
+require 'flexmock'
+
+class StdoutFilterTest < Test::Unit::TestCase
+  include Fluent
+  include FlexMock::TestCase
+
+  def setup
+    Fluent::Test.setup
+    Timecop.freeze
+  end
+
+  def teardown
+    Timecop.return
+  end
+
+  CONFIG = %[
+  ]
+
+  def create_driver(conf = CONFIG)
+    Test::FilterTestDriver.new(StdoutFilter, 'filter.test').configure(conf)
+  end
+
+  def emit(d, msg, time)
+    d.run {
+      d.emit(msg, time)
+    }.filtered_as_array[0][2]
+  end
+
+  def test_through_record
+    d = create_driver
+    time = Time.now
+    filtered = emit(d, {'test' => 'test'}, time)
+    assert_equal({'test' => 'test'}, filtered)
+  end
+
+  def test_configure_default
+    d = create_driver
+    assert_equal 'json', d.instance.formatter.output_type
+  end
+
+  def test_configure_output_type
+    d = create_driver(CONFIG + "\noutput_type json")
+    assert_equal 'json', d.instance.formatter.output_type
+
+    d = create_driver(CONFIG + "\noutput_type hash")
+    assert_equal 'hash', d.instance.formatter.output_type
+
+    d = create_driver(CONFIG + "\noutput_type ltsv")
+    assert_equal 'ltsv', d.instance.formatter.output_type
+
+    assert_raise(Fluent::ConfigError) do
+      d = create_driver(CONFIG + "\noutput_type foo")
+    end
+  end
+
+  def test_output_type_json
+    d = create_driver(CONFIG + "\noutput_type json")
+    time = Time.now
+    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
+
+    # NOTE: Float::NAN is not jsonable
+    d = create_driver(CONFIG + "\noutput_type json")
+    flexmock(d.instance.router).should_receive(:emit_error_event)
+    emit(d, {'test' => Float::NAN}, time)
+  end
+
+  def test_output_type_hash
+    d = create_driver(CONFIG + "\noutput_type hash")
+    time = Time.now
+    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
+
+    # NOTE: Float::NAN is not jsonable, but hash string can output it.
+    d = create_driver(CONFIG + "\noutput_type hash")
+    out = capture_log(d) { emit(d, {'test' => Float::NAN}, time) }
+    assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
+  end
+
+  # Use include_time_key to output the message's time
+  def test_include_time_key
+    d = create_driver(CONFIG + "\noutput_type json\ninclude_time_key true\nutc")
+    time = Time.now
+    message_time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    out = capture_log(d) { emit(d, {'test' => 'test'}, message_time) }
+    assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
+  end
+
+  # out_stdout formatter itself can also be replaced
+  def test_format_json
+    d = create_driver(CONFIG + "\nformat json")
+    time = Time.now
+    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    assert_equal "{\"test\":\"test\"}\n", out
+  end
+
+  private
+
+  # Capture the log output of the block given
+  def capture_log(d, &block)
+    tmp = d.instance.log
+    d.instance.log = StringIO.new
+    yield
+    return d.instance.log.string
+  ensure
+    d.instance.log = tmp
+  end
+end
+

data/test/plugin/test_in_tail.rb

@@ -40,6 +40,7 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal "t1", d.instance.tag
     assert_equal 2, d.instance.rotate_wait
     assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
+    assert_equal 1000, d.instance.read_lines_limit
   end
 
   # TODO: Should using more better approach instead of sleep wait
@@ -66,6 +67,30 @@ class TailInputTest < Test::Unit::TestCase
     assert_equal(true, emits.length > 0)
     assert_equal({"message" => "test3"}, emits[0][2])
     assert_equal({"message" => "test4"}, emits[1][2])
+    assert_equal(1, d.emit_streams.size)
+  end
+
+  data('1' => [1, 2], '10' => [10, 1])
+  def test_emit_with_read_lines_limit(data)
+    limit, num_emits = data
+    d = create_driver(CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + "read_lines_limit #{limit}")
+    msg = 'test' * 500 # in_tail reads 2048 bytes at once.
+
+    d.run do
+      sleep 1
+
+      File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+        f.puts msg
+        f.puts msg
+      }
+      sleep 1
+    end
+
+    emits = d.emits
+    assert_equal(true, emits.length > 0)
+    assert_equal({"message" => msg}, emits[0][2])
+    assert_equal({"message" => msg}, emits[1][2])
+    assert_equal(num_emits, d.emit_streams.size)
   end
 
   def test_emit_with_read_from_head
@@ -409,7 +434,7 @@ class TailInputTest < Test::Unit::TestCase
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
         EX_PATHS.each do |path|
-          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, any, any).once.and_return do
+          watcherclass.should_receive(:new).with(path, EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
             flexmock('TailWatcher') { |watcher|
               watcher.should_receive(:attach).once
               watcher.should_receive(:unwatched=).zero_or_more_times
@@ -425,7 +450,7 @@ class TailInputTest < Test::Unit::TestCase
       end
 
       flexstub(Fluent::NewTailInput::TailWatcher) do |watcherclass|
-        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, any, any).once.and_return do
+        watcherclass.should_receive(:new).with('test/plugin/data/2010/01/20100102-030406.log', EX_RORATE_WAIT, Fluent::NewTailInput::FilePositionEntry, any, true, 1000, any, any).once.and_return do
           flexmock('TailWatcher') do |watcher|
             watcher.should_receive(:attach).once
             watcher.should_receive(:unwatched=).zero_or_more_times

data/test/plugin/test_out_file.rb

@@ -45,8 +45,16 @@ class FileOutputTest < Test::Unit::TestCase
       create_driver %[path #{TMP_DIR}/test_path]
     end
 
+    assert_nothing_raised do
+      FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
+      File.chmod(0777, "#{TMP_DIR}/test_dir")
+      create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
+    end
+
     assert_raise(Fluent::ConfigError) do
-      create_driver %[path #{TMP_DIR}/does_not_exist/test_path]
+      FileUtils.mkdir_p("#{TMP_DIR}/test_dir")
+      File.chmod(0555, "#{TMP_DIR}/test_dir")
+      create_driver %[path #{TMP_DIR}/test_dir/foo/bar/baz]
     end
   end
 

data/test/test_parser.rb

@@ -131,6 +131,31 @@ module ParserTest
         assert_nil time, "parser returns nil if configured so"
       }
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::RegexpParser.new(
+        Regexp.new(%q!(?<time>.*)!),
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = '28/Feb/2013:12:00:00 +0900'
+      parser.parse(text) do |time, record|
+        assert_equal text, record['time']
+      end
+    end
+
+    def test_parse_with_keep_time_key_with_typecast
+      parser = TextParser::RegexpParser.new(
+        Regexp.new(%q!(?<time>.*)!),
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+        'types'=>'time:time:%d/%b/%Y:%H:%M:%S %z',
+      )
+      text = '28/Feb/2013:12:00:00 +0900'
+      parser.parse(text) do |time, record|
+        assert_equal 1362020400, record['time']
+      end
+    end
   end
 
   class ApacheParserTest < ::Test::Unit::TestCase
@@ -155,6 +180,18 @@ module ParserTest
         }, record)
       }
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::ApacheParser.new
+      parser.configure(
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777'
+      parser.parse(text) do |time, record|
+        assert_equal "28/Feb/2013:12:00:00 +0900", record['time']
+      end
+    end
   end
 
   class ApacheErrorParserTest < ::Test::Unit::TestCase
@@ -279,6 +316,17 @@ module ParserTest
       assert_equal(TextParser::SyslogParser::REGEXP, @parser.patterns['format'])
       assert_equal("%b %d %H:%M:%S", @parser.patterns['time_format'])
     end
+
+    def test_parse_with_keep_time_key
+      @parser.configure(
+        'time_format' => '%b %d %M:%S:%H',
+        'keep_time_key'=>'true',
+      )
+      text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
+      @parser.parse(text) do |time, record|
+        assert_equal "Feb 28 00:00:12", record['time']
+      end
+    end
   end
 
   class JsonParserTest < ::Test::Unit::TestCase
@@ -329,6 +377,18 @@ module ParserTest
         @parser.parse('{"time":[],"k":"v"}') { |time, record| }
       end
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::JSONParser.new
+      parser.configure(
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = "28/Feb/2013:12:00:00 +0900"
+      parser.parse("{\"time\":\"#{text}\"}") do |time, record|
+        assert_equal text, record['time']
+      end
+    end
   end
 
   class NginxParserTest < ::Test::Unit::TestCase
@@ -374,22 +434,25 @@ module ParserTest
   class TSVParserTest < ::Test::Unit::TestCase
     include ParserTest
 
-    def test_config_params
+    data('array param' => '["a","b"]', 'string param' => 'a,b')
+    def test_config_params(param)
       parser = TextParser::TSVParser.new
 
       assert_equal "\t", parser.delimiter
 
       parser.configure(
-        'keys' => 'a,b',
+        'keys' => param,
         'delimiter' => ',',
       )
 
+      assert_equal ['a', 'b'], parser.keys
       assert_equal ",", parser.delimiter
     end
 
-    def test_parse
+    data('array param' => '["time","a","b"]', 'string param' => 'time,a,b')
+    def test_parse(param)
       parser = TextParser::TSVParser.new
-      parser.configure('keys' => 'time,a,b', 'time_key' => 'time')
+      parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
         assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
@@ -440,14 +503,29 @@ module ParserTest
         assert_equal(expected, record)
       }
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::TSVParser.new
+      parser.configure(
+        'keys'=>'time',
+        'time_key'=>'time',
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = '28/Feb/2013:12:00:00 +0900'
+      parser.parse(text) do |time, record|
+        assert_equal text, record['time']
+      end
+    end
   end
 
   class CSVParserTest < ::Test::Unit::TestCase
     include ParserTest
 
-    def test_parse
+    data('array param' => '["time","c","d"]', 'string param' => 'time,c,d')
+    def test_parse(param)
       parser = TextParser::CSVParser.new
-      parser.configure('keys' => 'time,c,d', 'time_key' => 'time')
+      parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
         assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
@@ -457,11 +535,12 @@ module ParserTest
       }
     end
 
-    def test_parse_without_time
+    data('array param' => '["c","d"]', 'string param' => 'c,d')
+    def test_parse_without_time(param)
       time_at_start = Time.now.to_i
 
       parser = TextParser::CSVParser.new
-      parser.configure('keys' => 'c,d')
+      parser.configure('keys' => param)
       parser.parse("192.168.0.1,111") { |time, record|
         assert time && time >= time_at_start, "parser puts current time without time input"
         assert_equal({
@@ -472,7 +551,7 @@ module ParserTest
 
       parser = TextParser::CSVParser.new
       parser.estimate_current_event = false
-      parser.configure('keys' => 'c,d', 'time_key' => 'time')
+      parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("192.168.0.1,111") { |time, record|
         assert_equal({
           'c' => '192.168.0.1',
@@ -481,6 +560,20 @@ module ParserTest
         assert_nil time, "parser returns nil w/o time and if configured so"
       }
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::CSVParser.new
+      parser.configure(
+        'keys'=>'time',
+        'time_key'=>'time',
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = '28/Feb/2013:12:00:00 +0900'
+      parser.parse(text) do |time, record|
+        assert_equal text, record['time']
+      end
+    end
   end
 
   class LabeledTSVParserTest < ::Test::Unit::TestCase
@@ -567,7 +660,19 @@ module ParserTest
         assert_nil time, "parser returns nil w/o time and if configured so"
       }
     end
- end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::LabeledTSVParser.new
+      parser.configure(
+        'time_format'=>"%d/%b/%Y:%H:%M:%S %z",
+        'keep_time_key'=>'true',
+      )
+      text = '28/Feb/2013:12:00:00 +0900'
+      parser.parse("time:#{text}") do |time, record|
+        assert_equal text, record['time']
+      end
+    end
+  end
 
   class NoneParserTest < ::Test::Unit::TestCase
     include ParserTest
@@ -700,6 +805,18 @@ EOS
         }, record)
       }
     end
+
+    def test_parse_with_keep_time_key
+      parser = TextParser::MultilineParser.new
+      parser.configure(
+        'format1' => '/^(?<time>\d{4}-\d{1,2}-\d{1,2} \d{1,2}:\d{1,2}:\d{1,2})/',
+        'keep_time_key' => 'true'
+      )
+      text = '2013-3-03 14:27:33'
+      parser.parse(text) { |time, record|
+        assert_equal text, record['time']
+      }
+    end
   end
 
   class TextParserTest < ::Test::Unit::TestCase

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.12.8
+  version: 0.12.9
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-05-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -291,6 +291,7 @@ files:
 - bin/fluent-debug
 - bin/fluent-gem
 - bin/fluentd
+- example/filter_stdout.conf
 - example/in_http.conf
 - example/in_syslog.conf
 - example/in_tail.conf
@@ -342,6 +343,7 @@ files:
 - lib/fluent/plugin/file_util.rb
 - lib/fluent/plugin/filter_grep.rb
 - lib/fluent/plugin/filter_record_transformer.rb
+- lib/fluent/plugin/filter_stdout.rb
 - lib/fluent/plugin/in_debug_agent.rb
 - lib/fluent/plugin/in_dummy.rb
 - lib/fluent/plugin/in_exec.rb
@@ -370,6 +372,7 @@ files:
 - lib/fluent/process.rb
 - lib/fluent/registry.rb
 - lib/fluent/root_agent.rb
+- lib/fluent/rpc.rb
 - lib/fluent/status.rb
 - lib/fluent/supervisor.rb
 - lib/fluent/test.rb
@@ -400,6 +403,7 @@ files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_record_transformer.rb
+- test/plugin/test_filter_stdout.rb
 - test/plugin/test_in_debug_agent.rb
 - test/plugin/test_in_dummy.rb
 - test/plugin/test_in_exec.rb
@@ -484,6 +488,7 @@ test_files:
 - test/plugin/test_file_util.rb
 - test/plugin/test_filter_grep.rb
 - test/plugin/test_filter_record_transformer.rb
+- test/plugin/test_filter_stdout.rb
 - test/plugin/test_in_debug_agent.rb
 - test/plugin/test_in_dummy.rb
 - test/plugin/test_in_exec.rb