fluentd 0.10.48 → 0.10.49

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b08526fa1f26ff8869f1ddd41a411b856923004
-  data.tar.gz: 9ab2b8a889c2706be4d886a346d587a83f72f461
+  metadata.gz: 9d9ea1a003d545a18baf0c61a85fb216d183cb07
+  data.tar.gz: c3ccc3a4665acdf5726287ce6b29e2e6af5c5194
 SHA512:
-  metadata.gz: d56f16dfb2a401b961ec0f8761efcd677e1fb6f4b38e7b2d903e35747cded12adbf1fbc213276e42cef1b4a4ef43e128fe772849f152bf319ea30ad320184a38
-  data.tar.gz: 7ff524422dbee95283c98ab6e7414bffda324883ca1a582eb8f7cd74a0dbfdf7066138f37cc9eb16b1160c8df2f478f20140c41247876eb5894bb3120dcc59e6
+  metadata.gz: cd8c5f25fc9ca7cf31ac5d0fb2de908cee79204deb11e821e31774b8aecc112557ebcc0f0d2ddc033fcb4b189e31b05ff9fbe1a07d04ad4630a2b39980038f8b
+  data.tar.gz: af93420f7ed26fdf703d11723aa27e76a43528b90b67879687e8bb5fa94bb9bba55fbad61dbeaa10dc2218de776cc39df3e13372204a50359008a4abcebdf6f3

data/ChangeLog

@@ -1,3 +1,15 @@
+Release 0.10.49 - 2014/06/05
+
+* in_http: Add format option to support various input data format
+* in_http: Accept json / msgpack array in default format
+* in_tail: Print warning message when file not exist with 'read_from_head true'
+* out_file: Add append option to disable path increment
+* out_file: Add format option to support various output data format
+* config: Fix broken 'include' processing in V1 configuration. Now add @include
+* fluentd-debug: Fix undefined method 'usage' error when invalid option passed
+* supervisor: Fix incorrect --group option handling
+* Add TextFormatter module for output plugins
+
 Release 0.10.48 - 2014/05/18
 
 * config: Add inspect method to Section for dumping the status

data/lib/fluent/command/debug.rb

@@ -36,6 +36,14 @@ op.on('-u', '--unix PATH', "use unix socket instead of tcp") {|b|
   unix = b
 }
 
+(class<<self;self;end).module_eval do
+  define_method(:usage) do |msg|
+    puts op.to_s
+    puts "error: #{msg}" if msg
+    exit 1
+  end
+end
+
 begin
   op.parse!(ARGV)
 

data/lib/fluent/config/v1_parser.rb

@@ -94,10 +94,11 @@ module Fluent
             e_attrs, e_elems = parse_element(false, e_name)
             elems << Element.new(e_name, e_arg, e_attrs, e_elems)
 
-          elsif root_element && skip(/\@include#{SPACING}/)
-            uri = scan_string(LINE_END)
-            eval_include(attrs, elems, uri)
-            line_end
+          elsif root_element && skip(/(\@include|include)#{SPACING}/)
+            if !prev_match.start_with?('@')
+              $log.warn "'include' is deprecated. Use '@include' instead"
+            end
+            parse_include(attrs, elems)
 
           else
             k = scan_string(SPACING)
@@ -105,11 +106,15 @@ module Fluent
             if prev_match.include?("\n") # support 'tag_mapped' like "without value" configuration
               attrs[k] = ""
             else
-              v = parse_literal
-              unless line_end
-                parse_error! "expected end of line"
+              if k == '@include'
+                parse_include(attrs, elems)
+              else
+                v = parse_literal
+                unless line_end
+                  parse_error! "expected end of line"
+                end
+                attrs[k] = v
               end
-              attrs[k] = v
             end
           end
         end
@@ -117,6 +122,12 @@ module Fluent
         return attrs, elems
       end
 
+      def parse_include(attrs, elems)
+        uri = scan_string(LINE_END)
+        eval_include(attrs, elems, uri)
+        line_end
+      end
+
       def eval_include(attrs, elems, uri)
         u = URI.parse(uri)
         if u.scheme == 'file' || u.path == uri  # file path
@@ -127,12 +138,12 @@ module Fluent
             pattern = path
           end
 
-          Dir.glob(pattern).each { |path|
+          Dir.glob(pattern).sort.each { |path|
             basepath = File.dirname(path)
             fname = File.basename(path)
             data = File.read(path)
             ss = StringScanner.new(data)
-            V1Parser.new(ss, basepath, fname, @eval_context).parse(true, nil, attrs, elems)
+            V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
           }
 
         else
@@ -140,9 +151,9 @@ module Fluent
           fname = path
           require 'open-uri'
           data = nil
-          open(uri) { |f| read = f.read }
+          open(uri) { |f| data = f.read }
           ss = StringScanner.new(data)
-          V1Parser.new(ss, basepath, fname, @eval_context).parse(true, nil, attrs, elems)
+          V1Parser.new(ss, basepath, fname, @eval_context).parse_element(true, nil, attrs, elems)
         end
 
       rescue SystemCallError => e

data/lib/fluent/formatter.rb

@@ -0,0 +1,173 @@
+#
+# Fluent
+#
+# Copyright (C) 2014 Fluentd project
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+  require 'fluent/registry'
+
+  module TextFormatter
+    module HandleTagAndTimeMixin
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :include_time_key, :bool, :default => false
+          config_param :time_key, :string, :default => 'time'
+          config_param :time_format, :string, :default => nil
+          config_param :include_tag_key, :bool, :default => false
+          config_param :tag_key, :string, :default => 'tag'
+        }
+      end
+
+      def configure(conf)
+        super
+
+        if conf['utc']
+          @localtime = false
+        elsif conf['localtime']
+          @localtime = true
+        end
+        @timef = TimeFormatter.new(@time_format, @localtime)
+      end
+
+      def filter_record(tag, time, record)
+        if @include_tag_key
+          record[@tag_key] = tag
+        end
+        if @include_time_key
+          record[@time_key] = @timef.format(time)
+        end
+      end
+    end
+
+    class OutFileFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+
+      config_param :output_time, :bool, :default => true
+      config_param :output_tag, :bool, :default => true
+      config_param :delimiter, :default => "\t" do |val|
+        case val
+        when /SPACE/i then ' '
+        when /COMMA/i then ','
+        else "\t"
+        end
+      end
+
+      def configure(conf)
+        super
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        header = ''
+        header << "#{@timef.format(time)}#{@delimiter}" if @output_time
+        header << "#{tag}#{@delimiter}" if @output_tag
+        "#{header}#{Yajl.dump(record)}\n"
+      end
+    end
+
+    class JSONFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+
+      config_param :time_as_epoch, :bool, :default => false
+
+      def configure(conf)
+        super
+
+        if @time_as_epoch
+          if @include_time_key
+            @include_time_key = false
+          else
+            $log.warn "include_time_key is false so ignore time_as_epoch"
+            @time_as_epoch = false
+          end
+        end
+      end
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        record[@time_key] = time if @time_as_epoch
+        "#{Yajl.dump(record)}\n"
+      end
+    end
+
+    class LabeledTSVFormatter
+      include Configurable
+      include HandleTagAndTimeMixin
+
+      config_param :delimiter, :string, :default => "\t"
+      config_param :label_delimiter, :string, :default =>  ":"
+
+      def format(tag, time, record)
+        filter_record(tag, time, record)
+        formatted = record.inject('') { |result, pair|
+          result << @delimiter if result.length.nonzero?
+          result << "#{pair.first}#{@label_delimiter}#{pair.last}"
+        }
+        formatted << "\n"
+        formatted
+      end
+    end
+
+    class SingleValueFormatter
+      include Configurable
+
+      config_param :message_key, :string, :default => 'message'
+
+      def format(tag, time, record)
+        record[@message_key]
+      end
+    end
+
+    TEMPLATE_REGISTRY = Registry.new(:formatter_type, 'fluent/plugin/formatter_')
+    {
+      'out_file' => Proc.new { OutFileFormatter.new },
+      'json' => Proc.new { JSONFormatter.new },
+      'ltsv' => Proc.new { LabeledTSVFormatter.new },
+      'single_value' => Proc.new { SingleValueFormatter.new },
+    }.each { |name, factory|
+      TEMPLATE_REGISTRY.register(name, factory)
+    }
+
+    def self.register_template(name, factory_or_proc)
+      factory = if factory_or_proc.arity == 3
+                  Proc.new { factory_or_proc }
+                else
+                  factory_or_proc
+                end
+
+      TEMPLATE_REGISTRY.register(name, factory)
+    end
+
+    def self.create(conf)
+      format = conf['format']
+      if format.nil?
+        raise ConfigError, "'format' parameter is required"
+      end
+
+      # built-in template
+      begin
+        factory = TEMPLATE_REGISTRY.lookup(format)
+      rescue ConfigError => e
+        raise ConfigError, "unknown format: '#{format}'"
+      end
+
+      formatter = factory.call
+      formatter.configure(conf)
+      formatter
+    end
+  end
+end

data/lib/fluent/load.rb

@@ -28,6 +28,7 @@ require 'fluent/mixin'
 require 'fluent/process'
 require 'fluent/plugin'
 require 'fluent/parser'
+require 'fluent/formatter'
 require 'fluent/event'
 require 'fluent/buffer'
 require 'fluent/input'

data/lib/fluent/plugin/in_http.rb

@@ -35,9 +35,22 @@ module Fluent
     config_param :keepalive_timeout, :time, :default => 10   # TODO default
     config_param :backlog, :integer, :default => nil
     config_param :add_http_headers, :bool, :default => false
+    config_param :format, :string, :default => 'default'
 
     def configure(conf)
       super
+
+      m = if @format == 'default'
+            method(:parse_params_default)
+          else
+            parser = TextParser.new
+            parser.configure(conf)
+            @parser = parser
+            method(:parse_params_with_parser)
+          end
+      (class << self; self; end).module_eval do
+        define_method(:parse_params, m)
+      end
     end
 
     class KeepaliveManager < Coolio::TimerWatcher
@@ -79,7 +92,7 @@ module Fluent
         super
         @km = KeepaliveManager.new(@keepalive_timeout)
         #@lsock = Coolio::TCPServer.new(@bind, @port, Handler, @km, method(:on_request), @body_size_limit)
-        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, log)
+        @lsock = Coolio::TCPServer.new(lsock, nil, Handler, @km, method(:on_request), @body_size_limit, @format, log)
         @lsock.listen(@backlog) unless @backlog.nil?
 
         @loop = Coolio::Loop.new
@@ -108,22 +121,13 @@ module Fluent
       begin
         path = path_info[1..-1]  # remove /
         tag = path.split('/').join('.')
-
-        if msgpack = params['msgpack']
-          record = MessagePack.unpack(msgpack)
-
-        elsif js = params['json']
-          record = JSON.parse(js)
-
-        else
-          raise "'json' or 'msgpack' parameter is required"
-        end
+        record_time, record = parse_params(params)
 
         # Skip nil record
         if record.nil?
           return ["200 OK", {'Content-type'=>'text/plain'}, ""]
         end
-        
+
         if @add_http_headers
           params.each_pair { |k,v|
             if k.start_with?("HTTP_")
@@ -132,19 +136,29 @@ module Fluent
           }
         end
 
-        time = params['time']
-        time = time.to_i
-        if time == 0
-          time = Engine.now
-        end
-
+        time = if param_time = params['time']
+                 param_time = param_time.to_i
+                 param_time.zero? ? Engine.now : param_time
+               else
+                 record_time.nil? ? Engine.now : record_time
+               end
       rescue
         return ["400 Bad Request", {'Content-type'=>'text/plain'}, "400 Bad Request\n#{$!}\n"]
       end
 
       # TODO server error
       begin
-        Engine.emit(tag, time, record)
+        # Support batched requests
+        if record.is_a?(Array)           
+          mes = MultiEventStream.new
+          record.each do |single_record|
+            single_time = single_record.delete("time") || time 
+            mes.add(single_time, single_record)
+          end
+          Engine.emit_stream(tag, mes)
+	else
+          Engine.emit(tag, time, record)
+        end
       rescue
         return ["500 Internal Server Error", {'Content-type'=>'text/plain'}, "500 Internal Server Error\n#{$!}\n"]
       end
@@ -152,14 +166,40 @@ module Fluent
       return ["200 OK", {'Content-type'=>'text/plain'}, ""]
     end
 
+    private
+
+    def parse_params_default(params)
+      record = if msgpack = params['msgpack']
+                 MessagePack.unpack(msgpack)
+               elsif js = params['json']
+                 JSON.parse(js)
+               else
+                 raise "'json' or 'msgpack' parameter is required"
+               end
+      return nil, record
+    end
+
+    EVENT_RECORD_PARAMETER = '_event_record'
+
+    def parse_params_with_parser(params)
+      if content = params[EVENT_RECORD_PARAMETER]
+        time, record = @parser.parse(content)
+        raise "Received event is not #{@format}: #{content}" if record.nil?
+        return time, record
+      else
+        raise "'#{EVENT_RECORD_PARAMETER}' parameter is required"
+      end
+    end
+
     class Handler < Coolio::Socket
-      def initialize(io, km, callback, body_size_limit, log)
+      def initialize(io, km, callback, body_size_limit, format, log)
         super(io)
         @km = km
         @callback = callback
         @body_size_limit = body_size_limit
         @content_type = ""
         @next_close = false
+        @format = format
         @log = log
 
         @idle = 0
@@ -250,7 +290,9 @@ module Fluent
         uri = URI.parse(@parser.request_url)
         params = WEBrick::HTTPUtils.parse_query(uri.query)
 
-        if @content_type =~ /^application\/x-www-form-urlencoded/
+        if @format != 'default'
+          params[EVENT_RECORD_PARAMETER] = @body
+        elsif @content_type =~ /^application\/x-www-form-urlencoded/
           params.update WEBrick::HTTPUtils.parse_query(@body)
         elsif @content_type =~ /^multipart\/form-data; boundary=(.+)/
           boundary = WEBrick::HTTPUtils.dequote($1)

data/lib/fluent/plugin/in_tail.rb

@@ -140,7 +140,11 @@ module Fluent
         if @pf
           pe = @pf[path]
           if @read_from_head && pe.read_inode.zero?
-            pe.update(File::Stat.new(path).ino, 0)
+            begin
+              pe.update(File::Stat.new(path).ino, 0)
+            rescue Errno::ENOENT
+              $log.warn "#{path} not found. Continuing without tailing it."
+            end
           end
         end
 

data/lib/fluent/plugin/out_file.rb

@@ -25,9 +25,8 @@ module Fluent
     }
 
     config_param :path, :string
-
-    config_param :time_format, :string, :default => nil
-
+    config_param :format, :string, :default => 'out_file'
+    config_param :append, :bool, :default => false
     config_param :compress, :default => nil do |val|
       c = SUPPORTED_COMPRESS[val]
       unless c
@@ -35,7 +34,6 @@ module Fluent
       end
       c
     end
-
     config_param :symlink_path, :string, :default => nil
 
     def initialize
@@ -64,29 +62,18 @@ module Fluent
 
       super
 
-      @timef = TimeFormatter.new(@time_format, @localtime)
+      conf['format'] = @format
+      @formatter = TextFormatter.create(conf)
 
       @buffer.symlink_path = @symlink_path if @symlink_path
     end
 
     def format(tag, time, record)
-      time_str = @timef.format(time)
-      "#{time_str}\t#{tag}\t#{Yajl.dump(record)}\n"
+      @formatter.format(tag, time, record)
     end
 
     def write(chunk)
-      case @compress
-      when nil
-        suffix = ''
-      when :gz
-        suffix = ".gz"
-      end
-
-      i = 0
-      begin
-        path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
-        i += 1
-      end while File.exist?(path)
+      path = generate_path(chunk)
       FileUtils.mkdir_p File.dirname(path)
 
       case @compress
@@ -95,8 +82,10 @@ module Fluent
           chunk.write_to(f)
         }
       when :gz
-        Zlib::GzipWriter.open(path) {|f|
-          chunk.write_to(f)
+        File.open(path, "a", DEFAULT_FILE_PERMISSION) {|f|
+          gz = Zlib::GzipWriter.new(f)
+          chunk.write_to(gz)
+          gz.close
         }
       end
 
@@ -106,5 +95,28 @@ module Fluent
     def secondary_init(primary)
       # don't warn even if primary.class is not FileOutput
     end
+
+    private
+
+    def generate_path(chunk)
+      case @compress
+      when nil
+        suffix = ''
+      when :gz
+        suffix = ".gz"
+      end
+
+      if @append
+        "#{@path_prefix}#{chunk.key}#{@path_suffix}#{suffix}"
+      else
+        path = nil
+        i = 0
+        begin
+          path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
+          i += 1
+        end while File.exist?(path)
+        path
+      end
+    end
   end
 end

data/lib/fluent/supervisor.rb

@@ -18,9 +18,26 @@
 
 require 'fluent/env'
 require 'fluent/log'
+require 'etc'
 
 module Fluent
   class Supervisor
+    def self.get_etc_passwd(user)
+      if user.to_i.to_s == user
+        Etc.getpwuid(user.to_i)
+      else
+        Etc.getpwnam(user)
+      end
+    end
+
+    def self.get_etc_group(group)
+      if group.to_i.to_s == group
+        Etc.getgrgid(group.to_i)
+      else
+        Etc.getgrnam(group)
+      end
+    end
+
     class LoggerInitializer
       def initialize(path, level, chuser, chgroup, opts)
         @path = path
@@ -34,8 +51,8 @@ module Fluent
         if @path && @path != "-"
           @io = File.open(@path, "a")
           if @chuser || @chgroup
-            chuid = @chuser ? `id -u #{@chuser}`.to_i : nil
-            chgid = @chgroup ? `id -g #{@chgroup}`.to_i : nil
+            chuid = @chuser ? Supervisor.get_etc_passwd(@chuser).uid : nil
+            chgid = @chgroup ? Supervisor.get_etc_group(@chgroup).gid : nil
             File.chown(chuid, chgid, @path)
           end
         else
@@ -318,32 +335,18 @@ module Fluent
 
     def change_privilege
       if @chgroup
-        chgid = @chgroup.to_i
-        if chgid.to_s != @chgroup
-          chgid = `id -g #{@chgroup}`.to_i
-          if $?.to_i != 0
-            exit 1
-          end
-        end
-        Process::GID.change_privilege(chgid)
+        etc_group = Supervisor.get_etc_group(@chgroup)
+        Process::GID.change_privilege(etc_group.gid)
       end
 
       if @chuser
-        chuid = @chuser.to_i
-        if chuid.to_s != @chuser
-          chuid = `id -u #{@chuser}`.to_i
-          if $?.to_i != 0
-            exit 1
-          end
-        end
-
-        user_groups = `id -G #{@chuser}`.split.map(&:to_i)
-        if $?.to_i != 0
-          exit 1
-        end
+        etc_pw = Supervisor.get_etc_passwd(@chuser)
+        user_groups = [etc_pw.gid]
+        Etc.setgrent
+        Etc.group { |gr| user_groups << gr.gid if gr.mem.include?(etc_pw.name) } # emulate 'id -G'
 
         Process.groups = Process.groups | user_groups
-        Process::UID.change_privilege(chuid)
+        Process::UID.change_privilege(etc_pw.uid)
       end
     end
 

data/lib/fluent/version.rb

@@ -1,5 +1,5 @@
 module Fluent
 
-  VERSION = '0.10.48'
+  VERSION = '0.10.49'
 
 end

data/spec/config/config_parser_spec.rb

@@ -177,7 +177,105 @@ describe Fluent::Config::V1Parser do
     end
   end
 
+  # port from test_config.rb
   describe '@include parsing' do
-    # TODO
+    TMP_DIR = File.dirname(__FILE__) + "/tmp/v1_config#{ENV['TEST_ENV_NUMBER']}"
+
+    def write_config(path, data)
+      FileUtils.mkdir_p(File.dirname(path))
+      File.open(path, "w") { |f| f.write data }
+    end
+
+    def prepare_config
+      write_config "#{TMP_DIR}/config_test_1.conf", %[
+        k1 root_config
+        include dir/config_test_2.conf  #
+        @include #{TMP_DIR}/config_test_4.conf
+        include file://#{TMP_DIR}/config_test_5.conf
+        @include config.d/*.conf
+      ]
+      write_config "#{TMP_DIR}/dir/config_test_2.conf", %[
+        k2 relative_path_include
+        @include ../config_test_3.conf
+      ]
+      write_config "#{TMP_DIR}/config_test_3.conf", %[
+        k3 relative_include_in_included_file
+      ]
+      write_config "#{TMP_DIR}/config_test_4.conf", %[
+        k4 absolute_path_include
+      ]
+      write_config "#{TMP_DIR}/config_test_5.conf", %[
+        k5 uri_include
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_6.conf", %[
+        k6 wildcard_include_1
+        <elem1 name>
+          include normal_parameter
+        </elem1>
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_7.conf", %[
+        k7 wildcard_include_2
+      ]
+      write_config "#{TMP_DIR}/config.d/config_test_8.conf", %[
+        <elem2 name>
+          @include ../dir/config_test_9.conf
+        </elem2>
+      ]
+      write_config "#{TMP_DIR}/dir/config_test_9.conf", %[
+        k9 embeded
+        <elem3 name>
+          nested nested_value
+          include hoge
+        </elem3>
+      ]
+      write_config "#{TMP_DIR}/config.d/00_config_test_8.conf", %[
+        k8 wildcard_include_3
+        <elem4 name>
+          include normal_parameter
+        </elem4>
+      ]
+    end
+
+    it 'parses @include / include correctly' do
+      prepare_config
+      c = Fluent::Config.read("#{TMP_DIR}/config_test_1.conf", true)
+      expect(c['k1']).to eq('root_config')
+      expect(c['k2']).to eq('relative_path_include')
+      expect(c['k3']).to eq('relative_include_in_included_file')
+      expect(c['k4']).to eq('absolute_path_include')
+      expect(c['k5']).to eq('uri_include')
+      expect(c['k6']).to eq('wildcard_include_1')
+      expect(c['k7']).to eq('wildcard_include_2')
+      expect(c['k8']).to eq('wildcard_include_3')
+      expect(c.keys).to eq([
+        'k1',
+        'k2',
+        'k3',
+        'k4',
+        'k5',
+        'k8', # Because of the file name this comes first.
+        'k6',
+        'k7',
+      ])
+
+      elem1 = c.elements.find { |e| e.name == 'elem1' }
+      expect(elem1).to be
+      expect(elem1.arg).to eq('name')
+      expect(elem1['include']).to eq('normal_parameter')
+
+      elem2 = c.elements.find { |e| e.name == 'elem2' }
+      expect(elem2).to be
+      expect(elem2.arg).to eq('name')
+      expect(elem2['k9']).to eq('embeded')
+      expect(elem2.has_key?('include')).to be(false)
+
+      elem3 = elem2.elements.find { |e| e.name == 'elem3' }
+      expect(elem3).to be
+      expect(elem3['nested']).to eq('nested_value')
+      expect(elem3['include']).to eq('hoge')
+    end
+
+    # TODO: Add uri based include spec
   end
 end
+

data/test/plugin/test_in_http.rb

@@ -65,6 +65,25 @@ class HttpInputTest < Test::Unit::TestCase
     }
   end
 
+  def test_multi_json
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    events.each { |ev|
+      d.expect_emit tag, time, ev
+    }
+
+    d.run do
+      res = post("/#{tag}", {"json"=>events.to_json, "time"=>time.to_s})
+      assert_equal "200", res.code
+    end
+
+  end
+
   def test_json_with_add_http_headers
     d = create_driver(CONFIG + "add_http_headers true")
 
@@ -95,10 +114,7 @@ class HttpInputTest < Test::Unit::TestCase
 
     d.run do
       d.expected_emits.each {|tag,time,record|
-        http = Net::HTTP.new("127.0.0.1", PORT)
-        req = Net::HTTP::Post.new("/#{tag}?time=#{time.to_s}", {"content-type"=>"application/json; charset=utf-8"})
-        req.body = record.to_json
-        res = http.request(req)
+        res = post("/#{tag}?time=#{time.to_s}", record.to_json, {"content-type"=>"application/json; charset=utf-8"})
         assert_equal "200", res.code
       }
     end
@@ -120,10 +136,77 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
-  def post(path, params)
+  def test_multi_msgpack
+    d = create_driver
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    events = [{"a"=>1},{"a"=>2}]
+    tag = "tag1"
+
+    events.each { |ev|
+      d.expect_emit tag, time, ev
+    }
+
+    d.run do
+      res = post("/#{tag}", {"msgpack"=>events.to_msgpack, "time"=>time.to_s})
+      assert_equal "200", res.code
+    end
+
+  end
+
+  def test_with_regexp
+    d = create_driver(CONFIG + %[
+      format /^(?<field_1>\\d+):(?<field_2>\\w+)$/
+      types field_1:integer
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"field_1" => 1, "field_2" => 'str'}
+    d.expect_emit "tag2", time, {"field_1" => 2, "field_2" => 'str'}
+
+    d.run do
+      d.expected_emits.each { |tag, time, record|
+        body = record.map { |k, v|
+          v.to_s
+        }.join(':')
+        res = post("/#{tag}?time=#{time.to_s}", body)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def test_with_csv
+    require 'csv'
+
+    d = create_driver(CONFIG + %[
+      format csv
+      keys foo,bar
+    ])
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.expect_emit "tag1", time, {"foo" => "1", "bar" => 'st"r'}
+    d.expect_emit "tag2", time, {"foo" => "2", "bar" => 'str'}
+
+    d.run do
+      d.expected_emits.each { |tag, time, record|
+        body = record.map { |k, v| v }.to_csv
+        res = post("/#{tag}?time=#{time.to_s}", body)
+        assert_equal "200", res.code
+      }
+    end
+  end
+
+  def post(path, params, header = {})
     http = Net::HTTP.new("127.0.0.1", PORT)
-    req = Net::HTTP::Post.new(path, {})
-    req.set_form_data(params)
+    req = Net::HTTP::Post.new(path, header)
+    if params.is_a?(String)
+      req.body = params
+    else
+      req.set_form_data(params)
+    end
     http.request(req)
   end
 

data/test/plugin/test_in_tail.rb

@@ -337,4 +337,39 @@ class TailInputTest < Test::Unit::TestCase
       plugin.receive_lines(['foo', 'bar'], DummyWatcher.new('foo.bar.log'))
     end
   end
+
+  # Ensure that no fatal exception is raised when a file is missing and that
+  # files that do exist are still tailed as expected.
+  def test_missing_file
+    File.open("#{TMP_DIR}/tail.txt", "w") {|f|
+      f.puts "test1"
+      f.puts "test2"
+    }
+
+    # Try two different configs - one with read_from_head and one without,
+    # since their interactions with the filesystem differ.
+    config1 = %[
+      tag t1
+      path #{TMP_DIR}/non_existent_file.txt,#{TMP_DIR}/tail.txt
+      format none
+      rotate_wait 2s
+      pos_file #{TMP_DIR}/tail.pos
+    ]
+    config2 = config1 + '  read_from_head true'
+    [config1, config2].each do |config|
+      d = create_driver(config, false)
+      d.run do
+        sleep 1
+        File.open("#{TMP_DIR}/tail.txt", "a") {|f|
+          f.puts "test3"
+          f.puts "test4"
+        }
+        sleep 1
+      end
+      emits = d.emits
+      assert_equal(2, emits.length)
+      assert_equal({"message"=>"test3"}, emits[0][2])
+      assert_equal({"message"=>"test4"}, emits[1][2])
+    end
+  end
 end

data/test/plugin/test_out_file.rb

@@ -44,6 +44,24 @@ class FileOutputTest < Test::Unit::TestCase
     d.run
   end
 
+  def check_gzipped_result(path, expect)
+    # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
+    # Following code from https://www.ruby-forum.com/topic/971591#979520
+    result = ''
+    File.open(path) { |io|
+      loop do
+        gzr = Zlib::GzipReader.new(io)
+        result << gzr.read
+        unused = gzr.unused
+        gzr.finish
+        break if unused.nil?
+        io.pos -= unused.length
+      end
+    }
+
+    assert_equal expect, result
+  end
+
   def test_write
     d = create_driver
 
@@ -56,10 +74,19 @@ class FileOutputTest < Test::Unit::TestCase
     expect_path = "#{TMP_DIR}/out_file_test._0.log.gz"
     assert_equal expect_path, path
 
-    data = Zlib::GzipReader.open(expect_path) {|f| f.read }
-    assert_equal %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] +
-                    %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n],
-                 data
+    check_gzipped_result(path, %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+  end
+
+  def test_write_with_format_json
+    d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    # FileOutput#write returns path
+    path = d.run
+    check_gzipped_result(path, %[#{Yajl.dump({"a" => 1, 'time' => time})}\n] + %[#{Yajl.dump({"a" => 2, 'time' => time})}\n])
   end
 
   def test_write_path_increment
@@ -68,16 +95,43 @@ class FileOutputTest < Test::Unit::TestCase
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
     d.emit({"a"=>1}, time)
     d.emit({"a"=>2}, time)
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
 
     # FileOutput#write returns path
     path = d.run
     assert_equal "#{TMP_DIR}/out_file_test._0.log.gz", path
-
+    check_gzipped_result(path, formatted_lines)
     path = d.run
     assert_equal "#{TMP_DIR}/out_file_test._1.log.gz", path
-
+    check_gzipped_result(path, formatted_lines)
     path = d.run
     assert_equal "#{TMP_DIR}/out_file_test._2.log.gz", path
+    check_gzipped_result(path, formatted_lines)
+  end
+
+  def test_write_with_append
+    d = create_driver %[
+      path #{TMP_DIR}/out_file_test
+      compress gz
+      utc
+      append true
+    ]
+
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+    formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n]
+
+    # FileOutput#write returns path
+    path = d.run
+    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    check_gzipped_result(path, formatted_lines)
+    path = d.run
+    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    check_gzipped_result(path, formatted_lines * 2)
+    path = d.run
+    assert_equal "#{TMP_DIR}/out_file_test..log.gz", path
+    check_gzipped_result(path, formatted_lines * 3)
   end
 
   def test_write_with_symlink

data/test/scripts/fluent/plugin/formatter_known.rb

@@ -0,0 +1,5 @@
+module Fluent
+  TextFormatter.register_template('known', Proc.new { |tag, time, record|
+      "#{tag}:#{time}:#{record.size}"
+    })
+end

data/test/test_formatter.rb

@@ -0,0 +1,208 @@
+require 'helper'
+require 'fluent/test'
+require 'fluent/formatter'
+
+module FormatterTest
+  include Fluent
+
+  def time2str(time, localtime = false, format = nil)
+    if format
+      if localtime
+        Time.at(time).strftime(format)
+      else
+        Time.at(time).utc.strftime(format)
+      end
+    else
+      if localtime
+        Time.at(time).iso8601
+      else
+        Time.at(time).utc.iso8601
+      end
+    end
+  end
+
+  def tag
+    'tag'
+  end
+
+  def record
+    {'message' => 'awesome'}
+  end
+
+  class OutFileFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def setup
+      @formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('out_file').call
+      @time = Engine.now
+    end
+
+    def test_format
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("#{time2str(@time)}\t#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    end
+
+    def test_format_without_time
+      @formatter.configure('output_time' => 'false')
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("#{tag}\t#{Yajl.dump(record)}\n", formatted)
+    end
+
+    def test_format_without_tag
+      @formatter.configure('output_tag' => 'false')
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("#{time2str(@time)}\t#{Yajl.dump(record)}\n", formatted)
+    end
+
+    def test_format_without_time_and_tag
+      @formatter.configure('output_tag' => 'false', 'output_time' => 'false')
+      formatted = @formatter.format('tag', @time, record)
+
+      assert_equal("#{Yajl.dump(record)}\n", formatted)
+    end
+  end
+
+  class JsonFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def setup
+      @formatter = TextFormatter::JSONFormatter.new
+      @time = Engine.now
+    end
+
+    def test_format
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("#{Yajl.dump(record)}\n", formatted)
+    end
+
+    def test_format_with_include_tag
+      @formatter.configure('include_tag_key' => 'true', 'tag_key' => 'foo')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['foo'] = tag
+      assert_equal("#{Yajl.dump(r)}\n", formatted)
+    end
+
+    def test_format_with_include_time
+      @formatter.configure('include_time_key' => 'true', 'localtime' => '')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['time'] = time2str(@time, true)
+      assert_equal("#{Yajl.dump(r)}\n", formatted)
+    end
+
+    def test_format_with_include_time_as_number
+      @formatter.configure('include_time_key' => 'true', 'time_as_epoch' => 'true', 'time_key' => 'epoch')
+      formatted = @formatter.format(tag, @time, record.dup)
+
+      r = record
+      r['epoch'] = @time
+      assert_equal("#{Yajl.dump(r)}\n", formatted)
+    end
+  end
+
+  class LabeledTSVFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def setup
+      @formatter = TextFormatter::LabeledTSVFormatter.new
+      @time = Engine.now
+    end
+
+    def test_config_params
+      assert_equal "\t", @formatter.delimiter
+      assert_equal  ":", @formatter.label_delimiter
+
+      @formatter.configure(
+        'delimiter'       => ',',
+        'label_delimiter' => '=',
+      )
+
+      assert_equal ",", @formatter.delimiter
+      assert_equal "=", @formatter.label_delimiter
+    end
+
+    def test_format
+      @formatter.configure({})
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message:awesome\n", formatted)
+    end
+
+    def test_format_with_tag
+      @formatter.configure('include_tag_key' => 'true')
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message:awesome\ttag:tag\n", formatted)
+    end
+
+    def test_format_with_time
+      @formatter.configure('include_time_key' => 'true', 'time_format' => '%Y')
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message:awesome\ttime:#{Time.now.year}\n", formatted)
+    end
+
+    def test_format_with_customized_delimiters
+      @formatter.configure(
+        'include_tag_key' => 'true',
+        'delimiter'       => ',',
+        'label_delimiter' => '=',
+      )
+      formatted = @formatter.format(tag, @time, record)
+
+      assert_equal("message=awesome,tag=tag\n", formatted)
+    end
+  end
+
+  class SingleValueFormatterTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def test_config_params
+      formatter = TextFormatter::SingleValueFormatter.new
+      assert_equal "message", formatter.message_key
+
+      formatter.configure('message_key' => 'foobar')
+      assert_equal "foobar", formatter.message_key
+    end
+
+    def test_format
+      formatter = TextFormatter::TEMPLATE_REGISTRY.lookup('single_value').call
+      formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
+      assert_equal('awesome', formatted)
+    end
+
+    def test_format_with_message_key
+      formatter = TextFormatter::SingleValueFormatter.new
+      formatter.configure('message_key' => 'foobar')
+      formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
+
+      assert_equal('foo', formatted)
+    end
+  end
+
+  class FormatterLookupTest < ::Test::Unit::TestCase
+    include FormatterTest
+
+    def test_unknown_format
+      assert_raise ConfigError do
+        TextFormatter::TEMPLATE_REGISTRY.lookup('unknown')
+      end
+    end
+
+    def test_find_formatter
+      $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))
+      assert_nothing_raised ConfigError do
+        TextFormatter::TEMPLATE_REGISTRY.lookup('known')
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluentd
 version: !ruby/object:Gem::Version
-  version: 0.10.48
+  version: 0.10.49
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-05-18 00:00:00.000000000 Z
+date: 2014-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -287,6 +287,7 @@ files:
 - lib/fluent/engine.rb
 - lib/fluent/env.rb
 - lib/fluent/event.rb
+- lib/fluent/formatter.rb
 - lib/fluent/input.rb
 - lib/fluent/load.rb
 - lib/fluent/log.rb
@@ -363,9 +364,11 @@ files:
 - test/plugin/test_out_stdout.rb
 - test/plugin/test_out_stream.rb
 - test/scripts/exec_script.rb
+- test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/parser_known.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_formatter.rb
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_output.rb
@@ -427,9 +430,11 @@ test_files:
 - test/plugin/test_out_stdout.rb
 - test/plugin/test_out_stream.rb
 - test/scripts/exec_script.rb
+- test/scripts/fluent/plugin/formatter_known.rb
 - test/scripts/fluent/plugin/parser_known.rb
 - test/test_config.rb
 - test/test_configdsl.rb
+- test/test_formatter.rb
 - test/test_match.rb
 - test/test_mixin.rb
 - test/test_output.rb