fluentd 0.10.36 → 0.10.37

data/lib/fluent/env.rb

@@ -1,7 +1,7 @@
 module Fluent
-DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
-DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
-DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
-DEFAULT_LISTEN_PORT = 24224
-DEFAULT_FILE_PERMISSION = 0644
+  DEFAULT_CONFIG_PATH = ENV['FLUENT_CONF'] || '/etc/fluent/fluent.conf'
+  DEFAULT_PLUGIN_DIR = ENV['FLUENT_PLUGIN'] || '/etc/fluent/plugin'
+  DEFAULT_SOCKET_PATH = ENV['FLUENT_SOCKET'] || '/var/run/fluent/fluent.sock'
+  DEFAULT_LISTEN_PORT = 24224
+  DEFAULT_FILE_PERMISSION = 0644
 end

data/lib/fluent/event.rb

@@ -16,169 +16,165 @@
 #    limitations under the License.
 #
 module Fluent
+  class EventStream
+    include Enumerable
 
+    def repeatable?
+      false
+    end
 
-class EventStream
-  include Enumerable
-
-  def repeatable?
-    false
-  end
-
-  #def each(&block)
-  #end
-
-  def to_msgpack_stream
-    out = ''
-    each {|time,record|
-      [time,record].to_msgpack(out)
-    }
-    out
-  end
-end
-
-
-class OneEventStream < EventStream
-  def initialize(time, record)
-    @time = time
-    @record = record
-  end
-
-  def repeatable?
-    true
-  end
-
-  def each(&block)
-    block.call(@time, @record)
-    nil
-  end
-end
-
-
-class ArrayEventStream < EventStream
-  def initialize(entries)
-    @entries = entries
-  end
-
-  def repeatable?
-    true
-  end
-
-  def empty?
-    @entries.empty?
-  end
-
-  def each(&block)
-    @entries.each(&block)
-    nil
-  end
-
-  #attr_reader :entries
-  #
-  #def to_a
-  #  @entries
-  #end
-end
-
+    #def each(&block)
+    #end
 
-class MultiEventStream < EventStream
-  def initialize
-    @time_array = []
-    @record_array = []
+    def to_msgpack_stream
+      out = ''
+      each {|time,record|
+        [time,record].to_msgpack(out)
+      }
+      out
+    end
   end
 
-  def add(time, record)
-    @time_array << time
-    @record_array << record
-  end
 
-  def repeatable?
-    true
-  end
+  class OneEventStream < EventStream
+    def initialize(time, record)
+      @time = time
+      @record = record
+    end
 
-  def empty?
-    @time_array.empty?
-  end
+    def repeatable?
+      true
+    end
 
-  def each(&block)
-    time_array = @time_array
-    record_array = @record_array
-    for i in 0..time_array.length-1
-      block.call(time_array[i], record_array[i])
+    def each(&block)
+      block.call(@time, @record)
+      nil
     end
-    nil
   end
-end
 
-if $use_msgpack_5
 
-  class MessagePackEventStream < EventStream
-    def initialize(data, cached_unpacker=nil)
-      @data = data
+  class ArrayEventStream < EventStream
+    def initialize(entries)
+      @entries = entries
     end
 
     def repeatable?
       true
     end
 
+    def empty?
+      @entries.empty?
+    end
+
     def each(&block)
-      # TODO format check
-      unpacker = MessagePack::Unpacker.new
-      unpacker.feed_each(@data, &block)
+      @entries.each(&block)
       nil
     end
 
-    def to_msgpack_stream
-      @data
-    end
+    #attr_reader :entries
+    #
+    #def to_a
+    #  @entries
+    #end
   end
 
-else # for 0.4.x. Will be removed after 0.5.x is stable
 
-  class MessagePackEventStream < EventStream
-    def initialize(data, cached_unpacker=nil)
-      @data = data
-      @unpacker = cached_unpacker || MessagePack::Unpacker.new
+  class MultiEventStream < EventStream
+    def initialize
+      @time_array = []
+      @record_array = []
+    end
+
+    def add(time, record)
+      @time_array << time
+      @record_array << record
     end
 
     def repeatable?
       true
     end
 
+    def empty?
+      @time_array.empty?
+    end
+
     def each(&block)
-      @unpacker.reset
-      # TODO format check
-      @unpacker.feed_each(@data, &block)
+      time_array = @time_array
+      record_array = @record_array
+      for i in 0..time_array.length-1
+        block.call(time_array[i], record_array[i])
+      end
       nil
     end
+  end
 
-    def to_msgpack_stream
-      @data
+  if $use_msgpack_5
+
+    class MessagePackEventStream < EventStream
+      def initialize(data, cached_unpacker=nil)
+        @data = data
+      end
+
+      def repeatable?
+        true
+      end
+
+      def each(&block)
+        # TODO format check
+        unpacker = MessagePack::Unpacker.new
+        unpacker.feed_each(@data, &block)
+        nil
+      end
+
+      def to_msgpack_stream
+        @data
+      end
     end
-  end
 
-end
+  else # for 0.4.x. Will be removed after 0.5.x is stable
 
-#class IoEventStream < EventStream
-#  def initialize(io, num)
-#    @io = io
-#    @num = num
-#    @u = MessagePack::Unpacker.new(@io)
-#  end
-#
-#  def repeatable?
-#    false
-#  end
-#
-#  def each(&block)
-#    return nil if @num == 0
-#    @u.each {|obj|
-#      block.call(obj[0], obj[1])
-#      break if @array.size >= @num
-#    }
-#  end
-#end
+    class MessagePackEventStream < EventStream
+      def initialize(data, cached_unpacker=nil)
+        @data = data
+        @unpacker = cached_unpacker || MessagePack::Unpacker.new
+      end
 
+      def repeatable?
+        true
+      end
 
+      def each(&block)
+        @unpacker.reset
+        # TODO format check
+        @unpacker.feed_each(@data, &block)
+        nil
+      end
+
+      def to_msgpack_stream
+        @data
+      end
+    end
+
+  end
+
+  #class IoEventStream < EventStream
+  #  def initialize(io, num)
+  #    @io = io
+  #    @num = num
+  #    @u = MessagePack::Unpacker.new(@io)
+  #  end
+  #
+  #  def repeatable?
+  #    false
+  #  end
+  #
+  #  def each(&block)
+  #    return nil if @num == 0
+  #    @u.each {|obj|
+  #      block.call(obj[0], obj[1])
+  #      break if @array.size >= @num
+  #    }
+  #  end
+  #end
 end
 

data/lib/fluent/input.rb

@@ -16,27 +16,23 @@
 #    limitations under the License.
 #
 module Fluent
+  class Input
+    include Configurable
+    include PluginId
 
+    def initialize
+      super
+    end
 
-class Input
-  include Configurable
-  include PluginId
+    def configure(conf)
+      super
+    end
 
-  def initialize
-    super
-  end
-
-  def configure(conf)
-    super
-  end
-
-  def start
-  end
+    def start
+    end
 
-  def shutdown
+    def shutdown
+    end
   end
 end
 
-
-end
-

data/lib/fluent/load.rb

@@ -18,6 +18,7 @@ require 'fluent/version'
 require 'fluent/log'
 require 'fluent/status'
 require 'fluent/config'
+require 'fluent/config_dsl'
 require 'fluent/engine'
 require 'fluent/mixin'
 require 'fluent/process'

data/lib/fluent/log.rb

@@ -16,281 +16,277 @@
 #    limitations under the License.
 #
 module Fluent
+  class Log
+    module TTYColor
+      RESET   = "\033]R"
+      CRE     = "\033[K"
+      CLEAR   = "\033c"
+      NORMAL  = "\033[0;39m"
+      RED     = "\033[1;31m"
+      GREEN   = "\033[1;32m"
+      YELLOW  = "\033[1;33m"
+      BLUE    = "\033[1;34m"
+      MAGENTA = "\033[1;35m"
+      CYAN    = "\033[1;36m"
+      WHITE   = "\033[1;37m"
+    end
 
+    LEVEL_TRACE = 0
+    LEVEL_DEBUG = 1
+    LEVEL_INFO  = 2
+    LEVEL_WARN  = 3
+    LEVEL_ERROR = 4
+    LEVEL_FATAL = 5
+
+    LEVEL_TEXT = %w(trace debug info warn error fatal)
+
+    def initialize(out=STDERR, level=LEVEL_TRACE)
+      @out = out
+      @level = level
+      @debug_mode = false
+      @self_event = false
+      @tag = 'fluent'
+      @time_format = '%Y-%m-%d %H:%M:%S %z '
+      enable_color out.tty?
+      # TODO: This variable name is unclear so we should change to better name.
+      @threads_exclude_events = []
+    end
 
-class Log
-  module TTYColor
-    RESET   = "\033]R"
-    CRE     = "\033[K"
-    CLEAR   = "\033c"
-    NORMAL  = "\033[0;39m"
-    RED     = "\033[1;31m"
-    GREEN   = "\033[1;32m"
-    YELLOW  = "\033[1;33m"
-    BLUE    = "\033[1;34m"
-    MAGENTA = "\033[1;35m"
-    CYAN    = "\033[1;36m"
-    WHITE   = "\033[1;37m"
-  end
+    attr_accessor :out
+    attr_accessor :level
+    attr_accessor :tag
+    attr_accessor :time_format
 
-  LEVEL_TRACE = 0
-  LEVEL_DEBUG = 1
-  LEVEL_INFO  = 2
-  LEVEL_WARN  = 3
-  LEVEL_ERROR = 4
-  LEVEL_FATAL = 5
-
-  LEVEL_TEXT = %w(trace debug info warn error fatal)
-
-  def initialize(out=STDERR, level=LEVEL_TRACE)
-    @out = out
-    @level = level
-    @debug_mode = false
-    @self_event = false
-    @tag = 'fluent'
-    @time_format = '%Y-%m-%d %H:%M:%S %z '
-    enable_color out.tty?
-    # TODO: This variable name is unclear so we should change to better name.
-    @threads_exclude_events = []
-  end
+    def enable_debug(b=true)
+      @debug_mode = b
+      self
+    end
 
-  attr_accessor :out
-  attr_accessor :level
-  attr_accessor :tag
-  attr_accessor :time_format
+    def enable_event(b=true)
+      @self_event = b
+      self
+    end
 
-  def enable_debug(b=true)
-    @debug_mode = b
-    self
-  end
+    def enable_color(b=true)
+      if b
+        @color_trace = TTYColor::BLUE
+        @color_debug = TTYColor::WHITE
+        @color_info  = TTYColor::GREEN
+        @color_warn  = TTYColor::YELLOW
+        @color_error = TTYColor::MAGENTA
+        @color_fatal = TTYColor::RED
+        @color_reset = TTYColor::NORMAL
+      else
+        @color_trace = ''
+        @color_debug = ''
+        @color_info  = ''
+        @color_warn  = ''
+        @color_error = ''
+        @color_fatal = ''
+        @color_reset = ''
+      end
+      self
+    end
 
-  def enable_event(b=true)
-    @self_event = b
-    self
-  end
+    # If you want to suppress event emitting in specific thread, please use this method.
+    # Events in passed thread are never emitted.
+    def disable_events(thread)
+      @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
+    end
 
-  def enable_color(b=true)
-    if b
-      @color_trace = TTYColor::BLUE
-      @color_debug = TTYColor::WHITE
-      @color_info  = TTYColor::GREEN
-      @color_warn  = TTYColor::YELLOW
-      @color_error = TTYColor::MAGENTA
-      @color_fatal = TTYColor::RED
-      @color_reset = TTYColor::NORMAL
-    else
-      @color_trace = ''
-      @color_debug = ''
-      @color_info  = ''
-      @color_warn  = ''
-      @color_error = ''
-      @color_fatal = ''
-      @color_reset = ''
+    def on_trace(&block)
+      return if @level > LEVEL_TRACE
+      block.call if block
     end
-    self
-  end
 
-  # If you want to suppress event emitting in specific thread, please use this method.
-  # Events in passed thread are never emitted.
-  def disable_events(thread)
-    @threads_exclude_events.push(thread) unless @threads_exclude_events.include?(thread)
-  end
+    def trace(*args, &block)
+      return if @level > LEVEL_TRACE
+      args << block.call if block
+      time, msg = event(:trace, args)
+      puts [@color_trace, caller_line(time, 1, LEVEL_TRACE), msg, @color_reset].join
+    end
+    alias TRACE trace
 
-  def on_trace(&block)
-    return if @level > LEVEL_TRACE
-    block.call if block
-  end
+    def trace_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_TRACE
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_TRACE), msg].join
+      }
+      nil
+    end
 
-  def trace(*args, &block)
-    return if @level > LEVEL_TRACE
-    args << block.call if block
-    time, msg = event(:trace, args)
-    puts [@color_trace, caller_line(time, 1, LEVEL_TRACE), msg, @color_reset].join
-  end
-  alias TRACE trace
-
-  def trace_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_TRACE
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_TRACE), msg].join
-    }
-    nil
-  end
+    def on_debug(&block)
+      return if @level > LEVEL_DEBUG
+      block.call if block
+    end
 
-  def on_debug(&block)
-    return if @level > LEVEL_DEBUG
-    block.call if block
-  end
+    def debug(*args, &block)
+      return if @level > LEVEL_DEBUG
+      args << block.call if block
+      time, msg = event(:debug, args)
+      puts [@color_debug, caller_line(time, 1, LEVEL_DEBUG), msg, @color_reset].join
+    end
+    alias DEBUG debug
 
-  def debug(*args, &block)
-    return if @level > LEVEL_DEBUG
-    args << block.call if block
-    time, msg = event(:debug, args)
-    puts [@color_debug, caller_line(time, 1, LEVEL_DEBUG), msg, @color_reset].join
-  end
-  alias DEBUG debug
-
-  def debug_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_DEBUG
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_DEBUG), msg].join
-    }
-    nil
-  end
+    def debug_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_DEBUG
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_DEBUG), msg].join
+      }
+      nil
+    end
 
-  def on_info(&block)
-    return if @level > LEVEL_INFO
-    block.call if block
-  end
+    def on_info(&block)
+      return if @level > LEVEL_INFO
+      block.call if block
+    end
 
-  def info(*args, &block)
-    return if @level > LEVEL_INFO
-    args << block.call if block
-    time, msg = event(:info, args)
-    puts [@color_info, caller_line(time, 1, LEVEL_INFO), msg, @color_reset].join
-  end
-  alias INFO info
-
-  def info_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_INFO
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_INFO), msg].join
-    }
-    nil
-  end
+    def info(*args, &block)
+      return if @level > LEVEL_INFO
+      args << block.call if block
+      time, msg = event(:info, args)
+      puts [@color_info, caller_line(time, 1, LEVEL_INFO), msg, @color_reset].join
+    end
+    alias INFO info
 
-  def on_warn(&block)
-    return if @level > LEVEL_WARN
-    block.call if block
-  end
+    def info_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_INFO
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_INFO), msg].join
+      }
+      nil
+    end
 
-  def warn(*args, &block)
-    return if @level > LEVEL_WARN
-    args << block.call if block
-    time, msg = event(:warn, args)
-    puts [@color_warn, caller_line(time, 1, LEVEL_WARN), msg, @color_reset].join
-  end
-  alias WARN warn
-
-  def warn_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_WARN
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_WARN), msg].join
-    }
-    nil
-  end
+    def on_warn(&block)
+      return if @level > LEVEL_WARN
+      block.call if block
+    end
 
-  def on_error(&block)
-    return if @level > LEVEL_ERROR
-    block.call if block
-  end
+    def warn(*args, &block)
+      return if @level > LEVEL_WARN
+      args << block.call if block
+      time, msg = event(:warn, args)
+      puts [@color_warn, caller_line(time, 1, LEVEL_WARN), msg, @color_reset].join
+    end
+    alias WARN warn
 
-  def error(*args, &block)
-    return if @level > LEVEL_ERROR
-    args << block.call if block
-    time, msg = event(:error, args)
-    puts [@color_error, caller_line(time, 1, LEVEL_ERROR), msg, @color_reset].join
-  end
-  alias ERROR error
-
-  def error_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_ERROR
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_ERROR), msg].join
-    }
-    nil
-  end
+    def warn_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_WARN
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_WARN), msg].join
+      }
+      nil
+    end
 
-  def on_fatal(&block)
-    return if @level > LEVEL_FATAL
-    block.call if block
-  end
+    def on_error(&block)
+      return if @level > LEVEL_ERROR
+      block.call if block
+    end
 
-  def fatal(*args, &block)
-    return if @level > LEVEL_FATAL
-    args << block.call if block
-    time, msg = event(:fatal, args)
-    puts [@color_fatal, caller_line(time, 1, LEVEL_FATAL), msg, @color_reset].join
-  end
-  alias FATAL fatal
-
-  def fatal_backtrace(backtrace=$!.backtrace)
-    return if @level > LEVEL_FATAL
-    time = Time.now
-    backtrace.each {|msg|
-      puts ["  ", caller_line(time, 4, LEVEL_FATAL), msg].join
-    }
-    nil
-  end
+    def error(*args, &block)
+      return if @level > LEVEL_ERROR
+      args << block.call if block
+      time, msg = event(:error, args)
+      puts [@color_error, caller_line(time, 1, LEVEL_ERROR), msg, @color_reset].join
+    end
+    alias ERROR error
 
-  def puts(msg)
-    @out.puts(msg)
-    @out.flush
-    msg
-  rescue
-    # FIXME
-    nil
-  end
+    def error_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_ERROR
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_ERROR), msg].join
+      }
+      nil
+    end
 
-  def write(data)
-    @out.write(data)
-  end
+    def on_fatal(&block)
+      return if @level > LEVEL_FATAL
+      block.call if block
+    end
 
-  def flush
-    @out.flush
-  end
+    def fatal(*args, &block)
+      return if @level > LEVEL_FATAL
+      args << block.call if block
+      time, msg = event(:fatal, args)
+      puts [@color_fatal, caller_line(time, 1, LEVEL_FATAL), msg, @color_reset].join
+    end
+    alias FATAL fatal
 
-  private
-  def event(level, args)
-    time = Time.now
-    message = ''
-    map = {}
-    args.each {|a|
-      if a.is_a?(Hash)
-        a.each_pair {|k,v|
-          map[k.to_s] = v
+    def fatal_backtrace(backtrace=$!.backtrace)
+      return if @level > LEVEL_FATAL
+      time = Time.now
+      backtrace.each {|msg|
+        puts ["  ", caller_line(time, 4, LEVEL_FATAL), msg].join
+      }
+      nil
+    end
+
+    def puts(msg)
+      @out.puts(msg)
+      @out.flush
+      msg
+    rescue
+      # FIXME
+      nil
+    end
+
+    def write(data)
+      @out.write(data)
+    end
+
+    def flush
+      @out.flush
+    end
+
+    private
+    def event(level, args)
+      time = Time.now
+      message = ''
+      map = {}
+      args.each {|a|
+        if a.is_a?(Hash)
+          a.each_pair {|k,v|
+            map[k.to_s] = v
+          }
+        else
+          message << a.to_s
+        end
+      }
+
+      unless @threads_exclude_events.include?(Thread.current)
+        record = map.dup
+        record.keys.each {|key|
+          record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
         }
-      else
-        message << a.to_s
+        record['message'] = message.dup
+        Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
       end
-    }
 
-    unless @threads_exclude_events.include?(Thread.current)
-      record = map.dup
-      record.keys.each {|key|
-        record[key] = record[key].inspect unless record[key].respond_to?(:to_msgpack)
+      map.each_pair {|k,v|
+        message << " #{k}=#{v.inspect}"
       }
-      record['message'] = message.dup
-      Engine.push_log_event("#{@tag}.#{level}", time.to_i, record)
-    end
 
-    map.each_pair {|k,v|
-      message << " #{k}=#{v.inspect}"
-    }
-
-    return time, message
-  end
+      return time, message
+    end
 
-  def caller_line(time, depth, level)
-    line = caller(depth+1)[0]
-    log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: "
-    if @debug_mode
-      if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
-        file = match[1].split('/')[-2,2].join('/')
-        line = match[2]
-        method = match[3]
-        return "#{log_msg}#{file}:#{line}:#{method}: "
+    def caller_line(time, depth, level)
+      line = caller(depth+1)[0]
+      log_msg = "#{time.strftime(@time_format)}[#{LEVEL_TEXT[level]}]: "
+      if @debug_mode
+        if match = /^(.+?):(\d+)(?::in `(.*)')?/.match(line)
+          file = match[1].split('/')[-2,2].join('/')
+          line = match[2]
+          method = match[3]
+          return "#{log_msg}#{file}:#{line}:#{method}: "
+        end
       end
+      return log_msg
     end
-    return log_msg
   end
 end
 
-
-end
-