fluentd 0.10.36 → 0.10.37

data/lib/fluent/config_dsl.rb

@@ -0,0 +1,74 @@
+require 'fluent/config'
+
+module Fluent
+  module Config
+    module DSL
+      module DSLParser
+        def self.read(path)
+          path = File.expand_path(path)
+          data = File.read(path)
+          parse(data, path)
+        end
+
+        def self.parse(source, source_path="config.rb")
+          DSLElement.new('ROOT', nil).__eval(source, source_path).__to_config_element
+        end
+      end
+
+      class DSLElement < BasicObject
+        def initialize(name, arg)
+          @name = name
+          @arg = arg || ''
+          @attrs = {}
+          @elements = []
+        end
+
+        def __eval(source, source_path)
+          instance_eval(source, source_path)
+          self
+        end
+
+        def __to_config_element
+          Config::Element.new(@name, @arg, @attrs, @elements)
+        end
+
+        def method_missing(name, *args, &block)
+          raise ArgumentError, "Configuration DSL Syntax Error: only one argument allowed" if args.size > 1
+          value = args.first
+          if block
+            element = DSLElement.new(name.to_s, value)
+            element.instance_exec(&block)
+            @elements.push(element.__to_config_element)
+          else
+            # @attrs[name.to_s] = value.is_a?(::Symbol) ? value.to_s : value
+            @attrs[name.to_s] = value.to_s
+          end
+          self
+        end
+
+        def source(&block)
+          __element('source', nil, block)
+        end
+
+        def match(*args, &block)
+          __need_arg('match',  args); __element('match',  args.first, block)
+        end
+
+        private
+
+        def __element(name, arg, block)
+          raise ArgumentError, "#{name} block must be specified" if block.nil?
+          element = DSLElement.new(name.to_s, arg)
+          element.instance_exec(&block)
+          @elements.push(element.__to_config_element)
+          self
+        end
+
+        def __need_arg(name, args)
+          raise ArgumentError, "#{name} block requires arguments for match pattern" if args.nil? || args.size != 1
+          true
+        end
+      end
+    end
+  end
+end

data/lib/fluent/engine.rb

@@ -16,318 +16,321 @@
 #    limitations under the License.
 #
 module Fluent
+  class EngineClass
+    def initialize
+      @matches = []
+      @sources = []
+      @match_cache = {}
+      @match_cache_keys = []
+      @started = []
+      @default_loop = nil
 
+      @log_emit_thread = nil
+      @log_event_loop_stop = false
+      @log_event_queue = []
 
-class EngineClass
-  def initialize
-    @matches = []
-    @sources = []
-    @match_cache = {}
-    @match_cache_keys = []
-    @started = []
-    @default_loop = nil
-
-    @log_emit_thread = nil
-    @log_event_loop_stop = false
-    @log_event_queue = []
-
-    @suppress_emit_error_log_interval = 0
-    @next_emit_error_log_time = nil
-  end
+      @suppress_emit_error_log_interval = 0
+      @next_emit_error_log_time = nil
+    end
 
-  MATCH_CACHE_SIZE = 1024
+    MATCH_CACHE_SIZE = 1024
 
-  LOG_EMIT_INTERVAL = 0.1
+    LOG_EMIT_INTERVAL = 0.1
 
-  attr_reader :matches, :sources
+    attr_reader :matches, :sources
 
-  def init
-    BasicSocket.do_not_reverse_lookup = true
-    Plugin.load_plugins
-    if defined?(Encoding)
-      Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
-      Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
+    def init
+      BasicSocket.do_not_reverse_lookup = true
+      Plugin.load_plugins
+      if defined?(Encoding)
+        Encoding.default_internal = 'ASCII-8BIT' if Encoding.respond_to?(:default_internal)
+        Encoding.default_external = 'ASCII-8BIT' if Encoding.respond_to?(:default_external)
+      end
+      self
     end
-    self
-  end
 
-  def suppress_interval(interval_time)
-    @suppress_emit_error_log_interval = interval_time
-    @next_emit_error_log_time = Time.now.to_i
-  end
+    def suppress_interval(interval_time)
+      @suppress_emit_error_log_interval = interval_time
+      @next_emit_error_log_time = Time.now.to_i
+    end
 
-  def read_config(path)
-    $log.info "reading config file", :path=>path
-    File.open(path) {|io|
-      parse_config(io, File.basename(path), File.dirname(path))
-    }
-  end
+    def read_config(path)
+      $log.info "reading config file", :path=>path
+      File.open(path) {|io|
+        parse_config(io, File.basename(path), File.dirname(path))
+      }
+    end
 
-  def parse_config(io, fname, basepath=Dir.pwd)
-    conf = Config.parse(io, fname, basepath)
-    configure(conf)
-    conf.check_not_fetched {|key,e|
-      $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
-    }
-  end
+    def parse_config(io, fname, basepath=Dir.pwd)
+      conf = if fname =~ /\.rb$/
+               Config::DSL::DSLParser.parse(io, fname)
+             else
+               Config.parse(io, fname, basepath)
+             end
+      configure(conf)
+      conf.check_not_fetched {|key,e|
+        $log.warn "parameter '#{key}' in #{e.to_s.strip} is not used."
+      }
+    end
 
-  def configure(conf)
-    $log.info "using configuration file: #{conf.to_s.rstrip}"
+    def configure(conf)
+      $log.info "using configuration file: #{conf.to_s.rstrip}"
 
-    conf.elements.select {|e|
-      e.name == 'source'
-    }.each {|e|
-      type = e['type']
-      unless type
-        raise ConfigError, "Missing 'type' parameter on <source> directive"
-      end
-      $log.info "adding source type=#{type.dump}"
+      conf.elements.select {|e|
+        e.name == 'source'
+      }.each {|e|
+        type = e['type']
+        unless type
+          raise ConfigError, "Missing 'type' parameter on <source> directive"
+        end
+        $log.info "adding source type=#{type.dump}"
 
-      input = Plugin.new_input(type)
-      input.configure(e)
+        input = Plugin.new_input(type)
+        input.configure(e)
 
-      @sources << input
-    }
+        @sources << input
+      }
 
-    conf.elements.select {|e|
-      e.name == 'match'
-    }.each {|e|
-      type = e['type']
-      pattern = e.arg
-      unless type
-        raise ConfigError, "Missing 'type' parameter on <match #{e.arg}> directive"
-      end
-      $log.info "adding match", :pattern=>pattern, :type=>type
+      conf.elements.select {|e|
+        e.name == 'match'
+      }.each {|e|
+        type = e['type']
+        pattern = e.arg
+        unless type
+          raise ConfigError, "Missing 'type' parameter on <match #{e.arg}> directive"
+        end
+        $log.info "adding match", :pattern=>pattern, :type=>type
 
-      output = Plugin.new_output(type)
-      output.configure(e)
+        output = Plugin.new_output(type)
+        output.configure(e)
 
-      match = Match.new(pattern, output)
-      @matches << match
-    }
-  end
+        match = Match.new(pattern, output)
+        @matches << match
+      }
+    end
 
-  def load_plugin_dir(dir)
-    Plugin.load_plugin_dir(dir)
-  end
+    def load_plugin_dir(dir)
+      Plugin.load_plugin_dir(dir)
+    end
 
-  def emit(tag, time, record)
-    emit_stream tag, OneEventStream.new(time, record)
-  end
+    def emit(tag, time, record)
+      unless record.nil?
+        emit_stream tag, OneEventStream.new(time, record)
+      end
+    end
 
-  def emit_array(tag, array)
-    emit_stream tag, ArrayEventStream.new(array)
-  end
+    def emit_array(tag, array)
+      emit_stream tag, ArrayEventStream.new(array)
+    end
 
-  def emit_stream(tag, es)
-    target = @match_cache[tag]
-    unless target
-      target = match(tag) || NoMatchMatch.new
-      # this is not thread-safe but inconsistency doesn't
-      # cause serious problems while locking causes.
-      if @match_cache_keys.size >= MATCH_CACHE_SIZE
-        @match_cache_keys.delete @match_cache_keys.shift
+    def emit_stream(tag, es)
+      target = @match_cache[tag]
+      unless target
+        target = match(tag) || NoMatchMatch.new
+        # this is not thread-safe but inconsistency doesn't
+        # cause serious problems while locking causes.
+        if @match_cache_keys.size >= MATCH_CACHE_SIZE
+          @match_cache_keys.delete @match_cache_keys.shift
+        end
+        @match_cache[tag] = target
+        @match_cache_keys << tag
       end
-      @match_cache[tag] = target
-      @match_cache_keys << tag
+      target.emit(tag, es)
+    rescue => e
+      if @suppress_emit_error_log_interval == 0 || now > @next_emit_error_log_time
+        $log.warn "emit transaction failed ", :error_class=>e.class, :error=>e
+        $log.warn_backtrace
+        # $log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
+        @next_emit_error_log_time = Time.now.to_i + @suppress_emit_error_log_interval
+        # $log.debug "next emit failure log suppressed"
+        # $log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
+      end
+      raise
     end
-    target.emit(tag, es)
-  rescue => e
-    if @suppress_emit_error_log_interval == 0 || now > @next_emit_error_log_time
-      $log.warn "emit transaction failed ", :error_class=>e.class, :error=>e
-      $log.warn_backtrace
-      # $log.debug "current next_emit_error_log_time: #{Time.at(@next_emit_error_log_time)}"
-      @next_emit_error_log_time = Time.now.to_i + @suppress_emit_error_log_interval
-      # $log.debug "next emit failure log suppressed"
-      # $log.debug "next logged time is #{Time.at(@next_emit_error_log_time)}"
+
+    def match(tag)
+      @matches.find {|m| m.match(tag) }
     end
-    raise
-  end
 
-  def match(tag)
-    @matches.find {|m| m.match(tag) }
-  end
+    def match?(tag)
+      !!match(tag)
+    end
 
-  def match?(tag)
-    !!match(tag)
-  end
+    def flush!
+      flush_recursive(@matches)
+    end
 
-  def flush!
-    flush_recursive(@matches)
-  end
+    def now
+      # TODO thread update
+      Time.now.to_i
+    end
 
-  def now
-    # TODO thread update
-    Time.now.to_i
-  end
+    def log_event_loop
+      $log.disable_events(Thread.current)
+
+      while sleep(LOG_EMIT_INTERVAL)
+        break if @log_event_loop_stop
+        next if @log_event_queue.empty?
 
-  def log_event_loop
-    $log.disable_events(Thread.current)
+        # NOTE: thead-safe of slice! depends on GVL
+        events = @log_event_queue.slice!(0..-1)
+        next if events.empty?
 
-    while sleep(LOG_EMIT_INTERVAL)
-      break if @log_event_loop_stop
-      next if @log_event_queue.empty?
+        events.each {|tag,time,record|
+          begin
+            Engine.emit(tag, time, record)
+          rescue => e
+            $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error_class => e.class, :error => e
+          end
+        }
+      end
+    end
 
-      # NOTE: thead-safe of slice! depends on GVL
-      events = @log_event_queue.slice!(0..-1)
-      next if events.empty?
+    def run
+      begin
+        start
 
-      events.each {|tag,time,record|
-        begin
-          Engine.emit(tag, time, record)
-        rescue => e
-          $log.error "failed to emit fluentd's log event", :tag => tag, :event => record, :error_class => e.class, :error => e
+        if match?($log.tag)
+          $log.enable_event
+          @log_emit_thread = Thread.new(&method(:log_event_loop))
         end
-      }
-    end
-  end
 
-  def run
-    begin
-      start
+        # for empty loop
+        @default_loop = Coolio::Loop.default
+        @default_loop.attach Coolio::TimerWatcher.new(1, true)
+        # TODO attach async watch for thread pool
+        @default_loop.run
 
-      if match?($log.tag)
-        $log.enable_event
-        @log_emit_thread = Thread.new(&method(:log_event_loop))
+      rescue => e
+        $log.error "unexpected error", :error_class=>e.class, :error=>e
+        $log.error_backtrace
+      ensure
+        $log.info "shutting down fluentd"
+        shutdown
+        if @log_emit_thread
+          @log_event_loop_stop = true
+          @log_emit_thread.join
+        end
       end
+    end
 
-      # for empty loop
-      @default_loop = Coolio::Loop.default
-      @default_loop.attach Coolio::TimerWatcher.new(1, true)
-      # TODO attach async watch for thread pool
-      @default_loop.run
-
-    rescue => e
-      $log.error "unexpected error", :error_class=>e.class, :error=>e
-      $log.error_backtrace
-    ensure
-      $log.info "shutting down fluentd"
-      shutdown
-      if @log_emit_thread
-        @log_event_loop_stop = true
-        @log_emit_thread.join
+    def stop
+      if @default_loop
+        @default_loop.stop
+        @default_loop = nil
       end
+      nil
     end
-  end
 
-  def stop
-    if @default_loop
-      @default_loop.stop
-      @default_loop = nil
+    def push_log_event(tag, time, record)
+      return if @log_emit_thread.nil?
+      @log_event_queue.push([tag, time, record])
     end
-    nil
-  end
 
-  def push_log_event(tag, time, record)
-    return if @log_emit_thread.nil?
-    @log_event_queue.push([tag, time, record])
-  end
+    private
+    def start
+      @matches.each {|m|
+        m.start
+        @started << m
+      }
+      @sources.each {|s|
+        s.start
+        @started << s
+      }
+    end
 
-  private
-  def start
-    @matches.each {|m|
-      m.start
-      @started << m
-    }
-    @sources.each {|s|
-      s.start
-      @started << s
-    }
-  end
+    def shutdown
+      @started.map {|s|
+        Thread.new do
+          begin
+            s.shutdown
+          rescue => e
+            $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
+            $log.warn_backtrace
+          end
+        end
+      }.each {|t|
+        t.join
+      }
+    end
 
-  def shutdown
-    @started.map {|s|
-      Thread.new do
+    def flush_recursive(array)
+      array.each {|m|
         begin
-          s.shutdown
+          if m.is_a?(Match)
+            m = m.output
+          end
+          if m.is_a?(BufferedOutput)
+            m.force_flush
+          elsif m.is_a?(MultiOutput)
+            flush_recursive(m.outputs)
+          end
         rescue => e
-          $log.warn "unexpected error while shutting down", :error_class=>e.class, :error=>e
-          $log.warn_backtrace
+          $log.debug "error while force flushing", :error_class=>e.class, :error=>e
+          $log.debug_backtrace
         end
-      end
-    }.each {|t|
-      t.join
-    }
-  end
+      }
+    end
 
-  def flush_recursive(array)
-    array.each {|m|
-      begin
-        if m.is_a?(Match)
-          m = m.output
-        end
-        if m.is_a?(BufferedOutput)
-          m.force_flush
-        elsif m.is_a?(MultiOutput)
-          flush_recursive(m.outputs)
-        end
-      rescue => e
-        $log.debug "error while force flushing", :error_class=>e.class, :error=>e
-        $log.debug_backtrace
+    class NoMatchMatch
+      def initialize
+        @count = 0
       end
-    }
-  end
-
-  class NoMatchMatch
-    def initialize
-      @count = 0
-    end
 
-    def emit(tag, es)
-      # TODO use time instead of num of records
-      c = (@count += 1)
-      if c < 512
-        if Math.log(c) / Math.log(2) % 1.0 == 0
-          $log.warn "no patterns matched", :tag=>tag
-          return
-        end
-      else
-        if c % 512 == 0
-          $log.warn "no patterns matched", :tag=>tag
-          return
+      def emit(tag, es)
+        # TODO use time instead of num of records
+        c = (@count += 1)
+        if c < 512
+          if Math.log(c) / Math.log(2) % 1.0 == 0
+            $log.warn "no patterns matched", :tag=>tag
+            return
+          end
+        else
+          if c % 512 == 0
+            $log.warn "no patterns matched", :tag=>tag
+            return
+          end
         end
+        $log.on_trace { $log.trace "no patterns matched", :tag=>tag }
       end
-      $log.on_trace { $log.trace "no patterns matched", :tag=>tag }
-    end
 
-    def start
-    end
+      def start
+      end
 
-    def shutdown
-    end
+      def shutdown
+      end
 
-    def match(tag)
-      false
+      def match(tag)
+        false
+      end
     end
   end
-end
 
-Engine = EngineClass.new
+  Engine = EngineClass.new
 
 
-module Test
-  @@test = false
+  module Test
+    @@test = false
 
-  def test?
-    @@test
-  end
+    def test?
+      @@test
+    end
 
-  def self.setup
-    @@test = true
+    def self.setup
+      @@test = true
 
-    Fluent.__send__(:remove_const, :Engine)
-    engine = Fluent.const_set(:Engine, EngineClass.new).init
+      Fluent.__send__(:remove_const, :Engine)
+      engine = Fluent.const_set(:Engine, EngineClass.new).init
 
-    engine.define_singleton_method(:now=) {|n|
-      @now = n.to_i
-    }
-    engine.define_singleton_method(:now) {
-      @now || super()
-    }
+      engine.define_singleton_method(:now=) {|n|
+        @now = n.to_i
+      }
+      engine.define_singleton_method(:now) {
+        @now || super()
+      }
 
-    nil
+      nil
+    end
   end
 end
 
-end
-