fluentd 0.10.36 → 0.10.37

data/lib/fluent/plugin.rb

@@ -16,149 +16,145 @@
 #    limitations under the License.
 #
 module Fluent
+  class PluginClass
+    def initialize
+      @input = {}
+      @output = {}
+      @buffer = {}
+    end
 
+    def register_input(type, klass)
+      register_impl('input', @input, type, klass)
+    end
 
-class PluginClass
-  def initialize
-    @input = {}
-    @output = {}
-    @buffer = {}
-  end
-
-  def register_input(type, klass)
-    register_impl('input', @input, type, klass)
-  end
-
-  def register_output(type, klass)
-    register_impl('output', @output, type, klass)
-  end
-
-  def register_buffer(type, klass)
-    register_impl('buffer', @buffer, type, klass)
-  end
-
-  def new_input(type)
-    new_impl('input', @input, type)
-  end
-
-  def new_output(type)
-    new_impl('output', @output, type)
-  end
+    def register_output(type, klass)
+      register_impl('output', @output, type, klass)
+    end
 
-  def new_buffer(type)
-    new_impl('buffer', @buffer, type)
-  end
+    def register_buffer(type, klass)
+      register_impl('buffer', @buffer, type, klass)
+    end
 
-  def load_plugins
-    dir = File.join(File.dirname(__FILE__), "plugin")
-    load_plugin_dir(dir)
-    load_gem_plugins
-  end
+    def new_input(type)
+      new_impl('input', @input, type)
+    end
 
-  def load_plugin_dir(dir)
-    dir = File.expand_path(dir)
-    Dir.entries(dir).sort.each {|fname|
-      if fname =~ /\.rb$/
-        require File.join(dir, fname)
-      end
-    }
-    nil
-  end
+    def new_output(type)
+      new_impl('output', @output, type)
+    end
 
-  def load_plugin(type, name)
-    try_load_plugin(name, type)
-  end
+    def new_buffer(type)
+      new_impl('buffer', @buffer, type)
+    end
 
-  private
-  def load_gem_plugins
-    return unless defined? Gem
-    plugins = Gem.find_files('fluent_plugin')
-
-    plugins.each {|plugin|
-      begin
-        load plugin
-      rescue ::Exception => e
-        msg = "#{plugin.inspect}: #{e.message} (#{e.class})"
-        $log.warn "Error loading Fluent plugin #{msg}"
-      end
-    }
-  end
+    def load_plugins
+      dir = File.join(File.dirname(__FILE__), "plugin")
+      load_plugin_dir(dir)
+      load_gem_plugins
+    end
 
-  def register_impl(name, map, type, klass)
-    map[type] = klass
-    $log.trace { "registered #{name} plugin '#{type}'" }
-    nil
-  end
+    def load_plugin_dir(dir)
+      dir = File.expand_path(dir)
+      Dir.entries(dir).sort.each {|fname|
+        if fname =~ /\.rb$/
+          require File.join(dir, fname)
+        end
+      }
+      nil
+    end
 
-  def new_impl(name, map, type)
-    if klass = map[type]
-      return klass.new
+    def load_plugin(type, name)
+      try_load_plugin(name, type)
     end
-    try_load_plugin(name, type)
-    if klass = map[type]
-      return klass.new
+
+    private
+    def load_gem_plugins
+      return unless defined? Gem
+      plugins = Gem.find_files('fluent_plugin')
+
+      plugins.each {|plugin|
+        begin
+          load plugin
+        rescue ::Exception => e
+          msg = "#{plugin.inspect}: #{e.message} (#{e.class})"
+          $log.warn "Error loading Fluent plugin #{msg}"
+        end
+      }
     end
-    raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"
-  end
 
-  def try_load_plugin(name, type)
-    case name
-    when 'input'
-      path = "fluent/plugin/in_#{type}"
-    when 'output'
-      path = "fluent/plugin/out_#{type}"
-    when 'buffer'
-      path = "fluent/plugin/buf_#{type}"
-    else
-      return
+    def register_impl(name, map, type, klass)
+      map[type] = klass
+      $log.trace { "registered #{name} plugin '#{type}'" }
+      nil
     end
 
-    # prefer LOAD_PATH than gems
-    files = $LOAD_PATH.map {|lp|
-      lpath = File.join(lp, "#{path}.rb")
-      File.exist?(lpath) ? lpath : nil
-    }.compact
-    unless files.empty?
-      # prefer newer version
-      require File.expand_path(files.sort.last)
-      return
+    def new_impl(name, map, type)
+      if klass = map[type]
+        return klass.new
+      end
+      try_load_plugin(name, type)
+      if klass = map[type]
+        return klass.new
+      end
+      raise ConfigError, "Unknown #{name} plugin '#{type}'. Run 'gem search -rd fluent-plugin' to find plugins"
     end
 
-    # search gems
-    if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
-      specs = Gem::Specification.find_all {|spec|
-        spec.contains_requirable_file? path
-      }
+    def try_load_plugin(name, type)
+      case name
+      when 'input'
+        path = "fluent/plugin/in_#{type}"
+      when 'output'
+        path = "fluent/plugin/out_#{type}"
+      when 'buffer'
+        path = "fluent/plugin/buf_#{type}"
+      else
+        return
+      end
 
-      # prefer newer version
-      specs = specs.sort_by {|spec| spec.version }
-      if spec = specs.last
-        spec.require_paths.each {|lib|
-          file = "#{spec.full_gem_path}/#{lib}/#{path}"
-          require file
-        }
+      # prefer LOAD_PATH than gems
+      files = $LOAD_PATH.map {|lp|
+        lpath = File.join(lp, "#{path}.rb")
+        File.exist?(lpath) ? lpath : nil
+      }.compact
+      unless files.empty?
+        # prefer newer version
+        require File.expand_path(files.sort.last)
+        return
       end
 
-    # backward compatibility for rubygems < 1.8
-    elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
-      #files = Gem.find_files(path).sort
-      specs = Gem.searcher.find_all(path)
-
-      # prefer newer version
-      specs = specs.sort_by {|spec| spec.version }
-      specs.reverse_each {|spec|
-        files = Gem.searcher.matching_files(spec, path)
-        unless files.empty?
-          require files.first
-          break
+      # search gems
+      if defined?(::Gem::Specification) && ::Gem::Specification.respond_to?(:find_all)
+        specs = Gem::Specification.find_all {|spec|
+          spec.contains_requirable_file? path
+        }
+
+        # prefer newer version
+        specs = specs.sort_by {|spec| spec.version }
+        if spec = specs.last
+          spec.require_paths.each {|lib|
+            file = "#{spec.full_gem_path}/#{lib}/#{path}"
+            require file
+          }
         end
-      }
+
+        # backward compatibility for rubygems < 1.8
+      elsif defined?(::Gem) && ::Gem.respond_to?(:searcher)
+        #files = Gem.find_files(path).sort
+        specs = Gem.searcher.find_all(path)
+
+        # prefer newer version
+        specs = specs.sort_by {|spec| spec.version }
+        specs.reverse_each {|spec|
+          files = Gem.searcher.matching_files(spec, path)
+          unless files.empty?
+            require files.first
+            break
+          end
+        }
+      end
     end
   end
-end
-
-Plugin = PluginClass.new
-
 
+  Plugin = PluginClass.new
 end
 

data/lib/fluent/plugin/buf_file.rb

@@ -75,6 +75,8 @@ module Fluent
   class FileBuffer < BasicBuffer
     Plugin.register_buffer('file', self)
 
+    @@buffer_paths = {}
+
     def initialize
       require 'uri'
       super
@@ -85,6 +87,12 @@ module Fluent
     def configure(conf)
       super
 
+      if @@buffer_paths.has_key?(@buffer_path)
+        raise ConfigError, "Other '#{@@buffer_paths[@buffer_path]}' plugin already use same buffer_path: type = #{conf['type']}, buffer_path = #{@buffer_path}"
+      else
+        @@buffer_paths[@buffer_path] = conf['type']
+      end
+
       if pos = @buffer_path.index('*')
         @buffer_path_prefix = @buffer_path[0,pos]
         @buffer_path_suffix = @buffer_path[pos+1..-1]

data/lib/fluent/plugin/exec_util.rb

@@ -0,0 +1,49 @@
+module Fluent
+  module ExecUtil
+    class Parser
+      def initialize(on_message)
+        @on_message = on_message
+      end
+    end
+
+    class TSVParser < Parser
+      def initialize(keys, on_message)
+        @keys = keys
+        super(on_message)
+      end
+
+      def call(io)
+        io.each_line(&method(:each_line))
+      end
+
+      def each_line(line)
+        line.chomp!
+        vals = line.split("\t")
+
+        record = Hash[@keys.zip(vals)]
+
+        @on_message.call(record)
+      end
+    end
+
+    class JSONParser < Parser
+      def call(io)
+        y = Yajl::Parser.new
+        y.on_parse_complete = @on_message
+        y.parse(io)
+      end
+    end
+
+    class MessagePackParser < Parser
+      def call(io)
+        @u = MessagePack::Unpacker.new(io)
+        begin
+          @u.each(&@on_message)
+        rescue EOFError
+        end
+      end
+    end
+  end
+end
+
+

data/lib/fluent/plugin/in_exec.rb

@@ -21,10 +21,24 @@ module Fluent
 
     def initialize
       super
+      require 'fluent/plugin/exec_util'
     end
 
+    SUPPORTED_FORMAT = {
+      'tsv' => :tsv,
+      'json' => :json,
+      'msgpack' => :msgpack,
+    }
+
     config_param :command, :string
-    config_param :keys, :string
+    config_param :format, :default => :tsv do |val|
+      f = SUPPORTED_FORMAT[val]
+      raise ConfigError, "Unsupported format '#{val}'" unless f
+      f
+    end
+    config_param :keys, :default => [] do |val|
+      val.split(',')
+    end
     config_param :tag, :string, :default => nil
     config_param :tag_key, :string, :default => nil
     config_param :time_key, :string, :default => nil
@@ -44,8 +58,6 @@ module Fluent
         raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
       end
 
-      @keys = @keys.split(',')
-
       if @time_key
         if @time_format
           f = @time_format
@@ -54,6 +66,18 @@ module Fluent
           @time_parse_proc = Proc.new {|str| str.to_i }
         end
       end
+
+      case @format
+      when :tsv
+        if @keys.empty?
+          raise ConfigError, "keys option is required on exec input for tsv format"
+        end
+        @parser = ExecUtil::TSVParser.new(@keys, method(:on_message))
+      when :json
+        @parser = ExecUtil::JSONParser.new(method(:on_message))
+      when :msgpack
+        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
+      end
     end
 
     def start
@@ -82,47 +106,37 @@ module Fluent
     end
 
     def run
-      @io.each_line(&method(:each_line))
+      @parser.call(@io)
     end
 
     def run_periodic
       until @finished
         sleep @run_interval
         io = IO.popen(@command, "r")
-        io.each_line(&method(:each_line))
+        @parser.call(io)
         Process.waitpid(io.pid)
       end
     end
 
     private
-    def each_line(line)
-      begin
-        line.chomp!
-        vals = line.split("\t")
 
-        tag = @tag
-        time = nil
-        record = {}
-        for i in 0..@keys.length-1
-          key = @keys[i]
-          val = vals[i]
-          if key == @time_key
-            time = @time_parse_proc.call(val)
-          elsif key == @tag_key
-            tag = val
-          else
-            record[key] = val
-          end
-        end
+    def on_message(record)
+      if val = record.delete(@time_key)
+        time = @time_parse_proc.call(val)
+      else
+        time = Engine.now
+      end
 
-        if tag
-          time ||= Engine.now
-          Engine.emit(tag, time, record)
-        end
-      rescue
-        $log.error "exec failed to emit", :error=>$!.to_s, :line=>line
-        $log.warn_backtrace $!.backtrace
+      if val = record.delete(@tag_key)
+        tag = val
+      else
+        tag = @tag
       end
+
+      Engine.emit(tag, time, record)
+    rescue
+      $log.error "exec failed to emit", :error => $!.to_s, :error_class => $!.class.to_s, :record => Yajl.dump(record)
+      $log.warn_backtrace $!.backtrace
     end
   end
 end