fluentd 0.10.4 → 0.10.5

data/ChangeLog

@@ -1,4 +1,11 @@
 
+Release 0.10.5 - 2011/11/08
+
+* Added out_forward, in_forward
+* Added out_exec, in_exec
+* Added out_exec_filter
+
+
 Release 0.10.4 - 2011/11/07
 
 * TestDriver uses klass.dup.module_eval instead of inheriting class to

data/VERSION

@@ -1 +1 @@
-0.10.4
+0.10.5

data/fluent.conf

@@ -2,7 +2,7 @@
 ## built-in TCP input
 ## $ echo <json> | fluent-cat <tag>
 <source>
-  type tcp
+  type forward
 </source>
 
 ## built-in UNIX socket input
@@ -40,7 +40,7 @@
 
 ## match tag=system.** and forward to another fluent server
 #<match system.**>
-#  type tcp
+#  type forward
 #  host 192.168.0.11
 #  <secondary>
 #    host 192.168.0.12
@@ -51,7 +51,7 @@
 #<match myapp.**>
 #  type copy
 #  <store>
-#    type tcp
+#    type forward
 #    host 192.168.0.13
 #    buffer_type file
 #    buffer_path /var/log/fluent/myapp-forward

data/lib/fluent/plugin/in_exec.rb

@@ -0,0 +1,132 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class ExecInput < Input
+  Plugin.register_input('exec', self)
+
+  def initialize
+    super
+  end
+
+  config_param :command, :string
+  config_param :keys, :string
+  config_param :tag, :string, :default => nil
+  config_param :tag_key, :string, :default => nil
+  config_param :time_key, :string, :default => nil
+  config_param :time_format, :string, :default => nil
+  config_param :run_interval, :time, :default => nil
+
+  def configure(conf)
+    super
+
+    if localtime = conf['localtime']
+      @localtime = true
+    elsif utc = conf['utc']
+      @localtime = false
+    end
+
+    if !@tag && !@tag_key
+      raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
+    end
+
+    @keys = @keys.split(',')
+
+    if @time_key
+      if @time_format
+        f = @time_format
+        @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
+      else
+        @time_parse_proc = Proc.new {|str| str.to_i }
+      end
+    end
+  end
+
+  def start
+    if @run_interval
+      @finished = false
+      @thread = Thread.new(&method(:run_periodic))
+    else
+      @io = IO.popen(@command, "r")
+      @pid = @io.pid
+      @thread = Thread.new(&method(:run))
+    end
+  end
+
+  def shutdown
+    if @run_interval
+      @finished = true
+      @thread.join
+    else
+      Process.kill(:TERM, @pid)
+      if @thread.join(60)  # TODO wait time
+        return
+      end
+      Process.kill(:KILL, @pid)
+      @thread.join
+    end
+  end
+
+  def run
+    @io.each_line(&method(:each_line))
+  end
+
+  def run_periodic
+    until @finished
+      sleep @run_interval
+      io = IO.popen(@command, "r")
+      io.each_line(&method(:each_line))
+      Process.waitpid(io.pid)
+    end
+  end
+
+  private
+  def each_line(line)
+    begin
+      line.chomp!
+      vals = line.split("\t")
+
+      tag = @tag
+      time = nil
+      record = {}
+      for i in 0..@keys.length-1
+        key = @keys[i]
+        val = vals[i]
+        if key == @time_key
+          time = @time_parse_proc.call(val)
+        elsif key == @tag_key
+          tag = val
+        else
+          record[key] = val
+        end
+      end
+
+      if tag
+        time ||= Engine.now
+        Engine.emit(tag, time, record)
+      end
+    rescue
+      $log.error "exec failed to emit", :error=>$!, :line=>line
+      $log.warn_backtrace $!.backtrace
+    end
+  end
+end
+
+
+end

data/lib/fluent/plugin/in_forward.rb

@@ -0,0 +1,199 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class ForwardInput < Input
+  Plugin.register_input('forward', self)
+
+  def initialize
+    require 'socket'
+    require 'yajl'
+    super
+  end
+
+  config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+  config_param :bind, :string, :default => '0.0.0.0'
+
+  def configure(conf)
+    super
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+
+    @lsock = listen
+    @loop.attach(@lsock)
+
+    @usock = UDPSocket.new
+    @usock.bind(@bind, @port)
+    @hbr = HeartbeatRequestHandler.new(@usock, method(:on_heartbeat_request))
+    @loop.attach(@hbr)
+
+    @thread = Thread.new(&method(:run))
+    @cached_unpacker = MessagePack::Unpacker.new
+  end
+
+  def shutdown
+    @lsock.close
+    @loop.stop
+    @thread.join
+    @usock.close
+  end
+
+  def listen
+    $log.info "listening fluent socket on #{@bind}:#{@port}"
+    Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
+  end
+
+  #config_param :path, :string, :default => DEFAULT_SOCKET_PATH
+  #def listen
+  #  if File.exist?(@path)
+  #    File.unlink(@path)
+  #  end
+  #  FileUtils.mkdir_p File.dirname(@path)
+  #  $log.debug "listening fluent socket on #{@path}"
+  #  Coolio::UNIXServer.new(@path, Handler, method(:on_message))
+  #end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  protected
+  # message Entry {
+  #   1: long time
+  #   2: object record
+  # }
+  #
+  # message Forward {
+  #   1: string tag
+  #   2: list<Entry> entries
+  # }
+  #
+  # message PackedForward {
+  #   1: string tag
+  #   2: raw entries  # msgpack stream of Entry
+  # }
+  #
+  # message Message {
+  #   1: string tag
+  #   2: long? time
+  #   3: object record
+  # }
+  def on_message(msg)
+    # TODO format error
+    tag = msg[0].to_s
+    entries = msg[1]
+
+    if entries.class == String
+      # PackedForward
+      es = MessagePackEventStream.new(entries, @cached_unpacker)
+      Engine.emit_stream(tag, es)
+
+    elsif entries.class == Array
+      # Forward
+      es = MultiEventStream.new
+      entries.each {|e|
+        time = e[0].to_i
+        time = (now ||= Engine.now) if time == 0
+        record = e[1]
+        es.add(time, record)
+      }
+      Engine.emit_stream(tag, es)
+
+    else
+      # Message
+      time = msg[1]
+      time = Engine.now if time == 0
+      record = msg[2]
+      Engine.emit(tag, time, record)
+    end
+  end
+
+  class Handler < Coolio::Socket
+    def initialize(io, on_message)
+      super(io)
+      if io.is_a?(TCPSocket)
+        opt = [1, @timeout.to_i].pack('I!I!')  # { int l_onoff; int l_linger; }
+        io.setsockopt(Socket::SOL_SOCKET, Socket::SO_LINGER, opt)
+      end
+      $log.trace { "accepted fluent socket object_id=#{self.object_id}" }
+      @on_message = on_message
+    end
+
+    def on_connect
+    end
+
+    def on_read(data)
+      first = data[0]
+      if first == '{' || first == '['
+        m = method(:on_read_json)
+        @y = Yajl::Parser.new
+        @y.on_parse_complete = @on_message
+      else
+        m = method(:on_read_msgpack)
+        @u = MessagePack::Unpacker.new
+      end
+
+      (class<<self;self;end).module_eval do
+        define_method(:on_read, m)
+      end
+      m.call(data)
+    end
+
+    def on_read_json(data)
+      @y << data
+    end
+
+    def on_read_msgpack(data)
+      @u.feed_each(data, &@on_message)
+    end
+
+    def on_close
+      $log.trace { "closed fluent socket object_id=#{self.object_id}" }
+    end
+  end
+
+  class HeartbeatRequestHandler < Coolio::IO
+    def initialize(io, callback)
+      super(io)
+      @io = io
+      @callback = callback
+    end
+
+    def on_readable
+      msg, addr = @io.recvfrom(1024)
+      host = addr[3]
+      port = addr[1]
+      @callback.call(host, port, msg)
+    end
+  end
+
+  def on_heartbeat_request(host, port, msg)
+    $log.trace "heartbeat request from #{host}:#{port}"
+    @usock.send "", 0, host, port
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/in_stream.rb

@@ -18,6 +18,7 @@
 module Fluent
 
 
+# obsolete
 class StreamInput < Input
   def initialize
     require 'socket'
@@ -146,23 +147,35 @@ class StreamInput < Input
 end
 
 
-class TcpInput < StreamInput
+# obsolete
+# ForwardInput is backward compatible with TcpInput
+#class TcpInput < StreamInput
+#  Plugin.register_input('tcp', self)
+#
+#  config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
+#  config_param :bind, :string, :default => '0.0.0.0'
+#
+#  def configure(conf)
+#    super
+#  end
+#
+#  def listen
+#    $log.debug "listening fluent socket on #{@bind}:#{@port}"
+#    Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
+#  end
+#end
+class TcpInput < ForwardInput
   Plugin.register_input('tcp', self)
 
-  config_param :port, :integer, :default => DEFAULT_LISTEN_PORT
-  config_param :bind, :string, :default => '0.0.0.0'
-
-  def configure(conf)
+  def initialize
     super
-  end
-
-  def listen
-    $log.debug "listening fluent socket on #{@bind}:#{@port}"
-    Coolio::TCPServer.new(@bind, @port, Handler, method(:on_message))
+    ## TODO
+    #$log.warn "'tcp' input is obsoleted and will be removed. Use 'forward' instead."
   end
 end
 
 
+# obsolete
 class UnixInput < StreamInput
   Plugin.register_input('unix', self)
 
@@ -170,6 +183,8 @@ class UnixInput < StreamInput
 
   def configure(conf)
     super
+    ## TODO
+    #$log.warn "'unix' input is obsoleted and will be removed. Use 'forward' instead."
   end
 
   def listen

data/lib/fluent/plugin/out_exec.rb

@@ -0,0 +1,91 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class ExecOutput < TimeSlicedOutput
+  Plugin.register_output('exec', self)
+
+  def initialize
+    super
+    require 'tempfile'
+    @localtime = false
+  end
+
+  config_param :command, :string
+  config_param :keys, :string
+  config_param :tag_key, :string, :default => nil
+  config_param :time_key, :string, :default => nil
+  config_param :time_format, :string, :default => nil
+
+  def configure(conf)
+    super
+
+    @keys = @keys.split(',')
+
+    if @time_key
+      if @time_format
+        tf = TimeFormatter.new(@time_format, @localtime)
+        @time_format_proc = tf.method(:format)
+      else
+        @time_format_proc = Proc.new {|time| time.to_s }
+      end
+    end
+  end
+
+  def format(tag, time, record)
+    out = ''
+    last = @keys.length-1
+    for i in 0..last
+      key = @keys[i]
+      if key == @time_key
+        out << @time_format_proc.call(time)
+      elsif key == @tag_key
+        out << tag
+      else
+        out << record[key].to_s
+      end
+      out << "\t" if i != last
+    end
+    out << "\n"
+    out
+  end
+
+  def write(chunk)
+    if chunk.respond_to?(:path)
+      prog = "#{@command} #{chunk.path}"
+    else
+      tmpfile = Tempfile.new("fluent-plugin-exec-")
+      chunk.write_to(tmpfile)
+      tmpfile.close
+      prog = "#{@command} #{tmpfile.path}"
+    end
+
+    system(prog)
+    ecode = $?.to_i
+    tmpfile.delete if tmpfile
+
+    if ecode != 0
+      raise "command returns #{ecode}: #{prog}"
+    end
+  end
+end
+
+
+end
+