fluentd 0.10.0

data/lib/fluent/plugin/in_syslog.rb

@@ -0,0 +1,174 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class SyslogInput < Input
+  Plugin.register_input('syslog', self)
+
+  SYSLOG_REGEXP = /^\<([0-9]+)\>(.*)/
+  SYSLOG_ALL_REGEXP = /^\<(?<pri>[0-9]+)\>(?<time>[^ ]* [^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/
+  TIME_FORMAT = "%b %d %H:%M:%S"
+
+  FACILITY_MAP = {
+    0   => 'kern',
+    1   => 'user',
+    2   => 'mail',
+    3   => 'daemon',
+    4   => 'auth',
+    5   => 'syslog',
+    6   => 'lpr',
+    7   => 'news',
+    8   => 'uucp',
+    9   => 'cron',
+    10  => 'authpriv',
+    11  => 'ftp',
+    12  => 'ntp',
+    13  => 'audit',
+    14  => 'alert',
+    15  => 'at',
+    16  => 'local0',
+    17  => 'local1',
+    18  => 'local2',
+    19  => 'local3',
+    20  => 'local4',
+    21  => 'local5',
+    22  => 'local6',
+    23  => 'local7'
+  }
+
+  PRIORITY_MAP = {
+    0  => 'emerg',
+    1  => 'alert',
+    2  => 'crit',
+    3  => 'err',
+    4  => 'warn',
+    5  => 'notice',
+    6  => 'info',
+    7  => 'debug'
+  }
+
+  def initialize
+    super
+  end
+
+  config_param :port, :integer, :default => 5140
+  config_param :bind, :string, :default => '0.0.0.0'
+  config_param :tag, :string
+
+  def configure(conf)
+    super
+
+    parser = TextParser.new
+    if parser.configure(conf, false)
+      parser.use_template('syslog')
+      @parser = parser
+    else
+      @parser = nil
+    end
+  end
+
+  def start
+    if @parser
+      callback = method(:receive_data_parser)
+    else
+      callback = method(:receive_data)
+    end
+    EventMachine.open_datagram_socket(@bind, @port, UdpHandler, callback)
+  end
+
+  def receive_data_parser(data)
+    m = SYSLOG_REGEXP.match(data)
+    unless m
+      $log.debug "invalid syslog message: #{data.dump}"
+      return
+    end
+    pri = m[1].to_i
+    text = m[2]
+
+    time, record = @parser.parse(text)
+    unless time && record
+      return
+    end
+
+    emit(pri, time, record)
+
+  rescue
+    $log.warn data.dump, :error=>$!.to_s
+    $log.debug_backtrace
+  end
+
+  def receive_data(data)
+    m = SYSLOG_ALL_REGEXP.match(data)
+    unless m
+      $log.debug "invalid syslog message", :data=>data
+      return
+    end
+
+    pri = nil
+    time = nil
+    record = {}
+
+    m.names.each {|name|
+      if value = m[name]
+        case name
+        when "pri"
+          pri = value.to_i
+        when "time"
+          time = Time.strptime(value, TIME_FORMAT).to_i
+        else
+          record[name] = value
+        end
+      end
+    }
+
+    time ||= Engine.now
+
+    emit(pri, time, record)
+
+  rescue
+    $log.warn data.dump, :error=>$!.to_s
+    $log.debug_backtrace
+  end
+
+  private
+  def emit(pri, time, record)
+    facility = FACILITY_MAP[pri >> 3]
+    priority = PRIORITY_MAP[pri & 0b111]
+
+    tag = "#{@tag}.#{facility}.#{priority}"
+
+    Engine.emit(tag, time, record)
+  end
+
+  class UdpHandler < EventMachine::Connection
+    def initialize(callback)
+      super()
+      @callback = callback
+    end
+
+    def receive_data(data)
+      EventMachine.defer {
+        @callback.call(data)
+      }
+    end
+  end
+end
+
+
+end

data/lib/fluent/plugin/in_tail.rb

@@ -0,0 +1,150 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class TailInput < Input
+  Plugin.register_input('tail', self)
+
+  def initialize
+    super
+    @paths = []
+  end
+
+  config_param :path, :string
+  config_param :tag, :string
+
+  def configure(conf)
+    super
+
+    @paths = @path.split(',').map {|path| path.strip }
+    if @paths.empty?
+      raise ConfigError, "tail: 'path' parameter is required on tail input"
+    end
+
+    configure_parser(conf)
+  end
+
+  def configure_parser(conf)
+    @parser = TextParser.new
+    @parser.configure(conf)
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+    @paths.each {|path|
+      $log.debug "following tail of #{path}"
+      @loop.attach Handler.new(path, method(:receive_lines))
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @thread.join
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  def receive_lines(lines)
+    es = MultiEventStream.new
+    lines.each {|line|
+      begin
+        line.rstrip!  # remove \n
+        time, record = parse_line(line)
+        if time && record
+          es.add(time, record)
+        end
+      rescue
+        $log.warn line.dump, :error=>$!.to_s
+        $log.debug_backtrace
+      end
+    }
+
+    unless es.empty?
+      Engine.emit_stream(@tag, es)
+    end
+  end
+
+  def parse_line(line)
+    return @parser.parse(line)
+  end
+
+  # seek to the end of file first.
+  # logs never duplicate but may be lost if fluent is down.
+  class Handler < Coolio::StatWatcher
+    def initialize(path, callback)
+      @pos = File.stat(path).size
+      @buffer = ''
+      @callback = callback
+      super(path)
+    end
+
+    def on_change
+      lines = []
+
+      File.open(path) {|f|
+        if f.lstat.size < @pos
+          # moved or deleted
+          @pos = 0
+        else
+          f.seek(@pos)
+        end
+
+        line = f.gets
+        unless line
+          return
+        end
+
+        @buffer << line
+        unless line[line.length-1] == ?\n
+          @pos = f.pos
+          return
+        end
+
+        lines << @buffer
+        @buffer = ''
+
+        while line = f.gets
+          unless line[line.length-1] == ?\n
+            @buffer = line
+            break
+          end
+          lines << line
+        end
+
+        @pos = f.pos
+      }
+
+      @callback.call(lines)
+
+    rescue Errno::ENOENT
+      # moved or deleted
+      @pos = 0
+    end
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/out_copy.rb

@@ -0,0 +1,72 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class CopyOutput < MultiOutput
+  Plugin.register_output('copy', self)
+
+  def initialize
+    @outputs = []
+  end
+
+  attr_reader :outputs
+
+  def configure(conf)
+    conf.elements.select {|e|
+      e.name == 'store'
+    }.each {|e|
+      type = e['type']
+      unless type
+        raise ConfigError, "Missing 'type' parameter on <store> directive"
+      end
+      $log.debug "adding store type=#{type.dump}"
+
+      output = Plugin.new_output(type)
+      output.configure(e)
+      @outputs << output
+    }
+  end
+
+  def start
+    @outputs.each {|o|
+      o.start
+    }
+  end
+
+  def shutdown
+    @outputs.each {|o|
+      o.shutdown
+    }
+  end
+
+  def emit(tag, es, chain)
+    unless es.repeatable?
+      es = MultiEventStream.new
+      array = es.map {|time,record|
+        es.add(time, record)
+      }
+    end
+    chain = OutputChain.new(@outputs, tag, es, chain)
+    chain.next
+  end
+end
+
+
+end
+

data/lib/fluent/plugin/out_file.rb

@@ -0,0 +1,111 @@
+#
+# Fluent
+#
+# Copyright (C) 2011 FURUHASHI Sadayuki
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+#
+module Fluent
+
+
+class FileOutput < TimeSlicedOutput
+  Plugin.register_output('file', self)
+
+  SUPPORTED_COMPRESS = {
+    :gz => :gz,
+    :gzip => :gz,
+  }
+
+  config_param :path, :string
+
+  config_param :time_format, :string, :default => nil
+
+  config_param :compress, :default => nil do |val|
+    c = SUPPORTED_COMPRESS[val.to_sym]
+    unless c
+      raise ConfigError, "Unsupported compression algorithm '#{compress}'"
+    end
+    c
+  end
+
+  def initialize
+    require 'zlib'
+    require 'time'
+    super
+  end
+
+  def configure(conf)
+    if path = conf['path']
+      @path = path
+    end
+    unless @path
+      raise ConfigError, "'path' parameter is required on file output"
+    end
+
+    if pos = @path.index('*')
+      @path_prefix = @path[0,pos]
+      @path_suffix = @path[pos+1..-1]
+      conf['buffer_path'] ||= "#{@path}"
+    else
+      @path_prefix = @path+"."
+      @path_suffix = ".log"
+      conf['buffer_path'] ||= "#{@path}.*"
+    end
+
+    super
+
+    @timef = TimeFormatter.new(@time_format, @localtime)
+  end
+
+  def format(tag, time, record)
+    time_str = @timef.format(time)
+    "#{time_str}\t#{tag}\t#{record.to_json}\n"
+  end
+
+  def write(chunk)
+    case @compress
+    when nil
+      suffix = ''
+    when :gz
+      suffix = ".gz"
+    end
+
+    i = 0
+    begin
+      path = "#{@path_prefix}#{chunk.key}_#{i}#{@path_suffix}#{suffix}"
+      i += 1
+    end while File.exist?(path)
+    FileUtils.mkdir_p File.dirname(path)
+
+    case @compress
+    when nil
+      File.open(path, "a") {|f|
+        chunk.write_to(f)
+      }
+    when :gz
+      Zlib::GzipWriter.open(path) {|f|
+        chunk.write_to(f)
+      }
+    end
+
+    return path  # for test
+  end
+
+  def secondary_init(primary)
+    # don't warn even if primary.class is not FileOutput
+  end
+end
+
+
+end
+